diff --git a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/resolve_ci_config.py b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/resolve_ci_config.py index 4dba6c129615..1fe7d2559f79 100644 --- a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/resolve_ci_config.py +++ b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/resolve_ci_config.py @@ -32,9 +32,9 @@ def load_ci_configs( sys.path.insert(0, str(config_path)) try: - from ci_config_api import load_config_v1 + from ci_config_api import load_config - config = load_config_v1(config_path) + config = load_config(version=2, config_path=config_path) all_families = config.get_gpu_families(TRIGGER_TYPES) gpu_configs = { family: all_families.get(family, {}).get("linux", {}) diff --git a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/test_resolve_ci_config.py b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/test_resolve_ci_config.py index ac4fb1db4808..c2ff8c633c9b 100644 --- a/.github/actions/rocprofiler-sdk-util/resolve-ci-config/test_resolve_ci_config.py +++ b/.github/actions/rocprofiler-sdk-util/resolve-ci-config/test_resolve_ci_config.py @@ -92,7 +92,7 @@ def _write_config(tmpdir: Path, config: dict) -> None: from typing import Any @dataclass - class ConfigV1: + class Config: build_runners: dict gpu_families: dict _raw: dict @@ -104,12 +104,12 @@ def get_gpu_families(self, trigger_types): result[name] = cfg return result - def load_config_v1(config_path=None): + def load_config(version=2, config_path=None): if config_path is None: config_path = Path(__file__).parent with open(config_path / "runner-config.json") as f: raw = json.load(f) - return ConfigV1( + return Config( build_runners=raw["build_runners"], gpu_families=raw["gpu_families"], _raw=raw, diff --git a/.github/scripts/rccl_check_default_behavior.py b/.github/scripts/rccl_check_default_behavior.py new file mode 100644 index 000000000000..ef37dbd3c4e8 --- /dev/null +++ b/.github/scripts/rccl_check_default_behavior.py @@ -0,0 +1,879 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Require changelog and documentation updates for RCCL env-var changes. + +Tracks declarations, exact ``NCCL_`` / ``RCCL_`` string reads, and new +``ncclParam`` / ``rcclParam`` uses under ``projects/rccl/src``. Documentation +must change the entry's prose, not only name the variable. NCCL-only findings +may use either env-var page; findings containing an RCCL name require the RCCL +page. + +``[no-behavior-change]`` overrides the check. ``[internal-env]`` waives only +the documentation update. +""" + +import argparse +import http.client +import json +import os +import re +import subprocess +import sys +import urllib.request +from dataclasses import dataclass +from dataclasses import replace +from pathlib import Path +from typing import Literal + +SRC_TREE = "projects/rccl/src/" +CHANGELOG = "projects/rccl/CHANGELOG.md" +NCCL_ENV_DOC = "projects/rccl/docs/userguide/source/env.rst" +RCCL_ENV_DOC = "projects/rccl/docs/api-reference/env-variables.rst" +ENV_DOCS = (NCCL_ENV_DOC, RCCL_ENV_DOC) +COMMENT_MARKER = "" +COMMENT_AUTHOR = "github-actions[bot]" +PASS_MESSAGE = "No undocumented RCCL or NCCL environment-variable changes." +SKIP_TOKEN = "[no-behavior-change]" +INTERNAL_TOKEN = "[internal-env]" +SKIPPED_MESSAGE = f"{SKIP_TOKEN} recorded; the RCCL env-var check was skipped." +UNREADABLE_MESSAGE = ( + "The check cannot parse this declaration, so it cannot tell whether an env var changed. " + f"Use the same argument layout as other declarations of this macro, or put {SKIP_TOKEN} " + "in the PR title or body." +) +_MAX_CONTINUATION = 40 +_PAGE_SIZE = 100 + +_MACROS = ("DEFINE_NCCL_PARAM", "RCCL_PARAM_NCCL_ALIAS", "RCCL_PARAM", "NCCL_PARAM") +_OPENER_RE = re.compile(rf"^(?:{'|'.join(_MACROS)})\s*\(") +_ENV_EDGE = r"[A-Za-z0-9_]" +_FULL_ENV_RE = re.compile(rf"(?:NCCL|RCCL)_[A-Za-z0-9_]+") +_ENV_NAME_RE = re.compile(rf"(? str: + return self.env_names[0] + + +def calls_in(source: str, path: str) -> tuple[ParamCall, ...]: + """Env declarations and quoted ``NCCL_`` / ``RCCL_`` names in ``source``. + + A line that continues a ``/* */`` block with ``* `` is a comment and is skipped. + """ + lines = source.splitlines() + found: list[ParamCall] = [] + index = 0 + while index < len(lines): + stripped = lines[index].strip() + if _OPENER_RE.match(stripped): + calls, index = _call_at(lines, index, path) + found.extend(calls) + continue + if not stripped.startswith("* "): + found.extend(_quoted_envs(lines[index], path, index + 1)) + index += 1 + return tuple(found) + + +def _call_at(lines: list[str], index: int, path: str) -> tuple[list[ParamCall], int]: + """Parse the declaration opening at ``lines[index]``. Return its calls and the next line.""" + start = index + parts = [_strip_comments(lines[index]).strip()] + while _still_open(parts) and index + 1 < len(lines) and len(parts) <= _MAX_CONTINUATION: + index += 1 + parts.append(_strip_comments(lines[index]).strip()) + joined = " ".join(parts) + call = parse_declaration(joined, path, start + 1) + found = [call] if call is not None else [] + found.extend(_quoted_envs(joined, path, start + 1)) + return found, index + 1 + + +def _still_open(parts: list[str]) -> bool: + code = _STRING_RE.sub("", " ".join(parts)) + return code.count("(") > code.count(")") + + +def parse_declaration(code: str, path: str, line_no: int) -> ParamCall | None: + """The declaration in comment-free ``code``, or None when it does not parse. + + ``DEFINE_NCCL_PARAM(name, type, KEY, default, ...)`` names the variable + ``KEY``. The other macros take ``(name, "ENV", default)`` and prefix ``ENV``. + """ + head, _, rest = code.removesuffix(";").rstrip().partition("(") + macro = head.strip() + if not rest.endswith(")"): + return None + args = _split_args(rest[:-1]) + if macro == "DEFINE_NCCL_PARAM": + if len(args) < 4 or _FULL_ENV_RE.fullmatch(args[2]) is None: + return None + names, default = (args[2],), args[3] + elif len(args) == 3 and _STRING_RE.fullmatch(args[1]): + names, default = tuple(prefix + args[1][1:-1] for prefix in _PREFIXES[macro]), args[2] + else: + return None + return ParamCall( + env_names=names, + default=default, + path=path, + line=line_no, + accessor=_accessor_symbol(macro, args[0]), + ) + + +def _accessor_symbol(macro: str, c_name: str) -> str | None: + """Derive the accessor symbol, preserving lowercase C names.""" + if _IDENT_RE.fullmatch(c_name) is None: + return None + if macro == "DEFINE_NCCL_PARAM": + return c_name if _ACCESSOR_RE.fullmatch(c_name) else None + return _ACCESSOR_PREFIX[macro] + c_name + + +def _split_args(text: str) -> list[str]: + """Split macro arguments as the preprocessor does: commas inside strings or parentheses stay put.""" + args: list[str] = [] + buf: list[str] = [] + parens = 0 + quoted = False + for index, char in enumerate(text): + if char == '"' and not _escaped(text, index): + quoted = not quoted + elif not quoted and char == "(": + parens += 1 + elif not quoted and char == ")": + parens -= 1 + elif not quoted and char == "," and parens == 0: + args.append("".join(buf).strip()) + buf = [] + continue + buf.append(char) + if buf or args: + args.append("".join(buf).strip()) + return args + + +def _escaped(text: str, index: int) -> bool: + slashes = 0 + index -= 1 + while index >= 0 and text[index] == "\\": + slashes += 1 + index -= 1 + return slashes % 2 == 1 + + +def _quoted_envs(line: str, path: str, line_no: int) -> list[ParamCall]: + """``NCCL_`` / ``RCCL_`` names inside string literals, which carry no default.""" + if '"' not in line or ("NCCL_" not in line and "RCCL_" not in line): + return [] + found: list[ParamCall] = [] + for literal in _STRING_RE.findall(_strip_comments(line)): + text = literal[1:-1] + mention = _FULL_ENV_RE.fullmatch(text) is None + found.extend( + ParamCall(env_names=(name,), default=None, path=path, line=line_no, mention=mention) + for name in _FULL_ENV_RE.findall(text) + ) + return found + + +def _strip_comments(line: str) -> str: + """Drop ``//`` and ``/* */`` comments, leaving string literals intact.""" + out: list[str] = [] + quoted = False + index = 0 + while index < len(line): + char = line[index] + if char == '"' and not _escaped(line, index): + quoted = not quoted + out.append(char) + elif not quoted and line[index : index + 2] == "//": + break + elif not quoted and line[index : index + 2] == "/*": + end = line.find("*/", index + 2) + if end == -1: + break + index = end + 1 + else: + out.append(char) + index += 1 + return "".join(out) + + +def findings_between(before: dict[str, str], after: dict[str, str]) -> list[Finding]: + """Env names added, removed, or given a new default between two sets of files.""" + old = _by_name(before) + new = _by_name(after) + names = sorted(old.keys() | new.keys()) + found = (_pair_name(name, old.get(name, []), new.get(name, [])) for name in names) + return [item for item in found if item is not None] + + +def _by_name(sources: dict[str, str]) -> dict[str, list[ParamCall]]: + """Calls grouped under every name they carry, so adding or dropping an alias pairs on its own.""" + grouped: dict[str, list[ParamCall]] = {} + for path in sorted(sources): + for call in calls_in(sources[path], path): + for name in call.env_names: + grouped.setdefault(name, []).append(call) + return grouped + + +def _pair_name(name: str, old: list[ParamCall], new: list[ParamCall]) -> Finding | None: + read_before = any(not call.mention for call in old) + read_after = any(not call.mention for call in new) + old_values = _values(old) + new_values = _values(new) + kind: Kind + if not read_before and not read_after: + return None + if not read_before: + kind, changed = "added", new + elif not read_after: + kind, changed = "removed", old + elif old_values != new_values: + kind, changed = "default", [call for call in new if call.default is not None] or new + else: + return None + aliases = (alias for call in (*old, *new) for alias in call.env_names) + return Finding( + kind=kind, + env_names=tuple(dict.fromkeys([name, *aliases])), + old_default=old_values, + new_default=new_values, + places=tuple(sorted({Place(call.path, call.line) for call in changed})), + ) + + +def reference_findings( + before: dict[str, str], + after: dict[str, str], + accessors: dict[str, tuple[str, ...]], +) -> list[Finding]: + """Find declared accessors gained by more changed files than lose them.""" + gained: dict[str, list[Place]] = {} + lost: dict[str, int] = {} + for path in sorted(set(before) | set(after)): + old = {symbol for symbol, _line in _accessors_in(before.get(path, ""))} + by_symbol: dict[str, list[Place]] = {} + for symbol, line in _accessors_in(after.get(path, "")): + by_symbol.setdefault(symbol, []).append(Place(path, line)) + for symbol, places in by_symbol.items(): + if symbol not in old: + gained.setdefault(symbol, []).extend(places) + for symbol in old - by_symbol.keys(): + lost[symbol] = lost.get(symbol, 0) + 1 + found: list[Finding] = [] + for symbol, places in sorted(gained.items()): + env_names = accessors.get(symbol) + if not env_names: + continue + if len({place.path for place in places}) <= lost.get(symbol, 0): + continue + found.append( + Finding( + kind="referenced", + env_names=env_names, + old_default=None, + new_default=None, + places=tuple(sorted(set(places))), + ) + ) + return found + + +def _accessors_in(source: str) -> list[tuple[str, int]]: + """``ncclParamFoo`` / ``rcclParamFoo`` identifiers outside comments and strings.""" + found: list[tuple[str, int]] = [] + for number, line in enumerate(source.splitlines(), start=1): + if line.strip().startswith("* "): + continue + code = _STRING_RE.sub('""', _strip_comments(line)) + found.extend((match.group(0), number) for match in _ACCESSOR_RE.finditer(code)) + return found + + +def accessors_defined(sources: dict[str, str]) -> dict[str, tuple[str, ...]]: + index: dict[str, list[str]] = {} + for path in sorted(sources): + for call in calls_in(sources[path], path): + if call.accessor is None: + continue + bucket = index.setdefault(call.accessor, []) + for name in call.env_names: + if name not in bucket: + bucket.append(name) + return {symbol: tuple(names) for symbol, names in index.items()} + + +def _values(calls: list[ParamCall]) -> str | None: + values = sorted({call.default for call in calls if call.default is not None}) + if not values: + return None + return ", ".join(values) + + +def unreadable_declarations(sources: dict[str, str]) -> list[Place]: + """Declarations in ``sources`` that do not parse, so their variables cannot be checked.""" + places: list[Place] = [] + for path in sorted(sources): + parsed = {call.line for call in calls_in(sources[path], path) if call.default is not None} + places.extend( + Place(path, number) + for number, line in enumerate(sources[path].splitlines(), start=1) + if _OPENER_RE.match(line.strip()) and number not in parsed + ) + return places + + +def mentions(text: str, finding: Finding) -> bool: + """Whether ``text`` names one of the finding's variables as a whole token.""" + return any( + re.search(rf"(? str: + """A CHANGELOG bullet with the variable and default filled in.""" + name = f"`{finding.primary_env}`" + if finding.kind == "referenced": + return f"* {name}: " + if finding.kind == "default": + old = _tick(finding.old_default) or "none" + new = _tick(finding.new_default) or "none" + return f"* {name} default changed {old} -> {new}: " + if finding.kind == "added": + if finding.new_default is None: + return f"* {name}: " + return f"* {name} (default `{finding.new_default}`): " + if finding.old_default is None: + return f"* {name} removed: " + return f"* {name} removed (was default `{finding.old_default}`): " + + +def _annotations(finding: Finding) -> list[str]: + """One annotation per place, so each changed file shows it inline. + + A missing CHANGELOG line and a missing doc entry are both ``::error``. + A removal has no line, so it gets one per file. + """ + if finding.kind == "referenced": + detail = "newly referenced" + elif finding.kind == "added" and finding.new_default is None: + detail = "added" + elif finding.kind == "added": + detail = f"added with default {finding.new_default}" + elif finding.kind == "default": + detail = f"default changed {finding.old_default or 'none'} -> {finding.new_default or 'none'}" + elif finding.old_default is None: + detail = "removed" + else: + detail = f"removed (was default {finding.old_default})" + asks: list[tuple[str, str]] = [] + if finding.needs_changelog: + asks.append(("error", f"Name it in {CHANGELOG}, or put {SKIP_TOKEN} in the PR title or body.")) + if finding.needs_docs: + pages = " or ".join(doc_pages_for(finding)) + asks.append(( + "error", + f"Describe this change in {pages}, or put {INTERNAL_TOKEN} in the PR title or body " + "if it is not meant for users.", + )) + return list(dict.fromkeys( + f"::{level} {_where(finding, place)}::{finding.primary_env} {detail}. {ask}" + for level, ask in asks + for place in finding.places + )) + + +def _where(finding: Finding, place: Place) -> str: + """Annotation location. A removal's line is in the base revision, so it is left off.""" + if finding.kind == "removed": + return f"file={place.path}" + return f"file={place.path},line={place.line}" + + +def _tick(value: str | None) -> str: + if not value: + return "" + return f"`{value}`" + + +def _summary(undocumented: list[Finding], unreadable: list[Place]) -> str: + if _blocks(undocumented, unreadable): + sections = ["### Undocumented RCCL default-behavior changes\n"] + else: + sections = ["### New RCCL env vars without docs\n"] + if undocumented: + rows: list[str] = [] + for item in undocumented: + places = "
".join(f"`{place.path}:{place.line}`" for place in item.places) + rows.append( + f"| `{item.primary_env}` | {item.kind} | {_tick(item.old_default)} | " + f"{_tick(item.new_default)} | {_missing(item)} | {places} |" + ) + table = "\n".join(rows) + sections.append( + "| Variable | Trigger | Old default | New default | Missing | Locations |\n" + f"|---|---|---|---|---|---|\n{table}\n" + ) + bullets = [suggestion(item) for item in undocumented if item.needs_changelog] + if bullets: + sections.append(f"Paste into `{CHANGELOG}`:\n\n" + "\n".join(bullets) + "\n") + grouped: dict[tuple[str, ...], list[str]] = {} + for item in undocumented: + if item.needs_docs: + grouped.setdefault(doc_pages_for(item), []).append(suggestion(item).removesuffix(": ")) + for pages, entries in grouped.items(): + listed = " or ".join(f"`{path}`" for path in pages) + sections.append( + f"Describe these changes in {listed}, or put `{INTERNAL_TOKEN}` in the PR title " + "or body if the variable is not meant for users:\n\n" + "\n".join(entries) + "\n" + ) + if unreadable: + lines = "\n".join(f"* `{place.path}:{place.line}`" for place in unreadable) + sections.append(f"{UNREADABLE_MESSAGE}\n\n{lines}\n") + return "\n".join(sections) + + +def doc_pages_for(finding: Finding) -> tuple[str, ...]: + """NCCL-only findings may use either page; RCCL findings require its page.""" + nccl = any(name.startswith("NCCL_") for name in finding.env_names) + rccl = any(name.startswith("RCCL_") for name in finding.env_names) + if nccl and not rccl: + return ENV_DOCS + return (RCCL_ENV_DOC,) + + +def _doc_regions(text: str) -> dict[str, str]: + """Each env var's entry, from its name to the next, with whitespace collapsed.""" + collapsed = re.sub(r"\s+", " ", text).strip() + names = list(_ENV_NAME_RE.finditer(collapsed)) + regions: dict[str, str] = {} + for index, match in enumerate(names): + end = names[index + 1].start() if index + 1 < len(names) else len(collapsed) + name = match.group(0) + regions[name] = regions.get(name, "") + collapsed[match.start() : end].strip() + return regions + + +def _entry_prose(region: str | None, name: str) -> str: + """Words in an entry besides the variable name. Markup and spacing drop out.""" + if not region: + return "" + body = re.sub(rf"(? dict[str, dict[str, str]]: + return {path: _doc_regions(pages.get(path, "")) for path in ENV_DOCS} + + +def _docs_cover( + finding: Finding, + before: dict[str, dict[str, str]], + after: dict[str, dict[str, str]], +) -> bool: + """Accept changed entry prose; it need not repeat the exact code change.""" + for page in doc_pages_for(finding): + old = before.get(page, {}) + new = after.get(page, {}) + for name in finding.env_names: + if _entry_prose(old.get(name), name) != _entry_prose(new.get(name), name): + return True + return False + + +def _missing(finding: Finding) -> str: + needed = [("CHANGELOG", finding.needs_changelog), ("docs", finding.needs_docs)] + return ", ".join(label for label, missing in needed if missing) + + +def _blocks(undocumented: list[Finding], unreadable: list[Place]) -> bool: + return bool(unreadable) or any(item.needs_changelog or item.needs_docs for item in undocumented) + + +def report(undocumented: list[Finding], unreadable: list[Place], skipped: bool) -> int: + """Print annotations and a summary, sync the PR comment, and return the exit code.""" + if not undocumented and not unreadable: + print(PASS_MESSAGE) + sync_pull_request_comment(PASS_MESSAGE, create=False) + return 0 + if skipped: + print(f"::notice::{SKIPPED_MESSAGE}") + for finding in undocumented: + for place in finding.places: + print(f"::notice {_where(finding, place)}::{finding.primary_env} {finding.kind} (override)") + for place in unreadable: + print(f"::notice file={place.path},line={place.line}::unreadable declaration (override)") + sync_pull_request_comment(SKIPPED_MESSAGE, create=False) + return 0 + for finding in undocumented: + print(*_annotations(finding), sep="\n") + for place in unreadable: + print(f"::error file={place.path},line={place.line}::{UNREADABLE_MESSAGE}") + summary = _summary(undocumented, unreadable) + print(summary) + step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if step_summary: + with Path(step_summary).open("a", encoding="utf-8") as handle: + handle.write(summary) + sync_pull_request_comment(summary, create=True) + return 1 if _blocks(undocumented, unreadable) else 0 + + +@dataclass(frozen=True) +class PullRequest: + """The pull request this run checks and the token that may comment on it.""" + + token: str + comments_url: str + + +def sync_pull_request_comment(body: str, *, create: bool) -> None: + """Create or edit the one marker-tagged comment on this pull request. + + A passing check only edits a comment that already exists, so a clean PR + does not grow a new comment. The comment is edited in place and never + deleted. Without a token and pull request ref this does nothing, and + inside Actions it says so with a warning. + + The comment is advisory. A fork's read-only token, rate limiting, or a + network error becomes a warning and never changes the check result. + """ + pull = _pull_request() + if pull is None: + return + try: + _upsert_comment(pull, f"{COMMENT_MARKER}\n{body}", create=create) + except (OSError, http.client.HTTPException, ValueError) as exc: + print(f"::warning::Could not update the pull request comment: {exc}") + + +def _pull_request() -> PullRequest | None: + token = os.environ.get("GITHUB_TOKEN", "") + repo = os.environ.get("GITHUB_REPOSITORY", "") + match = re.fullmatch(r"refs/pull/(\d+)/merge", os.environ.get("GITHUB_REF", "")) + if token and "/" in repo and match is not None: + api = os.environ.get("GITHUB_API_URL", "https://api.github.com") + return PullRequest(token=token, comments_url=f"{api}/repos/{repo}/issues/{match.group(1)}/comments") + if os.environ.get("GITHUB_ACTIONS") == "true": + print("::warning::No GITHUB_TOKEN or pull request ref; the PR comment was not updated.") + return None + + +def _upsert_comment(pull: PullRequest, body: str, *, create: bool) -> None: + existing = _marked_comment_url(pull) + if existing is not None: + _github_json("PATCH", existing, pull.token, {"body": body}) + elif create: + _github_json("POST", pull.comments_url, pull.token, {"body": body}) + + +def _marked_comment_url(pull: PullRequest) -> str | None: + """URL of this check's own comment, searching every page of the thread.""" + page = 1 + while True: + url = f"{pull.comments_url}?per_page={_PAGE_SIZE}&page={page}" + listed = _github_json("GET", url, pull.token, None) + if not isinstance(listed, list): + raise ValueError(f"GitHub returned {type(listed).__name__} for {url}") + for item in listed: + if _is_marked(item): + return str(item["url"]) + if len(listed) < _PAGE_SIZE: + return None + page += 1 + + +def _is_marked(item: object) -> bool: + """A comment by the workflow bot that carries the marker. A person quoting it is skipped.""" + if not isinstance(item, dict) or "url" not in item: + return False + user = item.get("user") + author = user.get("login") if isinstance(user, dict) else None + return author == COMMENT_AUTHOR and COMMENT_MARKER in str(item.get("body", "")) + + +def _github_json(method: str, url: str, token: str, payload: dict[str, str] | None) -> object: + request = urllib.request.Request( + url, + data=None if payload is None else json.dumps(payload).encode(), + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: + raw = response.read().decode() + return json.loads(raw) if raw else None + + +def evaluate( + changed: list[Finding], + *, + changelog_added: str, + env_docs: dict[str, str], + internal: bool, + known_before: set[str], + known_after: set[str], + env_docs_before: dict[str, str] | None = None, +) -> list[Finding]: + """Mark findings lacking changelog or documentation coverage.""" + before_docs = {path: "" for path in ENV_DOCS} if env_docs_before is None else env_docs_before + old_regions = _region_maps(before_docs) + new_regions = _region_maps(env_docs) + pending: list[Finding] = [] + for item in _one_per_variable(_without_existing(changed, known_before, known_after)): + needs_changelog = not mentions(changelog_added, item) + needs_docs = not internal and not _docs_cover(item, old_regions, new_regions) + if needs_changelog or needs_docs: + pending.append(replace(item, needs_changelog=needs_changelog, needs_docs=needs_docs)) + return pending + + +def _without_existing(found: list[Finding], known_before: set[str], known_after: set[str]) -> list[Finding]: + return [ + item + for item in found + if not (item.kind == "added" and item.primary_env in known_before) + and not (item.kind == "removed" and item.primary_env in known_after) + ] + + +def _env_names(found: list[Finding], kind: Kind) -> set[str]: + return {item.primary_env for item in found if item.kind == kind} + + +def _one_per_variable(found: list[Finding]) -> list[Finding]: + """Merge findings whose names overlap, such as the two an alias declaration gives. + + The highest-ranked finding supplies the kind and defaults: a default edit, + then a removal, then a new name, preferring one that carries a default. + """ + merged: list[Finding] = [] + for item in sorted(found, key=_finding_rank): + names = set(item.env_names) + index = next((i for i, kept in enumerate(merged) if names & set(kept.env_names)), None) + if index is None: + merged.append(item) + continue + kept = merged[index] + merged[index] = replace( + kept, + env_names=tuple(dict.fromkeys((*kept.env_names, *item.env_names))), + places=tuple(sorted({*kept.places, *item.places})), + ) + return sorted(merged, key=lambda item: (item.places[0], item.primary_env)) + + +def _finding_rank(item: Finding) -> tuple[int, int, Place, bool]: + """Merge order. Of the two findings an alias declaration gives, the ``RCCL_`` one leads.""" + has_default = item.old_default is not None or item.new_default is not None + return ( + _KIND_RANK[item.kind], + 0 if has_default else 1, + item.places[0], + not item.primary_env.startswith("RCCL_"), + ) + + +def _git(repo: Path, args: list[str], *, ok: tuple[int, ...] = (0,)) -> str: + result = subprocess.run( + ["git", *args], cwd=repo, capture_output=True, encoding="utf-8", errors="replace" + ) + if result.returncode not in ok: + raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr) + return result.stdout + + +def changed_sources(repo: Path, base_ref: str) -> tuple[dict[str, str], dict[str, str]]: + """Base and HEAD text of the files under ``SRC_TREE`` that differ between them.""" + fields = _git( + repo, ["diff", "--no-renames", "--name-status", "-z", base_ref, "HEAD", "--", SRC_TREE] + ).split("\0") + statuses = list(zip(fields[0::2], fields[1::2])) + if statuses: + # Prefetch base blobs once; repeated `git show` calls fetch them individually. + _git(repo, ["diff", "--shortstat", base_ref, "HEAD", "--", SRC_TREE, CHANGELOG]) + before: dict[str, str] = {} + after: dict[str, str] = {} + for status, path in statuses: + if status != "A": + before[path] = _git(repo, ["show", f"{base_ref}:{path}"]) + if status != "D": + after[path] = _git(repo, ["show", f"HEAD:{path}"]) + return before, after + + +def known_names(repo: Path, rev: str, names: set[str]) -> set[str]: + """Which of ``names`` product source declares or reads at ``rev``. + + ``NCCL_PARAM`` and ``RCCL_PARAM`` spell the name without its prefix, so a + file holding either spelling is parsed the same way as a changed file. + """ + if not names: + return set() + needles = sorted({*names, *(f'"{name.partition("_")[2]}"' for name in names)}) + patterns = [part for needle in needles for part in ("-e", needle)] + listed = _git(repo, ["grep", "-I", "-l", "-z", "-F", *patterns, rev, "--", SRC_TREE], ok=(0, 1)) + found = { + name + for entry in listed.split("\0") + if entry + for call in calls_in(_git(repo, ["show", entry]), entry) + if not call.mention + for name in call.env_names + } + return names & found + + +def changelog_additions(repo: Path, base_ref: str) -> str: + """Lines the pull request adds to the changelog.""" + diff = _git(repo, ["diff", "-U0", "--no-color", base_ref, "HEAD", "--", CHANGELOG]) + return "\n".join( + line[1:] for line in diff.splitlines() if line.startswith("+") and not line.startswith("+++") + ) + + +def accessor_map(repo: Path) -> dict[str, tuple[str, ...]]: + """Map ``ncclParamFoo`` / ``rcclParamFoo`` to the env names declared at HEAD.""" + listed = _git(repo, ["grep", "-I", "-l", "-z", "-E", _DECL_GREP, "HEAD", "--", SRC_TREE], ok=(0, 1)) + sources: dict[str, str] = {} + for entry in listed.split("\0"): + if not entry: + continue + _rev, separator, path = entry.partition(":") + if not separator: + continue + sources[path] = _git(repo, ["show", entry]) + return accessors_defined(sources) + + +def env_doc_pair(repo: Path, base_ref: str) -> tuple[dict[str, str], dict[str, str]]: + """Treat pages missing at either revision as empty.""" + before: dict[str, str] = {} + after: dict[str, str] = {} + for path in ENV_DOCS: + before[path] = _git(repo, ["show", f"{base_ref}:{path}"], ok=(0, 128)) + after[path] = _git(repo, ["show", f"HEAD:{path}"], ok=(0, 128)) + return before, after + + +def git_root() -> Path: + """Repository root containing the working directory.""" + return Path(_git(Path("."), ["rev-parse", "--show-toplevel"]).strip()) + + +def _new_accessor(before: dict[str, str], after: dict[str, str]) -> bool: + for path in set(before) | set(after): + old = {symbol for symbol, _line in _accessors_in(before.get(path, ""))} + if any(symbol not in old for symbol, _line in _accessors_in(after.get(path, ""))): + return True + return False + + +def check(repo: Path, base_ref: str, override_texts: list[str]) -> int: + """Run the check against ``base_ref``. Return 0 when the PR may merge.""" + before, after = changed_sources(repo, base_ref) + changed = findings_between(before, after) + if _new_accessor(before, after): + changed.extend(reference_findings(before, after, accessor_map(repo))) + overrides = "\n".join([*override_texts, _git(repo, ["log", "--pretty=%s", f"{base_ref}..HEAD"])]) + added = _env_names(changed, "added") + docs_before, docs_after = env_doc_pair(repo, base_ref) if changed else ({}, {}) + pending = evaluate( + changed, + changelog_added=changelog_additions(repo, base_ref) if changed else "", + env_docs=docs_after, + env_docs_before=docs_before, + internal=INTERNAL_TOKEN in overrides, + known_before=known_names(repo, base_ref, added), + known_after=known_names(repo, "HEAD", _env_names(changed, "removed")), + ) + return report(pending, unreadable_declarations(after), skipped=SKIP_TOKEN in overrides) + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--base-ref", + default="HEAD^", + help="Git ref the pull request is diffed against (default: HEAD^).", + ) + args = parser.parse_args(argv) + texts = [os.environ.get("PR_TITLE", ""), os.environ.get("PR_BODY", "")] + try: + return check(git_root(), args.base_ref, texts) + except subprocess.CalledProcessError as exc: + sys.stderr.write(exc.stderr or str(exc)) + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/scripts/tests/rccl_check_default_behavior_test.py b/.github/scripts/tests/rccl_check_default_behavior_test.py new file mode 100644 index 000000000000..22b1bde042a2 --- /dev/null +++ b/.github/scripts/tests/rccl_check_default_behavior_test.py @@ -0,0 +1,821 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for rccl_check_default_behavior.py.""" + +from dataclasses import replace +from pathlib import Path + +import pytest + +import rccl_check_default_behavior as gate + +NEW_PARAM = 'NCCL_PARAM(RMADisable, "RMA_DISABLE", 0);' +SRC = "projects/rccl/src/rma/rma.cc" + + +@pytest.fixture(autouse=True) +def _outside_actions(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep the check off the real job summary and pull request when pytest runs in CI.""" + for name in ( + "GITHUB_ACTIONS", + "GITHUB_API_URL", + "GITHUB_REF", + "GITHUB_REPOSITORY", + "GITHUB_STEP_SUMMARY", + "GITHUB_TOKEN", + "PR_BODY", + "PR_TITLE", + ): + monkeypatch.delenv(name, raising=False) + + +def _pages(value: str | dict[str, str]) -> dict[str, str]: + if isinstance(value, str): + return {path: value for path in gate.ENV_DOCS} + return value + + +def _pending( + before: dict[str, str], + after: dict[str, str], + changelog_added: str = "", + env_docs: str | dict[str, str] = "", + *, + env_docs_before: str | dict[str, str] | None = None, + internal: bool = False, + known: tuple[str, ...] = (), +) -> list[gate.Finding]: + return gate.evaluate( + gate.findings_between(before, after), + changelog_added=changelog_added, + env_docs=_pages(env_docs), + env_docs_before=None if env_docs_before is None else _pages(env_docs_before), + internal=internal, + known_before=set(known), + known_after=set(known), + ) + + +def test_a_new_param_needs_a_changelog_line_and_a_doc_entry() -> None: + after = {SRC: f"int x;\n{NEW_PARAM}\n"} + changelog = "* `NCCL_RMA_DISABLE` (default `0`): off" + pending = _pending({}, after) + assert [(item.kind, item.primary_env, item.places) for item in pending] == [ + ("added", "NCCL_RMA_DISABLE", (gate.Place(SRC, 2),)) + ] + assert [(item.needs_changelog, item.needs_docs) for item in pending] == [(True, True)] + assert [(item.needs_changelog, item.needs_docs) for item in _pending({}, after, changelog)] == [(False, True)] + assert _pending({}, after, changelog, env_docs="``NCCL_RMA_DISABLE``\n") != [] + assert _pending({}, after, changelog, env_docs="``NCCL_RMA_DISABLE`` turns RMA off.\n") == [] + + +def test_the_nccl_userguide_covers_only_nccl_variables() -> None: + nccl = {SRC: f"{NEW_PARAM}\n"} + rccl = {SRC: 'RCCL_PARAM(Fast, "FAST", 1);\n'} + nccl_log = "* `NCCL_RMA_DISABLE` (default `0`): off" + rccl_log = "* `RCCL_FAST` (default `1`): on" + described = "NCCL_RMA_DISABLE disables RMA.\nRCCL_FAST enables the fast path.\n" + only_nccl_page = {gate.NCCL_ENV_DOC: described, gate.RCCL_ENV_DOC: ""} + only_rccl_page = {gate.NCCL_ENV_DOC: "", gate.RCCL_ENV_DOC: described} + + assert _pending({}, nccl, nccl_log, only_nccl_page) == [] + assert _pending({}, nccl, nccl_log, only_rccl_page) == [] + assert _pending({}, rccl, rccl_log, only_rccl_page) == [] + + pending = _pending({}, rccl, rccl_log, only_nccl_page) + assert [(item.needs_changelog, item.needs_docs, gate.doc_pages_for(item)) for item in pending] == [ + (False, True, (gate.RCCL_ENV_DOC,)) + ] + assert gate.NCCL_ENV_DOC not in gate._annotations(pending[0])[0] + assert gate.RCCL_ENV_DOC in gate._annotations(pending[0])[0] + + +def test_every_change_needs_a_doc_entry_whose_content_changed() -> None: + before = {SRC: f"{NEW_PARAM}\n"} + retuned = {SRC: NEW_PARAM.replace(", 0)", ", 1)") + "\n"} + unchanged = {gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE`` default 0\n", gate.RCCL_ENV_DOC: ""} + spaced = {gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE``\n\tdefault 0\n", gate.RCCL_ENV_DOC: ""} + updated = {gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE`` default 1\n", gate.RCCL_ENV_DOC: ""} + changelog = "* `NCCL_RMA_DISABLE` default changed `0` -> `1`" + + assert [(item.kind, item.needs_docs) for item in _pending(before, retuned)] == [("default", True)] + assert [(item.kind, item.needs_docs) for item in _pending(before, {SRC: "int x;\n"})] == [("removed", True)] + assert _pending(before, retuned, changelog, unchanged, env_docs_before=unchanged) != [] + assert _pending(before, retuned, changelog, spaced, env_docs_before=unchanged) != [] + assert _pending(before, retuned, changelog, updated, env_docs_before=unchanged) == [] + rewritten = { + gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE`` stays off unless the job asks for RMA.\n", + gate.RCCL_ENV_DOC: "", + } + assert _pending(before, retuned, changelog, rewritten, env_docs_before=unchanged) == [] + named_only = { + gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE`` default 0\n``NCCL_RMA_DISABLE``\n", + gate.RCCL_ENV_DOC: "", + } + assert _pending(before, retuned, changelog, named_only, env_docs_before=unchanged) != [] + + neighbor = { + gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE`` default 0\n``NCCL_OTHER`` default 2\n", + gate.RCCL_ENV_DOC: "", + } + neighbor_edit = { + gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE`` default 0\n``NCCL_OTHER`` default 3\n", + gate.RCCL_ENV_DOC: "", + } + assert _pending(before, retuned, changelog, neighbor_edit, env_docs_before=neighbor) != [] + + +def test_internal_env_waives_the_doc_entry_but_not_the_changelog() -> None: + pending = _pending({}, {SRC: f"{NEW_PARAM}\n"}, internal=True) + assert [(item.needs_changelog, item.needs_docs) for item in pending] == [(True, False)] + + +def test_a_missing_changelog_line_and_a_missing_doc_entry_are_errors() -> None: + finding = _pending({}, {SRC: f"{NEW_PARAM}\n"})[0] + pages = f"{gate.ENV_DOCS[0]} or {gate.ENV_DOCS[1]}" + changelog, docs = gate._annotations(finding) + assert changelog.startswith(f"::error file={SRC},line=1::") + assert f"Name it in {gate.CHANGELOG}, or put {gate.SKIP_TOKEN}" in changelog + assert docs.startswith(f"::error file={SRC},line=1::") + assert f"Describe this change in {pages}, or put {gate.INTERNAL_TOKEN}" in docs + assert gate._annotations(replace(finding, needs_changelog=False)) == [docs] + + +def test_default_edit_reports_old_and_new_values() -> None: + path = "projects/rccl/src/init.cc" + before = {path: 'RCCL_PARAM(CeAllreduce, "CE_ALLREDUCE", 0);\n'} + after = {path: 'RCCL_PARAM(CeAllreduce, "CE_ALLREDUCE", -1);\n'} + pending = _pending(before, after) + assert len(pending) == 1 + finding = pending[0] + assert finding.kind == "default" + assert finding.primary_env == "RCCL_CE_ALLREDUCE" + assert finding.old_default == "0" + assert finding.new_default == "-1" + assert "default changed `0` -> `-1`" in gate.suggestion(finding) + + +def test_multiline_default_edit_is_caught() -> None: + path = "projects/rccl/src/misc/signals.cc" + before = {path: 'RCCL_PARAM(EnableSignalHandler, "ENABLE_SIGNALHANDLER",\n 0);\n'} + after = {path: 'RCCL_PARAM(EnableSignalHandler, "ENABLE_SIGNALHANDLER",\n 1);\n'} + pending = _pending(before, after) + assert len(pending) == 1 + assert pending[0].kind == "default" + assert pending[0].places == (gate.Place(path, 1),) + assert pending[0].old_default == "0" + assert pending[0].new_default == "1" + + +def test_cast_and_block_comment_defaults_parse() -> None: + source = "\n".join( + [ + 'NCCL_PARAM(SocketInlineSize, "SOCKET_INLINE", /*128 B=*/1 << 7);', + 'NCCL_PARAM(P2pNetChunkSize, "P2P_NET_CHUNKSIZE", (1 << 17)); /* 128 kB */', + 'RCCL_PARAM(DdaLLOneShotThreshold, "DDA_LL_ONESHOT_THRESHOLD", (size_t)(1) * 1024 * 1024); // 1 MiB', + 'NCCL_PARAM(LsaTeamSize, "LSA_TEAM_SIZE", 0)', + 'RCCL_PARAM(DdaThreshold, "DDA_THRESHOLD", kDdaThresholdUnset);', + 'RCCL_PARAM(CeArMaxMsgBytes, "CE_AR_MAX_MSG_BYTES", -1); // -1 = from arch table (2-shot)', + ] + ) + calls = gate.calls_in(source, SRC) + assert [call.default for call in calls] == [ + "1 << 7", + "(1 << 17)", + "(size_t)(1) * 1024 * 1024", + "0", + "kDdaThresholdUnset", + "-1", + ] + + +def test_comments_are_not_part_of_a_declaration() -> None: + define = 'DEFINE_NCCL_PARAM(x, int, NCCL_FOO, // key\n 0, 0, parser, "desc");\n' + assert [(call.env_names, call.default) for call in gate.calls_in(define, SRC)] == [(("NCCL_FOO",), "0")] + + before = {SRC: 'NCCL_PARAM(X, "X", /* old note */ 0);\n'} + after = {SRC: 'NCCL_PARAM(X, "X", /* new note */ 0);\n'} + assert _pending(before, after) == [] + + +def test_only_block_comment_lines_are_skipped() -> None: + source = ' * Reads getenv("NCCL_DOC_ONLY") at init.\n*out = getenv("NCCL_NEW_KNOB");\n' + assert [item.primary_env for item in _pending({}, {SRC: source})] == ["NCCL_NEW_KNOB"] + + +def test_renaming_the_c_identifier_is_not_a_behavior_change() -> None: + before = {SRC: 'NCCL_PARAM(OldName, "FOO", 0);\n'} + after = {SRC: 'NCCL_PARAM(NewName, "FOO", 0);\n'} + assert _pending(before, after) == [] + + +def test_moving_a_declaration_is_not_a_change_but_moving_it_with_a_new_default_is() -> None: + other = "projects/rccl/src/rma/other.cc" + before = {SRC: f"{NEW_PARAM}\n", other: "int y;\n"} + assert _pending(before, {SRC: "int x;\n", other: f"int y;\n{NEW_PARAM}\n"}) == [] + + moved = NEW_PARAM.replace(", 0)", ", 1)") + pending = _pending(before, {SRC: "int x;\n", other: f"int y;\n{moved}\n"}) + assert [(item.kind, item.old_default, item.new_default, item.places) for item in pending] == [ + ("default", "0", "1", (gate.Place(other, 2),)) + ] + + +def test_removed_param_reports_only_its_old_default() -> None: + before = {SRC: f'{NEW_PARAM}\nWARN("NCCL_RMA_DISABLE=%d", v);\n'} + pending = _pending(before, {SRC: "int x;\n"}) + assert [(item.kind, item.primary_env, item.old_default, item.new_default) for item in pending] == [ + ("removed", "NCCL_RMA_DISABLE", "0", None) + ] + assert "| removed | `0` | |" in gate._summary(pending, []) + annotations = gate._annotations(pending[0]) + assert len(annotations) == 2 + assert annotations[0].startswith(f"::error file={SRC}::") + assert "Name it in" in annotations[0] + assert gate.NCCL_ENV_DOC in annotations[1] + + +def test_log_text_alone_is_not_an_env_change() -> None: + before = {SRC: 'WARN("QP sharing (NCCL_IB_QP_SHARING_ENABLE=1) is invalid");\n'} + after = {SRC: 'WARN("QP sharing (RCCL_IB_QP_SHARING_ENABLE=1) is invalid");\n'} + assert _pending(before, after) == [] + assert _pending({}, {SRC: 'INFO(NCCL_INIT, "type NCCL_GIN_CONNECTION_FULL");\n'}) == [] + + +def test_a_new_variable_lists_its_log_mentions_too() -> None: + other = "projects/rccl/src/rma/other.cc" + after = {SRC: f"{NEW_PARAM}\n", other: 'int y;\nWARN("NCCL_RMA_DISABLE=%d is ignored", v);\n'} + pending = _pending({}, after) + assert [item.places for item in pending] == [(gate.Place(other, 2), gate.Place(SRC, 1))] + + +def test_alias_is_covered_by_either_prefix() -> None: + after = {SRC: 'RCCL_PARAM_NCCL_ALIAS(IbCast, "IB_CAST", -1);\n'} + assert _pending({}, after)[0].env_names == ("RCCL_IB_CAST", "NCCL_IB_CAST") + assert _pending( + {}, after, "* NCCL_IB_CAST now aliases RCCL_IB_CAST", env_docs="``RCCL_IB_CAST`` accepts the NCCL name.\n" + ) == [] + + +def test_adding_or_dropping_the_nccl_alias_adds_or_removes_that_name() -> None: + plain = {SRC: 'RCCL_PARAM(IbCast, "IB_CAST", -1);\n'} + alias = {SRC: 'RCCL_PARAM_NCCL_ALIAS(IbCast, "IB_CAST", -1);\n'} + assert [(item.kind, item.primary_env) for item in _pending(plain, alias)] == [("added", "NCCL_IB_CAST")] + assert [(item.kind, item.primary_env) for item in _pending(alias, plain)] == [("removed", "NCCL_IB_CAST")] + + +def test_a_longer_variable_name_does_not_cover_a_shorter_one() -> None: + after = {SRC: 'NCCL_PARAM(Ib, "IB", 0);\n'} + pending = _pending({}, after, "* `NCCL_IB_CAST` (default `0`): unrelated") + assert [item.primary_env for item in pending] == ["NCCL_IB"] + + +def test_macro_definitions_are_ignored() -> None: + header = "projects/rccl/src/include/param.h" + source = "#define NCCL_PARAM(name, env, deftVal) \\\n" + assert _pending({header: source}, {header: source + "\n"}) == [] + + +def test_one_entry_per_variable_lists_every_file() -> None: + call = 'const char* v = ncclGetEnv("NCCL_IB_RECEIVER_SIDE_MATCHING_SCHEME");\n' + init = "projects/rccl/src/transport/net_ib_cast/init.cc" + scheduler = "projects/rccl/src/transport/net_ib_cast/scheduler.cc" + pending = _pending({}, {scheduler: call, init: call}) + assert [item.primary_env for item in pending] == ["NCCL_IB_RECEIVER_SIDE_MATCHING_SCHEME"] + assert pending[0].places == (gate.Place(init, 1), gate.Place(scheduler, 1)) + + summary = gate._summary(pending, []) + assert f"| CHANGELOG, docs | `{init}:1`
`{scheduler}:1` |" in summary + assert summary.count("* `NCCL_IB_RECEIVER_SIDE_MATCHING_SCHEME`: ") == 1 + assert summary.count("* `NCCL_IB_RECEIVER_SIDE_MATCHING_SCHEME`\n") == 1 + assert [line.split("::")[1] for line in gate._annotations(pending[0])] == [ + f"error file={init},line=1", + f"error file={scheduler},line=1", + f"error file={init},line=1", + f"error file={scheduler},line=1", + ] + + +def _in_pull_request(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_TOKEN", "token") + monkeypatch.setenv("GITHUB_REPOSITORY", "ROCm/rocm-systems") + monkeypatch.setenv("GITHUB_REF", "refs/pull/42/merge") + + +def _comment(body: str, url: str, author: str = gate.COMMENT_AUTHOR) -> dict[str, object]: + return {"body": body, "url": url, "user": {"login": author}} + + +def _fake_github(monkeypatch: pytest.MonkeyPatch, comments: list[dict[str, object]]) -> list[tuple[str, str]]: + """Serve ``comments`` as the PR thread, 100 per page. Return each (method, url) called.""" + _in_pull_request(monkeypatch) + calls: list[tuple[str, str]] = [] + + def request(method: str, url: str, token: str, payload: dict[str, str] | None) -> object: + assert token == "token" + calls.append((method, url)) + if method == "GET": + page = int(url.rsplit("page=", 1)[1]) + return comments[(page - 1) * 100 : page * 100] + assert payload is not None + if method == "POST": + comments.append(_comment(payload["body"], f"{url}/{len(comments)}")) + else: + next(item for item in comments if item["url"] == url)["body"] = payload["body"] + return {} + + monkeypatch.setattr(gate, "_github_json", request) + return calls + + +def test_failure_comment_is_created_then_edited_in_place(monkeypatch: pytest.MonkeyPatch) -> None: + comments: list[dict[str, object]] = [] + calls = _fake_github(monkeypatch, comments) + gate.sync_pull_request_comment("first failure\n", create=True) + gate.sync_pull_request_comment("second failure\n", create=True) + assert [method for method, _ in calls] == ["GET", "POST", "GET", "PATCH"] + assert [item["body"] for item in comments] == [f"{gate.COMMENT_MARKER}\nsecond failure\n"] + + +def test_passing_check_edits_an_existing_comment_and_never_creates_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + comments: list[dict[str, object]] = [] + calls = _fake_github(monkeypatch, comments) + gate.sync_pull_request_comment(gate.PASS_MESSAGE, create=False) + assert comments == [] + + comments.append(_comment(f"{gate.COMMENT_MARKER}\nfailure", "https://api.github.com/c/1")) + gate.sync_pull_request_comment(gate.PASS_MESSAGE, create=False) + assert [method for method, _ in calls] == ["GET", "GET", "PATCH"] + assert comments[0]["body"] == f"{gate.COMMENT_MARKER}\n{gate.PASS_MESSAGE}" + + +def test_comment_search_reads_every_page_and_skips_a_quoted_marker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + marked = f"{gate.COMMENT_MARKER}\nfailure" + comments = [_comment("lgtm", f"https://api.github.com/c/{n}", author="dev") for n in range(99)] + comments.append(_comment(marked, "https://api.github.com/c/quoted", author="dev")) + comments.append(_comment(marked, "https://api.github.com/c/bot")) + calls = _fake_github(monkeypatch, comments) + gate.sync_pull_request_comment("second failure\n", create=True) + assert [method for method, _ in calls] == ["GET", "GET", "PATCH"] + assert calls[-1] == ("PATCH", "https://api.github.com/c/bot") + + +def test_comment_errors_become_a_warning_not_a_failure( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + _in_pull_request(monkeypatch) + + def request(method: str, url: str, token: str, payload: dict[str, str] | None) -> object: + raise ConnectionResetError("reset by peer") + + monkeypatch.setattr(gate, "_github_json", request) + gate.sync_pull_request_comment("failure\n", create=True) + assert "::warning::Could not update the pull request comment" in capsys.readouterr().out + + +def test_comment_needs_a_token_and_warns_in_actions_without_one( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def request(method: str, url: str, token: str, payload: dict[str, str] | None) -> object: + raise AssertionError("no GitHub call without a token") + + monkeypatch.setattr(gate, "_github_json", request) + gate.sync_pull_request_comment("failure\n", create=True) + assert capsys.readouterr().out == "" + + monkeypatch.setenv("GITHUB_ACTIONS", "true") + monkeypatch.setenv("GITHUB_REF", "refs/pull/42/merge") + gate.sync_pull_request_comment("failure\n", create=True) + assert "::warning::No GITHUB_TOKEN" in capsys.readouterr().out + + +def test_suggestion_for_a_new_variable() -> None: + finding = _pending({}, {SRC: f"{NEW_PARAM}\n"})[0] + assert gate.suggestion(finding) == "* `NCCL_RMA_DISABLE` (default `0`): " + + +def test_new_getenv_and_define_param_are_env_changes() -> None: + getenv_only = {SRC: 'const char* arch = getenv("RCCL_EP_FAKE_ARCH");\n'} + added = _pending({}, getenv_only) + assert [(item.primary_env, item.new_default) for item in added] == [("RCCL_EP_FAKE_ARCH", None)] + assert gate.suggestion(added[0]) == "* `RCCL_EP_FAKE_ARCH`: " + + define = ( + "DEFINE_NCCL_PARAM(ncclParamDebugLevel, ncclDebugLogLevel, NCCL_DEBUG, NCCL_LOG_NONE,\n" + ' 0, parser, "level");\n' + ) + edited = define.replace("NCCL_LOG_NONE", "NCCL_LOG_WARN") + pending = _pending({SRC: define}, {SRC: edited}) + assert len(pending) == 1 + assert pending[0].primary_env == "NCCL_DEBUG" + assert pending[0].kind == "default" + assert pending[0].old_default == "NCCL_LOG_NONE" + assert pending[0].new_default == "NCCL_LOG_WARN" + + +def test_a_new_accessor_use_references_an_existing_variable() -> None: + decl = 'NCCL_PARAM(IbMatching, "IB_MATCHING", -2);\n' + call = "if (ncclParamIbMatching()) return;\n" + accessors = gate.accessors_defined({"projects/rccl/src/p2p.cc": decl}) + assert accessors == {"ncclParamIbMatching": ("NCCL_IB_MATCHING",)} + + init = "projects/rccl/src/init.cc" + refs = gate.reference_findings({init: "int x;\n"}, {init: f"int x;\n{call}"}, accessors) + pending = gate.evaluate( + refs, + changelog_added="", + env_docs={}, + internal=False, + known_before={"NCCL_IB_MATCHING"}, + known_after=set(), + ) + assert [(item.kind, item.primary_env, item.needs_changelog, item.needs_docs, item.places) for item in pending] == [ + ("referenced", "NCCL_IB_MATCHING", True, True, (gate.Place(init, 2),)) + ] + assert "newly referenced" in gate._annotations(pending[0])[0] + assert gate.suggestion(pending[0]) == "* `NCCL_IB_MATCHING`: " + assert ( + gate.evaluate( + refs, + changelog_added="* NCCL_IB_MATCHING now gates init", + env_docs={gate.NCCL_ENV_DOC: "NCCL_IB_MATCHING now gates init\n", gate.RCCL_ENV_DOC: ""}, + internal=False, + known_before={"NCCL_IB_MATCHING"}, + known_after=set(), + ) + == [] + ) + + +def test_rccl_param_and_alias_accessors_resolve() -> None: + plain = 'RCCL_PARAM(Fast, "FAST", 1);\n' + alias = 'RCCL_PARAM_NCCL_ALIAS(IbCast, "IB_CAST", -1);\n' + defined = 'DEFINE_NCCL_PARAM(ncclParamDebugLevel, int, NCCL_DEBUG, 0, 0, parser, "d");\n' + accessors = gate.accessors_defined({SRC: plain + alias + defined}) + assert accessors["rcclParamFast"] == ("RCCL_FAST",) + assert accessors["rcclParamIbCast"] == ("RCCL_IB_CAST", "NCCL_IB_CAST") + assert accessors["ncclParamDebugLevel"] == ("NCCL_DEBUG",) + + call = {SRC: "if (rcclParamIbCast()) return;\n"} + pending = gate.evaluate( + gate.reference_findings({}, call, accessors), + changelog_added="* `NCCL_IB_CAST`: shared", + env_docs={gate.NCCL_ENV_DOC: "", gate.RCCL_ENV_DOC: "RCCL_IB_CAST shared with NCCL\n"}, + internal=False, + known_before={"RCCL_IB_CAST", "NCCL_IB_CAST"}, + known_after=set(), + ) + assert pending == [] + + +def test_an_accessor_already_used_or_only_moved_is_not_new() -> None: + decl = 'NCCL_PARAM(IbMatching, "IB_MATCHING", -2);\n' + call = "if (ncclParamIbMatching()) return;\n" + accessors = gate.accessors_defined({SRC: decl}) + assert gate.reference_findings({SRC: call}, {SRC: call + call}, accessors) == [] + moved = gate.reference_findings({SRC: call}, {"projects/rccl/src/other.cc": call}, accessors) + assert moved == [] + + +def test_a_lowercase_param_name_is_still_an_accessor() -> None: + decl = 'RCCL_PARAM(disableReduceCopyPipelining, "DISABLE_REDUCE_COPY_PIPELINING", 0);\n' + accessors = gate.accessors_defined({SRC: decl}) + symbol = "rcclParamdisableReduceCopyPipelining" + assert accessors[symbol] == ("RCCL_DISABLE_REDUCE_COPY_PIPELINING",) + refs = gate.reference_findings({}, {SRC: f"if ({symbol}()) return;\n"}, accessors) + assert [item.primary_env for item in refs] == ["RCCL_DISABLE_REDUCE_COPY_PIPELINING"] + + +def test_param_machinery_and_comments_are_not_accessor_uses() -> None: + decl = 'NCCL_PARAM(IbMatching, "IB_MATCHING", -2);\n' + accessors = gate.accessors_defined({SRC: decl}) + source = "\n".join( + [ + "ncclParamOneOf parser;", + "struct ncclParamInterface;", + "rcclParamMutex##name", + " * ncclParamIbMatching() in a comment", + '// ncclParamIbMatching()', + 'WARN("ncclParamIbMatching() is not a call");', + ] + ) + assert gate.reference_findings({}, {SRC: source + "\n"}, accessors) == [] + + +def test_a_new_variable_and_its_accessor_call_are_one_finding() -> None: + source = 'NCCL_PARAM(IbMatching, "IB_MATCHING", -2);\nif (ncclParamIbMatching()) return;\n' + accessors = gate.accessors_defined({SRC: source}) + pending = gate.evaluate( + [*gate.findings_between({}, {SRC: source}), *gate.reference_findings({}, {SRC: source}, accessors)], + changelog_added="", + env_docs={}, + internal=False, + known_before=set(), + known_after=set(), + ) + assert [(item.kind, item.primary_env, item.places) for item in pending] == [ + ("added", "NCCL_IB_MATCHING", (gate.Place(SRC, 1), gate.Place(SRC, 2))) + ] + + +def test_a_second_read_of_an_existing_env_is_not_new() -> None: + first = 'const char* proto = ncclGetEnv("NCCL_PROTO");\n' + second = first + 'const char* again = getenv("NCCL_PROTO");\n' + assert _pending({SRC: first}, {SRC: second}) == [] + assert _pending({}, {"projects/rccl/src/other.cc": second}, known=("NCCL_PROTO",)) == [] + + +def test_a_bare_nccl_token_is_not_an_env_by_itself() -> None: + before = {SRC: "int flags = NCCL_INIT;\n"} + after = {SRC: "int flags = NCCL_INIT | NCCL_COLL;\n"} + assert _pending(before, after) == [] + + +DEBUG_CC = """\ +DEFINE_NCCL_PARAM(ncclParamDebugLevel, ncclDebugLogLevel, NCCL_DEBUG, NCCL_LOG_NONE, + NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, + ncclParamOneOf(makeOptions( + makeOption("WARN", NCCL_LOG_WARN, "Prints only messages indicating a fatal error."), + makeOption("ABORT", NCCL_LOG_ABORT, "") + )), "Set debug output level"); + +DEFINE_NCCL_PARAM(ncclParamDebugSubsys, uint64_t, NCCL_DEBUG_SUBSYS, + NCCL_INIT | NCCL_BOOTSTRAP | NCCL_ENV, + NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, + (ncclParamBitsetOf(makeOptions( + makeOption("INIT", NCCL_INIT, "NCCL and comm initialization (included in default)") + ))), "Filter debug output by (comma-separated)"); + +DEFINE_NCCL_PARAM(ncclParamWarnEnableDebugInfo, bool, NCCL_WARN_ENABLE_DEBUG_INFO, false, + NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, NCCL_PARAM_DEFAULT, + "If enabled, the debug level will be set to INFO after a WARN level debug message is logged."); + +DEFINE_NCCL_PARAM(ncclParamDebugTimestampLevel, uint32_t, NCCL_DEBUG_TIMESTAMP_LEVELS, (1u << NCCL_LOG_WARN), + NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, + ncclParamBitsetOf( + makeOptions(makeOption("ALL", + (1u << NCCL_LOG_VERSION | 1u << NCCL_LOG_WARN), + "on All messages"))), + "Set which log lines get a timestamp depending upon the level of the log"); + +DEFINE_NCCL_PARAM(ncclParamDebugTsFormat, const char*, NCCL_DEBUG_TIMESTAMP_FORMAT, "[%F %T] ", + NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, NCCL_PARAM_DEFAULT, + "Set the format used when printing debug log messages"); + +DEFINE_NCCL_PARAM(ncclParamDebugFile, const char*, NCCL_DEBUG_FILE, nullptr, + NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, NCCL_PARAM_DEFAULT, + "Set the NCCL debug logging output to a file. " + "please convert to a relative or absolute path first."); +""" + + +def test_define_nccl_param_forms_from_debug_cc_parse() -> None: + defaults = { + call.env_names[0]: call.default + for call in gate.calls_in(DEBUG_CC, "projects/rccl/src/debug.cc") + if call.default is not None + } + assert defaults == { + "NCCL_DEBUG": "NCCL_LOG_NONE", + "NCCL_DEBUG_SUBSYS": "NCCL_INIT | NCCL_BOOTSTRAP | NCCL_ENV", + "NCCL_WARN_ENABLE_DEBUG_INFO": "false", + "NCCL_DEBUG_TIMESTAMP_LEVELS": "(1u << NCCL_LOG_WARN)", + "NCCL_DEBUG_TIMESTAMP_FORMAT": '"[%F %T] "', + "NCCL_DEBUG_FILE": "nullptr", + } + + +def test_accessor_map_resolves_old_params_and_skips_param_machinery() -> None: + repo = Path(__file__).resolve().parents[3] + if not (repo / gate.SRC_TREE).is_dir(): + pytest.skip(f"{gate.SRC_TREE} is not in this checkout") + mapped = gate.accessor_map(repo) + assert "NCCL_P2P_DISABLE" in mapped.get("ncclParamP2pDisable", ()) + assert mapped["rcclParamIbCastQpSchedEnable"] == ("RCCL_IB_QP_SCHED_ENABLE", "NCCL_IB_QP_SCHED_ENABLE") + assert mapped["rcclParamdisableReduceCopyPipelining"] == ("RCCL_DISABLE_REDUCE_COPY_PIPELINING",) + assert "ncclParamOneOf" not in mapped + assert "ncclParamInterface" not in mapped + assert "rcclParamMutex" not in mapped + + +def test_every_declaration_in_the_tree_parses() -> None: + repo = Path(__file__).resolve().parents[3] + if not (repo / gate.SRC_TREE).is_dir(): + pytest.skip(f"{gate.SRC_TREE} is not in this checkout") + paths = gate._git(repo, ["ls-files", gate.SRC_TREE]).splitlines() + sources = {rel: (repo / rel).read_text(encoding="utf-8", errors="replace") for rel in paths} + assert gate.unreadable_declarations(sources) == [] + + +def _commit(repo: Path, files: dict[str, str], message: str) -> None: + for rel, text in files.items(): + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + gate._git(repo, ["add", "-A"]) + gate._git(repo, ["commit", "-m", message]) + + +def _repo(tmp_path: Path) -> Path: + gate._git(tmp_path, ["init", "-b", "main"]) + gate._git(tmp_path, ["config", "user.email", "rccl-ci@example.com"]) + gate._git(tmp_path, ["config", "user.name", "RCCL CI"]) + pages = {gate.ENV_DOCS[0]: "Environment Variables\n", gate.ENV_DOCS[1]: "RCCL environment variables\n"} + _commit(tmp_path, {gate.CHANGELOG: "# Changelog\n", SRC: "int x;\n", **pages}, "base") + return tmp_path + + +def test_known_names_cover_reads_and_unprefixed_declarations(tmp_path: Path) -> None: + repo = _repo(tmp_path) + source = ( + 'getenv("NCCL_IB_CAST");\n' + 'NCCL_PARAM(IbMatching, "IB_MATCHING", -2);\n' + 'WARN("NCCL_IB_TEXT_ONLY=%d", v);\n' + ) + _commit(repo, {"projects/rccl/src/ib.cc": source}, "ib") + asked = {"NCCL_IB", "NCCL_IB_CAST", "NCCL_IB_MATCHING", "RCCL_IB_MATCHING", "NCCL_IB_TEXT_ONLY", "NCCL_ABSENT"} + assert gate.known_names(repo, "HEAD", asked) == {"NCCL_IB_CAST", "NCCL_IB_MATCHING"} + assert gate.known_names(repo, "HEAD^", asked) == set() + + +def test_check_fails_without_a_changelog_entry(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + repo = _repo(tmp_path) + _commit(repo, {SRC: f"int x;\n{NEW_PARAM}\n"}, "add RMA disable") + + assert gate.check(repo, "HEAD^", []) == 1 + captured = capsys.readouterr() + assert f"::error file={SRC},line=2::" in captured.out + assert "NCCL_RMA_DISABLE" in captured.out + + +def test_check_passes_when_the_changelog_and_either_env_page_name_it(tmp_path: Path) -> None: + repo = _repo(tmp_path) + fast = 'RCCL_PARAM(Fast, "FAST", 1);' + changelog = "# Changelog\n\n* `NCCL_RMA_DISABLE` (default `0`): off\n* `RCCL_FAST` (default `1`): on\n" + _commit( + repo, + { + SRC: f"int x;\n{NEW_PARAM}\n{fast}\n", + gate.CHANGELOG: changelog, + gate.ENV_DOCS[0]: "NCCL_RMA_DISABLE\n Turns RMA off.\n", + gate.ENV_DOCS[1]: " * - | ``RCCL_FAST``\n Enables the fast path.\n", + }, + "add RMA disable and fast", + ) + + assert gate.check(repo, "HEAD^", []) == 0 + + +def test_a_missing_doc_entry_fails_and_comments( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + repo = _repo(tmp_path) + changelog = "# Changelog\n\n* `NCCL_RMA_DISABLE` (default `0`): off\n" + _commit(repo, {SRC: f"int x;\n{NEW_PARAM}\n", gate.CHANGELOG: changelog}, "add RMA disable") + comments: list[dict[str, object]] = [] + _fake_github(monkeypatch, comments) + + assert gate.check(repo, "HEAD^", []) == 1 + out = capsys.readouterr().out + assert f"::error file={SRC},line=2::NCCL_RMA_DISABLE added with default 0. Describe this change in" in out + assert "::warning" not in out + assert len(comments) == 1 + body = str(comments[0]["body"]) + assert "### Undocumented RCCL default-behavior changes" in body + assert "Describe these changes in" in body + assert "Not blocking" not in body + + assert gate.check(repo, "HEAD^", [f"debug knob {gate.INTERNAL_TOKEN}"]) == 0 + assert "::error" not in capsys.readouterr().out + assert comments[0]["body"] == f"{gate.COMMENT_MARKER}\n{gate.PASS_MESSAGE}" + + +def test_check_passes_when_the_commit_subject_records_the_override( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repo = _repo(tmp_path) + _commit(repo, {SRC: f"int x;\n{NEW_PARAM}\n"}, "reformat params [no-behavior-change]") + + assert gate.check(repo, "HEAD^", []) == 0 + assert gate.SKIPPED_MESSAGE in capsys.readouterr().out + + +def test_main_takes_the_override_from_the_pr_body_and_reports_git_errors( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + repo = _repo(tmp_path) + _commit(repo, {SRC: f"int x;\n{NEW_PARAM}\n"}, "add RMA disable") + monkeypatch.chdir(repo) + + assert gate.main([]) == 1 + monkeypatch.setenv("PR_BODY", f"retune only {gate.SKIP_TOKEN}") + assert gate.main([]) == 0 + capsys.readouterr() + assert gate.main(["--base-ref", "no-such-ref"]) == 1 + assert "no-such-ref" in capsys.readouterr().err + + +def test_check_fails_on_an_unreadable_declaration_in_a_changed_file( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repo = _repo(tmp_path) + _commit(repo, {SRC: 'int x;\nNCCL_PARAM(Broken, "BROKEN");\n'}, "broken declaration") + + assert gate.check(repo, "HEAD^", []) == 1 + out = capsys.readouterr().out + assert f"::error file={SRC},line=2::{gate.UNREADABLE_MESSAGE}" in out + assert f"* `{SRC}:2`" in out + assert gate.check(repo, "HEAD^", [f"cleanup {gate.SKIP_TOKEN}"]) == 0 + + _commit(repo, {"projects/rccl/src/other.cc": "int y;\n"}, "unrelated") + assert gate.check(repo, "HEAD^", []) == 0 + + +def test_check_scans_only_product_source(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + repo = _repo(tmp_path) + mention = 'getenv("RCCL_ENABLE_HOST_GRAPH");\n' + _commit( + repo, + {"projects/rccl/test/host/fakes.cc": mention, "projects/rccl/tools/scripts/run.py": mention}, + "tests and tools", + ) + + assert gate.check(repo, "HEAD^", []) == 0 + assert gate.PASS_MESSAGE in capsys.readouterr().out + + +def test_check_fails_on_a_new_accessor_use_until_the_changelog_names_it(tmp_path: Path) -> None: + repo = _repo(tmp_path) + _commit(repo, {"projects/rccl/src/p2p.cc": 'NCCL_PARAM(IbMatching, "IB_MATCHING", -2);\n'}, "declare") + _commit(repo, {"projects/rccl/src/init.cc": "int x;\nif (ncclParamIbMatching()) return;\n"}, "use it") + + assert gate.check(repo, "HEAD^", []) == 1 + _commit( + repo, + { + gate.CHANGELOG: "# Changelog\n\n* `NCCL_IB_MATCHING`: init now reads it\n", + gate.NCCL_ENV_DOC: "Environment Variables\n\nNCCL_IB_MATCHING\n now read at init\n", + }, + "document the use", + ) + assert gate.check(repo, "HEAD~2", []) == 0 + + other = "projects/rccl/src/other.cc" + _commit( + repo, + {"projects/rccl/src/init.cc": "int x;\n", other: "if (ncclParamIbMatching()) return;\n"}, + "move the call", + ) + assert gate.check(repo, "HEAD^", []) == 0 + + +def test_check_rejects_a_whitespace_only_doc_edit(tmp_path: Path) -> None: + repo = _repo(tmp_path) + entry = "``NCCL_RMA_DISABLE``\n default 0\n" + changelog = "# Changelog\n\n* `NCCL_RMA_DISABLE` (default `0`): off\n" + _commit( + repo, + {SRC: f"int x;\n{NEW_PARAM}\n", gate.CHANGELOG: changelog, gate.NCCL_ENV_DOC: entry}, + "add RMA disable", + ) + retuned = NEW_PARAM.replace(", 0)", ", 1)") + _commit( + repo, + { + SRC: f"int x;\n{retuned}\n", + gate.CHANGELOG: changelog + "* `NCCL_RMA_DISABLE` default changed `0` -> `1`: retune\n", + gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE``\n\tdefault 0\n", + }, + "retune and rewrap the doc", + ) + assert gate.check(repo, "HEAD^", []) == 1 + + _commit(repo, {gate.NCCL_ENV_DOC: "``NCCL_RMA_DISABLE``\n default 1\n"}, "record the new default") + assert gate.check(repo, "HEAD~2", []) == 0 + + +def test_check_accepts_a_new_read_of_a_declared_variable(tmp_path: Path) -> None: + repo = _repo(tmp_path) + _commit(repo, {"projects/rccl/src/p2p.cc": 'NCCL_PARAM(IbMatching, "IB_MATCHING", -2);\n'}, "declare") + _commit(repo, {"projects/rccl/src/init.cc": 'const char* v = getenv("NCCL_IB_MATCHING");\n'}, "read") + + assert gate.check(repo, "HEAD^", []) == 0 + + +def test_check_reports_a_new_default_in_a_renamed_file( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + repo = _repo(tmp_path) + old, new = "projects/rccl/src/old.cc", "projects/rccl/src/new.cc" + body = "".join(f"int filler{n};\n" for n in range(20)) + _commit(repo, {old: f"{body}{NEW_PARAM}\n"}, "add param") + (repo / old).unlink() + _commit(repo, {new: f"{body}{NEW_PARAM.replace(', 0)', ', 1)')}\n"}, "rename and retune") + + assert gate.check(repo, "HEAD^", []) == 1 + assert f"::error file={new},line=21::NCCL_RMA_DISABLE default changed 0 -> 1." in capsys.readouterr().out diff --git a/.github/workflows/amdsmi-asan.yml b/.github/workflows/amdsmi-asan.yml new file mode 100644 index 000000000000..acf5311aab99 --- /dev/null +++ b/.github/workflows/amdsmi-asan.yml @@ -0,0 +1,230 @@ +# --------------------------------------------------------------------------- +# AMD-SMI ASAN CI. +# +# Builds AMD-SMI with AddressSanitizer (ASAN, including LeakSanitizer) and +# runs its tests, so memory errors fail CI. AMD-SMI has no GPU kernels, so +# host ASAN covers all of it. +# +# unit: hardware-free unit tests on a GitHub-hosted runner. +# gpu: full test suite plus an amd-smi CLI sweep on a self-hosted GPU runner. +# +# Both jobs build with GCC in the same pinned image, so findings reproduce +# locally with `docker run` on it. +# --------------------------------------------------------------------------- + +name: AMDSMI ASAN + +on: + pull_request: + branches: [develop] + paths: + - 'projects/amdsmi/**' + - '.github/workflows/amdsmi-asan.yml' + push: + branches: [develop] + paths: + - 'projects/amdsmi/**' + - '.github/workflows/amdsmi-asan.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Group by PR number so a new push supersedes the previous run. Pushes to + # develop fall back to the commit sha, which is unique, so post-merge runs + # never cancel each other. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +env: + DEBIAN_FRONTEND: noninteractive + BUILD_DIR: /tmp/amdsmi-asan-build + RESULTS_DIR: /tmp/amdsmi-asan-results + # Only instrumented processes read this. The flags enable checks that are + # off by default; the first error aborts the process and fails the step. + ASAN_OPTIONS: detect_leaks=1:strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1 + +defaults: + run: + shell: bash + +jobs: + unit: + name: ASAN unit tests + runs-on: ubuntu-24.04 + timeout-minutes: 30 + container: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 # noble, 2026-09-11 + + steps: + # The official image ships no git, which actions/checkout needs to clone + # rather than fall back to a tarball, and configure uses to fetch + # esmi_ib_library. + - name: Bootstrap git + run: | + apt-get update + apt-get install -y --no-install-recommends git ca-certificates + + # rocm-systems is a large monorepo; only the amdsmi project and this + # workflow are needed, so sparse-checkout keeps the fetch cheap. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/amdsmi + .github/workflows + + - name: Install build dependencies + run: | + set -euo pipefail + git config --global --add safe.directory "$GITHUB_WORKSPACE" + apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build pkg-config python3 \ + libdrm-dev libnl-3-dev libnl-genl-3-dev libmnl-dev libgtest-dev + + - name: Configure + run: | + set -euo pipefail + cmake -S "$GITHUB_WORKSPACE/projects/amdsmi" -B "$BUILD_DIR" -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DADDRESS_SANITIZER=ON \ + -DBUILD_TESTS=ON \ + -DENABLE_ESMI_LIB=ON + + - name: Build + run: cmake --build "$BUILD_DIR" --parallel "$(nproc)" + + - name: Run unit tests under ASAN + run: | + set -euo pipefail + mkdir -p "$RESULTS_DIR" + export LSAN_OPTIONS="suppressions=$GITHUB_WORKSPACE/projects/amdsmi/tests/amd_smi_test/lsan.supp:print_suppressions=1" + "$BUILD_DIR/tests/amd_smi_test/amdsmitst" --gtest_filter='*Unit*' \ + --gtest_output="xml:$RESULTS_DIR/amdsmitst-unit.xml" + + - name: Upload test results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: amdsmi-asan-unit-results + path: ${{ env.RESULTS_DIR }} + if-no-files-found: warn + retention-days: 7 + + gpu: + name: ASAN GPU tests + # Privileged self-hosted runner: never auto-run untrusted fork-PR code on + # it. Mirrors the guard in amdsmi-build.yml. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} + runs-on: + - self-hosted + - ${{ vars.RUNNER_TYPE }} + timeout-minutes: 45 + container: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 # noble, 2026-09-11 + options: --rm --privileged --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -u root + env: + # Skip the tests that change GPU settings: an ASAN abort mid-test would + # leave them changed on this shared runner. amdsmi-build.yml runs them. + AMDSMI_NON_PRIVILEGED: 1 + AMDSMI_PREFIX: /opt/amdsmi-asan + + steps: + # The official image ships no git, which actions/checkout needs to clone + # rather than fall back to a tarball, and configure uses to fetch + # esmi_ib_library. + - name: Bootstrap git + run: | + apt-get update + apt-get install -y --no-install-recommends git ca-certificates + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/amdsmi + .github/workflows + + - name: Install build dependencies + run: | + set -euo pipefail + git config --global --add safe.directory "$GITHUB_WORKSPACE" + apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build pkg-config python3 \ + libdrm-dev libnl-3-dev libnl-genl-3-dev libmnl-dev libgtest-dev + + - name: Configure + run: | + set -euo pipefail + cmake -S "$GITHUB_WORKSPACE/projects/amdsmi" -B "$BUILD_DIR" -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DADDRESS_SANITIZER=ON \ + -DBUILD_TESTS=ON \ + -DENABLE_ESMI_LIB=ON + + - name: Build + id: build + run: cmake --build "$BUILD_DIR" --parallel "$(nproc)" + + - name: Install + id: install + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: | + cmake --install "$BUILD_DIR" --prefix "$AMDSMI_PREFIX" + echo "$AMDSMI_PREFIX/bin" >> "$GITHUB_PATH" + + - name: Run full test suite under ASAN + run: | + set -euo pipefail + mkdir -p "$RESULTS_DIR" + export LSAN_OPTIONS="suppressions=$GITHUB_WORKSPACE/projects/amdsmi/tests/amd_smi_test/lsan.supp:print_suppressions=1" + cd "$GITHUB_WORKSPACE/projects/amdsmi/tests/amd_smi_test" + source amdsmitst.exclude + source detect_asic_filter.sh + "$BUILD_DIR/tests/amd_smi_test/amdsmitst" --gtest_filter="-${GTEST_EXCLUDE}" \ + --gtest_output="xml:$RESULTS_DIR/amdsmitst.xml" + + # The CLI sweep still runs when the test suite fails, so one run + # reports both. + + # Python is not instrumented: preload ASAN (with libstdc++, so it can + # intercept the library's C++ throws) and use PYTHONMALLOC=malloc so it + # sees overflows of ctypes buffers. The interpreter leaks, so no LSan. + - name: Run amd-smi CLI under ASAN + if: ${{ !cancelled() && steps.install.outcome == 'success' }} + run: | + set -euo pipefail + mkdir -p "$RESULTS_DIR/cli" + asan_rt="$(realpath "$(gcc -print-file-name=libasan.so)")" + cxx_rt="$(realpath "$(g++ -print-file-name=libstdc++.so)")" + export LD_PRELOAD="$asan_rt:$cxx_rt" + export PYTHONMALLOC=malloc + export ASAN_OPTIONS="$ASAN_OPTIONS:detect_leaks=0" + # Must pass, or the sweep below proves nothing. + "$AMDSMI_PREFIX/bin/amd-smi" version + # Same commands as amdsmi-build.yml, which judges their exit codes; + # only an ASAN report (lines prefixed ====) fails this step. + commands=(list static firmware ucode bad-pages metric process topology monitor dmon xgmi partition) + findings=() + for cmd in "${commands[@]}"; do + log="$RESULTS_DIR/cli/amd-smi_$cmd.log" + rc=0 + "$AMDSMI_PREFIX/bin/amd-smi" "$cmd" > "$log" 2>&1 || rc=$? + echo "amd-smi $cmd: exit $rc" + if grep -qE '^==[0-9]+==' "$log"; then + findings+=("$cmd") + cat "$log" + fi + done + if [ "${#findings[@]}" -gt 0 ]; then + echo "::error::ASAN reported errors in: amd-smi ${findings[*]}" + exit 1 + fi + + - name: Upload test results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: amdsmi-asan-gpu-results + path: ${{ env.RESULTS_DIR }} + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/amdsmi-python-versions.yml b/.github/workflows/amdsmi-python-versions.yml index e1ba036c8510..e6cb67ac298b 100644 --- a/.github/workflows/amdsmi-python-versions.yml +++ b/.github/workflows/amdsmi-python-versions.yml @@ -46,11 +46,10 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 15 # 3.6/3.7 run inside official python images because actions/setup-python no - # longer provides those EOL interpreters on a supported hosted runner (they - # were only on ubuntu-20.04, itself being retired). The images are archived, - # not deleted, so this stays green after the runner EOL. Newer minors run - # natively via setup-python. Empty matrix.container = no container. - container: ${{matrix.container}} + # longer provides those EOL interpreters on a supported hosted runner. The + # checkout happens on the host and the image only runs the tests, so nothing + # is installed from the (EOL, archived) Debian mirrors. Newer minors run + # natively via setup-python. Empty matrix.container = host interpreter. strategy: fail-fast: false matrix: @@ -66,13 +65,6 @@ jobs: - { python-version: '3.14', container: '' } steps: - # The slim python images ship without git; actions/checkout needs it. - - name: Install git (container legs only) - if: matrix.container != '' - run: | - apt-get update - apt-get install -y --no-install-recommends git ca-certificates - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: # rocm-systems is a large monorepo; only the amdsmi tests are needed. @@ -89,6 +81,13 @@ jobs: python-version: ${{matrix.python-version}} - name: Run loader tests under this interpreter + if: matrix.container == '' run: | python3 projects/amdsmi/tests/run_amdsmi_python_versions_test.py \ --interpreters "$(command -v python3)" + + - name: Run loader tests under this interpreter (container) + if: matrix.container != '' + run: | + docker run --rm -v "$PWD:/src" -w /src ${{matrix.container}} \ + sh -c 'python3 projects/amdsmi/tests/run_amdsmi_python_versions_test.py --interpreters "$(command -v python3)"' diff --git a/.github/workflows/cuid-codeql.yml b/.github/workflows/cuid-codeql.yml index e9b817be4f18..a81bc1c506e7 100644 --- a/.github/workflows/cuid-codeql.yml +++ b/.github/workflows/cuid-codeql.yml @@ -5,6 +5,7 @@ on: branches: [develop] paths: - 'projects/cuid/**' + - 'shared/sha256/**' - '.github/workflows/cuid-codeql.yml' schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/cuid-workflow.yml b/.github/workflows/cuid-workflow.yml index c47882c4723b..d305773294dd 100644 --- a/.github/workflows/cuid-workflow.yml +++ b/.github/workflows/cuid-workflow.yml @@ -5,7 +5,7 @@ on: branches: [develop] paths: - 'projects/cuid/**' - - 'projects/rocprofiler-sdk/source/lib/common/sha256.cpp' + - 'shared/sha256/**' - '.github/workflows/cuid-workflow.yml' workflow_dispatch: @@ -19,32 +19,6 @@ env: ROCM_DIR: /opt/rocm jobs: - sha256-drift-check: - name: SHA-256 Drift Check - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} - runs-on: - - self-hosted - - ${{ vars.RUNNER_TYPE }} - steps: - # Runner workspace (including the top-level dir itself) can be left root-owned by other - # container jobs; this job has no host sudo access, so use a throwaway root container - # (docker is already required here for build-and-test) to clear it and hand it back. - - name: Clean workspace - run: | - docker run --rm -v "$GITHUB_WORKSPACE":/workspace alpine sh -c \ - "find /workspace -mindepth 1 -delete && chown $(id -u):$(id -g) /workspace" - - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - sparse-checkout: | - projects/cuid/tests/check_sha256_drift.py - projects/cuid/lib/src/sha256.cc - projects/rocprofiler-sdk/source/lib/common/sha256.cpp - sparse-checkout-cone-mode: false - - - name: Check for sha256 drift - run: python3 projects/cuid/tests/check_sha256_drift.py - build-and-test: name: Build and Test if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} diff --git a/.github/workflows/hipfile-build.yml b/.github/workflows/hipfile-build.yml index 16ecc567c2bf..aaa5683a300a 100644 --- a/.github/workflows/hipfile-build.yml +++ b/.github/workflows/hipfile-build.yml @@ -163,14 +163,14 @@ jobs: cmake \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_CXX_COMPILER="${_AIS_INPUT_CXX_COMPILER}" \ - -DAIS_CXX_STANDARD="${{ env.AIS_INPUT_CXX_STANDARD }}" \ + -DHIPFILE_CXX_STANDARD="${{ env.AIS_INPUT_CXX_STANDARD }}" \ -DCMAKE_CXX_FLAGS="-Werror" \ -DCMAKE_HIP_ARCHITECTURES="${{ env.AIS_HIP_ARCHITECTURES }}" \ -DCMAKE_HIP_PLATFORM=amd \ - -DAIS_BUILD_DOCS=ON \ - -DAIS_USE_CODE_COVERAGE=${{ env.AIS_USE_CODE_COVERAGE == 'true' && 'ON' || 'OFF' }} \ + -DHIPFILE_BUILD_DOCS=ON \ + -DHIPFILE_USE_CODE_COVERAGE=${{ env.AIS_USE_CODE_COVERAGE == 'true' && 'ON' || 'OFF' }} \ -DROCM_VERSION="${ROCM_VERSION}" \ - -DAIS_CAPABLE_DIR=/mnt/ais-fs \ + -DHIPFILE_CAPABLE_DIR=/mnt/ais-fs \ ../../rocm-systems/projects/hipfile ' - name: Build hipFile for the AMD platform (${{ inputs.cxx_compiler }}) @@ -206,12 +206,12 @@ jobs: cmake \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER="${_AIS_INPUT_CXX_COMPILER}" \ - -DAIS_CXX_STANDARD="${{ env.AIS_INPUT_CXX_STANDARD }}" \ + -DHIPFILE_CXX_STANDARD="${{ env.AIS_INPUT_CXX_STANDARD }}" \ -DCMAKE_CXX_FLAGS="-Werror" \ -DCMAKE_HIP_ARCHITECTURES="${{ env.AIS_HIP_ARCHITECTURES }}" \ -DCMAKE_HIP_PLATFORM=amd \ -DROCM_VERSION="${ROCM_VERSION}" \ - -DAIS_CAPABLE_DIR=/mnt/ais-fs \ + -DHIPFILE_CAPABLE_DIR=/mnt/ais-fs \ ../../rocm-systems/projects/hipfile ' - name: Build hipFile Release for the AMD platform (${{ inputs.cxx_compiler }}) diff --git a/.github/workflows/hipfile-ci-system.yml b/.github/workflows/hipfile-ci-system.yml index 2b7e7a712ab1..8bc8309ba3c0 100644 --- a/.github/workflows/hipfile-ci-system.yml +++ b/.github/workflows/hipfile-ci-system.yml @@ -178,6 +178,7 @@ jobs: ${GITHUB_WORKSPACE}/hipfile-build-dir/examples/basics/various-mem-rw \ ${GITHUB_WORKSPACE}/hipfile-build-dir/examples/basics/subregion-write \ ${GITHUB_WORKSPACE}/hipfile-build-dir/examples/basics/roundtrip-verify \ + ${GITHUB_WORKSPACE}/hipfile-build-dir/examples/batch/batch-roundtrip \ ${GITHUB_WORKSPACE}/hipfile-build-dir/examples/async/roundtrip-async \ ${GITHUB_WORKSPACE}/hipfile-build-dir/examples/async/roundtrip-async-nonblocking-stream \ ${GITHUB_WORKSPACE}/hipfile-build-dir/examples/async/roundtrip-async-multi-stream \ diff --git a/.github/workflows/hipfile-ci.yml b/.github/workflows/hipfile-ci.yml index b48f843ea3a4..63ad11783bd9 100644 --- a/.github/workflows/hipfile-ci.yml +++ b/.github/workflows/hipfile-ci.yml @@ -55,6 +55,11 @@ jobs: cxx_standard: ${{ inputs.cxx_standard }} platform: ${{ inputs.platform }} rocm_version: ${{ inputs.rocm_version }} + # python/tests/main.py is an end-to-end hardware exercise of the bindings, and + # is platform/compiler/standard independent, so run it once on an AIS + # runner for a single representative leg rather than fanning it out. This + # is the same ubuntu-7.14.0 leg codeql/sanitizers/clang_tidy target. + run_hardware_test: ${{ inputs.platform == 'ubuntu' && inputs.cxx_compiler == 'amdclang++' && inputs.cxx_standard == 20 && inputs.rocm_version == '7.14.0' }} AIS_system_tests: needs: [build_and_test] @@ -70,17 +75,31 @@ jobs: platform: ${{ inputs.platform }} rocm_version: ${{ inputs.rocm_version }} - AIS_system_lvm_tests: + AIS_system_lvm16_tests: needs: [build_and_test] uses: ./.github/workflows/hipfile-ci-system.yml with: ais_hipfile_pkg_dev_filename: ${{ needs.build_and_test.outputs.ais_hipfile_pkg_dev_filename }} ais_hipfile_pkg_filename: ${{ needs.build_and_test.outputs.ais_hipfile_pkg_filename }} - ais_mount_path: /mnt/ais/lvm + ais_mount_path: /mnt/ais/lvm16 ci_image: ${{ needs.build_and_test.outputs.ci_image }} cxx_compiler: ${{ inputs.cxx_compiler }} cxx_standard: ${{ inputs.cxx_standard }} - mount_label: system_lvm + mount_label: system_lvm16 + platform: ${{ inputs.platform }} + rocm_version: ${{ inputs.rocm_version }} + + AIS_system_lvm_mirror_tests: + needs: [build_and_test] + uses: ./.github/workflows/hipfile-ci-system.yml + with: + ais_hipfile_pkg_dev_filename: ${{ needs.build_and_test.outputs.ais_hipfile_pkg_dev_filename }} + ais_hipfile_pkg_filename: ${{ needs.build_and_test.outputs.ais_hipfile_pkg_filename }} + ais_mount_path: /mnt/ais/lvm_mirror + ci_image: ${{ needs.build_and_test.outputs.ci_image }} + cxx_compiler: ${{ inputs.cxx_compiler }} + cxx_standard: ${{ inputs.cxx_standard }} + mount_label: system_lvm_mirror platform: ${{ inputs.platform }} rocm_version: ${{ inputs.rocm_version }} diff --git a/.github/workflows/hipfile-clang-tidy.yml b/.github/workflows/hipfile-clang-tidy.yml index c91c43246425..c6da26035393 100644 --- a/.github/workflows/hipfile-clang-tidy.yml +++ b/.github/workflows/hipfile-clang-tidy.yml @@ -113,13 +113,13 @@ jobs: -DCMAKE_CXX_COMPILER=amdclang++ \ -DCMAKE_HIP_ARCHITECTURES="${{ env.AIS_HIP_ARCHITECTURES }}" \ -DCMAKE_HIP_PLATFORM=amd \ - -DAIS_USE_CLANG_TIDY=ON \ - -DAIS_CLANG_TIDY_EXECUTABLE=clang-tidy-${AIS_CLANG_TIDY_VERSION} \ + -DHIPFILE_USE_CLANG_TIDY=ON \ + -DHIPFILE_CLANG_TIDY_EXECUTABLE=clang-tidy-${AIS_CLANG_TIDY_VERSION} \ ../../rocm-systems/projects/hipfile ' - name: Compile hipFile with clang-tidy # clang-tidy runs as part of compilation via CMAKE_CXX_CLANG_TIDY. - # AISClangTidy.cmake passes -warnings-as-errors=*, so any finding fails + # HipFileClangTidy.cmake passes -warnings-as-errors=*, so any finding fails # the build. No tests are run; we only care about clang-tidy diagnostics. run: | docker exec \ diff --git a/.github/workflows/hipfile-profiler-abi.yml b/.github/workflows/hipfile-profiler-abi.yml index 197da3036c51..d44ff5c17c6d 100644 --- a/.github/workflows/hipfile-profiler-abi.yml +++ b/.github/workflows/hipfile-profiler-abi.yml @@ -47,6 +47,7 @@ jobs: sparse-checkout: | projects/hipfile projects/rocprofiler-sdk + shared/sha256 .github/workflows - name: Authenticating to GitHub Container Registry uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 diff --git a/.github/workflows/hipfile-python-bindings.yml b/.github/workflows/hipfile-python-bindings.yml index dab2f7dd9085..14d7949efd64 100644 --- a/.github/workflows/hipfile-python-bindings.yml +++ b/.github/workflows/hipfile-python-bindings.yml @@ -10,6 +10,9 @@ env: AIS_INPUT_HIPFILE_PKG_FILENAME: ${{ inputs.ais_hipfile_pkg_filename }} AIS_INPUT_PLATFORM: ${{ inputs.platform }} AIS_INPUT_ROCM_VERSION: ${{ inputs.rocm_version }} + AIS_INPUT_RUN_HARDWARE_TEST: ${{ inputs.run_hardware_test }} + # Host path of the AIS-capable filesystem the hardware test does its IO on. + AIS_MOUNT_PATH: /mnt/ais/ext4 AIS_PKG_MGR: >- ${{ startsWith(inputs.platform, 'rocky') && 'dnf' || @@ -44,6 +47,17 @@ on: description: "Target ROCm MAJOR.MINOR.PATCH Version" required: true type: string + run_hardware_test: + description: >- + Run python/tests/main.py against real hardware. Also selects the runner: + when true the job moves off the GitHub-hosted runner onto a + self-hosted AIS one. Off by default: it needs a GPU and an + AIS-capable filesystem, and it is platform/compiler independent, so + the caller enables it for a single representative matrix leg rather + than all of them. + required: false + default: false + type: boolean outputs: hipfile_python_sdist_filename: description: "Filename of the hipFile Python sdist artifact" @@ -55,7 +69,10 @@ permissions: jobs: build_python_bindings: - runs-on: [ubuntu-24.04] + # The sdist build itself is platform-agnostic and happy on a GitHub-hosted + # runner, so only the leg that also runs the hardware test claims one of the + # scarce self-hosted AIS runners (shared with the AIS_system_* jobs). + runs-on: ["${{ inputs.run_hardware_test && 'AIS' || 'ubuntu-24.04' }}"] outputs: hipfile_python_sdist_filename: ${{ steps.sdist-metadata.outputs.HIPFILE_PYTHON_SDIST_FILENAME }} steps: @@ -89,11 +106,23 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Starting Docker Container + # The GPU devices and the AIS filesystem mount only exist on the AIS + # runner, so they are spliced in solely for the hardware-test leg. run: | + AIS_HARDWARE_ARGS=() + if [[ "${AIS_INPUT_RUN_HARDWARE_TEST}" == "true" ]]; then + AIS_HARDWARE_ARGS=( + --device=/dev/kfd + --device=/dev/dri + --security-opt seccomp=unconfined + -v "${AIS_MOUNT_PATH}:/mnt/ais-fs" + ) + fi docker run \ -dt \ --rm \ --pull always \ + "${AIS_HARDWARE_ARGS[@]}" \ -v ${GITHUB_WORKSPACE}:/mnt/ais:ro \ -v "${AIS_GHA_CMD_FILES_DIR}:${AIS_GHA_CMD_FILES_DOCKER_DIR}" \ --name "${AIS_CONTAINER_NAME}" \ @@ -208,6 +237,32 @@ jobs: if-no-files-found: error retention-days: 1 compression-level: 0 + # End-to-end exercise of the *real* Cython extension, and the reason this + # job claims an AIS runner for one matrix leg: the pytest suite above is + # deliberately hermetic (it fakes hipfile._hipfile), so nothing there + # touches a GPU or AIS storage. main.py does. It runs after the artifact + # upload so a hardware failure still leaves the sdist published for + # downstream consumers, while the red job keeps gating them. + - name: Run python/tests/main.py against AIS hardware + if: ${{ inputs.run_hardware_test }} + run: | + docker exec \ + -t \ + -e "_AIS_IO_DIR=/mnt/ais-fs/${AIS_CONTAINER_NAME}" \ + -w /ais/rocm-systems/projects/hipfile \ + "${AIS_CONTAINER_NAME}" \ + /bin/bash -c ' + set -e + mkdir -p "${_AIS_IO_DIR}" + head --bytes 2M /dev/urandom > "${_AIS_IO_DIR}/random_2MiB.bin" + .venv/bin/python python/tests/main.py \ + "${_AIS_IO_DIR}/random_2MiB.bin" \ + "${_AIS_IO_DIR}/output.bin" + ' + - name: Destroy hipfile IO test directory + if: ${{ always() && inputs.run_hardware_test }} + run: | + docker exec -t "${AIS_CONTAINER_NAME}" /bin/bash -c "rm -fr /mnt/ais-fs/${AIS_CONTAINER_NAME}" - name: Cleanup & Stop the Docker container if: ${{ always() }} run: | @@ -216,3 +271,12 @@ jobs: else echo "Container ${AIS_CONTAINER_NAME} does not exist; skipping stop" fi + # Only the self-hosted AIS runner has a persistent workspace to clean. + - name: Cleanup self-hosted runner workspace + if: ${{ always() && inputs.run_hardware_test }} + shell: bash + # ${VAR:?} aborts if GITHUB_WORKSPACE is unset/empty, so a missing value + # can never let the glob expand to `/*` and wipe the persistent runner. + run: | + shopt -s dotglob + rm -rf "${GITHUB_WORKSPACE:?}"/* diff --git a/.github/workflows/hipfile-sanitizers.yml b/.github/workflows/hipfile-sanitizers.yml index b496c73aec0f..39058fa2ad8c 100644 --- a/.github/workflows/hipfile-sanitizers.yml +++ b/.github/workflows/hipfile-sanitizers.yml @@ -31,9 +31,9 @@ jobs: # with it, so each runs as its own matrix leg with the appropriate flag. sanitizer: - name: asan - cmake_flag: -DAIS_USE_SANITIZERS=ON + cmake_flag: -DHIPFILE_USE_SANITIZERS=ON - name: tsan - cmake_flag: -DAIS_USE_THREAD_SANITIZER=ON + cmake_flag: -DHIPFILE_USE_THREAD_SANITIZER=ON steps: - name: Set AIS CI container name # On non-pull_request triggering workflows, AIS_PR_NUMBER may be empty. diff --git a/.github/workflows/hrr-skills.yml b/.github/workflows/hrr-skills.yml new file mode 100644 index 000000000000..ff42af4b2d23 --- /dev/null +++ b/.github/workflows/hrr-skills.yml @@ -0,0 +1,168 @@ +# Checks for the HRR agent skills. These are the checks that need no GPU: the +# unit suites each skill ships, a syntax pass over their scripts, and the +# structural rules the AMD skills catalogue enforces on anything it federates. +# The behavioural evals need an agent and a language-model gateway, so they are +# not run here. + +name: "HRR skills" + +on: + push: + branches: [develop] + paths: + - 'projects/hrr/skills/**' + # Checked for broken references by the structural job, so a change to it + # has to run this workflow too. + - 'projects/hrr/README.md' + - '.github/workflows/hrr-skills.yml' + pull_request: + paths: + - 'projects/hrr/skills/**' + - 'projects/hrr/README.md' + - '.github/workflows/hrr-skills.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + unit-tests: + name: "HRR skills: unit tests" + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + sparse-checkout: | + projects/hrr/skills + .github/workflows + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install pytest + run: python -m pip install --disable-pip-version-check 'pytest==9.1.1' + + - name: Compile every Python file + run: python -m compileall -q projects/hrr/skills + + - name: Run the tests each skill ships + run: | + set -euo pipefail + # Discovered rather than listed, so a new skill is covered the day it + # lands, and an empty result is a failure rather than a silent pass. + mapfile -t suites < <(find projects/hrr/skills -name 'test_*.py' -print | sort) + if [ ${#suites[@]} -eq 0 ]; then + echo "No skill test suites found under projects/hrr/skills." + exit 1 + fi + printf 'Found %s\n' "${suites[@]}" + # One pytest process per skill: each suite puts its own skill's + # scripts/ on sys.path, and a module one suite imports stays in + # sys.modules, so two skills naming a script the same would otherwise + # test the first one's copy. importlib mode lets test files share a name. + mapfile -t skills < <(printf '%s\n' "${suites[@]}" | cut -d/ -f1-4 | sort -u) + rc=0 + for skill in "${skills[@]}"; do + echo "::group::$skill" + python -m pytest "$skill" -q -rs --import-mode=importlib || rc=1 + echo "::endgroup::" + done + exit "$rc" + + script-syntax: + name: "HRR skills: scripts parse" + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + sparse-checkout: | + projects/hrr/skills + .github/workflows + persist-credentials: false + + - name: Parse every shell script + run: | + set -euo pipefail + mapfile -t scripts < <(find projects/hrr/skills -name '*.sh' -print | sort) + if [ ${#scripts[@]} -eq 0 ]; then + echo "No shell scripts found under projects/hrr/skills." + exit 1 + fi + printf '%s\0' "${scripts[@]}" | xargs -0 -n1 bash -n + echo "${#scripts[@]} shell script(s) parse." + + # The Windows replay path is PowerShell, and nothing else in the tree + # checks it. pwsh ships on the runner image. The errors are written out + # as text: left as objects they are buffered for a table, and exit 1 + # discards them unprinted. + - name: Parse every PowerShell script + run: | + set -euo pipefail + mapfile -t scripts < <(find projects/hrr/skills -name '*.ps1' -print | sort) + if [ ${#scripts[@]} -eq 0 ]; then + echo "No PowerShell scripts found under projects/hrr/skills." + exit 0 + fi + # The path goes in as data, so a quote in a file name cannot end the string. + for script in "${scripts[@]}"; do + HRR_PS1="$script" pwsh -NoProfile -Command ' + $errors = $null + [void][System.Management.Automation.Language.Parser]::ParseFile($env:HRR_PS1, [ref]$null, [ref]$errors) + if ($errors) { $errors | ForEach-Object { Write-Host $_ }; exit 1 } + ' + done + echo "${#scripts[@]} PowerShell script(s) parse." + + structural: + name: "HRR skills: catalogue structural checks" + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + # The whole project, not just the skills: this job checks the references + # the markdown makes, and a file left out of the checkout is + # indistinguishable from a broken link. + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + sparse-checkout: | + projects/hrr + .github/workflows + persist-credentials: false + + # The action pins itself, not the uv it installs, which is otherwise + # whatever is latest on the day. + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + version: '0.12.21' + + - name: Check there is something to check + run: | + set -euo pipefail + # skillscope suppresses its own "no skills found" error when --docs is + # given, so a rename that stops matching would leave this job green + # having graded nothing. + mapfile -t skills < <(find projects/hrr/skills -mindepth 2 -maxdepth 2 -name SKILL.md -print | sort) + if [ ${#skills[@]} -eq 0 ]; then + echo "No skills found under projects/hrr/skills." + exit 1 + fi + printf 'Found %s\n' "${skills[@]}" + + # skillscope is the harness the AMD skills catalogue runs. Pinned to a + # commit, like every action here: it grades the same skills we ship, so + # its rules changing under us is a thing to adopt deliberately rather + # than discover on a red run. + - name: Run skillscope structural checks + run: | + uvx --from git+https://github.com/amd/skillscope@68b69e2eff60c41f5d5482bb89f24e07ce4fbebb skillscope \ + --repo . structural \ + --skills-dir 'projects/hrr/skills/*' \ + --skill-files skill-card.md \ + --skill-sections Description,Owner,License \ + --docs 'projects/hrr/README.md' diff --git a/.github/workflows/profiler-hub-ci.yml b/.github/workflows/profiler-hub-ci.yml index af8894300ece..1c6fdda38e61 100644 --- a/.github/workflows/profiler-hub-ci.yml +++ b/.github/workflows/profiler-hub-ci.yml @@ -54,20 +54,16 @@ jobs: run: | sudo apt-get update sudo apt-get install -y \ - cmake \ ${{ matrix.compiler.pkg }} \ libsqlite3-dev libspdlog-dev libfmt-dev nlohmann-json3-dev \ - libgtest-dev libgmock-dev - # libbenchmark-dev on ubuntu-22.04 ships LTO bytecode from gcc 11.2, - # incompatible with the current runner's gcc 11.4 (lto1: bytecode - # stream ... version 11.2 instead of the expected 11.3). The runner - # image pre-installs libbenchmark-dev, so actively purge it on 22.04 - # and let the build fall back to FetchContent. Install on other OSes. - if [[ "${{ matrix.os }}" == "ubuntu-22.04" ]]; then - sudo apt-get purge -y libbenchmark-dev libbenchmark1 || true - else - sudo apt-get install -y libbenchmark-dev - fi + libgtest-dev libgmock-dev libbenchmark-dev + + - name: Pin cmake to the project's minimum + run: | + python3 -m pip install 'cmake==3.25' + # The runner image ships its own cmake in /usr/local/bin; prepend the + # pinned one so it wins regardless of the image's PATH order. + python3 -c 'import cmake; print(cmake.CMAKE_BIN_DIR)' >> "$GITHUB_PATH" - name: Configure working-directory: profilers/profiler-hub @@ -77,15 +73,11 @@ jobs: run: | cmake --version ${CXX} --version - EXTRA_CMAKE_ARGS=() - if [[ "${{ matrix.os }}" == "ubuntu-22.04" ]]; then - EXTRA_CMAKE_ARGS+=(-DCMAKE_DISABLE_FIND_PACKAGE_benchmark=ON) - fi cmake -S . -B build \ -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ -DPROFILER_HUB_BUILD_TESTS=ON \ -DPROFILER_HUB_BUILD_BENCHMARKS=ON \ - "${EXTRA_CMAKE_ARGS[@]}" + -DPROFILER_HUB_FETCH_DEPENDENCIES=ON - name: Build working-directory: profilers/profiler-hub diff --git a/.github/workflows/profiler-hub-coverage.yml b/.github/workflows/profiler-hub-coverage.yml index 825f57b2e59a..1ec3bebd77bd 100644 --- a/.github/workflows/profiler-hub-coverage.yml +++ b/.github/workflows/profiler-hub-coverage.yml @@ -47,12 +47,18 @@ jobs: run: | sudo apt-get update sudo apt-get install -y \ - cmake \ gcc g++ \ lcov gcovr \ libsqlite3-dev libspdlog-dev libfmt-dev nlohmann-json3-dev \ libgtest-dev libgmock-dev libbenchmark-dev + - name: Pin cmake to the project's minimum + run: | + python3 -m pip install 'cmake==3.25' + # The runner image ships its own cmake in /usr/local/bin; prepend the + # pinned one so it wins regardless of the image's PATH order. + python3 -c 'import cmake; print(cmake.CMAKE_BIN_DIR)' >> "$GITHUB_PATH" + - name: Configure working-directory: profilers/profiler-hub run: | @@ -62,7 +68,8 @@ jobs: -DCMAKE_BUILD_TYPE=Debug \ -DPROFILER_HUB_ENABLE_COVERAGE=ON \ -DPROFILER_HUB_BUILD_TESTS=ON \ - -DPROFILER_HUB_BUILD_BENCHMARKS=OFF + -DPROFILER_HUB_BUILD_BENCHMARKS=OFF \ + -DPROFILER_HUB_FETCH_DEPENDENCIES=ON - name: Build working-directory: profilers/profiler-hub diff --git a/.github/workflows/profiler-hub-install.yml b/.github/workflows/profiler-hub-install.yml index bef67af5fb0f..847ad9b95abb 100644 --- a/.github/workflows/profiler-hub-install.yml +++ b/.github/workflows/profiler-hub-install.yml @@ -52,18 +52,26 @@ jobs: run: | sudo apt-get update sudo apt-get install -y \ - cmake \ gcc g++ \ libsqlite3-dev libspdlog-dev libfmt-dev nlohmann-json3-dev + - name: Pin cmake to the project's minimum + run: | + python3 -m pip install 'cmake==3.25' + # The runner image ships its own cmake in /usr/local/bin; prepend the + # pinned one so it wins regardless of the image's PATH order. + python3 -c 'import cmake; print(cmake.CMAKE_BIN_DIR)' >> "$GITHUB_PATH" + - name: Configure profiler-hub working-directory: profilers/profiler-hub run: | + cmake --version cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DPROFILER_HUB_BUILD_TESTS=OFF \ - -DPROFILER_HUB_BUILD_BENCHMARKS=OFF + -DPROFILER_HUB_BUILD_BENCHMARKS=OFF \ + -DPROFILER_HUB_FETCH_DEPENDENCIES=ON - name: Build profiler-hub working-directory: profilers/profiler-hub diff --git a/.github/workflows/profiler-hub-sanitizers.yml b/.github/workflows/profiler-hub-sanitizers.yml index 29cd695aa9b2..bdb9a55c23a3 100644 --- a/.github/workflows/profiler-hub-sanitizers.yml +++ b/.github/workflows/profiler-hub-sanitizers.yml @@ -61,17 +61,6 @@ jobs: profilers/profiler-hub .github/workflows - # cmake, gcc/g++, util-linux are preinstalled on ubuntu-24.04 runners. - # The sqlite3/spdlog/fmt/nlohmann_json system packages are intentionally - # omitted: the project's cmake/{sqlite3,spdlog,nlohmann_json}.cmake fall - # back to FetchContent when find_package fails, so deps build with the - # sanitizer flags applied. System libs built without -fsanitize would - # produce false-positive reports. - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libgtest-dev libgmock-dev libbenchmark-dev - - name: Cache ccache uses: actions/cache@v6 with: @@ -82,10 +71,21 @@ jobs: - name: Setup ccache run: | + sudo apt-get update sudo apt-get install -y ccache ccache --max-size=1G ccache --zero-stats + - name: Pin cmake to the project's minimum + run: | + python3 -m pip install 'cmake==3.25' + # The runner image ships its own cmake in /usr/local/bin; prepend the + # pinned one so it wins regardless of the image's PATH order. + python3 -c 'import cmake; print(cmake.CMAKE_BIN_DIR)' >> "$GITHUB_PATH" + + # A library compiled without -fsanitize is invisible to ASan, and its + # synchronisation is missing from TSan's happens-before model, producing + # both missed races and spurious ones. - name: Configure working-directory: profilers/profiler-hub env: @@ -98,6 +98,8 @@ jobs: -DCMAKE_BUILD_TYPE=Debug \ -DPROFILER_HUB_BUILD_TESTS=ON \ -DPROFILER_HUB_BUILD_BENCHMARKS=OFF \ + -DPROFILER_HUB_FETCH_DEPENDENCIES=ON \ + -DFETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_C_FLAGS="${{ matrix.sanitizer.cflags }}" \ diff --git a/.github/workflows/profiler-hub-static-analysis.yml b/.github/workflows/profiler-hub-static-analysis.yml index 6c22fb9e6363..b9aaf2e7a3f7 100644 --- a/.github/workflows/profiler-hub-static-analysis.yml +++ b/.github/workflows/profiler-hub-static-analysis.yml @@ -42,16 +42,23 @@ jobs: profilers/profiler-hub .github/workflows + # No gtest/gmock/benchmark: this job configures with tests and benchmarks + # off, and cmake/{gtest,benchmark}.cmake are reached only from tests/. - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y \ - cmake \ gcc g++ clang-tidy-18 \ - libsqlite3-dev libspdlog-dev libfmt-dev nlohmann-json3-dev \ - libgtest-dev libgmock-dev libbenchmark-dev + libsqlite3-dev libspdlog-dev libfmt-dev nlohmann-json3-dev sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-18 100 + - name: Pin cmake to the project's minimum + run: | + python3 -m pip install 'cmake==3.25' + # The runner image ships its own cmake in /usr/local/bin; prepend the + # pinned one so it wins regardless of the image's PATH order. + python3 -c 'import cmake; print(cmake.CMAKE_BIN_DIR)' >> "$GITHUB_PATH" + - name: Configure working-directory: profilers/profiler-hub env: @@ -64,7 +71,8 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DPROFILER_HUB_BUILD_TESTS=OFF \ - -DPROFILER_HUB_BUILD_BENCHMARKS=OFF + -DPROFILER_HUB_BUILD_BENCHMARKS=OFF \ + -DPROFILER_HUB_FETCH_DEPENDENCIES=ON - name: Build working-directory: profilers/profiler-hub diff --git a/.github/workflows/rccl-default-behavior-test.yml b/.github/workflows/rccl-default-behavior-test.yml new file mode 100644 index 000000000000..62aea0b09ad9 --- /dev/null +++ b/.github/workflows/rccl-default-behavior-test.yml @@ -0,0 +1,47 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +name: RCCL default behavior tests + +# Tests for .github/scripts/rccl_check_default_behavior.py. Do not make this a +# required check: it installs pytest from PyPI, and neither an outage nor a +# failing test should block PRs that do not touch the checker. + +on: + pull_request: + paths: + - .github/scripts/rccl_check_default_behavior.py + - .github/scripts/tests/conftest.py + - .github/scripts/tests/rccl_check_default_behavior_test.py + - .github/workflows/rccl-default-behavior.yml + - .github/workflows/rccl-default-behavior-test.yml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + test: + name: Test the RCCL env-var changelog check + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + # The tree-wide parse test reads every file under projects/rccl/src. + sparse-checkout: | + .github/scripts + projects/rccl/src + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install pytest + run: python3 -m pip install --disable-pip-version-check pytest + + - name: Run the tests + run: python3 -m pytest .github/scripts/tests/rccl_check_default_behavior_test.py diff --git a/.github/workflows/rccl-default-behavior.yml b/.github/workflows/rccl-default-behavior.yml new file mode 100644 index 000000000000..1e8dbf349ca4 --- /dev/null +++ b/.github/workflows/rccl-default-behavior.yml @@ -0,0 +1,55 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +name: RCCL default behavior + +# Every PR into develop waits for this check, so the job runs only git and the +# script on the runner's python3, and the script must stay standard-library +# only. Its tests run in rccl-default-behavior-test.yml, which is not required. +# +# No `paths:` filter: a path-skipped workflow reports no check, so a required +# status would never pass on a PR that does not touch RCCL. The script exits 0 +# when the diff has no env-var change. +# +# pull-requests: write lets the script keep one marker-tagged PR comment up to +# date with the workflow token; it grants no write access to repository +# contents. A fork's token is read-only, so there the comment fails with a +# warning and the check result stands. + +on: + pull_request: + branches: [develop, main, rccl-develop, rccl_develop] + +permissions: + contents: read + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + default-behavior: + # A ruleset requires this check by name. Renaming the job leaves every PR + # waiting on a check that never reports until the ruleset is updated. + name: RCCL env-var changelog + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + # HEAD^ on a pull_request merge commit is the base branch tip. + fetch-depth: 2 + # Broad source scans would otherwise fetch blobs individually. + sparse-checkout: | + .github/scripts + projects/rccl/src + projects/rccl/docs/api-reference + projects/rccl/docs/userguide/source + + - name: Check env-var changes are in the changelog + env: + GITHUB_TOKEN: ${{ github.token }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_BODY: ${{ github.event.pull_request.body }} + run: python3 .github/scripts/rccl_check_default_behavior.py --base-ref HEAD^ diff --git a/.github/workflows/rocjitsu-benchmarks.yml b/.github/workflows/rocjitsu-benchmarks.yml index d0ededc4b510..c3ecbf0e3619 100644 --- a/.github/workflows/rocjitsu-benchmarks.yml +++ b/.github/workflows/rocjitsu-benchmarks.yml @@ -170,7 +170,7 @@ jobs: repository: ROCm/rocm-systems path: dashboard persist-credentials: false - sparse-checkout: emulation/rocjitsu/website + sparse-checkout: emulation/rocjitsu/website/dashboard - name: Set up publisher Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -233,7 +233,7 @@ jobs: --is-beta ) - node "${GITHUB_WORKSPACE}/dashboard/emulation/rocjitsu/website/scripts/validate-dashboard-data.mjs" \ + node "${GITHUB_WORKSPACE}/dashboard/emulation/rocjitsu/website/dashboard/scripts/validate-dashboard-data.mjs" \ "${GITHUB_WORKSPACE}/publication/rocjitsu-dashboard/data" git add rocjitsu-dashboard/data diff --git a/.github/workflows/rocjitsu-corpus-tests.yml b/.github/workflows/rocjitsu-corpus-tests.yml index bbb64978391d..8ef0e8f90eb2 100644 --- a/.github/workflows/rocjitsu-corpus-tests.yml +++ b/.github/workflows/rocjitsu-corpus-tests.yml @@ -9,6 +9,7 @@ on: pull_request: paths: - "emulation/rocjitsu/**" + - "emulation/mirage/**" - ".github/scripts/rocjitsu_corpus_translation_comment*.py" - ".github/workflows/rocjitsu-*.yml" push: @@ -16,6 +17,7 @@ on: - develop paths: - "emulation/rocjitsu/**" + - "emulation/mirage/**" - ".github/workflows/rocjitsu-*.yml" workflow_dispatch: inputs: @@ -24,10 +26,15 @@ on: default: "ROCm/rocjitsu-test-corpus" required: true corpus_ref: - description: "Branch, tag, or SHA used by all corpus tests" + description: "Branch, tag, or SHA used by the HIP and DBT corpus tests" # Last updated on 2026/08/11. default: "cc781dd88cabd1638d1beeb4539c521ccf27de60" required: true + vulkan_corpus_ref: + description: "Branch, tag, or SHA used by the Vulkan compute/buffer CTS tests" + # Last updated on 2026/09/25. + default: "4c7fdd08d55bc1801cf1525732d6cabad06e05de" + required: true concurrency: # Preserve every non-PR run, including manual reruns of one develop commit. @@ -43,6 +50,8 @@ env: ROCJITSU_SOURCE_DIR: ${{ github.workspace }}/rocm-systems/emulation/rocjitsu ROCJITSU_BUILD_DIR: ${{ github.workspace }}/build ROCJITSU_CORPUS_DIR: ${{ github.workspace }}/rocjitsu-test-corpus + ROCJITSU_VULKAN_CORPUS_DIR: ${{ github.workspace }}/rocjitsu-vulkan-corpus + ROCJITSU_VULKAN_CORPUS_REF: 4c7fdd08d55bc1801cf1525732d6cabad06e05de ROCJITSU_DBT_VENV: ${{ github.workspace }}/gfx1250-dbt-venv-${{ github.run_id }}-${{ github.run_attempt }} ROCJITSU_HSACO_CORPUS: ${{ github.workspace }}/gfx1250-packaged-hsacos-${{ github.run_id }}-${{ github.run_attempt }} ROCJITSU_DBT_CORPUS_REF: cc781dd88cabd1638d1beeb4539c521ccf27de60 @@ -59,7 +68,14 @@ jobs: name: test (${{ matrix.name }}) container: image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98 - options: --user 0:0 + # `--init` gives the container a PID 1 that reaps orphans, as every + # real machine has. Without it a process whose parent exits is + # reparented to a PID 1 that never calls `wait`, so it stays a + # zombie for the life of the job: `/proc/` remains and + # `kill(pid, 0)` still succeeds. Mirage's teardown tests assert a + # process tree is *gone* after a reap, and they cannot distinguish + # that from a leak. + options: --user 0:0 --init env: DEBIAN_FRONTEND: noninteractive continue-on-error: ${{ matrix.allow_failure }} @@ -92,8 +108,10 @@ jobs: corpus_sanitizer: none - name: asan-ubsan allow_failure: true - # Sanitized simulator cases need extra interpreter headroom. - # A failed-test rerun preserves extra diagnostics. + # Sanitized simulator cases need extra interpreter headroom. Long + # gfx1100 FPSAN cases can cross four minutes under shared-runner + # load, so give the first pass the same six-minute budget as + # Release and retain a longer failed-test diagnostic rerun. verify_dbt_idempotence: 0 dbt_timeout: 180 dbt_memory_limit_mib: 4096 @@ -107,8 +125,8 @@ jobs: sanitizer_runtime: SHARED run_simulator_corpus: true corpus_worker_limit: 4 - corpus_soft_timeout: 240 - corpus_hard_timeout: 480 + corpus_soft_timeout: 360 + corpus_hard_timeout: 600 corpus_rerun_timeout: 1200 corpus_sanitizer: clang-asan - name: tsan @@ -162,6 +180,7 @@ jobs: build-essential \ ca-certificates \ cmake \ + curl \ git \ libclang-rt-dev \ libdrm-dev \ @@ -174,6 +193,7 @@ jobs: path: rocm-systems sparse-checkout: | emulation/rocjitsu + emulation/mirage .github shared/machine-readable-isa/isa @@ -370,6 +390,38 @@ jobs: exit 1 fi + - name: Install Rust for Mirage + if: matrix.name == 'release' + shell: bash + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --profile minimal --default-toolchain 1.88.0 + echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" + + - name: Test Mirage against this RocJITsu build + if: matrix.name == 'release' + working-directory: ${{ github.workspace }}/rocm-systems/emulation/mirage + # A cold, uncached debug build of the whole workspace plus ~270 + # process-spawning integration tests, several of which carry + # 60-90s readiness deadlines of their own. A kill here produces no + # libtest output at all, so there is nothing to triage. + timeout-minutes: 30 + shell: bash + env: + ROCJITSU_LIB: ${{ env.ROCJITSU_BUILD_DIR }}/librocjitsu.so + ROCJITSU_HOOKS_LIB: ${{ env.ROCJITSU_BUILD_DIR }}/lib/rocjitsu/src/rocjitsu/hooks/librocjitsu_hooks.so + ROCJITSU_INCLUDE_DIR: ${{ env.ROCJITSU_SOURCE_DIR }}/lib/rocjitsu/include + run: | + test -f "${ROCJITSU_LIB}" + test -f "${ROCJITSU_HOOKS_LIB}" + # Setting ROCJITSU_INCLUDE_DIR is a hard override: the ABI + # layout test early-returns on it instead of falling back to + # its own search, so a wrong path here does not fail — it + # silently skips the one check that catches header drift. + test -d "${ROCJITSU_INCLUDE_DIR}" + unset MIRAGE_E2E_ALLOW_SKIP + cargo test --locked --workspace --no-fail-fast -- --test-threads=4 --nocapture + - name: Test static ASan launcher preload if: matrix.run_static_asan_launch working-directory: ${{ env.ROCJITSU_SOURCE_DIR }} @@ -394,6 +446,7 @@ jobs: --output-on-failure - name: Install corpus test dependencies + id: install_corpus_dependencies if: matrix.run_simulator_corpus working-directory: ${{ env.ROCJITSU_CORPUS_DIR }} shell: bash @@ -418,10 +471,13 @@ jobs: python -m pytest lib/python/amdisa/tests - name: Run corpus tests - if: matrix.run_simulator_corpus - # Clean runs on this change completed in 4-13 minutes. The sanitizer - # rerun budget is capped at 20 minutes within this step-level failsafe. - timeout-minutes: 60 + # Keep the HIP corpus signal even if an independent test step fails. + if: ${{ !cancelled() && matrix.run_simulator_corpus && steps.build_rocjitsu.outcome == 'success' && steps.install_corpus_dependencies.outcome == 'success' }} + # A clean sanitizer sweep currently takes about 38 minutes. Each target + # may then consume its own 20-minute failed-test rerun budget, so leave + # enough headroom for two independently flaky targets before the + # step-level failsafe stops the whole corpus. + timeout-minutes: 90 working-directory: ${{ env.ROCJITSU_CORPUS_DIR }} shell: bash env: @@ -673,3 +729,70 @@ jobs: path: ${{ env.ROCJITSU_CORPUS_DIR }}/.pytest-artifacts/gfx1250-dbt if-no-files-found: ignore retention-days: 7 + + # Finish existing HIP/DBT coverage before starting Vulkan-only setup. + - name: Install Mesa RADV for Vulkan compute CTS + id: install_vulkan_dependencies + if: ${{ !cancelled() && matrix.name == 'release' && steps.build_rocjitsu.outcome == 'success' }} + shell: bash + run: | + apt-get install -y --no-install-recommends \ + mesa-vulkan-drivers \ + libvulkan1 \ + libdrm-amdgpu1 \ + libpng-dev \ + pkg-config + # The recorded RDNA3/RDNA4 selection used these versions. Fail here + # if an old container mirror cannot provide RDNA4-capable userspace. + dpkg --compare-versions "$(dpkg-query -W -f='${Version}' mesa-vulkan-drivers)" ge 25.2.8 + dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libdrm-amdgpu1)" ge 2.4.125 + dpkg-query -W mesa-vulkan-drivers libvulkan1 libdrm-amdgpu1 + + # Keep the existing simulator/DBT pin and its translation baseline stable. + - name: Checkout Vulkan corpus + id: checkout_vulkan_corpus + if: ${{ !cancelled() && matrix.name == 'release' && steps.install_vulkan_dependencies.outcome == 'success' }} + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: ${{ github.event.inputs.corpus_repository || 'ROCm/rocjitsu-test-corpus' }} + ref: ${{ github.event.inputs.vulkan_corpus_ref || env.ROCJITSU_VULKAN_CORPUS_REF }} + path: rocjitsu-vulkan-corpus + + - name: Build pinned Vulkan CTS + id: build_vulkan_cts + if: ${{ !cancelled() && matrix.name == 'release' && steps.install_vulkan_dependencies.outcome == 'success' && steps.checkout_vulkan_corpus.outcome == 'success' }} + # Includes dependency downloads and a fresh Release build. + timeout-minutes: 30 + working-directory: ${{ env.ROCJITSU_VULKAN_CORPUS_DIR }} + shell: bash + run: | + uv pip install --system -r corpus/vulkan/requirements.txt + python3 -m pytest tests/test_vulkan_suite.py -q + bash scripts/build_vulkan_cts.sh + + # Allow runner headroom for the full compute/buffer selection. + - name: Run Vulkan compute and buffer CTS on RDNA3 + if: ${{ !cancelled() && matrix.name == 'release' && steps.build_rocjitsu.outcome == 'success' && steps.build_vulkan_cts.outcome == 'success' }} + timeout-minutes: 30 + shell: bash + env: + ROCJITSU_CORPUS_DIR: ${{ env.ROCJITSU_VULKAN_CORPUS_DIR }} + run: bash "${ROCJITSU_SOURCE_DIR}/tests/corpus/run-vulkan-cts.sh" gfx1100 --case ci + + - name: Run Vulkan compute and buffer CTS on RDNA4 + if: ${{ !cancelled() && matrix.name == 'release' && steps.build_rocjitsu.outcome == 'success' && steps.build_vulkan_cts.outcome == 'success' }} + timeout-minutes: 30 + shell: bash + env: + ROCJITSU_CORPUS_DIR: ${{ env.ROCJITSU_VULKAN_CORPUS_DIR }} + run: bash "${ROCJITSU_SOURCE_DIR}/tests/corpus/run-vulkan-cts.sh" gfx1201 --case ci + + - name: Upload Vulkan CTS diagnostics + if: ${{ always() && matrix.name == 'release' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rocjitsu-vulkan-cts-release + path: ${{ env.ROCJITSU_VULKAN_CORPUS_DIR }}/.pytest-artifacts/vulkan-* + include-hidden-files: true + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/rocjitsu-formatting.yml b/.github/workflows/rocjitsu-formatting.yml index c007ae25642c..6646bc818d82 100644 --- a/.github/workflows/rocjitsu-formatting.yml +++ b/.github/workflows/rocjitsu-formatting.yml @@ -1,4 +1,4 @@ -# Formatting checks for emulation/rocjitsu/. +# Formatting checks for the emulation/ subtrees. name: "rocjitsu-formatting" on: @@ -8,7 +8,12 @@ on: # `pre-commit run --all-files` locally to catch issues outside this # scope. We should incrementally add more paths here as projects # adopt pre-commit. + # This job is named `pre-commit`, which is the one entry in + # `checks.required_check_runs` in tools/systems_pr_bot/policy.yml, so a + # path that is missing here leaves therock-pr-bot waiting out its timeout + # on a check run that is never created. paths: + - "emulation/mirage/**" - "emulation/rocjitsu/**" - ".github/workflows/rocjitsu-*.yml" - ".pre-commit-config.yaml" @@ -31,6 +36,7 @@ jobs: # which files pre-commit can check. To add a new project to this # CI job, add its path here. That's the only change needed. sparse-checkout: | + emulation/mirage emulation/rocjitsu .github diff --git a/.github/workflows/rocjitsu-handbook-test.yml b/.github/workflows/rocjitsu-handbook-test.yml new file mode 100644 index 000000000000..896932372576 --- /dev/null +++ b/.github/workflows/rocjitsu-handbook-test.yml @@ -0,0 +1,66 @@ +name: "rocjitsu-handbook" + +on: + pull_request: + paths: + # Source links can target any file within Rocjitsu. + - "emulation/rocjitsu/**" + - "!emulation/rocjitsu/docs/sphinx/**" + - ".github/workflows/rocjitsu-handbook-test.yml" + - ".github/workflows/rocjitsu-publish-website.yml" + push: + branches: + - develop + paths: + - "emulation/rocjitsu/**" + - "!emulation/rocjitsu/docs/sphinx/**" + - ".github/workflows/rocjitsu-handbook-test.yml" + - ".github/workflows/rocjitsu-publish-website.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + verify: + name: Verify rocjitsu handbook + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: emulation/rocjitsu/website/handbook + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: emulation/rocjitsu/website/handbook/requirements.txt + + - name: Install dependencies + run: | + python -m venv .venv + .venv/bin/python -m pip install -r requirements.txt + + - name: Build handbook + run: .venv/bin/python -m mkdocs build --strict -f mkdocs.yml + + - name: Test canonical documentation publishing + run: .venv/bin/python -m unittest discover -s tests + + - name: Set up Node.js for browser-script tests + uses: actions/setup-node@v4 + with: + node-version: "22.13.0" + + - name: Test repository counts + run: node --test tests/repository-stats.test.cjs diff --git a/.github/workflows/rocjitsu-publish-website.yml b/.github/workflows/rocjitsu-publish-website.yml new file mode 100644 index 000000000000..6d14af65790e --- /dev/null +++ b/.github/workflows/rocjitsu-publish-website.yml @@ -0,0 +1,165 @@ +# Publish the Rocjitsu handbook and dashboard from develop. +# Replace only gh-pages/rocjitsu/ and gh-pages/rocjitsu-dashboard/. + +name: "rocjitsu-publish-website" + +on: + push: + branches: + - develop + paths: + # Handbook source links can target any file within Rocjitsu. + - "emulation/rocjitsu/**" + - "!emulation/rocjitsu/docs/sphinx/**" + - ".github/workflows/rocjitsu-publish-website.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: rocjitsu-publish-website + cancel-in-progress: false + +jobs: + publish: + name: Publish rocjitsu websites + if: github.ref == 'refs/heads/develop' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # A delayed run must publish current develop, not roll back to its event SHA. + ref: develop + persist-credentials: false + + - name: Read published source revisions + id: published + shell: bash + run: | + echo "head=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + if git ls-remote --exit-code origin refs/heads/gh-pages > /dev/null; then + git fetch --no-tags --depth=1 origin gh-pages + for entry in dashboard:rocjitsu-dashboard handbook:rocjitsu; do + name=${entry%%:*} + destination=${entry#*:} + revision=$(git show "FETCH_HEAD:$destination/.source-revision" 2>/dev/null || true) + if [[ -n "$revision" && ! "$revision" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid source revision for $name" >&2 + exit 1 + fi + echo "$name=$revision" >> "$GITHUB_OUTPUT" + done + else + status=$? + # No Pages branch means neither site has been published yet. + if [[ "$status" -ne 2 ]]; then exit "$status"; fi + fi + + - name: Detect dashboard changes since publication + id: dashboard + if: github.event_name == 'push' && steps.published.outputs.dashboard != '' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3.0.4 + with: + base: ${{ steps.published.outputs.dashboard }} + ref: ${{ steps.published.outputs.head }} + filters: | + changed: + - 'emulation/rocjitsu/website/dashboard/**' + - '.github/workflows/rocjitsu-publish-website.yml' + + - name: Detect handbook changes since publication + id: handbook + if: github.event_name == 'push' && steps.published.outputs.handbook != '' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3.0.4 + with: + base: ${{ steps.published.outputs.handbook }} + ref: ${{ steps.published.outputs.head }} + predicate-quantifier: every + filters: | + changed: + - '{emulation/rocjitsu/**,.github/workflows/rocjitsu-publish-website.yml}' + - '!emulation/rocjitsu/docs/sphinx/**' + + - name: Select websites to publish + id: changes + env: + DASHBOARD: ${{ github.event_name == 'workflow_dispatch' || steps.published.outputs.dashboard == '' || steps.dashboard.outputs.changed == 'true' }} + HANDBOOK: ${{ github.event_name == 'workflow_dispatch' || steps.published.outputs.handbook == '' || steps.handbook.outputs.changed == 'true' }} + run: | + echo "dashboard=$DASHBOARD" >> "$GITHUB_OUTPUT" + echo "handbook=$HANDBOOK" >> "$GITHUB_OUTPUT" + + - name: Set up Node.js + if: steps.changes.outputs.dashboard == 'true' || steps.changes.outputs.handbook == 'true' + uses: actions/setup-node@v4 + with: + node-version: "22.13.0" + cache: ${{ (steps.changes.outputs.dashboard == 'true') && 'npm' || '' }} + cache-dependency-path: emulation/rocjitsu/website/dashboard/package-lock.json + + - name: Verify and build rocjitsu dashboard + if: steps.changes.outputs.dashboard == 'true' + working-directory: emulation/rocjitsu/website/dashboard + run: | + npm ci + npm run test:e2e:install + npm run verify + git rev-parse HEAD > dist/.source-revision + + - name: Set up Python + if: steps.changes.outputs.handbook == 'true' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: emulation/rocjitsu/website/handbook/requirements.txt + + - name: Build rocjitsu handbook + if: steps.changes.outputs.handbook == 'true' + working-directory: emulation/rocjitsu/website/handbook + run: | + python -m venv .venv + .venv/bin/python -m pip install -r requirements.txt + .venv/bin/python -m mkdocs build --strict -f mkdocs.yml + .venv/bin/python -m unittest discover -s tests + node --test tests/repository-stats.test.cjs + git rev-parse HEAD > site/.source-revision + + - name: Generate publishing token + if: steps.changes.outputs.dashboard == 'true' || steps.changes.outputs.handbook == 'true' + id: app-token + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + + - name: Publish rocjitsu dashboard + if: steps.changes.outputs.dashboard == 'true' + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 + with: + # GITHUB_TOKEN pushes do not trigger branch-based Pages builds. + github_token: ${{ steps.app-token.outputs.token }} + publish_branch: gh-pages + publish_dir: ./emulation/rocjitsu/website/dashboard/dist + destination_dir: rocjitsu-dashboard + commit_message: "Publish rocjitsu dashboard" + # Delete stale files only within this destination. + keep_files: false + # Preserve the existing Pages root, including its Jekyll configuration. + enable_jekyll: true + + - name: Publish rocjitsu handbook + if: steps.changes.outputs.handbook == 'true' + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 + with: + github_token: ${{ steps.app-token.outputs.token }} + publish_branch: gh-pages + publish_dir: ./emulation/rocjitsu/website/handbook/site + destination_dir: rocjitsu + commit_message: "Publish rocjitsu handbook" + keep_files: false + enable_jekyll: true diff --git a/.github/workflows/rocjitsu-website-test.yml b/.github/workflows/rocjitsu-website-test.yml index caf6838f871d..e7316ec8a9af 100644 --- a/.github/workflows/rocjitsu-website-test.yml +++ b/.github/workflows/rocjitsu-website-test.yml @@ -3,14 +3,16 @@ name: "rocjitsu-website" on: pull_request: paths: - - "emulation/rocjitsu/website/**" + - "emulation/rocjitsu/website/dashboard/**" - ".github/workflows/rocjitsu-website-test.yml" + - ".github/workflows/rocjitsu-publish-website.yml" push: branches: - develop paths: - - "emulation/rocjitsu/website/**" + - "emulation/rocjitsu/website/dashboard/**" - ".github/workflows/rocjitsu-website-test.yml" + - ".github/workflows/rocjitsu-publish-website.yml" workflow_dispatch: permissions: @@ -27,7 +29,7 @@ jobs: timeout-minutes: 10 defaults: run: - working-directory: emulation/rocjitsu/website + working-directory: emulation/rocjitsu/website/dashboard steps: - name: Check out repository uses: actions/checkout@v4 @@ -37,7 +39,7 @@ jobs: with: node-version: "22.13.0" cache: npm - cache-dependency-path: emulation/rocjitsu/website/package-lock.json + cache-dependency-path: emulation/rocjitsu/website/dashboard/package-lock.json - name: Install dependencies run: npm ci @@ -45,23 +47,8 @@ jobs: - name: Install Playwright browser run: npm run test:e2e:install - - name: Lint - run: npm run lint - - - name: Build - run: npm run build - - - name: Run unit tests - run: npm run test:unit - - - name: Run e2e tests - run: npm run test:e2e - - - name: Smoke test production hosting and caching - run: npm run test:e2e:production - - - name: Run chart race tests - run: npm run test:e2e:chart-race + - name: Verify website + run: npm run verify - name: Upload Playwright diagnostics if: ${{ !cancelled() && failure() }} @@ -69,7 +56,7 @@ jobs: with: name: rocjitsu-website-playwright-results path: | - emulation/rocjitsu/website/test-results - emulation/rocjitsu/website/playwright-report + emulation/rocjitsu/website/dashboard/test-results + emulation/rocjitsu/website/dashboard/playwright-report if-no-files-found: ignore retention-days: 7 diff --git a/.github/workflows/rocprof-trace-decoder-continuous_integration.yml b/.github/workflows/rocprof-trace-decoder-continuous_integration.yml index 375a3856ae72..669b13b4bdd3 100644 --- a/.github/workflows/rocprof-trace-decoder-continuous_integration.yml +++ b/.github/workflows/rocprof-trace-decoder-continuous_integration.yml @@ -176,6 +176,7 @@ jobs: sparse-checkout: | .github/actions/rocprofiler-sdk-util projects/rocprofiler-sdk + shared/sha256 projects/aqlprofile projects/rocprofiler-register projects/rocprof-trace-decoder diff --git a/.github/workflows/rocprofiler-sdk-code_coverage.yml b/.github/workflows/rocprofiler-sdk-code_coverage.yml index 55ce5fa51dad..0b4d075e3df2 100644 --- a/.github/workflows/rocprofiler-sdk-code_coverage.yml +++ b/.github/workflows/rocprofiler-sdk-code_coverage.yml @@ -10,6 +10,7 @@ on: - develop paths: - 'projects/rocprofiler-sdk/**' + - 'shared/sha256/**' - '.github/actions/rocprofiler-sdk-util/**' - '.github/workflows/rocprofiler-sdk-code_coverage.yml' - '.github/actions/rocprofiler-sdk-util/action.yml' @@ -26,6 +27,7 @@ on: pull_request: paths: - 'projects/rocprofiler-sdk/**' + - 'shared/sha256/**' - '.github/actions/rocprofiler-sdk-util/**' - '.github/workflows/rocprofiler-sdk-code_coverage.yml' - '.github/actions/rocprofiler-sdk-util/action.yml' @@ -137,6 +139,7 @@ jobs: sparse-checkout: | .github/actions/rocprofiler-sdk-util projects/rocprofiler-sdk + shared/sha256 projects/aqlprofile projects/rocprofiler-register projects/rocr-runtime diff --git a/.github/workflows/rocprofiler-sdk-codeql.yml b/.github/workflows/rocprofiler-sdk-codeql.yml index 99bc625ae829..6452f62b6860 100644 --- a/.github/workflows/rocprofiler-sdk-codeql.yml +++ b/.github/workflows/rocprofiler-sdk-codeql.yml @@ -7,6 +7,7 @@ on: - '.github/workflows/rocprofiler-sdk-codeql.yml' - '.github/actions/rocprofiler-sdk-util/action.yml' - 'projects/rocprofiler-sdk/**' + - 'shared/sha256/**' - '!**/*.md' - '!**/*.rtf' - '!**/*.rst' @@ -24,6 +25,7 @@ on: - '.github/workflows/rocprofiler-sdk-codeql.yml' - '.github/actions/rocprofiler-sdk-util/action.yml' - 'projects/rocprofiler-sdk/**' + - 'shared/sha256/**' - '!**/*.md' - '!**/*.rtf' - '!**/*.rst' @@ -104,6 +106,7 @@ jobs: sparse-checkout: | .github/actions/rocprofiler-sdk-util projects/rocprofiler-sdk + shared/sha256 projects/clr projects/hip .github/workflows/rocprofiler-sdk-codeql.yml diff --git a/.github/workflows/rocprofiler-sdk-continuous_integration.yml b/.github/workflows/rocprofiler-sdk-continuous_integration.yml index 337a33d2a665..e26d80c867cd 100644 --- a/.github/workflows/rocprofiler-sdk-continuous_integration.yml +++ b/.github/workflows/rocprofiler-sdk-continuous_integration.yml @@ -8,6 +8,7 @@ on: branches: [ develop ] paths: - 'projects/rocprofiler-sdk/**' + - 'shared/sha256/**' - '.github/actions/rocprofiler-sdk-util/**' - '.github/workflows/rocprofiler-sdk-continuous_integration.yml' - '.github/actions/rocprofiler-sdk-util/action.yml' @@ -24,6 +25,7 @@ on: pull_request: paths: - 'projects/rocprofiler-sdk/**' + - 'shared/sha256/**' - '.github/actions/rocprofiler-sdk-util/**' - '.github/workflows/rocprofiler-sdk-continuous_integration.yml' - '.github/actions/rocprofiler-sdk-util/action.yml' @@ -151,6 +153,7 @@ jobs: sparse-checkout: | .github/actions/rocprofiler-sdk-util projects/rocprofiler-sdk + shared/sha256 projects/aqlprofile projects/rocprofiler-register projects/rocprof-trace-decoder @@ -397,6 +400,7 @@ jobs: sparse-checkout: | .github/actions/rocprofiler-sdk-util projects/rocprofiler-sdk + shared/sha256 projects/aqlprofile projects/rocprofiler-register projects/rocprof-trace-decoder @@ -546,6 +550,7 @@ jobs: sparse-checkout: | .github/actions/rocprofiler-sdk-util projects/rocprofiler-sdk + shared/sha256 projects/aqlprofile projects/rocprofiler-register projects/rocprof-trace-decoder diff --git a/.github/workflows/rocprofiler-sdk-docs.yml b/.github/workflows/rocprofiler-sdk-docs.yml index 30115d14f502..05ac623a311c 100644 --- a/.github/workflows/rocprofiler-sdk-docs.yml +++ b/.github/workflows/rocprofiler-sdk-docs.yml @@ -76,7 +76,9 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: - sparse-checkout: projects/rocprofiler-sdk + sparse-checkout: | + projects/rocprofiler-sdk + shared/sha256 submodules: false set-safe-directory: true - name: Init/Update submodules @@ -142,6 +144,7 @@ jobs: sparse-checkout: | .github/actions/rocprofiler-sdk-util projects/rocprofiler-sdk + shared/sha256 projects/rocr-runtime projects/clr projects/hip diff --git a/.github/workflows/rocprofiler-sdk-formatting.yml b/.github/workflows/rocprofiler-sdk-formatting.yml index 2c61c1723617..1fed58591c43 100644 --- a/.github/workflows/rocprofiler-sdk-formatting.yml +++ b/.github/workflows/rocprofiler-sdk-formatting.yml @@ -9,6 +9,7 @@ on: pull_request: paths: - 'projects/rocprofiler-sdk/**' + - 'shared/sha256/**' - '!**/*.md' - '!**/*.rtf' - '!**/*.rst' @@ -33,7 +34,9 @@ jobs: steps: - uses: actions/checkout@v6 with: - sparse-checkout: projects/rocprofiler-sdk + sparse-checkout: | + projects/rocprofiler-sdk + shared/sha256 - name: Extract branch name shell: bash @@ -73,7 +76,9 @@ jobs: steps: - uses: actions/checkout@v6 with: - sparse-checkout: projects/rocprofiler-sdk + sparse-checkout: | + projects/rocprofiler-sdk + shared/sha256 - name: Extract branch name shell: bash @@ -92,7 +97,7 @@ jobs: - name: Run clang-format working-directory: projects/rocprofiler-sdk run: | - FORMAT_FILES=$(find samples source tests benchmark -type f | egrep '\.(h|hpp|hh|c|cc|cpp)(|\.in)$') + FORMAT_FILES=$(find samples source tests benchmark ../../shared/sha256 -type f | egrep '\.(h|hpp|hh|c|cc|cpp)(|\.in)$') command -v git command -v clang-format git --version @@ -120,7 +125,8 @@ jobs: steps: - uses: actions/checkout@v6 with: - sparse-checkout: projects/rocprofiler-sdk + sparse-checkout: | + projects/rocprofiler-sdk - name: Extract branch name shell: bash @@ -159,7 +165,9 @@ jobs: steps: - uses: actions/checkout@v6 with: - sparse-checkout: projects/rocprofiler-sdk + sparse-checkout: | + projects/rocprofiler-sdk + shared/sha256 - name: Extract branch name shell: bash @@ -172,7 +180,7 @@ jobs: working-directory: projects/rocprofiler-sdk run: | OUTFILE=missing_newline.txt - for i in $(find source tests samples benchmark docker cmake -type f | egrep -v '\.(bin|png|csv)$|source/docs/_(build|doxygen)'); do VAL=$(tail -c 1 ${i}); if [ -n "${VAL}" ]; then echo "- ${i}" >> ${OUTFILE}; fi; done + for i in $(find source tests samples benchmark docker cmake ../../shared/sha256 -type f | egrep -v '\.(bin|png|csv)$|source/docs/_(build|doxygen)'); do VAL=$(tail -c 1 ${i}); if [ -n "${VAL}" ]; then echo "- ${i}" >> ${OUTFILE}; fi; done if [[ -f ${OUTFILE} && $(cat ${OUTFILE} | wc -l) -gt 0 ]]; then echo -e "\nError! Source code missing new line at end of file...\n" echo -e "\nFiles:\n" @@ -186,7 +194,8 @@ jobs: steps: - uses: actions/checkout@v6 with: - sparse-checkout: projects/rocprofiler-sdk + sparse-checkout: | + projects/rocprofiler-sdk - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/rocprofiler-sdk-python.yml b/.github/workflows/rocprofiler-sdk-python.yml index 5aece30e4c47..307a7206404b 100644 --- a/.github/workflows/rocprofiler-sdk-python.yml +++ b/.github/workflows/rocprofiler-sdk-python.yml @@ -46,7 +46,8 @@ jobs: steps: - uses: actions/checkout@v6 with: - sparse-checkout: projects/rocprofiler-sdk + sparse-checkout: | + projects/rocprofiler-sdk - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: diff --git a/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml b/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml index 0ab9053557a0..af791edb4a54 100644 --- a/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml +++ b/.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml @@ -20,6 +20,7 @@ on: - '!**/.wordlist.txt' - '!projects/rocprofiler-sdk/CODEOWNERS' - '!projects/rocprofiler-sdk/source/docs/**' + - 'shared/sha256/**' pull_request: paths: - '.github/workflows/rocprofiler-sdk-rocm_release_compatibility.yml' @@ -33,6 +34,7 @@ on: - '!**/.wordlist.txt' - '!projects/rocprofiler-sdk/CODEOWNERS' - '!projects/rocprofiler-sdk/source/docs/**' + - 'shared/sha256/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/rocprofiler-sdk-wsl.yml b/.github/workflows/rocprofiler-sdk-wsl.yml index d5ee1af0afe4..06643bb6cd9c 100644 --- a/.github/workflows/rocprofiler-sdk-wsl.yml +++ b/.github/workflows/rocprofiler-sdk-wsl.yml @@ -21,6 +21,7 @@ on: - 'projects/rocr-runtime/libhsakmt/**' - 'projects/rocr-runtime/runtime/hsa-runtime/core/util/**' - 'shared/amdgpu-windows-interop/wkmi/**' + - 'shared/sha256/**' - '!**/*.md' - '!**/*.rst' - '!**/*.rtf' @@ -34,6 +35,7 @@ on: - 'projects/rocr-runtime/libhsakmt/**' - 'projects/rocr-runtime/runtime/hsa-runtime/core/util/**' - 'shared/amdgpu-windows-interop/wkmi/**' + - 'shared/sha256/**' - '!**/*.md' - '!**/*.rst' - '!**/*.rtf' @@ -115,6 +117,7 @@ jobs: projects/rocprofiler-sdk projects/rocr-runtime shared/amdgpu-windows-interop/wkmi + shared/sha256 .dvc sparse-checkout-cone-mode: true submodules: false diff --git a/.github/workflows/rocprofiler-systems-build-group.yml b/.github/workflows/rocprofiler-systems-build-group.yml index f15455c6a4bf..812ab0feb528 100644 --- a/.github/workflows/rocprofiler-systems-build-group.yml +++ b/.github/workflows/rocprofiler-systems-build-group.yml @@ -74,7 +74,6 @@ jobs: -DROCPROFSYS_BUILD_EXAMPLES=ON -DROCPROFSYS_USE_PYTHON=ON -DROCPROFSYS_BUILD_DYNINST=ON - -DROCPROFSYS_BUILD_TBB=ON -DROCPROFSYS_BUILD_ELFUTILS=ON -DROCPROFSYS_BUILD_LIBIBERTY=ON -DROCPROFSYS_MAX_THREADS=64 @@ -369,7 +368,7 @@ jobs: # ────────────────────────────────────────────────────────────────────────── # System-dependency builds: validate that system-provided libs substitute - # for the bundled ones (BUILD_{TBB,ELFUTILS,LIBIBERTY}=OFF). + # for the bundled ones (BUILD_{ELFUTILS,LIBIBERTY}=OFF). # Matrix entries are filtered to the selected distro group. # ────────────────────────────────────────────────────────────────────────── system-deps: @@ -403,7 +402,6 @@ jobs: -DROCPROFSYS_BUILD_EXAMPLES=ON -DROCPROFSYS_USE_PYTHON=ON -DROCPROFSYS_BUILD_DYNINST=ON - -DROCPROFSYS_BUILD_TBB=OFF -DROCPROFSYS_BUILD_ELFUTILS=OFF -DROCPROFSYS_BUILD_LIBIBERTY=OFF -DROCPROFSYS_BUILD_HIDDEN_VISIBILITY=ON @@ -469,7 +467,7 @@ jobs: max_attempts: 5 shell: bash command: | - dnf install -y --enablerepo=crb tbb-devel fmt-devel spdlog-devel + dnf install -y --enablerepo=crb fmt-devel spdlog-devel dnf install -y binutils-devel dnf clean all @@ -496,7 +494,7 @@ jobs: git config --global --add safe.directory ${PWD} echo "CMake: $(cmake --version | head -n 1)" echo "Compiler: $(${{ matrix.compiler }} --version | head -n 1)" - echo "Testing build with SYSTEM dependencies (ROCPROFSYS_BUILD_{TBB,ELFUTILS,LIBIBERTY}=OFF)" + echo "Testing build with SYSTEM dependencies (ROCPROFSYS_BUILD_{ELFUTILS,LIBIBERTY}=OFF)" python3 ./scripts/run-ci.py --stage generate \ --name "${{ github.repository_owner }}-${{ github.ref_name }}-${{ matrix.cdash_suffix }}" \ --build-jobs $(nproc) \ diff --git a/.github/workflows/rocprofiler-systems-build.yml b/.github/workflows/rocprofiler-systems-build.yml index f0dce8051752..a7a4a9667640 100644 --- a/.github/workflows/rocprofiler-systems-build.yml +++ b/.github/workflows/rocprofiler-systems-build.yml @@ -291,7 +291,6 @@ jobs: -DROCPROFSYS_BUILD_TESTING=OFF \ -DROCPROFSYS_USE_PYTHON=ON \ -DROCPROFSYS_BUILD_DYNINST=ON \ - -DROCPROFSYS_BUILD_TBB=ON \ -DROCPROFSYS_BUILD_ELFUTILS=ON \ -DROCPROFSYS_BUILD_LIBIBERTY=ON \ -DROCPROFSYS_PYTHON_PREFIX=/opt/conda/envs \ diff --git a/.github/workflows/rocprofiler-systems-code-coverage.yml b/.github/workflows/rocprofiler-systems-code-coverage.yml index 605c4a658ddb..ae8547fde89e 100644 --- a/.github/workflows/rocprofiler-systems-code-coverage.yml +++ b/.github/workflows/rocprofiler-systems-code-coverage.yml @@ -36,6 +36,7 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + fetch-depth: 0 sparse-checkout: | projects/rocprofiler-systems/ .gitmodules @@ -47,6 +48,8 @@ jobs: echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=/opt/rocm/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + # libtbb-dev is required by the baseline build below, which compiles the + # base branch. It can be dropped once the base branch no longer needs TBB. - name: Install Packages timeout-minutes: 25 uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 #v4.0.0 @@ -77,6 +80,36 @@ jobs: update-alternatives \ --install /usr/bin/gcov gcov /usr/bin/gcov-${GCC_VER} 500 + - name: Resolve Coverage Baseline Ref + timeout-minutes: 5 + shell: bash + env: + GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + PR_SHA=$(git rev-parse HEAD) + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_REF="${{ github.event.pull_request.base.ref }}" + else + BASE_REF="develop" + fi + git fetch origin "${BASE_REF}" + BASE_SHA=$(git rev-parse FETCH_HEAD) + echo "PR_SHA=${PR_SHA}" >> $GITHUB_ENV + echo "BASE_SHA=${BASE_SHA}" >> $GITHUB_ENV + echo "Baseline: ${BASE_REF} @ ${BASE_SHA}" + echo "PR head: ${PR_SHA}" + + - name: Checkout Baseline (base branch) + timeout-minutes: 5 + shell: bash + run: git checkout ${{ env.BASE_SHA }} + + - name: Use PR Version of Coverage Script + timeout-minutes: 5 + shell: bash + run: git checkout ${{ env.PR_SHA }} -- projects/rocprofiler-systems/scripts/generate-coverage.py + - name: Configure timeout-minutes: 15 shell: bash @@ -94,6 +127,43 @@ jobs: "-DROCPROFSYS_DISABLE_EXAMPLES=transpose;rccl;videodecode;jpegdecode;openmp;roctx;scratch-memory;unified-memory;transferBench;hpc;sdma_test;annotate" \ -DROCPROFSYS_BUILD_NUMBER=${{ github.run_attempt }} + - name: Build Baseline + timeout-minutes: 60 + shell: bash + working-directory: projects/rocprofiler-systems/ + run: | + cmake --build build/coverage --parallel $(nproc) + + - name: Unit Tests (Baseline) + timeout-minutes: 10 + shell: bash + working-directory: projects/rocprofiler-systems/ + run: | + ./build/coverage/bin/rocprof-sys-unit-tests + + - name: Generate Baseline Unit Test Coverage + if: always() + timeout-minutes: 15 + shell: bash + working-directory: projects/rocprofiler-systems/ + run: | + python3 scripts/generate-coverage.py \ + --build-dir build/coverage \ + --source-dir . \ + --dir source \ + --output-dir .codecov \ + --label unit-tests-baseline + + # git checkout updates the gitlink but leaves the submodule work tree at + # whatever the baseline build checked out, so sync it explicitly. Otherwise + # the PR is measured against the base branch's Dyninst. + - name: Checkout PR Branch + timeout-minutes: 10 + shell: bash + run: | + git checkout ${{ env.PR_SHA }} + git submodule update --init --recursive projects/rocprofiler-systems/external/dyninst + - name: Build timeout-minutes: 60 shell: bash @@ -101,6 +171,11 @@ jobs: run: | cmake --build build/coverage --parallel $(nproc) + - name: Clear Baseline Coverage Data + shell: bash + working-directory: projects/rocprofiler-systems/ + run: find build/coverage -name '*.gcda' -delete + - name: Unit Tests timeout-minutes: 10 shell: bash @@ -117,8 +192,26 @@ jobs: python3 scripts/generate-coverage.py \ --build-dir build/coverage \ --source-dir . \ + --dir source \ --output-dir .codecov \ - --label unit-tests + --label unit-tests \ + --baseline .codecov/unit-tests-baseline.json + + - name: Generate Diff-Only Unit Test Coverage + if: always() + timeout-minutes: 15 + shell: bash + working-directory: projects/rocprofiler-systems/ + run: | + python3 scripts/generate-coverage.py \ + --build-dir build/coverage \ + --source-dir . \ + --dir source \ + --output-dir .codecov \ + --label unit-tests-diff \ + --diff-only \ + --base ${{ env.BASE_SHA }} \ + --baseline .codecov/unit-tests-baseline.json - name: E2E Tests timeout-minutes: 60 @@ -140,6 +233,7 @@ jobs: python3 scripts/generate-coverage.py \ --build-dir build/coverage \ --source-dir . \ + --dir source \ --output-dir .codecov \ --label e2e-tests @@ -150,14 +244,22 @@ jobs: working-directory: projects/rocprofiler-systems/ run: | get-md-contents() { if [ -f ".codecov/${1}.md" ]; then cat ".codecov/${1}.md"; else echo "_No report generated._"; fi; } + section() { + echo "
" + echo "

${1}

" + echo "" + get-md-contents "${2}" + echo "" + echo "
" + } { echo "# Code Coverage Report" echo "" - echo "## Unit Tests" - get-md-contents unit-tests + section "Unit Tests" unit-tests + echo "" + section "Unit Tests (PR Diff Files Only)" unit-tests-diff echo "" - echo "## E2E Tests" - get-md-contents e2e-tests + section "E2E Tests" e2e-tests echo "" echo "" } > .codecov/report.md diff --git a/.github/workflows/rocprofiler-systems-continuous-integration.yml b/.github/workflows/rocprofiler-systems-continuous-integration.yml index a29308e33db7..138c852a2862 100644 --- a/.github/workflows/rocprofiler-systems-continuous-integration.yml +++ b/.github/workflows/rocprofiler-systems-continuous-integration.yml @@ -251,7 +251,6 @@ jobs: -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }} \ -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-systems \ -DROCPROFSYS_BUILD_DYNINST=ON \ - -DROCPROFSYS_BUILD_TBB=ON \ -DROCPROFSYS_BUILD_ELFUTILS=ON \ -DROCPROFSYS_BUILD_LIBIBERTY=ON \ -DROCPROFSYS_BUILD_CI=ON \ diff --git a/.github/workflows/rocprofiler-systems-ubuntu-noble-sanitizers.yml b/.github/workflows/rocprofiler-systems-ubuntu-noble-sanitizers.yml index db731f1a6c23..422340b5bdf3 100644 --- a/.github/workflows/rocprofiler-systems-ubuntu-noble-sanitizers.yml +++ b/.github/workflows/rocprofiler-systems-ubuntu-noble-sanitizers.yml @@ -60,7 +60,7 @@ jobs: build-essential bzip2 chrpath cmake curl environment-modules flex gettext \ git-core gnupg2 gzip iproute2 libboost-all-dev libdw-dev libdwarf-dev \ libelf-dev libgmock-dev libgtest-dev libiberty-dev libpapi-dev libpfm4-dev \ - libsqlite3-dev libtbb-dev libtool locales lsb-release m4 ninja-build \ + libsqlite3-dev libtool locales lsb-release m4 ninja-build \ nlohmann-json3-dev python3-pip software-properties-common unzip wget \ vim zip zlib1g-dev libucx-dev ucx-utils libomp-dev libopenmpi-dev && apt-get autoclean @@ -178,7 +178,6 @@ jobs: ${SANITIZER_CMAKE_FLAGS} \ -DROCPROFSYS_BUILD_DYNINST=${{ matrix.build-dyninst }} \ -DROCPROFSYS_BUILD_BOOST=OFF \ - -DROCPROFSYS_BUILD_TBB=OFF \ -DROCPROFSYS_BUILD_ELFUTILS=OFF \ -DROCPROFSYS_BUILD_LIBIBERTY=OFF \ -DROCPROFSYS_MAX_THREADS=64 \ diff --git a/.github/workflows/therock-build-linux.yml b/.github/workflows/therock-build-linux.yml index ed4d4d3595b3..7883cf91245a 100644 --- a/.github/workflows/therock-build-linux.yml +++ b/.github/workflows/therock-build-linux.yml @@ -48,7 +48,7 @@ jobs: with: repository: "ROCm/TheRock" path: "TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Install python deps run: | diff --git a/.github/workflows/therock-ci-linux.yml b/.github/workflows/therock-ci-linux.yml index 9f38d22a17cf..1e983ab3d9be 100644 --- a/.github/workflows/therock-ci-linux.yml +++ b/.github/workflows/therock-ci-linux.yml @@ -50,7 +50,7 @@ jobs: with: repository: "ROCm/TheRock" path: "TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Install python deps run: | diff --git a/.github/workflows/therock-ci-windows.yml b/.github/workflows/therock-ci-windows.yml index b93075e0d3da..860cf9efa258 100644 --- a/.github/workflows/therock-ci-windows.yml +++ b/.github/workflows/therock-ci-windows.yml @@ -52,7 +52,7 @@ jobs: with: repository: "ROCm/TheRock" path: "TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Set up Python uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 diff --git a/.github/workflows/therock-ci.yml b/.github/workflows/therock-ci.yml index aa99f0b5a1d2..a1dbe549c853 100644 --- a/.github/workflows/therock-ci.yml +++ b/.github/workflows/therock-ci.yml @@ -95,7 +95,7 @@ jobs: with: repository: "ROCm/TheRock" path: TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Checkout CI config uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/therock-multi-arch-ci-asan-nightly.yml b/.github/workflows/therock-multi-arch-ci-asan-nightly.yml index e78e672c7ab5..79d00c37adae 100644 --- a/.github/workflows/therock-multi-arch-ci-asan-nightly.yml +++ b/.github/workflows/therock-multi-arch-ci-asan-nightly.yml @@ -28,7 +28,7 @@ concurrency: jobs: setup: - uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_variant: "asan" # Python/PyTorch/JAX builds are not supported with ASAN yet @@ -37,7 +37,7 @@ jobs: build_jax: false linux_amdgpu_families: "gfx94X,gfx950,gfx125X" repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 external_repo: '{"repository":"${{ github.repository }}","ref":"${{ github.sha }}"}' log_therock_ref: @@ -64,7 +64,7 @@ jobs: needs.setup.outputs.linux_build_config != '' && needs.setup.outputs.enable_build_jobs == 'true' }} - uses: ROCm/TheRock/.github/workflows/multi_arch_ci_linux.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/multi_arch_ci_linux.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_config: ${{ needs.setup.outputs.linux_build_config }} test_labels: ${{ needs.setup.outputs.linux_test_labels }} @@ -74,7 +74,7 @@ jobs: test_type: ${{ needs.setup.outputs.test_type }} external_repo_config: ${{ needs.setup.outputs.external_repo_config }} repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 permissions: contents: read id-token: write @@ -91,7 +91,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 sparse-checkout: build_tools/github_actions sparse-checkout-cone-mode: true diff --git a/.github/workflows/therock-multi-arch-ci-asan.yml b/.github/workflows/therock-multi-arch-ci-asan.yml index 0fe7b3c0d03d..9be89b66e412 100644 --- a/.github/workflows/therock-multi-arch-ci-asan.yml +++ b/.github/workflows/therock-multi-arch-ci-asan.yml @@ -57,7 +57,7 @@ concurrency: jobs: setup: - uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_variant: "asan" # Python/PyTorch/JAX builds are not supported with ASAN yet @@ -70,7 +70,7 @@ jobs: baseline_run_id: ${{ inputs.baseline_run_id || '' }} baseline_repository: ${{ inputs.baseline_repository || (inputs.baseline_run_id != '' && github.repository) || '' }} repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 external_repo: '{"repository":"${{ github.repository }}","ref":"${{ github.sha }}"}' linux_build_and_test: @@ -81,7 +81,7 @@ jobs: needs.setup.outputs.linux_build_config != '' && needs.setup.outputs.enable_build_jobs == 'true' }} - uses: ROCm/TheRock/.github/workflows/multi_arch_ci_linux.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/multi_arch_ci_linux.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_config: ${{ needs.setup.outputs.linux_build_config }} test_labels: ${{ needs.setup.outputs.linux_test_labels }} @@ -91,7 +91,7 @@ jobs: test_type: ${{ needs.setup.outputs.test_type }} external_repo_config: ${{ needs.setup.outputs.external_repo_config }} repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 permissions: contents: read id-token: write @@ -109,7 +109,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 sparse-checkout: build_tools/github_actions sparse-checkout-cone-mode: true diff --git a/.github/workflows/therock-multi-arch-ci-nightly.yml b/.github/workflows/therock-multi-arch-ci-nightly.yml index 634a9312a218..868dbf454682 100644 --- a/.github/workflows/therock-multi-arch-ci-nightly.yml +++ b/.github/workflows/therock-multi-arch-ci-nightly.yml @@ -10,7 +10,7 @@ name: TheRock Multi-Arch Nightly CI -# TheRock ref: pinned to ROCm/TheRock commit e0e238c0323f76aeabdd348cfd3798a653820fdc. +# TheRock ref: pinned to ROCm/TheRock commit dd6e405832348bac0babaf2dfd8ebb2ffa587cb8. # ============================================================================= # TEMPORARY: MI455 (gfx125X) bringup configuration @@ -40,7 +40,7 @@ concurrency: jobs: setup: - uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_variant: "release" # Limit GPU families for nightly CI: gfx94X, gfx950, gfx125X (Linux) and gfx1151 (Windows) @@ -51,7 +51,7 @@ jobs: prebuilt_stages: '' baseline_run_id: '' repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 # TEMPORARY: MI455 bringup - pass family_overrides to configure test runner and limited tests # TODO(geomin12): Remove family_overrides once MI455 is in therock-ci-config external_repo: >- @@ -93,7 +93,7 @@ jobs: needs.setup.outputs.linux_build_config != '' && needs.setup.outputs.enable_build_jobs == 'true' }} - uses: ROCm/TheRock/.github/workflows/multi_arch_ci_linux.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/multi_arch_ci_linux.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_config: ${{ needs.setup.outputs.linux_build_config }} test_labels: ${{ needs.setup.outputs.linux_test_labels }} @@ -105,7 +105,7 @@ jobs: # Empty string triggers full test run in downstream workflow changed_projects: '' repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 permissions: contents: read id-token: write @@ -118,7 +118,7 @@ jobs: needs.setup.outputs.windows_build_config != '' && needs.setup.outputs.enable_build_jobs == 'true' }} - uses: ROCm/TheRock/.github/workflows/multi_arch_ci_windows.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/multi_arch_ci_windows.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_config: ${{ needs.setup.outputs.windows_build_config }} test_labels: ${{ needs.setup.outputs.windows_test_labels }} @@ -128,7 +128,7 @@ jobs: # Empty string triggers full test run in downstream workflow changed_projects: '' repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 permissions: contents: read id-token: write @@ -146,7 +146,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 sparse-checkout: build_tools/github_actions sparse-checkout-cone-mode: true diff --git a/.github/workflows/therock-multi-arch-ci.yml b/.github/workflows/therock-multi-arch-ci.yml index 2772a4d975a6..e7e4df0a9275 100644 --- a/.github/workflows/therock-multi-arch-ci.yml +++ b/.github/workflows/therock-multi-arch-ci.yml @@ -12,7 +12,7 @@ # - Supports prebuilt stages to skip certain build stages # - Better parallelization across GPU architectures -# TheRock ref: pinned to ROCm/TheRock commit e0e238c0323f76aeabdd348cfd3798a653820fdc. +# TheRock ref: pinned to ROCm/TheRock commit dd6e405832348bac0babaf2dfd8ebb2ffa587cb8. # NOTE: update all `@` refs and `ref:` inputs together when switching # ============================================================================= @@ -121,7 +121,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 path: _therock # _therock_utils + (cone-mode) root BUILD_TOPOLOGY.toml and # build_tools/project_mappings.json are needed to map changed projects @@ -169,7 +169,7 @@ jobs: setup: needs: configure - uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_variant: "release" # Limit GPU families for rocm-systems CI @@ -195,7 +195,7 @@ jobs: # and determines which stages to rebuild based on changed files stage_reuse_mode: reuse-stage repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 # TEMPORARY: MI455 bringup - pass family_overrides to configure test runner and limited tests # TODO(geomin12): Remove family_overrides once MI455 is in therock-ci-config # NOTE: MI455 machine is down - test-runs-on set to "" until back online (was: linux-mi455-gpu-rocm) @@ -223,7 +223,7 @@ jobs: needs.setup.outputs.enable_build_jobs == 'true' && needs.configure.outputs.skip_tests != 'true' }} - uses: ROCm/TheRock/.github/workflows/multi_arch_ci_linux.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/multi_arch_ci_linux.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_config: ${{ needs.setup.outputs.linux_build_config }} test_labels: ${{ needs.setup.outputs.linux_test_labels }} @@ -235,7 +235,7 @@ jobs: # Empty string when run_all_tests=true triggers full test run in downstream workflow changed_projects: ${{ needs.configure.outputs.run_all_tests == 'true' && '' || needs.configure.outputs.changed_projects }} repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 permissions: contents: read id-token: write @@ -249,7 +249,7 @@ jobs: needs.setup.outputs.enable_build_jobs == 'true' && needs.configure.outputs.skip_tests != 'true' }} - uses: ROCm/TheRock/.github/workflows/multi_arch_ci_windows.yml@e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + uses: ROCm/TheRock/.github/workflows/multi_arch_ci_windows.yml@dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 with: build_config: ${{ needs.setup.outputs.windows_build_config }} test_labels: ${{ needs.setup.outputs.windows_test_labels }} @@ -259,7 +259,7 @@ jobs: # Empty string when run_all_tests=true triggers full test run in downstream workflow changed_projects: ${{ needs.configure.outputs.run_all_tests == 'true' && '' || needs.configure.outputs.changed_projects }} repository: ROCm/TheRock - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 permissions: contents: read id-token: write @@ -278,7 +278,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 sparse-checkout: build_tools/github_actions sparse-checkout-cone-mode: true diff --git a/.github/workflows/therock-rccl-ci-linux.yml b/.github/workflows/therock-rccl-ci-linux.yml index d7eabe0e7613..fbe26106b8f8 100644 --- a/.github/workflows/therock-rccl-ci-linux.yml +++ b/.github/workflows/therock-rccl-ci-linux.yml @@ -65,7 +65,7 @@ jobs: with: repository: "ROCm/TheRock" path: "TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Install python deps run: | diff --git a/.github/workflows/therock-rccl-test-jax-collective.yml b/.github/workflows/therock-rccl-test-jax-collective.yml index 4ea25b7ec7fe..f33f27c335bd 100644 --- a/.github/workflows/therock-rccl-test-jax-collective.yml +++ b/.github/workflows/therock-rccl-test-jax-collective.yml @@ -72,7 +72,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Checkout rocm-systems scripts uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/therock-rccl-test-madengine.yml b/.github/workflows/therock-rccl-test-madengine.yml index 6f6495bcff31..54db5d388825 100644 --- a/.github/workflows/therock-rccl-test-madengine.yml +++ b/.github/workflows/therock-rccl-test-madengine.yml @@ -84,7 +84,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Checkout rocm-systems scripts uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -154,6 +154,8 @@ jobs: echo "ARTIFACT_RUN_ID=${run_id}" >> $GITHUB_ENV - name: Fetch RCCL artifacts + env: + GITHUB_TOKEN: ${{ github.token }} run: | python3 build_tools/install_rocm_from_artifacts.py \ --run-id="${ARTIFACT_RUN_ID}" \ diff --git a/.github/workflows/therock-rccl-test-packages-multi-node.yml b/.github/workflows/therock-rccl-test-packages-multi-node.yml index db2cc79c623c..fcb26790ff8f 100644 --- a/.github/workflows/therock-rccl-test-packages-multi-node.yml +++ b/.github/workflows/therock-rccl-test-packages-multi-node.yml @@ -49,7 +49,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 # Centralized arch-specific configuration - name: Set arch-specific paths diff --git a/.github/workflows/therock-rccl-test-packages-single-node.yml b/.github/workflows/therock-rccl-test-packages-single-node.yml index 4e29d840b855..57aeaf644b1a 100644 --- a/.github/workflows/therock-rccl-test-packages-single-node.yml +++ b/.github/workflows/therock-rccl-test-packages-single-node.yml @@ -55,7 +55,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Run setup test environment workflow uses: './.github/actions/setup_test_environment' diff --git a/.github/workflows/therock-rccl-test-pytorch-distributed.yml b/.github/workflows/therock-rccl-test-pytorch-distributed.yml index c758452ad67e..fbc7e0e9fcf6 100644 --- a/.github/workflows/therock-rccl-test-pytorch-distributed.yml +++ b/.github/workflows/therock-rccl-test-pytorch-distributed.yml @@ -79,7 +79,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Checkout rocm-systems scripts uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/therock-rccl-test-rocprof.yml b/.github/workflows/therock-rccl-test-rocprof.yml index e7e5b0085a10..8b45ab49413f 100644 --- a/.github/workflows/therock-rccl-test-rocprof.yml +++ b/.github/workflows/therock-rccl-test-rocprof.yml @@ -77,7 +77,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Checkout rocm-systems scripts uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/therock-test-component.yml b/.github/workflows/therock-test-component.yml index caa7780145de..bd8adf2fa563 100644 --- a/.github/workflows/therock-test-component.yml +++ b/.github/workflows/therock-test-component.yml @@ -61,7 +61,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 sparse-checkout: build_tools path: "prejob" @@ -79,7 +79,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Run setup test environment workflow uses: './.github/actions/setup_test_environment' diff --git a/.github/workflows/therock-test-packages.yml b/.github/workflows/therock-test-packages.yml index f2eaf428f689..9b74d3f0ff16 100644 --- a/.github/workflows/therock-test-packages.yml +++ b/.github/workflows/therock-test-packages.yml @@ -68,7 +68,7 @@ jobs: sparse-checkout: build_tools path: "prejob" repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 # Checkout failure is possible on Windows, as it's the first job on a GPU test runner. # Post-job cleanup isn't necessary since no executables are launched in this job. @@ -81,7 +81,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: "ROCm/TheRock" - ref: e0e238c0323f76aeabdd348cfd3798a653820fdc # 2026-09-23 + ref: dd6e405832348bac0babaf2dfd8ebb2ffa587cb8 # 2026-10-05 - name: Checkout CI config uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.gitmodules b/.gitmodules index cd158b84e818..d5bee9b94b02 100644 --- a/.gitmodules +++ b/.gitmodules @@ -108,10 +108,6 @@ [submodule "projects/rocprofiler-compute/src/vendored/pyyaml"] path = projects/rocprofiler-compute/src/vendored/pyyaml url = https://github.com/yaml/pyyaml.git -[submodule "projects/rocprofiler-systems/external/onetbb"] - path = projects/rocprofiler-systems/external/onetbb - url = https://github.com/uxlfoundation/oneTBB.git - tag = v2022.3.0 [submodule "projects/rocprofiler-sdk/external/abseil-cpp"] path = projects/rocprofiler-sdk/external/abseil-cpp url = https://github.com/abseil/abseil-cpp.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c00f37dd1ec5..edee03cea062 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,3 +56,22 @@ repos: rev: 0.25.1 hooks: - id: gersemi + # rocprofiler-sdk is formatted by the cmake-format hook below. The two + # tools disagree about the same files, so whichever runs second reports + # the other's output as unformatted. + exclude: ^projects/rocprofiler-sdk/ + + # rocprofiler-sdk formats its CMake with cmake-format, enforced by the cmake job + # in rocprofiler-sdk-formatting.yml. Running it here is what makes a root + # pre-commit produce what that job expects; dropping the project from CMake + # formatting altogether would leave contributors to discover the difference in + # CI. cmake-format reads projects/rocprofiler-sdk/.cmake-format.yaml, which it + # finds by walking up from each file it is given. + - repo: local + hooks: + - id: cmake-format + name: cmake-format (rocprofiler-sdk) + language: python + additional_dependencies: ["cmake-format==0.6.13", "pyyaml"] + entry: cmake-format -i + files: ^projects/rocprofiler-sdk/.*(CMakeLists\.txt|\.cmake)$ diff --git a/emulation/mirage/Cargo.lock b/emulation/mirage/Cargo.lock index 728e77cb08b3..f5ea358aabec 100644 --- a/emulation/mirage/Cargo.lock +++ b/emulation/mirage/Cargo.lock @@ -167,7 +167,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -308,7 +308,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -353,6 +353,17 @@ dependencies = [ "wasip3", ] +[[package]] +name = "goblin" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17582616a7718cca54cec18e534a76c7c4aec11a8b9a85695712f262fd15a4c8" +dependencies = [ + "log", + "plain", + "scroll", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -568,8 +579,10 @@ name = "mirage_core" version = "0.2.0" dependencies = [ "chrono", + "goblin", "inventory", "nix", + "rustix", "serde", "serde_json", "tempfile", @@ -606,6 +619,7 @@ version = "0.2.0" dependencies = [ "inventory", "mirage_core", + "serde_json", ] [[package]] @@ -717,6 +731,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "prettyplease" version = "0.2.37" @@ -724,7 +744,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -811,6 +831,26 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scroll" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1257cd4248b4132760d6524d6dda4e053bc648c9070b960929bf50cfb1e7add" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1a36a382ed65dbcc0ab47fd5e9a94112417ccd34560a392ef3b7b0f0ec39148" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "semver" version = "1.0.28" @@ -844,7 +884,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -930,6 +970,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -969,7 +1020,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -980,7 +1031,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1017,7 +1068,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1063,7 +1114,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1191,7 +1242,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1259,7 +1310,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1270,7 +1321,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1415,7 +1466,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -1431,7 +1482,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] diff --git a/emulation/mirage/Cargo.toml b/emulation/mirage/Cargo.toml index d20da48d6bd8..dbf1ab97c4d8 100644 --- a/emulation/mirage/Cargo.toml +++ b/emulation/mirage/Cargo.toml @@ -158,6 +158,8 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } tempfile = "3" inventory = "0.3.24" libloading = "0.8" +goblin = "0.10.7" +rustix = { version = "1", features = ["fs"] } mirage_core = { path = "core" } mirage_ctl = { path = "ctl" } diff --git a/emulation/mirage/builtin/Cargo.toml b/emulation/mirage/builtin/Cargo.toml index 99fc3446033b..bfac10faa1e9 100644 --- a/emulation/mirage/builtin/Cargo.toml +++ b/emulation/mirage/builtin/Cargo.toml @@ -13,6 +13,7 @@ serde = { workspace = true } serde_json = { workspace = true } [build-dependencies] +mirage_core = { workspace = true } serde_json = { workspace = true } [dev-dependencies] diff --git a/emulation/mirage/builtin/build.rs b/emulation/mirage/builtin/build.rs index aa62697f44af..cbdfbff73599 100644 --- a/emulation/mirage/builtin/build.rs +++ b/emulation/mirage/builtin/build.rs @@ -1,122 +1,218 @@ -//! Build script: read the per-CU limits and the arch name out of the -//! rocjitsu presets the builtin agents mirror. +//! Embed the rocjitsu configs mirage ships as agents, validating each +//! one against [`mirage_core::agent::AgentDef`] here rather than at run +//! time: the configs are source, not installed data — a released mirage +//! cannot go looking for them — and doing it here means a config that +//! stops parsing fails the build instead of every run. //! -//! These four values and the arch are not mirage's to choose. They -//! describe what the emulator will accept: `cdna5` caps `num_wf_slots` -//! at 64, and rocjitsu refuses a config that asks for more — in -//! `rj_vm_create_from_string`, which means the session dies at daemon -//! start rather than at profile validation, where a user could act on -//! it. So they are read from `configs/*.json` rather than copied into -//! `agents.rs`, and a preset that changes takes the agent with it. -//! -//! They were copied, and they drifted exactly as copies do. `gfx1250.json` -//! became `gfx1250_mi455x.json` with `arch` renamed `cdna5` and its wave -//! slots cut from 80 to 64; the rename reached `agents.rs` and the wave -//! slots did not, so every MI450X session failed at daemon start until -//! somebody read the error and the two files side by side. -//! -//! Read at build time rather than at run time on purpose. The presets are -//! source, not installed data — a released mirage cannot go looking for -//! them — and doing it here means a preset that loses a key fails the -//! build instead of every run. -//! -//! What is deliberately *not* taken from the preset is the shape of the -//! machine: the component tree, the IOD fan-out and the KFD device -//! identity are mirage's own, and all three builtins are a uniform 256 -//! CUs where the presets are 320, 288 and 256. See `agents.rs`. +//! The whole directory is read, rather than a list kept here by hand. +//! That list was three entries long while `rocjitsu/configs` held eleven +//! usable ones, so mirage could emulate an MI210 or a W7900 and had no +//! name for either. Adding a GPU to rocjitsu now adds it to mirage. -// A build script may panic: a preset that is missing, unparseable, or -// has lost a key is not something the crate can be built without, and +// A build script may panic: a config that is missing, unparseable or no +// longer an agent is not something the crate can be built without, and // there is no caller to return an error to. #![allow(clippy::expect_used, clippy::panic)] +use std::collections::BTreeMap; use std::fmt::Write as _; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; -/// The presets the builtin agents mirror, in `agents()` order: the Rust -/// constant to emit, and the preset it is read from. -const PRESETS: [(&str, &str); 3] = [ - ("MI300X", "gfx942_cdna3"), - ("MI350X", "gfx950_mi355x"), - ("MI450X", "gfx1250_mi455x"), +/// The names mirage addressed its first three agents by, before it read +/// the whole directory. +/// +/// They are not derivable — `gfx1250_mi455x` describes an MI455X and +/// mirage has always called it `mi450x` — and they are load-bearing: +/// `--profile` defaults to `mi350x`, the shipped topologies name +/// `MI350X` and `MI300X`, and every agent and profile a user already has +/// on disk was written under them. Everything else is addressed by its +/// config's file stem, which is unique and needs no table. +const ESTABLISHED_NAMES: [(&str, &str); 3] = [ + ("gfx942_cdna3", "mi300x"), + ("gfx950_mi355x", "mi350x"), + ("gfx1250_mi455x", "mi450x"), ]; -/// The per-CU config keys a builtin agent carries. Order is the order -/// they are written into the generated struct literal. -const CU_KEYS: [&str; 4] = [ - "num_wf_slots", - "sgprs_per_wf", - "vgprs_per_wf", - "lds_size_kb", -]; +/// One config that could become an agent, before the pick below. +struct Candidate { + stem: String, + config: serde_json::Value, +} fn main() { println!("cargo:rerun-if-changed=build.rs"); let configs = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../rocjitsu/configs"); + // The directory as well as each file in it: a config that is *added* + // changes no file mirage already reads, and without this the new GPU + // would not appear until something else forced a rebuild. + println!("cargo:rerun-if-changed={}", configs.display()); - let mut out = String::from( - "// @generated by build.rs from the rocjitsu presets — do not edit.\n\ - // Regenerate by touching the preset or `build.rs`.\n\n", - ); - for (name, stem) in PRESETS { - let path = configs.join(format!("{stem}.json")); + let mut devices: BTreeMap> = BTreeMap::new(); + for path in json_files(&configs) { println!("cargo:rerun-if-changed={}", path.display()); + let stem = path + .file_stem() + .expect("a *.json path has a stem") + .to_string_lossy() + .into_owned(); let text = std::fs::read_to_string(&path) - .unwrap_or_else(|e| panic!("cannot read the rocjitsu preset {}: {e}", path.display())); - let json: serde_json::Value = serde_json::from_str(&text) - .unwrap_or_else(|e| panic!("{} is not valid JSON: {e}", path.display())); + .unwrap_or_else(|error| panic!("cannot read {}: {error}", path.display())); + let config: serde_json::Value = serde_json::from_str(&text) + .unwrap_or_else(|error| panic!("invalid JSON in {}: {error}", path.display())); + // A DBT guest config (`guest_*.json`) describes which guest runs + // on which host, not a machine — there is no agent in it to take. + if !config["vm"].is_object() || !config["topology"].is_object() { + continue; + } + devices + .entry(device_key(&stem, &config)) + .or_default() + .push(Candidate { stem, config }); + } - let arch = json - .pointer("/vm/arch") - .and_then(serde_json::Value::as_str) - .unwrap_or_else(|| panic!("{} has no `vm.arch`", path.display())); - let cu = first_compute_unit( - json.pointer("/topology/root") - .unwrap_or(&serde_json::Value::Null), - ) - .unwrap_or_else(|| panic!("{} has no `compute_unit` in its topology", path.display())); + let mut presets: BTreeMap = BTreeMap::new(); + for (key, mut candidates) in devices { + // One agent per device, so `mirage agent list` is a list of + // machines rather than of files. rocjitsu ships up to three + // configs for the same GPU — the standalone one, a `_kmd` one, + // and an `_Ngpu` one — and the differences between them are not + // differences mirage has to express: it synthesises its own + // config for every session and writes `vm.gpu.num_gpus` from the + // profile's `--gpus-per-node`, so an agent that bakes in a GPU + // count would have that count overwritten. The plainest config + // for the device is the one taken; `gfx90a_mi210_kmd` is the + // MI210's only config and is therefore taken as it is. + candidates.sort_by_key(|c| (variant_rank(&c.stem), c.stem.len(), c.stem.clone())); + let chosen = candidates.remove(0); + let name = ESTABLISHED_NAMES + .iter() + .find_map(|(stem, name)| (*stem == chosen.stem).then_some((*name).to_string())) + .unwrap_or_else(|| chosen.stem.clone()); + check_name(&name, &chosen.stem); + if let Some((taken, _)) = presets.get(&name) { + panic!( + "rocjitsu configs {taken}.json and {}.json both want the mirage name \ + {name:?} (device key {key:?})", + chosen.stem + ); + } + presets.insert(name, (chosen.stem, chosen.config)); + } + + // A rocjitsu config that is renamed or retired takes a mirage name + // with it, and the three below are the ones nothing on a user's disk + // survives losing. Fail here, where the fix is to add the new stem to + // the table above, rather than shipping a mirage whose default + // `--profile mi350x` names nothing. + for (stem, name) in ESTABLISHED_NAMES { + assert!( + presets.get(name).is_some_and(|(chosen, _)| chosen == stem), + "no rocjitsu config maps to the mirage name {name:?}: it came from \ + configs/{stem}.json, which is no longer the config chosen for its \ + device. Point ESTABLISHED_NAMES at the config that replaced it." + ); + } + let mut output = String::from("pub(crate) static PRESETS: &[Preset] = &[\n"); + for (name, (stem, config)) in &presets { + let agent = serde_json::json!({"vm": config["vm"], "topology": config["topology"]}); + serde_json::from_value::(agent.clone()) + .unwrap_or_else(|error| panic!("invalid agent in {stem}.json: {error}")); + let json = serde_json::to_string(&agent).expect("agent JSON is serializable"); + let config_json = serde_json::to_string(&config).expect("config JSON is serializable"); + let device = &config["vm"]["gpu"]["device"]; writeln!( - out, - "/// Read from `rocjitsu/configs/{stem}.json` at build time.\n\ - pub(crate) const {name}: Preset = Preset {{\n \ - preset: \"{stem}.json\",\n arch: \"{arch}\"," + output, + " Preset {{ name: {name:?}, stem: {stem:?}, marketing_name: {:?}, \ + arch: {:?}, agent: {json:?}, config: {config_json:?} }},", + device["marketing_name"].as_str().unwrap_or_default(), + config["vm"]["arch"].as_str().unwrap_or_default(), ) .expect("writing to a String cannot fail"); - for key in CU_KEYS { - let value = cu_value(cu, key) - .unwrap_or_else(|| panic!("{} has no `{key}` on its compute unit", path.display())); - writeln!(out, " {key}: \"{value}\",").expect("writing to a String cannot fail"); - } - out.push_str("};\n\n"); } + output.push_str("];\n"); - let dest = + let destination = PathBuf::from(std::env::var_os("OUT_DIR").expect("cargo sets OUT_DIR")).join("presets.rs"); - std::fs::write(&dest, out).unwrap_or_else(|e| panic!("cannot write {}: {e}", dest.display())); + std::fs::write(&destination, output) + .unwrap_or_else(|error| panic!("cannot write {}: {error}", destination.display())); +} + +/// Every `*.json` in `dir`, in a fixed order. +/// +/// Sorted, because `read_dir` is not: an unsorted walk would reorder the +/// generated table between builds on the same sources, and with it the +/// pick below whenever two candidates tie. +fn json_files(dir: &Path) -> Vec { + let mut out: Vec = std::fs::read_dir(dir) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display())) + .map(|entry| { + entry + .unwrap_or_else(|error| panic!("cannot read {}: {error}", dir.display())) + .path() + }) + .filter(|path| path.extension().is_some_and(|e| e == "json")) + .collect(); + out.sort(); + out } -/// The first `compute_unit` node in a preset's component tree. +/// What makes two configs the same machine. /// -/// Found by walking rather than by the path `soc -> xcd -> se -> cu`, -/// which differs between presets and would be a second thing to keep in -/// step with them. -fn first_compute_unit(node: &serde_json::Value) -> Option<&serde_json::Value> { - if node.get("type").and_then(serde_json::Value::as_str) == Some("compute_unit") { - return Some(node); +/// `gfx_target_version` is the GPU's ISA version and is exactly the +/// grouping wanted: rocjitsu's `_kmd` and `_Ngpu` variants of a device +/// carry the same one, and no two devices share it. A config that does +/// not name it stands alone under its own stem rather than being pooled +/// with every other config that also omits it. +fn device_key(stem: &str, config: &serde_json::Value) -> String { + match config["vm"]["gpu"]["device"]["gfx_target_version"].as_u64() { + Some(version) => format!("gfx_target_version:{version}"), + None => format!("stem:{stem}"), } - node.get("children")? - .as_array()? - .iter() - .find_map(first_compute_unit) } -/// One `{"key": …, "value": …}` entry from a component's `config`. -fn cu_value<'a>(node: &'a serde_json::Value, key: &str) -> Option<&'a str> { - node.get("config")? - .as_array()? - .iter() - .find(|e| e.get("key").and_then(serde_json::Value::as_str) == Some(key))? - .get("value")? - .as_str() +/// How far a config is from being the plain one for its device. +/// +/// Lexical, from the naming convention `rocjitsu/docs/npi.md` lays down +/// (`configs/.json` plus `_kmd` and any multi-GPU variant), and +/// deliberately so: it is the convention that makes `gfx950_mi355x` the +/// config for the MI350X and `gfx950_mi355x_kmd_2gpu` a variant of it, +/// and nothing inside the documents says which is which. A config +/// following no convention scores 0 and simply competes on its stem. +fn variant_rank(stem: &str) -> u32 { + let mut rank = 0; + for part in stem.split('_') { + if part == "kmd" { + rank += 1; + } + // `2gpu`, `4gpu`: a GPU count baked into a config mirage would + // overwrite from the profile anyway, so these lose hardest. + if let Some(count) = part.strip_suffix("gpu") + && !count.is_empty() + && count.chars().all(|c| c.is_ascii_digit()) + { + rank += 2; + } + } + rank +} + +/// Refuse a config whose stem mirage could not address a document by. +/// +/// Agents and profiles are stored lowercase under `.json` and +/// restricted to `[A-Za-z0-9._+-]`, so a stem outside that set would +/// produce a builtin that no command could name. Caught here, against +/// the same rule the store enforces, rather than as a run-time refusal +/// on a document mirage itself wrote. +fn check_name(name: &str, stem: &str) { + use mirage_core::store::{DocKind, validate_name}; + if let Err(error) = validate_name(DocKind::Agent, name) { + panic!("rocjitsu config {stem}.json cannot be a mirage agent: {error}"); + } + assert!( + DocKind::Agent.canonical(name) == name, + "rocjitsu config {stem}.json would be stored as \ + {:?} rather than {name:?}; give it a lowercase name in ESTABLISHED_NAMES", + DocKind::Agent.canonical(name) + ); } diff --git a/emulation/mirage/builtin/legacy/mi300x.json b/emulation/mirage/builtin/legacy/mi300x.json new file mode 100644 index 000000000000..60aa253e4f9e --- /dev/null +++ b/emulation/mirage/builtin/legacy/mi300x.json @@ -0,0 +1,243 @@ +{ + "topology": { + "links": [ + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 8, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].cp.req_[j*8+k] -> xcd[i].se[j].cu[k].cpl", + "src": "", + "weight": 2, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 8, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k].req -> xcd[i].l2.cpl_[j*8+k]", + "src": "", + "weight": 10, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + } + ], + "latency": 1, + "pattern": "xcd[i].l2.req -> iod[i/2].msc.cpl_[i%2]", + "src": "", + "weight": 3, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 4, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + } + ], + "latency": 1, + "pattern": "iod[i].peer_req -> iod[j].peer_cpl", + "src": "", + "weight": 1, + "where_expr": "i != j" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 7, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k].adj_req -> xcd[i].se[j].cu[k+1].adj_cpl", + "src": "", + "weight": 2, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 7, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k+1].adj_req_r -> xcd[i].se[j].cu[k].adj_cpl_r", + "src": "", + "weight": 2, + "where_expr": "" + } + ], + "root": { + "children": [ + { + "name": "vram", + "type": "gpu_memory" + }, + { + "config": [ + { + "key": "num_hbm_stacks", + "value": "2" + } + ], + "name": "iod[0:4]", + "type": "iod" + }, + { + "children": [ + { + "name": "l2", + "type": "l2_cache" + }, + { + "name": "cp", + "type": "command_processor" + }, + { + "children": [ + { + "config": [ + { + "key": "num_wf_slots", + "value": "32" + }, + { + "key": "sgprs_per_wf", + "value": "104" + }, + { + "key": "vgprs_per_wf", + "value": "512" + }, + { + "key": "lds_size_kb", + "value": "64" + } + ], + "name": "cu[0:8]", + "type": "compute_unit" + } + ], + "name": "se[0:4]", + "type": "shader_engine" + } + ], + "name": "xcd[0:8]", + "type": "xcd" + } + ], + "name": "soc", + "type": "soc" + } + }, + "vm": { + "arch": "cdna3", + "gpu": { + "device": { + "device_id": 29856, + "drm_render_minor": 128, + "family_id": 146, + "gfx_target_version": 90402, + "gpu_id": 50148, + "l1_assoc": 0, + "l1_line_size": 0, + "l1_size_kb": 0, + "l2_assoc": 0, + "l2_line_size": 0, + "l2_size_kb": 4096, + "lds_size_kb": 64, + "local_mem_size": 206158430208, + "marketing_name": "AMD Instinct MI300X", + "max_engine_clk_fcompute": 2100, + "max_slots_scratch_cu": 0, + "max_waves_per_simd": 8, + "mem_clk_max": 1300, + "mem_width": 8192, + "num_cp_queues": 128, + "num_cu_per_sh": 5, + "num_sdma_engines": 4, + "num_sdma_xgmi_engines": 6, + "num_shader_arrays_per_engine": 2, + "num_shader_engines": 4, + "simd_count": 1216, + "simd_per_cu": 4, + "unique_id": 0, + "vendor_id": 4098, + "wave_front_size": 64 + }, + "num_gpus": 1, + "num_iods": 0, + "num_xcds": 0 + } + } +} diff --git a/emulation/mirage/builtin/legacy/mi350x.json b/emulation/mirage/builtin/legacy/mi350x.json new file mode 100644 index 000000000000..83cdcbba9d08 --- /dev/null +++ b/emulation/mirage/builtin/legacy/mi350x.json @@ -0,0 +1,243 @@ +{ + "topology": { + "links": [ + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 8, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].cp.req_[j*8+k] -> xcd[i].se[j].cu[k].cpl", + "src": "", + "weight": 2, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 8, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k].req -> xcd[i].l2.cpl_[j*8+k]", + "src": "", + "weight": 10, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + } + ], + "latency": 1, + "pattern": "xcd[i].l2.req -> iod[i/4].msc.cpl_[i%4]", + "src": "", + "weight": 3, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 2, + "start": 0, + "var_name": "i" + }, + { + "end": 2, + "start": 0, + "var_name": "j" + } + ], + "latency": 1, + "pattern": "iod[i].peer_req -> iod[j].peer_cpl", + "src": "", + "weight": 1, + "where_expr": "i != j" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 7, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k].adj_req -> xcd[i].se[j].cu[k+1].adj_cpl", + "src": "", + "weight": 2, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 7, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k+1].adj_req_r -> xcd[i].se[j].cu[k].adj_cpl_r", + "src": "", + "weight": 2, + "where_expr": "" + } + ], + "root": { + "children": [ + { + "name": "vram", + "type": "gpu_memory" + }, + { + "config": [ + { + "key": "num_hbm_stacks", + "value": "4" + } + ], + "name": "iod[0:2]", + "type": "iod" + }, + { + "children": [ + { + "name": "l2", + "type": "l2_cache" + }, + { + "name": "cp", + "type": "command_processor" + }, + { + "children": [ + { + "config": [ + { + "key": "num_wf_slots", + "value": "32" + }, + { + "key": "sgprs_per_wf", + "value": "104" + }, + { + "key": "vgprs_per_wf", + "value": "512" + }, + { + "key": "lds_size_kb", + "value": "160" + } + ], + "name": "cu[0:8]", + "type": "compute_unit" + } + ], + "name": "se[0:4]", + "type": "shader_engine" + } + ], + "name": "xcd[0:8]", + "type": "xcd" + } + ], + "name": "soc", + "type": "soc" + } + }, + "vm": { + "arch": "cdna4", + "gpu": { + "device": { + "device_id": 5892, + "drm_render_minor": 128, + "family_id": 160, + "gfx_target_version": 90500, + "gpu_id": 38144, + "l1_assoc": 0, + "l1_line_size": 0, + "l1_size_kb": 0, + "l2_assoc": 0, + "l2_line_size": 0, + "l2_size_kb": 4096, + "lds_size_kb": 160, + "local_mem_size": 309237645312, + "marketing_name": "AMD Instinct MI350X", + "max_engine_clk_fcompute": 2700, + "max_slots_scratch_cu": 0, + "max_waves_per_simd": 8, + "mem_clk_max": 1600, + "mem_width": 8192, + "num_cp_queues": 128, + "num_cu_per_sh": 4, + "num_sdma_engines": 5, + "num_sdma_xgmi_engines": 12, + "num_shader_arrays_per_engine": 2, + "num_shader_engines": 4, + "simd_count": 1024, + "simd_per_cu": 4, + "unique_id": 5929628898254127105, + "vendor_id": 4098, + "wave_front_size": 64 + }, + "num_gpus": 1, + "num_iods": 0, + "num_xcds": 0 + } + } +} diff --git a/emulation/mirage/builtin/legacy/mi450x.json b/emulation/mirage/builtin/legacy/mi450x.json new file mode 100644 index 000000000000..fa367025d3c7 --- /dev/null +++ b/emulation/mirage/builtin/legacy/mi450x.json @@ -0,0 +1,243 @@ +{ + "topology": { + "links": [ + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 8, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].cp.req_[j*8+k] -> xcd[i].se[j].cu[k].cpl", + "src": "", + "weight": 2, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 8, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k].req -> xcd[i].l2.cpl_[j*8+k]", + "src": "", + "weight": 10, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + } + ], + "latency": 1, + "pattern": "xcd[i].l2.req -> iod[i/4].msc.cpl_[i%4]", + "src": "", + "weight": 3, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 2, + "start": 0, + "var_name": "i" + }, + { + "end": 2, + "start": 0, + "var_name": "j" + } + ], + "latency": 1, + "pattern": "iod[i].peer_req -> iod[j].peer_cpl", + "src": "", + "weight": 1, + "where_expr": "i != j" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 7, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k].adj_req -> xcd[i].se[j].cu[k+1].adj_cpl", + "src": "", + "weight": 2, + "where_expr": "" + }, + { + "dst": "", + "for_ranges": [ + { + "end": 8, + "start": 0, + "var_name": "i" + }, + { + "end": 4, + "start": 0, + "var_name": "j" + }, + { + "end": 7, + "start": 0, + "var_name": "k" + } + ], + "latency": 1, + "pattern": "xcd[i].se[j].cu[k+1].adj_req_r -> xcd[i].se[j].cu[k].adj_cpl_r", + "src": "", + "weight": 2, + "where_expr": "" + } + ], + "root": { + "children": [ + { + "name": "vram", + "type": "gpu_memory" + }, + { + "config": [ + { + "key": "num_hbm_stacks", + "value": "4" + } + ], + "name": "iod[0:2]", + "type": "iod" + }, + { + "children": [ + { + "name": "l2", + "type": "l2_cache" + }, + { + "name": "cp", + "type": "command_processor" + }, + { + "children": [ + { + "config": [ + { + "key": "num_wf_slots", + "value": "64" + }, + { + "key": "sgprs_per_wf", + "value": "128" + }, + { + "key": "vgprs_per_wf", + "value": "1024" + }, + { + "key": "lds_size_kb", + "value": "320" + } + ], + "name": "cu[0:8]", + "type": "compute_unit" + } + ], + "name": "se[0:4]", + "type": "shader_engine" + } + ], + "name": "xcd[0:8]", + "type": "xcd" + } + ], + "name": "soc", + "type": "soc" + } + }, + "vm": { + "arch": "cdna5", + "gpu": { + "device": { + "device_id": 1250, + "drm_render_minor": 128, + "family_id": 0, + "gfx_target_version": 120500, + "gpu_id": 1250, + "l1_assoc": 0, + "l1_line_size": 0, + "l1_size_kb": 0, + "l2_assoc": 0, + "l2_line_size": 0, + "l2_size_kb": 4096, + "lds_size_kb": 160, + "local_mem_size": 309237645312, + "marketing_name": "gfx1250", + "max_engine_clk_fcompute": 2700, + "max_slots_scratch_cu": 0, + "max_waves_per_simd": 20, + "mem_clk_max": 1600, + "mem_width": 8192, + "num_cp_queues": 128, + "num_cu_per_sh": 4, + "num_sdma_engines": 5, + "num_sdma_xgmi_engines": 12, + "num_shader_arrays_per_engine": 2, + "num_shader_engines": 4, + "simd_count": 1024, + "simd_per_cu": 4, + "unique_id": 1250, + "vendor_id": 4098, + "wave_front_size": 32 + }, + "num_gpus": 1, + "num_iods": 0, + "num_xcds": 0 + } + } +} diff --git a/emulation/mirage/builtin/src/agents.rs b/emulation/mirage/builtin/src/agents.rs index ee5cabeabaad..079d3bf51e5f 100644 --- a/emulation/mirage/builtin/src/agents.rs +++ b/emulation/mirage/builtin/src/agents.rs @@ -1,323 +1,67 @@ -//! Strongly-typed builtin [`AgentDef`]s. +//! The builtin [`AgentDef`]s: one per rocjitsu config mirage ships. //! -//! Each agent mirrors one of the rocjitsu `configs/*.json` files: -//! `MI300X` follows `gfx942_cdna3.json`, `MI350X` follows -//! `gfx950_mi355x.json`, and `MI450X` follows `gfx1250_mi455x.json`. -//! All three share the same `soc -> {vram, iod, xcd -> {l2, cp, -//! se -> cu}}` component tree and six link patterns; they differ -//! only in their device identity, the IOD fan-out, and the per-CU -//! wavefront/register/LDS config. -//! -//! # What comes from the preset and what does not -//! -//! The arch name and the four per-CU limits are read out of the preset -//! at build time (see [`crate::presets`]) rather than written here. -//! They are not mirage's to choose: they say what the emulator will -//! accept, and `cdna5` refusing more than 64 wave slots is a fact about -//! rocjitsu, not a preference. Copied, they drifted — `gfx1250.json` -//! became `gfx1250_mi455x.json` with 80 wave slots cut to 64, the arch -//! rename landed here and the wave slots did not, and every MI450X -//! session died at daemon start. -//! -//! The *shape* of the machine is mirage's own and stays here: the -//! component tree, the IOD fan-out and the KFD device identity. All -//! three builtins are a deliberate uniform 256 CUs, where the presets -//! they mirror are 320, 288 and 256 — an emulated grid mirage picked, not -//! a transcription of the hardware, and `every_agent_is_256_cus` holds it -//! there so that a later widening of this sync cannot quietly change what -//! a profile emulates. +//! Each is the `{vm, topology}` of `rocjitsu/configs/.json`, +//! embedded and validated at build time — see `build.rs` for which +//! config becomes which agent, and why one GPU yields one agent when +//! rocjitsu ships several configs for it. -use mirage_core::agent::{ - AgentDef, AgentTopologyDef, AmdgpuConfig, ComponentDef, ConfigEntry, ForRange, KfdDeviceInfo, - LinkDef, VirtualMachineConfig, -}; +use mirage_core::agent::AgentDef; -use crate::presets::{MI300X, MI350X, MI450X, Preset}; +use crate::presets::PRESETS; /// All builtin agents, keyed by the name written to disk. -/// -/// Agent names are case-insensitive and always stored lowercase, so the -/// registry keys are lowercase to match the on-disk filenames. pub fn agents() -> Vec<(&'static str, AgentDef)> { - vec![ - ("mi300x", mi300x()), - ("mi350x", mi350x()), - ("mi450x", mi450x()), - // \NPI new GPU: add a builtin agent mirroring its configs/.json here. - ] -} - -/// `MI300X` builtin agent (registry key `MI300X`), mirroring the -/// rocjitsu `gfx942_cdna3.json` config: `arch = cdna3`, marketing -/// name "AMD Instinct MI300X", and an 8-XCD / 4-SE / 8-CU shader -/// fabric over a 4-IOD memory tier. -pub fn mi300x() -> AgentDef { - AgentDef { - vm: VirtualMachineConfig { - arch: MI300X.arch.to_string(), - gpu: AmdgpuConfig { - num_xcds: 0, - num_iods: 0, - memory: None, - num_gpus: 1, - device: KfdDeviceInfo { - gpu_id: 50148, - gfx_target_version: 90402, - vendor_id: 4098, - device_id: 29856, - family_id: 146, - unique_id: 0, - marketing_name: "AMD Instinct MI300X".to_string(), - // First DRM render node (`/dev/dri/renderD128`); the - // rocjitsu schema defaults this to 128 and a 0 here - // maps the emulated GPU onto a non-existent render - // node, so HSA aborts with OUT_OF_RESOURCES. - drm_render_minor: 128, - simd_count: 1216, - max_waves_per_simd: 8, - num_shader_engines: 4, - num_shader_arrays_per_engine: 2, - num_cu_per_sh: 5, - simd_per_cu: 4, - wave_front_size: 64, - local_mem_size: 206158430208, - lds_size_kb: 64, - mem_width: 8192, - mem_clk_max: 1300, - l2_size_kb: 4096, - num_sdma_engines: 4, - num_sdma_xgmi_engines: 6, - num_cp_queues: 128, - max_engine_clk_fcompute: 2100, - ..Default::default() - }, - }, - }, - topology: topology(4, "2", &MI300X), - } -} - -/// `MI350X` builtin agent (registry key `MI350X`), mirroring the -/// rocjitsu `gfx950_mi355x.json` config: `arch = cdna4`, marketing -/// name "AMD Instinct MI350X", and an 8-XCD / 4-SE / 8-CU shader -/// fabric over a 2-IOD memory tier. -pub fn mi350x() -> AgentDef { - AgentDef { - vm: VirtualMachineConfig { - arch: MI350X.arch.to_string(), - gpu: AmdgpuConfig { - num_xcds: 0, - num_iods: 0, - memory: None, - num_gpus: 1, - device: KfdDeviceInfo { - gpu_id: 38144, - gfx_target_version: 90500, - vendor_id: 4098, - device_id: 5892, - family_id: 160, - unique_id: 5929628898254127105, - marketing_name: "AMD Instinct MI350X".to_string(), - // First DRM render node (`/dev/dri/renderD128`); the - // rocjitsu schema defaults this to 128 and a 0 here - // maps the emulated GPU onto a non-existent render - // node, so HSA aborts with OUT_OF_RESOURCES. - drm_render_minor: 128, - simd_count: 1024, - max_waves_per_simd: 8, - num_shader_engines: 4, - num_shader_arrays_per_engine: 2, - num_cu_per_sh: 4, - simd_per_cu: 4, - wave_front_size: 64, - local_mem_size: 309237645312, - lds_size_kb: 160, - mem_width: 8192, - mem_clk_max: 1600, - l2_size_kb: 4096, - num_sdma_engines: 5, - num_sdma_xgmi_engines: 12, - num_cp_queues: 128, - max_engine_clk_fcompute: 2700, - ..Default::default() - }, - }, - }, - topology: topology(2, "4", &MI350X), - } + PRESETS + .iter() + .map(|preset| (preset.name, from_preset(preset.agent))) + .collect() } -/// `MI450X` builtin agent (registry key `MI450X`), mirroring the -/// rocjitsu `gfx1250_mi455x.json` config: `arch = cdna5`, an -/// 8-XCD / 4-SE / 8-CU shader fabric and a 2-IOD memory tier. -pub fn mi450x() -> AgentDef { - AgentDef { - vm: VirtualMachineConfig { - arch: MI450X.arch.to_string(), - gpu: AmdgpuConfig { - num_xcds: 0, - num_iods: 0, - memory: None, - num_gpus: 1, - device: KfdDeviceInfo { - gpu_id: 1250, - gfx_target_version: 120500, - vendor_id: 4098, - device_id: 1250, - family_id: 0, - unique_id: 1250, - marketing_name: "gfx1250".to_string(), - // First DRM render node (`/dev/dri/renderD128`); the - // rocjitsu schema defaults this to 128 and a 0 here - // maps the emulated GPU onto a non-existent render - // node, so HSA aborts with OUT_OF_RESOURCES. - drm_render_minor: 128, - simd_count: 1024, - max_waves_per_simd: 20, - num_shader_engines: 4, - num_shader_arrays_per_engine: 2, - num_cu_per_sh: 4, - simd_per_cu: 4, - wave_front_size: 32, - local_mem_size: 309237645312, - lds_size_kb: 160, - mem_width: 8192, - mem_clk_max: 1600, - l2_size_kb: 4096, - num_sdma_engines: 5, - num_sdma_xgmi_engines: 12, - num_cp_queues: 128, - max_engine_clk_fcompute: 2700, - ..Default::default() - }, - }, - }, - topology: topology(2, "4", &MI450X), - } -} - -/// Build the shared `soc -> {vram, iod, xcd -> {l2, cp, se -> cu}}` -/// component tree used by every builtin agent. +/// The builtin agent called `name`, if mirage ships one. /// -/// `num_iods` / `num_hbm_stacks` set the IOD memory-tier fan-out, which -/// is mirage's own; `preset` supplies the per-CU `num_wf_slots`, -/// `sgprs_per_wf`, `vgprs_per_wf` and `lds_size_kb`, which are rocjitsu's -/// and are read from `configs/*.json` at build time. -/// -/// The `cu[0:8]` here is deliberate and is *not* the preset's — see the -/// module docs on why the CU count is mirage's to pick. -fn topology(num_iods: u32, num_hbm_stacks: &str, preset: &Preset) -> AgentTopologyDef { - let cu = ComponentDef { - name: "cu[0:8]".to_string(), - r#type: "compute_unit".to_string(), - config: vec![ - entry("num_wf_slots", preset.num_wf_slots), - entry("sgprs_per_wf", preset.sgprs_per_wf), - entry("vgprs_per_wf", preset.vgprs_per_wf), - entry("lds_size_kb", preset.lds_size_kb), - ], - ..Default::default() - }; - let se = ComponentDef { - name: "se[0:4]".to_string(), - r#type: "shader_engine".to_string(), - children: vec![cu], - ..Default::default() - }; - let xcd = ComponentDef { - name: "xcd[0:8]".to_string(), - r#type: "xcd".to_string(), - children: vec![leaf("l2", "l2_cache"), leaf("cp", "command_processor"), se], - ..Default::default() - }; - let iod = ComponentDef { - name: format!("iod[0:{num_iods}]"), - r#type: "iod".to_string(), - config: vec![entry("num_hbm_stacks", num_hbm_stacks)], - ..Default::default() - }; - let root = ComponentDef { - name: "soc".to_string(), - r#type: "soc".to_string(), - children: vec![leaf("vram", "gpu_memory"), iod, xcd], - ..Default::default() - }; - AgentTopologyDef { - root, - links: links(num_iods), - } +/// Case-insensitively, as every other way of naming an agent is: +/// `mirage agent show MI350X` and `mirage agent show mi350x` are the +/// same document, and a lookup here that disagreed would be a second +/// spelling rule for the same names. +#[must_use] +pub fn agent(name: &str) -> Option { + PRESETS + .iter() + .find(|preset| preset.name.eq_ignore_ascii_case(name)) + .map(|preset| from_preset(preset.agent)) } -/// The six link patterns wiring the command processor to the CUs, -/// the CUs to the L2 and IOD memory tier, the IODs to each other, -/// and adjacent CUs together. Only the L2->IOD fan-out and the IOD -/// peer range depend on `num_iods`. -fn links(num_iods: u32) -> Vec { - let xcds_per_iod = 8 / num_iods; - let ijk = || vec![range("i", 0, 8), range("j", 0, 4), range("k", 0, 8)]; - let ijk_adj = || vec![range("i", 0, 8), range("j", 0, 4), range("k", 0, 7)]; - vec![ - LinkDef { - pattern: "xcd[i].cp.req_[j*8+k] -> xcd[i].se[j].cu[k].cpl".to_string(), - for_ranges: ijk(), - weight: 2, - ..Default::default() - }, - LinkDef { - pattern: "xcd[i].se[j].cu[k].req -> xcd[i].l2.cpl_[j*8+k]".to_string(), - for_ranges: ijk(), - weight: 10, - ..Default::default() - }, - LinkDef { - pattern: format!("xcd[i].l2.req -> iod[i/{xcds_per_iod}].msc.cpl_[i%{xcds_per_iod}]"), - for_ranges: vec![range("i", 0, 8)], - weight: 3, - ..Default::default() - }, - LinkDef { - pattern: "iod[i].peer_req -> iod[j].peer_cpl".to_string(), - for_ranges: vec![range("i", 0, num_iods), range("j", 0, num_iods)], - where_expr: "i != j".to_string(), - weight: 1, - ..Default::default() - }, - LinkDef { - pattern: "xcd[i].se[j].cu[k].adj_req -> xcd[i].se[j].cu[k+1].adj_cpl".to_string(), - for_ranges: ijk_adj(), - weight: 2, - ..Default::default() - }, - LinkDef { - pattern: "xcd[i].se[j].cu[k+1].adj_req_r -> xcd[i].se[j].cu[k].adj_cpl_r".to_string(), - for_ranges: ijk_adj(), - weight: 2, - ..Default::default() - }, - ] +#[must_use] +pub fn source_config(name: &str) -> Option { + PRESETS + .iter() + .find(|preset| preset.name.eq_ignore_ascii_case(name)) + .and_then(|preset| serde_json::from_str(preset.config).ok()) } -fn leaf(name: &str, r#type: &str) -> ComponentDef { - ComponentDef { - name: name.to_string(), - r#type: r#type.to_string(), - ..Default::default() - } -} - -fn entry(key: &str, value: &str) -> ConfigEntry { - ConfigEntry { - key: key.to_string(), - value: value.to_string(), - } +#[must_use] +pub fn source_config_for_agent(agent: &AgentDef) -> Option { + let target = agent.vm.gpu.device.gfx_target_version; + PRESETS + .iter() + .find(|preset| { + serde_json::from_str::(preset.agent) + .is_ok_and(|preset_agent| preset_agent.vm.gpu.device.gfx_target_version == target) + }) + .and_then(|preset| serde_json::from_str(preset.config).ok()) } -fn range(var_name: &str, start: u32, end: u32) -> ForRange { - ForRange { - var_name: var_name.to_string(), - start, - end, - } +/// One builtin, parsed from the config `build.rs` embedded. +/// +/// The `expect` is the workspace's one production opt-out of +/// `expect_used`, and it is here because there is nothing to report: +/// `build.rs` parses the same string into the same [`AgentDef`] and +/// fails the build if it cannot, so a panic here means the crate was +/// linked against a `mirage_core` it was not built against. There is no +/// user input on this path and no configuration that reaches it. +#[allow(clippy::expect_used)] +fn from_preset(json: &str) -> AgentDef { + serde_json::from_str(json).expect("builtin agent validated by build.rs") } #[cfg(test)] @@ -326,117 +70,111 @@ mod tests { use super::*; - #[test] - fn agents_have_expected_keys() { - let a = agents(); - assert_eq!(a.len(), 3); - assert_eq!(a[0].0, "mi300x"); - assert_eq!(a[1].0, "mi350x"); - assert_eq!(a[2].0, "mi450x"); - } - - /// The number of CUs a builtin emulates is mirage's, not the - /// preset's, and this is what says so. + /// Every agent is the config it names, in full and unchanged. /// - /// The presets these agents mirror are 320, 288 and 256 CUs — the - /// real parts. The builtins are a uniform 256, which is the grid - /// mirage chose to emulate, and changing it changes every result - /// every profile produces. Now that the per-CU limits are read from - /// the preset it would be a small and reasonable-looking step to read - /// the component tree from it too; this fails if anybody takes it. + /// Read off disk rather than through `PRESETS`, so this compares + /// what mirage ships against the source of truth rather than the + /// embedding against itself. #[test] - fn every_agent_is_256_cus() { - for (name, agent) in agents() { + fn every_builtin_matches_its_rocjitsu_config() { + for preset in PRESETS { + let path = format!( + "{}/../../rocjitsu/configs/{}.json", + env!("CARGO_MANIFEST_DIR"), + preset.stem + ); + let source = std::fs::read_to_string(&path).unwrap(); + let config: serde_json::Value = serde_json::from_str(&source).unwrap(); + let document = serde_json::json!({ + "vm": config["vm"], "topology": config["topology"] + }); + let expected: AgentDef = serde_json::from_value(document.clone()).unwrap(); + let agent = agent(preset.name).unwrap(); + + assert_eq!(agent, expected, "{}", preset.name); + // Round-trips to the config's own bytes: a field the config + // omits has to stay omitted, or rocjitsu's schema default + // for it is replaced by Rust's. See `AgentDef`. + assert_eq!( + serde_json::to_value(&agent).unwrap(), + document, + "{}", + preset.name + ); assert_eq!( - compute_units(&agent.topology.root), - 256, - "`{name}` no longer emulates 256 CUs; the preset's own CU \ - count is not mirage's to adopt — see the module docs" + serde_json::from_value::(serde_json::to_value(&agent).unwrap()).unwrap(), + agent, + "{}", + preset.name + ); + // rocjitsu refuses to load a device with SDMA engines and + // nowhere to queue to, so a config mirage ships as an agent + // must not be one. + let device = &agent.vm.gpu.device; + assert!( + device.num_sdma_engines == 0 || device.num_sdma_queues_per_engine > 0, + "{}", + preset.name ); } } - /// The per-CU limits really do come from the preset. - /// - /// Tautological against the current code and deliberately so: it is - /// here to fail the day somebody writes a literal back into - /// `topology()`, which is how these drifted from the preset the first - /// time. + /// The three names that were mirage's whole builtin set are still + /// there: `--profile` defaults to `mi350x`, the shipped topologies + /// name `MI350X` and `MI300X`, and users have all three on disk. #[test] - fn the_per_cu_limits_come_from_the_preset() { - for (agent, preset) in [ - (mi300x(), &MI300X), - (mi350x(), &MI350X), - (mi450x(), &MI450X), - ] { - assert_eq!(agent.vm.arch, preset.arch, "{}", preset.preset); - for (key, want) in [ - ("num_wf_slots", preset.num_wf_slots), - ("sgprs_per_wf", preset.sgprs_per_wf), - ("vgprs_per_wf", preset.vgprs_per_wf), - ("lds_size_kb", preset.lds_size_kb), - ] { - assert_eq!(cu_config(&agent, key), want, "{} {key}", preset.preset); - } + fn the_established_names_survive_reading_the_whole_directory() { + let names: Vec<&str> = agents().into_iter().map(|(name, _)| name).collect(); + for established in ["mi300x", "mi350x", "mi450x"] { + assert!(names.contains(&established), "{established} in {names:?}"); } } - /// Every compute unit under `node`, multiplying out the `[0:N]` - /// ranges on the way down. - fn compute_units(node: &ComponentDef) -> u32 { - let span = node - .name - .rsplit_once("[0:") - .and_then(|(_, rest)| rest.strip_suffix(']')) - .and_then(|n| n.parse::().ok()) - .unwrap_or(1); - if node.r#type == "compute_unit" { - return span; - } - span * node.children.iter().map(compute_units).sum::() - } - - /// One per-CU config value from an agent's component tree. - /// - /// Walks to the first compute unit rather than spelling out - /// `soc -> xcd -> se -> cu`, which would break whenever the tree - /// grows a level. - fn cu_config(agent: &AgentDef, key: &str) -> String { - fn find(node: &ComponentDef, key: &str) -> Option { - if node.r#type == "compute_unit" - && let Some(entry) = node.config.iter().find(|e| e.key == key) - { - return Some(entry.value.clone()); + /// One agent per GPU, not one per config file. + #[test] + fn no_two_agents_are_the_same_device() { + let mut seen = std::collections::BTreeMap::new(); + for (name, agent) in agents() { + let version = agent.vm.gpu.device.gfx_target_version; + if let Some(other) = seen.insert(version, name) { + panic!("agents {other} and {name} are both gfx_target_version {version}"); } - node.children.iter().find_map(|child| find(child, key)) } - find(&agent.topology.root, key).unwrap_or_else(|| panic!("no `{key}` on any compute unit")) } #[test] - fn mi300x_identity() { - let a = mi300x(); - let d = &a.vm.gpu.device; - assert_eq!(d.marketing_name, "AMD Instinct MI300X"); - assert_eq!(d.num_shader_engines, 4); - assert_eq!(d.simd_count, 1216); - assert_eq!(a.topology.links.len(), 6); + fn agents_are_addressed_case_insensitively() { + assert_eq!(agent("MI350X"), agent("mi350x")); + assert!(agent("mi350x").is_some()); + assert!(agent("no-such-gpu").is_none()); } #[test] - fn mi350x_identity() { - let d = mi350x().vm.gpu.device; - assert_eq!(d.marketing_name, "AMD Instinct MI350X"); - assert_eq!(d.num_shader_engines, 4); - assert_eq!(d.simd_count, 1024); + fn source_configs_follow_agent_lookup() { + let agent = agent("GFX1251_SYNTHETIC").unwrap(); + let by_name = source_config("gfx1251_synthetic").unwrap(); + let by_agent = source_config_for_agent(&agent).unwrap(); + + assert_eq!(by_name, by_agent); + assert_eq!(by_name["vm"]["target"], "gfx1251"); + assert!(source_config("no-such-gpu").is_none()); + assert!(source_config_for_agent(&AgentDef::default()).is_none()); } #[test] - fn mi450x_identity() { - let a = mi450x(); - assert_eq!(a.vm.arch, "cdna5"); - assert_eq!(a.vm.gpu.device.marketing_name, "gfx1250"); - assert_eq!(a.vm.gpu.device.gfx_target_version, 120500); - assert_eq!(a.topology.links.len(), 6); + fn source_configs_match_agents_with_omitted_fields() { + let mut config = source_config("mi350x").unwrap(); + config["vm"]["gpu"]["device"] + .as_object_mut() + .unwrap() + .remove("mem_clk_max"); + let agent: AgentDef = serde_json::from_value(serde_json::json!({ + "vm": config["vm"], + "topology": config["topology"] + })) + .unwrap(); + + let source = source_config_for_agent(&agent).unwrap(); + assert!(source["vm"]["gpu"]["device"].get("mem_clk_max").is_some()); } } diff --git a/emulation/mirage/builtin/src/lib.rs b/emulation/mirage/builtin/src/lib.rs index 6130d9606dcf..c125c12d3c5d 100644 --- a/emulation/mirage/builtin/src/lib.rs +++ b/emulation/mirage/builtin/src/lib.rs @@ -1,21 +1,38 @@ -//! Built-in agents, topologies and profiles that mirage preloads into -//! `/{agent,topology,profile}/`. +//! Built-in agents, topologies and profiles. //! -//! Historically these shipped as `agents/*.json` files embedded into -//! `mirage_core` at build time and parsed at runtime. They now live -//! here as strongly-typed [`mirage_core::agent::AgentDef`] / -//! [`mirage_core::topology::TopologyDef`] constructors so the data is -//! validated by the compiler instead of by a runtime parse. +//! Everything here comes from the rocjitsu configs mirage ships. Each +//! `rocjitsu/configs/.json` is embedded and validated against +//! [`mirage_core::agent::AgentDef`] at build time (see `build.rs`), and +//! becomes one builtin agent ([`mod@agents`]) and one builtin profile +//! pinning it ([`mod@profiles`]). System layouts ([`mod@topologies`]) +//! remain mirage-owned constructors over those agent names. +//! +//! # Two kinds of builtin +//! +//! Agents and topologies are **seeded**: mirage writes them into +//! `/{agent,topology}/` and writes back any that goes +//! missing, because they are documents a user edits, references by name, +//! and expects to find as files. +//! +//! Builtin profiles are **generated**: nothing is written, and +//! [`mirage_core::store`] derives one from this crate whenever the +//! profile directory has no file of that name. A profile is three fields +//! around an agent reference, so the file said nothing the agent did not +//! — and a stored copy went stale on upgrade, needing `mirage state +//! builtins` to pick up a definition mirage could just as well produce. +//! Writing your own profile under a builtin's name still shadows it, and +//! deleting that file still brings the builtin back. //! //! This crate owns both the builtin *data* and the policy for writing -//! it to disk. It relies on `mirage_core` only for the low-level path -//! resolution ([`mirage_core::paths`]) and JSON serialization -//! ([`mirage_core::state::write_json`]), and registers what it ships with -//! [`mirage_core::store`] so the store can tell a document mirage seeded -//! from one the user wrote. +//! the seeded half to disk. It relies on `mirage_core` only for the +//! low-level path resolution ([`mirage_core::paths`]) and JSON +//! serialization ([`mirage_core::state::write_json`]), and registers what +//! it ships with [`mirage_core::store`] so the store can tell a document +//! mirage seeded from one the user wrote, and can serve a profile mirage +//! never wrote at all. pub mod agents; -pub mod presets; +mod presets; pub mod profiles; pub mod topologies; @@ -24,10 +41,11 @@ use std::path::PathBuf; use serde::Serialize; use mirage_core::error::Result; -use mirage_core::store::{BuiltinDocuments, DocKind, is_pristine_builtin}; +use mirage_core::profile::ProfileDef; +use mirage_core::store::{BuiltinDocuments, BuiltinProfiles, DocKind, is_pristine_builtin}; -pub use agents::{agents, mi300x, mi350x, mi450x}; -pub use profiles::profiles; +pub use agents::{agent, agents}; +pub use profiles::{profile_named, profiles}; pub use topologies::{default_topology, topologies}; // Tell `mirage_core` what mirage ships. The core store has to be able to @@ -40,6 +58,22 @@ inventory::submit! { BuiltinDocuments { documents: builtin_documents } } +// And what it generates rather than ships. Registered separately because +// the two answer different questions: a *seeded* document on disk may be +// mirage's own untouched copy, and a generated one has no copy on disk at +// all, so every profile file that exists is the user's. +inventory::submit! { + BuiltinProfiles { profiles: builtin_profiles } +} + +/// The builtin profiles, for [`mirage_core::store`] to serve. +fn builtin_profiles() -> Vec<(String, ProfileDef)> { + profiles() + .into_iter() + .map(|(name, profile)| (name.to_string(), profile)) + .collect() +} + fn builtin_documents() -> Vec<(DocKind, String, serde_json::Value)> { fn collect( out: &mut Vec<(DocKind, String, serde_json::Value)>, @@ -62,7 +96,9 @@ fn builtin_documents() -> Vec<(DocKind, String, serde_json::Value)> { let mut out = Vec::new(); collect(&mut out, DocKind::Agent, agents()); collect(&mut out, DocKind::Topology, topologies()); - collect(&mut out, DocKind::Profile, profiles()); + // Not profiles: mirage writes none, so no profile file is ever + // mirage's own seed and none of the rules this list feeds applies to + // one. They are registered as [`BuiltinProfiles`] instead. out } @@ -73,8 +109,8 @@ fn builtin_documents() -> Vec<(DocKind, String, serde_json::Value)> { /// is a fact about that one document and says nothing about the other /// forty. Returning it lets the caller finish the other two kinds and /// then say everything it left alone at once — `mirage state builtins` -/// used to abandon the run at the first one, so repairing three edited -/// builtins took three invocations to even discover. +/// used to abandon the run at the first one, so repairing an edited +/// agent and an edited topology took two invocations to even discover. #[derive(Debug, Default)] pub struct Ensured { /// Every document of this kind, as its name and whether this pass @@ -118,21 +154,16 @@ pub fn ensure_topologies(force: bool) -> Result { ensure(DocKind::Topology, topologies(), force) } -/// Write all builtin profiles to disk. See `ensure`. -/// -/// # Errors -/// -/// Returns an error if a document cannot be written. -pub fn ensure_profiles(force: bool) -> Result { - ensure(DocKind::Profile, profiles(), force) -} - /// Materialise one kind of builtin, and report what happened to each /// document. /// /// Without `force` — the startup path, run before every command — only /// missing documents are written, so a fresh config directory fills -/// itself in and an existing one is left exactly as it is. +/// itself in and an existing one is left exactly as it is. The one +/// exception is [`refresh_agent_mirage_wrote`], which replaces an agent +/// file mirage can prove it wrote itself — a document an earlier release +/// seeded — with the one it ships now, because rocjitsu will not load +/// the older ones at all. It touches nothing else. /// /// With `force` — `mirage state builtins`, which exists so a mirage /// upgrade can bring its new definitions with it — every document that is @@ -147,7 +178,7 @@ pub fn ensure_profiles(force: bool) -> Result { /// user to fix unless they want the shipped version back. What they need /// is to be told which files those are — so they are named in /// [`Ensured::edited`] and reported by the caller, which is the only -/// place that can name all three kinds in one breath. +/// place that can name both kinds in one breath. fn ensure( kind: DocKind, documents: Vec<(&'static str, T)>, @@ -157,6 +188,7 @@ fn ensure( for (name, document) in documents { let path = kind.path(name); if path.exists() { + refresh_agent_mirage_wrote(kind, name, &path, &document)?; if !force { out.documents.push((name.to_string(), false)); continue; @@ -173,6 +205,132 @@ fn ensure( Ok(out) } +/// The agent documents previous releases of mirage wrote. +/// +/// These cannot be derived from anything: they are what the definitions +/// in `builtin/src/agents.rs` serialised to before mirage read its +/// agents out of the rocjitsu configs, and that code is gone. So they +/// are kept verbatim, generated by building the release that wrote them +/// and serialising its own `agents()`. A file that equals one of these +/// is mirage's own work, however little it now resembles what mirage +/// ships, and [`refresh_agent_mirage_wrote`] is what acts on that. +/// +/// The table is deliberately incomplete and appending to it is the +/// intended way to extend it. The serialised shape moved more than once +/// before this — `num_gpus` arrived in v0.1.0, `arch` went `gfx1250` to +/// `cdna5` — and a document from one of those older shapes is simply not +/// recognised: it is left alone, named by `mirage state builtins`, and +/// refused with its reason at profile validation, which is the same +/// place a user's own document ends up. Recognising one more release is +/// a matter of dumping its agents and adding the file. +const SUPERSEDED_AGENTS: [(&str, &str); 3] = [ + ("mi300x", include_str!("../legacy/mi300x.json")), + ("mi350x", include_str!("../legacy/mi350x.json")), + ("mi450x", include_str!("../legacy/mi450x.json")), +]; + +/// Bring a stored agent that mirage itself wrote up to what it ships. +/// +/// rocjitsu refuses a device that has SDMA engines and no queues on them +/// — `num_sdma_queues_per_engine must be nonzero when num_sdma_engines +/// is nonzero`, thrown while the config is loaded, which means the +/// session dies at daemon start. Mirage never wrote the field before +/// this release, so every agent file an older mirage left behind is one +/// rocjitsu will now reject — including `mi350x`, which is what +/// `--profile` defaults to, so an upgraded machine that does nothing +/// unusual gets a failed session out of the box. +/// +/// Which is still not licence to write the field into every file +/// bearing a builtin's name, because a *pristine* builtin already has +/// it: every document this can ever see is one mirage cannot claim from +/// its contents alone. Filling in just this GPU's queue count is the +/// wrong repair — an MI350X seeded before this release has five SDMA +/// engines and four CUs per shader array, and would come out of it with +/// five engines, today's eight queues each, and its own old shader +/// fabric, describing no GPU either party ships. +/// +/// So the question asked is whether the stored document is one mirage +/// wrote, and there are exactly two ways for it to be: +/// +/// * it is the shipped document with only the queue count missing, which +/// is what an install that last ran *this* release's predecessor by a +/// hair looks like; or +/// * it equals one of [`SUPERSEDED_AGENTS`], the documents earlier +/// releases wrote. +/// +/// Either way the answer is the whole shipped document, not a patched +/// copy of the old one — the same file a fresh install gets, which is +/// the only version mirage can vouch for. Anything else is somebody's +/// own document and is left exactly as it is: `mirage state builtins` +/// names it and says how to take the shipped version, and the rocjitsu +/// backend refuses an incomplete pair at profile validation, where the +/// document can still be edited. +fn refresh_agent_mirage_wrote( + kind: DocKind, + name: &str, + path: &std::path::Path, + document: &T, +) -> Result<()> { + if !matches!(kind, DocKind::Agent) { + return Ok(()); + } + let Ok(shipped) = serde_json::to_value(document) else { + return Ok(()); + }; + let Ok(stored) = mirage_core::state::read_json::(path) else { + return Ok(()); + }; + if stored == shipped { + return Ok(()); + } + if !completes_to_shipped(&stored, &shipped) && !is_superseded_agent(name, &stored) { + return Ok(()); + } + mirage_core::state::write_json(path, &shipped) +} + +/// Is `stored` the shipped document with only its queue count missing? +fn completes_to_shipped(stored: &serde_json::Value, shipped: &serde_json::Value) -> bool { + // A shipped agent whose own config omits the field has no value to + // complete anything with. + let Some(queues) = shipped + .pointer("/vm/gpu/device/num_sdma_queues_per_engine") + .cloned() + else { + return false; + }; + let mut candidate = stored.clone(); + let Some(device) = candidate + .pointer_mut("/vm/gpu/device") + .and_then(serde_json::Value::as_object_mut) + else { + return false; + }; + // An explicit zero is an answer, and mirage does not argue with it — + // rocjitsu will. + if device.contains_key("num_sdma_queues_per_engine") { + return false; + } + device.insert("num_sdma_queues_per_engine".to_string(), queues); + candidate == *shipped +} + +/// Does `stored` equal the document some earlier release wrote for +/// `name`? +/// +/// Compared as parsed JSON rather than as text, so a file that has been +/// reformatted — or written by a serialiser that ordered its keys +/// differently — is still recognised as the document it is. +fn is_superseded_agent(name: &str, stored: &serde_json::Value) -> bool { + SUPERSEDED_AGENTS + .iter() + .filter(|(superseded, _)| superseded.eq_ignore_ascii_case(name)) + .any(|(_, document)| { + serde_json::from_str::(document) + .is_ok_and(|superseded| superseded == *stored) + }) +} + #[cfg(test)] mod tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] @@ -221,6 +379,177 @@ mod tests { } } + /// An explicit zero is the user's answer, and mirage does not + /// second-guess it — even though rocjitsu will refuse the file. + #[test] + fn an_explicit_sdma_queue_count_is_left_alone() { + let _guard = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + for (name, agent) in agents() { + let path = mirage_core::paths::agent_path(name); + let mut edited = serde_json::to_value(&agent).unwrap(); + edited["vm"]["gpu"]["device"]["num_sdma_queues_per_engine"] = 0.into(); + mirage_core::state::write_json(&path, &edited).unwrap(); + for force in [false, true] { + ensure_agents(force).unwrap(); + assert_eq!( + mirage_core::state::read_json::(&path).unwrap(), + edited + ); + } + } + + mirage_core::paths::clear_test_root(); + } + + /// The document mirage may complete is the shipped one with that + /// single key taken out, and completing it produces the shipped one + /// whole. + #[test] + fn a_shipped_agent_missing_only_the_queue_count_is_completed() { + let _guard = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + for (name, agent) in agents() { + let path = mirage_core::paths::agent_path(name); + let shipped = serde_json::to_value(&agent).unwrap(); + let mut stored = shipped.clone(); + let device = stored["vm"]["gpu"]["device"].as_object_mut().unwrap(); + // An agent whose own rocjitsu config omits the field has + // nothing to complete and nothing to say here. + if device.remove("num_sdma_queues_per_engine").is_none() { + continue; + } + mirage_core::state::write_json(&path, &stored).unwrap(); + + ensure_agents(false).unwrap(); + let completed = mirage_core::state::read_json::(&path).unwrap(); + assert_eq!(completed, shipped, "{name}"); + + // Idempotent: a second pass has nothing left to do. + ensure_agents(false).unwrap(); + assert_eq!( + mirage_core::state::read_json::(&path).unwrap(), + shipped, + "{name}" + ); + } + + mirage_core::paths::clear_test_root(); + } + + /// An agent an earlier release wrote becomes the one mirage ships. + /// + /// This is the upgrade, and the documents are the real ones: the + /// previous release's `mi350x` has five SDMA engines, four CUs per + /// shader array and no queue count at all, against a shipped MI355X + /// of two engines and eight queues each. Patching the one field into + /// it would leave a machine neither party ships, and leaving it + /// alone leaves `--profile`'s own default unable to start a session, + /// so the whole document is replaced. + #[test] + fn an_agent_an_earlier_release_wrote_is_replaced_with_the_shipped_one() { + let _guard = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + for (name, superseded) in SUPERSEDED_AGENTS { + let shipped = agents() + .into_iter() + .find(|(shipped, _)| shipped.eq_ignore_ascii_case(name)) + .map(|(_, agent)| serde_json::to_value(&agent).unwrap()) + .unwrap_or_else(|| panic!("mirage still ships an agent called {name}")); + let superseded: serde_json::Value = serde_json::from_str(superseded).unwrap(); + assert_ne!( + superseded, shipped, + "{name}: a fixture equal to the shipped agent tests nothing" + ); + + let path = mirage_core::paths::agent_path(name); + mirage_core::state::write_json(&path, &superseded).unwrap(); + + ensure_agents(false).unwrap(); + assert_eq!( + mirage_core::state::read_json::(&path).unwrap(), + shipped, + "{name}" + ); + + // Idempotent: a second pass has nothing left to do. + ensure_agents(false).unwrap(); + assert_eq!( + mirage_core::state::read_json::(&path).unwrap(), + shipped, + "{name}" + ); + } + + mirage_core::paths::clear_test_root(); + } + + /// A document mirage cannot claim is not touched at all. + /// + /// Recognising the documents earlier releases wrote is not licence + /// to guess: one changed field is enough to make a file the user's, + /// and a file that merely *resembles* an older release is not one. + #[test] + fn an_agent_that_is_not_one_mirage_wrote_is_left_alone() { + let _guard = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + for (name, agent) in agents() { + let path = mirage_core::paths::agent_path(name); + let shipped = serde_json::to_value(&agent).unwrap(); + + // The user's own edit of the shipped document. + let mut edited = shipped.clone(); + let device = edited["vm"]["gpu"]["device"].as_object_mut().unwrap(); + device.remove("num_sdma_queues_per_engine"); + device.insert("marketing_name".to_string(), "custom GPU".into()); + + // A document shaped like an older release's but not one of + // them, which is also what every unrecognised older shape + // looks like from here. + let mut invented = shipped.clone(); + let device = invented["vm"]["gpu"]["device"].as_object_mut().unwrap(); + device.remove("num_sdma_queues_per_engine"); + device.insert("num_sdma_engines".to_string(), 5.into()); + device.insert("num_cu_per_sh".to_string(), 4.into()); + invented["topology"]["root"]["children"] = serde_json::json!([]); + + let mut theirs = vec![edited, invented]; + + // The previous release's own document with a single field + // changed: recognition has to be exact, so this is theirs. + if let Some((_, superseded)) = SUPERSEDED_AGENTS + .iter() + .find(|(superseded, _)| superseded.eq_ignore_ascii_case(name)) + { + let mut nearly: serde_json::Value = serde_json::from_str(superseded).unwrap(); + nearly["vm"]["gpu"]["device"]["num_cp_queues"] = 7.into(); + theirs.push(nearly); + } + + for document in theirs { + mirage_core::state::write_json(&path, &document).unwrap(); + for force in [false, true] { + ensure_agents(force).unwrap(); + assert_eq!( + mirage_core::state::read_json::(&path).unwrap(), + document, + "{name} (force={force})" + ); + } + } + } + + mirage_core::paths::clear_test_root(); + } + #[test] fn a_forced_rewrite_leaves_an_edited_builtin_alone() { // `mirage state builtins` used to overwrite a builtin the user had @@ -232,31 +561,32 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); mirage_core::paths::set_test_root(tmp.path()); - ensure_profiles(false).unwrap(); - let mut mine = mirage_core::store::profile_get("mi350x").unwrap(); - mine.description = Some("my own mi350x".to_string()); - mirage_core::state::write_json(&mirage_core::paths::profile_path("mi350x"), &mine).unwrap(); + ensure_topologies(false).unwrap(); + let mut mine = mirage_core::store::topology_get("MI350X-1x8").unwrap(); + mine.num_nodes = 7; + mirage_core::state::write_json(&mirage_core::paths::topology_path("MI350X-1x8"), &mine) + .unwrap(); - let forced = ensure_profiles(true).unwrap(); + let forced = ensure_topologies(true).unwrap(); assert_eq!( forced .edited .iter() .map(|(n, _)| n.as_str()) .collect::>(), - vec!["mi350x"], + vec!["MI350X-1x8"], "the edited builtin must be named" ); assert_eq!( - mirage_core::store::profile_get("mi350x").unwrap(), + mirage_core::store::topology_get("MI350X-1x8").unwrap(), mine, "the user's edits must survive" ); // And it is a report, not a failure: the pass carried on and // refreshed everything it safely could. Abandoning the run here - // was what made repairing three edited builtins take three runs. - for name in ["mi300x", "mi450x"] { + // was what made repairing several edited builtins take one run each. + for name in ["MI350X-1x1", "MI350X-2x8", "MI300X-1x8"] { assert!( forced.documents.contains(&(name.to_string(), true)), "{name} should have been refreshed" @@ -265,15 +595,15 @@ mod tests { // Deleting the edited one restores the shipped version, which is // what the report tells the user to do. - std::fs::remove_file(mirage_core::paths::profile_path("mi350x")).unwrap(); - let clean = ensure_profiles(true).unwrap(); + std::fs::remove_file(mirage_core::paths::topology_path("MI350X-1x8")).unwrap(); + let clean = ensure_topologies(true).unwrap(); assert!(clean.edited.is_empty()); assert!(clean.documents.iter().all(|(_, w)| *w)); assert_eq!( - mirage_core::store::profile_get("mi350x") + mirage_core::store::topology_get("MI350X-1x8") .unwrap() - .description, - None + .num_nodes, + 1 ); mirage_core::paths::clear_test_root(); @@ -288,17 +618,89 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); mirage_core::paths::set_test_root(tmp.path()); - ensure_profiles(false).unwrap(); - for name in ["mi300x", "mi350x", "mi450x"] { - let mut mine = mirage_core::store::profile_get(name).unwrap(); - mine.description = Some(format!("my own {name}")); - mirage_core::state::write_json(&mirage_core::paths::profile_path(name), &mine).unwrap(); + ensure_topologies(false).unwrap(); + let edited = ["MI300X-1x8", "MI350X-1x1", "MI350X-1x8"]; + for name in edited { + let mut mine = mirage_core::store::topology_get(name).unwrap(); + mine.num_nodes = 7; + mirage_core::state::write_json(&mirage_core::paths::topology_path(name), &mine) + .unwrap(); } - let forced = ensure_profiles(true).unwrap(); + let forced = ensure_topologies(true).unwrap(); let mut named: Vec<&str> = forced.edited.iter().map(|(n, _)| n.as_str()).collect(); named.sort_unstable(); - assert_eq!(named, vec!["mi300x", "mi350x", "mi450x"]); + assert_eq!(named, edited.to_vec()); + + mirage_core::paths::clear_test_root(); + } + + /// A builtin profile is usable without ever being written, and + /// stays unwritten however many commands run. + /// + /// This is the whole point of generating them: `mirage run --profile + /// mi350x` on a fresh machine, and on one whose config directory + /// mirage cannot write, has to find `mi350x`. Seeding them meant a + /// directory that could not be written read as a machine with no + /// profiles at all — `profile list` printed nothing and exited 0. + #[test] + fn builtin_profiles_are_served_without_being_stored() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + let shipped: Vec = profiles().iter().map(|(n, _)| (*n).to_string()).collect(); + assert!(shipped.contains(&"mi350x".to_string())); + + // Every startup pass there is, and then the whole set is still + // readable and still not on disk. + ensure_agents(false).unwrap(); + ensure_topologies(false).unwrap(); + + assert_eq!(mirage_core::store::profile_list().unwrap(), shipped); + for name in &shipped { + let profile = mirage_core::store::profile_get(name).unwrap(); + assert_eq!(&profile.name, name); + assert!( + !mirage_core::paths::profile_path(name).exists(), + "{name} must not have been written" + ); + } + assert!(!mirage_core::paths::profile_root().exists()); + + mirage_core::paths::clear_test_root(); + } + + /// Writing over a builtin's name shadows it; deleting that file + /// brings the builtin back, with nothing to re-seed. + #[test] + fn a_profile_file_shadows_the_builtin_of_that_name() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + let mut mine = mirage_core::store::profile_get("mi350x").unwrap(); + mine.description = Some("my own mi350x".to_string()); + mirage_core::state::write_json(&mirage_core::paths::profile_path("mi350x"), &mine).unwrap(); + + assert_eq!(mirage_core::store::profile_get("mi350x").unwrap(), mine); + // Shadowed, not duplicated. + let listed = mirage_core::store::profile_list().unwrap(); + assert_eq!(listed.iter().filter(|n| *n == "mi350x").count(), 1); + + mirage_core::store::profile_delete("mi350x").unwrap(); + assert_ne!(mirage_core::store::profile_get("mi350x").unwrap(), mine); + assert!( + mirage_core::store::profile_get("mi350x") + .unwrap() + .description + .is_some() + ); + + // And the builtin itself cannot be deleted: there is no file, and + // saying "deleted" would be a lie. + let error = mirage_core::store::profile_delete("mi350x").unwrap_err(); + assert!(error.to_string().contains("builtin"), "{error}"); mirage_core::paths::clear_test_root(); } diff --git a/emulation/mirage/builtin/src/presets.rs b/emulation/mirage/builtin/src/presets.rs index 1d3e6d767a62..97528aa3a700 100644 --- a/emulation/mirage/builtin/src/presets.rs +++ b/emulation/mirage/builtin/src/presets.rs @@ -1,32 +1,28 @@ -//! The parts of a rocjitsu preset the builtin agents must not drift from. +//! The rocjitsu configs mirage ships, embedded by `build.rs`. //! -//! Filled in by `build.rs`, which reads `rocjitsu/configs/*.json` — see -//! there for why these five values are read rather than written, and for -//! what is deliberately left to the [`agents`](mod@crate::agents) -//! module instead. +//! One entry per GPU rocjitsu has a config for. Everything mirage +//! preloads is derived from this table: the builtin agents are the +//! `{vm, topology}` in each config ([`mod@crate::agents`]), and the +//! builtin profiles are one per entry, generated on demand +//! ([`mod@crate::profiles`]). -/// The arch name and per-CU limits of one rocjitsu preset. -/// -/// Strings because that is what a component's `config` carries: rocjitsu -/// reads these back out of the JSON mirage writes, and parsing them to -/// integers here only to print them again would be a chance to change -/// them. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// One shipped rocjitsu config, as mirage addresses it. pub(crate) struct Preset { - /// The preset file this came from, for error messages and doc. - pub(crate) preset: &'static str, - /// `vm.arch` — which ISA rocjitsu emulates, and therefore which - /// limits it enforces on the values below. - pub(crate) arch: &'static str, - /// Wavefront slots per compute unit. The one that bites: `cdna5` - /// caps it at 64 and rocjitsu refuses a larger value outright. - pub(crate) num_wf_slots: &'static str, - /// Scalar registers per wavefront. - pub(crate) sgprs_per_wf: &'static str, - /// Vector registers per wavefront. - pub(crate) vgprs_per_wf: &'static str, - /// Local data share per compute unit, in KiB. - pub(crate) lds_size_kb: &'static str, + /// The name the agent and its profile are addressed by. + pub name: &'static str, + /// The `rocjitsu/configs/.json` it was read from, named in + /// the profile's description so a user can go and read the source. + pub stem: &'static str, + /// `vm.gpu.device.marketing_name` — the GPU this describes. + pub marketing_name: &'static str, + /// `vm.arch` — the architecture rocjitsu emulates it as. + pub arch: &'static str, + /// The `{vm, topology}` document, as JSON. Parsed rather than + /// constructed, which is what keeps a field the config omits + /// omitted; see [`mirage_core::agent::AgentDef`]. + pub agent: &'static str, + /// The full RocJITsu source config, as JSON. + pub config: &'static str, } include!(concat!(env!("OUT_DIR"), "/presets.rs")); diff --git a/emulation/mirage/builtin/src/profiles.rs b/emulation/mirage/builtin/src/profiles.rs index 0c30bdc49302..cf1031c45a9b 100644 --- a/emulation/mirage/builtin/src/profiles.rs +++ b/emulation/mirage/builtin/src/profiles.rs @@ -1,45 +1,86 @@ -//! Strongly-typed builtin [`ProfileDef`]s. +//! The builtin [`ProfileDef`]s: one per rocjitsu config mirage ships. //! -//! These are the ready-to-use presets mirage preloads into -//! `/profile/` so a fresh install can `mirage run -//! --profile ` without first hand-building a profile. -//! -//! Each profile pins a single GPU agent and is named after that GPU in -//! lowercase (e.g. `mi450x`). Every builtin profile targets the +//! These are the ready-to-use presets that let a fresh install `mirage +//! run --profile ` without first hand-building a profile. Each +//! pins a single GPU agent, is named after that agent, and targets the //! rocjitsu software emulator, which works on every builtin agent. +//! +//! # They are generated, not stored +//! +//! Nothing here is ever written to `/profile/`. A builtin +//! profile is derived from the agent it pins, every time it is asked +//! for, and [`mirage_core::store`] serves it from here when the profile +//! directory has no file of that name — see [`crate::builtin_profiles`]. +//! +//! Writing them out had no upside and two costs. A profile is three +//! fields around an agent reference, so the file added nothing the agent +//! did not already say, and it went stale the moment a mirage upgrade +//! changed how the derivation works — leaving the user to run `mirage +//! state builtins` to pick up a definition mirage could simply have +//! produced. And they had to be seeded before they could be listed, +//! which made `mirage profile list` on an unwritable config directory +//! report a machine with no profiles at all. +//! +//! Writing your own profile under a builtin's name still works and still +//! wins: the file shadows the generated one, and deleting it brings the +//! generated one back. use mirage_core::common::{MaybeRef, SimpleMap}; use mirage_core::emulator::{EmulatorDef, EmulatorKind, ExecMode}; use mirage_core::profile::ProfileDef; use mirage_core::topology::TopologyDef; -/// All builtin profiles, keyed by the name written to disk. -/// -/// Names are the target GPU in lowercase (e.g. `mi450x`) so the GPU -/// each profile targets is visible at a glance in `mirage profile -/// list`. Every builtin targets the rocjitsu emulator. +use crate::presets::{PRESETS, Preset}; + +/// The emulator every builtin profile targets. +const EMULATOR: &str = "rocjitsu"; + +/// All builtin profiles, keyed by the name they are addressed by — the +/// same name as the agent each one pins. pub fn profiles() -> Vec<(&'static str, ProfileDef)> { - vec![ - ("mi300x", profile("mi300x", "rocjitsu", "MI300X")), - ("mi350x", profile("mi350x", "rocjitsu", "MI350X")), - ("mi450x", profile("mi450x", "rocjitsu", "MI450X")), - ] + PRESETS + .iter() + .map(|preset| (preset.name, profile(preset))) + .collect() +} + +/// The builtin profile called `name`, if mirage ships one. +/// +/// Case-insensitive, as profile names are everywhere else. +#[must_use] +pub fn profile_named(name: &str) -> Option { + PRESETS + .iter() + .find(|preset| preset.name.eq_ignore_ascii_case(name)) + .map(profile) } -/// Build a single-GPU builtin profile pinning `agent`. -fn profile(name: &str, emulator: &str, agent: &str) -> ProfileDef { +/// Build the single-GPU profile for one shipped config. +fn profile(preset: &Preset) -> ProfileDef { ProfileDef { - name: name.to_string(), - description: None, + name: preset.name.to_string(), + // Seven profiles need telling apart, and the name alone does not + // do it: `gfx1151` names a file, not a machine. The GPU, the + // architecture rocjitsu emulates it as, and the config it comes + // from are the three things a user picking one wants, and the + // last is where they go to read the rest. + description: Some(format!( + "{} ({}), from RocJITsu's configs/{}.json", + preset.marketing_name, preset.arch, preset.stem + )), emulator: EmulatorDef { - emulator: EmulatorKind::from(emulator), + extra: Default::default(), + emulator: EmulatorKind::from(EMULATOR), plugins: Default::default(), exec_mode: ExecMode::default(), options: SimpleMap::default(), + // One GPU on one node. Wider runs come from `--num-nodes` + // and `--gpus-per-node`, which override this per run, so the + // profile does not have to guess at a shape. topology: MaybeRef::Owned(TopologyDef { num_nodes: 1, gpus_per_node: 1, - agent: MaybeRef::Ref(agent.to_string()), + agent: MaybeRef::Ref(preset.name.to_string()), }), }, containerize: None, @@ -52,22 +93,43 @@ mod tests { use super::*; + /// There is exactly one profile per builtin agent, and it pins that + /// agent. A profile naming an agent mirage does not ship is a + /// dangling reference the first `mirage run` would meet. #[test] - fn ships_expected_profiles() { - let names: Vec<&str> = profiles().iter().map(|(n, _)| *n).collect(); - assert_eq!(names, vec!["mi300x", "mi350x", "mi450x"]); + fn every_profile_pins_the_agent_it_is_named_after() { + let agents: Vec<&str> = crate::agents::agents() + .into_iter() + .map(|(name, _)| name) + .collect(); + let profiles = profiles(); + assert_eq!( + profiles.iter().map(|(n, _)| *n).collect::>(), + agents, + "one profile per agent" + ); + for (name, profile) in profiles { + let MaybeRef::Owned(topology) = &profile.emulator.topology else { + panic!("builtin profile {name} must carry its topology inline"); + }; + assert_eq!(topology.agent, MaybeRef::Ref(name.to_string())); + assert_eq!(topology.total_gpus(), 1); + assert_eq!(profile.name, name); + } } #[test] fn every_builtin_targets_rocjitsu() { for (name, p) in profiles() { assert_eq!( - p.emulator.emulator, "rocjitsu", + p.emulator.emulator, EMULATOR, "builtin profile {name} must target rocjitsu" ); } } + /// Profiles are stored lowercase, so a builtin whose name is not + /// already lowercase could never be written over by the user. #[test] fn names_are_lowercase() { for (name, _) in profiles() { @@ -92,4 +154,25 @@ mod tests { ); } } + + /// Each description says which GPU it is and where it came from — + /// the two questions `mirage profile list -l` exists to answer. + #[test] + fn descriptions_name_the_gpu_and_its_config() { + for preset in PRESETS { + let description = profile_named(preset.name).unwrap().description.unwrap(); + assert!(description.contains(preset.marketing_name), "{description}"); + assert!( + description.contains(&format!("configs/{}.json", preset.stem)), + "{description}" + ); + } + } + + #[test] + fn profiles_are_addressed_case_insensitively() { + assert_eq!(profile_named("MI350X"), profile_named("mi350x")); + assert!(profile_named("mi350x").is_some()); + assert!(profile_named("no-such-gpu").is_none()); + } } diff --git a/emulation/mirage/core/Cargo.toml b/emulation/mirage/core/Cargo.toml index 313f25eb65b6..378c83a52684 100644 --- a/emulation/mirage/core/Cargo.toml +++ b/emulation/mirage/core/Cargo.toml @@ -15,6 +15,8 @@ nix = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } inventory = { workspace = true } +goblin = { workspace = true } +rustix = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/emulation/mirage/core/src/agent.rs b/emulation/mirage/core/src/agent.rs index 1331f07fb4a2..e3edf6201b11 100644 --- a/emulation/mirage/core/src/agent.rs +++ b/emulation/mirage/core/src/agent.rs @@ -10,10 +10,8 @@ //! system-level layout that arranges agents into racks/nodes lives //! in [`crate::topology`]. //! -//! Every type here rejects unknown fields; see [`crate::profile`] for -//! why. It bites hardest on an agent, where the fields are the emulated -//! device's identity — a mistyped `l2_size_kb` that silently defaulted to -//! zero produced a GPU the workload could see and could not explain. +//! Additional fields are retained for the emulator, including fields in +//! nested device and topology objects. use serde::{Deserialize, Serialize}; @@ -23,8 +21,9 @@ fn one() -> u32 { /// Key-value pair for component configuration. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct ConfigEntry { + #[serde(flatten)] + pub extra: serde_json::Map, pub key: String, /// All values as strings, parsed by the factory. @@ -33,8 +32,9 @@ pub struct ConfigEntry { /// Port definition for dynamic ports. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct PortDef { + #[serde(flatten)] + pub extra: serde_json::Map, pub name: String, /// "in" or "out". @@ -46,8 +46,9 @@ pub struct PortDef { /// Component definition (recursive for hierarchy). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct ComponentDef { + #[serde(flatten)] + pub extra: serde_json::Map, /// Name or range pattern like `"xcd[0:7]"`. pub name: String, @@ -70,8 +71,9 @@ pub struct ComponentDef { /// Range variable for link pattern expansion. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct ForRange { + #[serde(flatten)] + pub extra: serde_json::Map, /// Variable name: "i", "j", "k". pub var_name: String, @@ -84,8 +86,9 @@ pub struct ForRange { /// Link definition (direct or pattern-based). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] pub struct LinkDef { + #[serde(flatten)] + pub extra: serde_json::Map, /// Direct source: "soc.xcd0.l2.hbm_out". #[serde(default)] pub src: String, @@ -116,6 +119,7 @@ pub struct LinkDef { impl Default for LinkDef { fn default() -> Self { Self { + extra: Default::default(), src: String::new(), dst: String::new(), pattern: String::new(), @@ -132,8 +136,9 @@ impl Default for LinkDef { /// Mirrors the flatbuffer `TopologyDef` in /// `rocjitsu/schemas/simulation_config.fbs`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct AgentTopologyDef { + #[serde(flatten)] + pub extra: serde_json::Map, pub root: ComponentDef, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -143,8 +148,9 @@ pub struct AgentTopologyDef { /// KFD device identity and topology properties for sysfs generation. /// Mirrors `KfdDeviceInfo` in the rocjitsu flatbuffer schema. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct KfdDeviceInfo { + #[serde(flatten)] + pub extra: serde_json::Map, #[serde(default)] pub gpu_id: u32, #[serde(default)] @@ -202,6 +208,8 @@ pub struct KfdDeviceInfo { #[serde(default)] pub num_sdma_xgmi_engines: u32, #[serde(default)] + pub num_sdma_queues_per_engine: u32, + #[serde(default)] pub num_cp_queues: u32, #[serde(default)] pub max_engine_clk_fcompute: u32, @@ -209,8 +217,9 @@ pub struct KfdDeviceInfo { /// AMDGPU memory configuration. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct GpuMemoryConfig { + #[serde(flatten)] + pub extra: serde_json::Map, #[serde(default)] pub size_mb: u32, #[serde(default)] @@ -219,8 +228,9 @@ pub struct GpuMemoryConfig { /// AMDGPU top-level configuration. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct AmdgpuConfig { + #[serde(flatten)] + pub extra: serde_json::Map, #[serde(default)] pub num_xcds: u32, #[serde(default)] @@ -240,31 +250,161 @@ pub struct AmdgpuConfig { } /// Virtual machine hardware model. Mirrors `VirtualMachineConfig` -/// in the rocjitsu flatbuffer schema. `programs` is intentionally -/// omitted: it's runtime workload configuration, not hardware. +/// in the rocjitsu flatbuffer schema. Fields without a typed accessor, +/// including `programs`, are retained in the passthrough map. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] pub struct VirtualMachineConfig { + #[serde(flatten)] + pub extra: serde_json::Map, #[serde(default)] pub arch: String, #[serde(default)] pub gpu: AmdgpuConfig, } +impl KfdDeviceInfo { + /// The gfx target this device presents, or `None` when it names no + /// GPU. + /// + /// The authority for the question, because this is where the answer + /// is written down: anything that needs the ISA a session emulates + /// reaches this device and asks, rather than keeping a second copy + /// that can disagree with it. + #[must_use] + pub fn gfx_target(&self) -> Option { + crate::hardware::GfxTarget::new(self.gfx_target_version) + } +} + /// Top-level agent (single-device hardware) definition. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] +/// +/// # Absent is not zero +/// +/// rocjitsu's schema has its own defaults, and several of them are not +/// Rust's: `drm_render_minor` defaults to 128 there and to 0 here, and a +/// 0 maps the emulated GPU onto a render node that does not exist, so +/// HSA aborts with `OUT_OF_RESOURCES`. A field the document did not +/// mention therefore has to stay unmentioned when mirage writes the +/// document back out, rather than being written as the Rust default. +/// +/// [`omitted`](Self::omitted) is how that is remembered: every path +/// `Deserialize` had to fill in, with the value it filled in, so +/// `Serialize` can take it back out again. Two consequences a caller +/// has to know about, because the type cannot enforce either: +/// +/// * The record is taken when the document is *parsed*. An `AgentDef` +/// built in Rust — [`Default::default`], a struct literal — has an +/// empty record and serializes every field explicitly, `drm_render_minor: 0` +/// included. Build agents by parsing, not by constructing, anywhere +/// the result reaches rocjitsu. +/// * The record is keyed by JSON pointer, so array paths are keyed by +/// *index*. Inserting into, removing from or reordering any `Vec` in +/// here after parsing re-points those paths at different elements, +/// and an element that happens to hold the recorded default loses +/// that field on the next serialize. Reparse rather than mutate a +/// `Vec`. +#[derive(Debug, Clone, Eq, Default)] pub struct AgentDef { + /// Document keys with no typed field of their own, kept so they + /// reach the emulator. `vm` and `topology` are not among them and + /// are overwritten on serialize by the fields below. + pub extra: serde_json::Map, pub vm: VirtualMachineConfig, pub topology: AgentTopologyDef, + /// `(pointer, value)` for every field `Deserialize` defaulted + /// because the document omitted it. See the type docs. + omitted: Vec<(String, serde_json::Value)>, +} + +/// Two agents are equal when they describe the same machine. +/// +/// Deliberately not derived: `omitted` is provenance, not content, so a +/// terse document and a fully spelled-out one describing the same GPU +/// would otherwise compare unequal while serializing identically. +impl PartialEq for AgentDef { + fn eq(&self, other: &Self) -> bool { + self.extra == other.extra && self.vm == other.vm && self.topology == other.topology + } +} + +impl Serialize for AgentDef { + fn serialize(&self, serializer: S) -> Result { + let mut value = serde_json::Value::Object(self.extra.clone()); + value["vm"] = serde_json::to_value(&self.vm).map_err(serde::ser::Error::custom)?; + value["topology"] = + serde_json::to_value(&self.topology).map_err(serde::ser::Error::custom)?; + for (path, default) in &self.omitted { + if value.pointer(path) == Some(default) + && let Some((parent, key)) = path.rsplit_once('/') + && let Some(object) = value + .pointer_mut(parent) + .and_then(serde_json::Value::as_object_mut) + { + object.remove(&key.replace("~1", "/").replace("~0", "~")); + } + } + value.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for AgentDef { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct Fields { + vm: VirtualMachineConfig, + topology: AgentTopologyDef, + #[serde(flatten)] + extra: serde_json::Map, + } + let original = serde_json::Value::deserialize(deserializer)?; + let fields: Fields = + serde_json::from_value(original.clone()).map_err(serde::de::Error::custom)?; + let mut agent = Self { + vm: fields.vm, + topology: fields.topology, + extra: fields.extra, + omitted: Vec::new(), + }; + let defaulted = serde_json::to_value(&agent).map_err(serde::de::Error::custom)?; + collect_omitted(&original, &defaulted, "", &mut agent.omitted); + Ok(agent) + } +} + +fn collect_omitted( + original: &serde_json::Value, + defaulted: &serde_json::Value, + path: &str, + omitted: &mut Vec<(String, serde_json::Value)>, +) { + if let Some(object) = defaulted.as_object() { + for (key, value) in object { + let child_path = format!("{path}/{}", key.replace('~', "~0").replace('/', "~1")); + if let Some(child) = original.get(key) { + collect_omitted(child, value, &child_path, omitted); + } else { + omitted.push((child_path, value.clone())); + } + } + } else if let (Some(original), Some(defaulted)) = (original.as_array(), defaulted.as_array()) { + for (index, (child, value)) in original.iter().zip(defaulted).enumerate() { + collect_omitted(child, value, &format!("{path}/{index}"), omitted); + } + } +} + +impl AgentDef { + /// The gfx target this agent's GPU presents, or `None` when it names + /// no GPU. See [`KfdDeviceInfo::gfx_target`]. + #[must_use] + pub fn gfx_target(&self) -> Option { + self.vm.gpu.device.gfx_target() + } } /// On-disk agent store backed by `/agent/`. /// -/// Agents may be stored as opaque JSON blobs by external backends -/// (e.g. rocjitsu's `cdna3`/`cdna4` flatbuffer configs). `get()` -/// will fail to parse those as [`AgentDef`]; callers that need raw -/// access should read the file at [`crate::paths::agent_path`]. +/// Additional emulator fields are retained when an agent is read or written. /// /// [`crate::store::agent_get`] is where a `MaybeRef::Ref` on a topology is /// followed, so it is diff --git a/emulation/mirage/core/src/emulator.rs b/emulation/mirage/core/src/emulator.rs index c02ad109389e..200440f9624a 100644 --- a/emulation/mirage/core/src/emulator.rs +++ b/emulation/mirage/core/src/emulator.rs @@ -31,29 +31,155 @@ pub type EmulatorKind = String; /// The emulator half of a profile: which backend, how it runs, and the /// system it emulates. /// -/// Unknown fields are rejected; see [`crate::profile`] for why. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +/// Additional fields are passed to the selected emulator's +/// configuration, or refused by a backend that has none — see +/// [`EmulatorDef::reject_extra`]. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] pub struct EmulatorDef { + /// Fields with no typed meaning to mirage, handed to the backend as + /// they were written. Deserialization never puts one of the field + /// names below in here; [`Serialize`] guarantees the same for a map + /// built in Rust, because a flattened map that shadows a typed + /// field would otherwise write a profile with duplicate JSON keys + /// that mirage could never read back. + #[serde(flatten)] + pub extra: serde_json::Map, + /// Which emulator backend runs this profile, by its canonical /// lowercase name. pub emulator: EmulatorKind, /// Which of the backend's plugins to enable, each with its argument /// object. See [`PluginsDef`] for what an empty object means. + #[serde(default)] pub plugins: PluginsDef, /// Functional or clocked emulation. + #[serde(default)] pub exec_mode: ExecMode, /// Backend-specific configuration overrides, e.g. /// `{"gpu_model": "cdna3"}`. + #[serde(default)] pub options: SimpleMap, /// System topology (rack/node/GPU layout plus the per-GPU agent). pub topology: MaybeRef, } +impl EmulatorDef { + /// Refuse passthrough fields on behalf of a backend that does not + /// forward them. + /// + /// [`extra`](Self::extra) is a promise the *rocjitsu* backend keeps: + /// it merges these keys into the simulation config it synthesises, + /// so a profile can reach a rocjitsu setting mirage has no typed + /// field for. No other backend reads them. Since the map lives on + /// the shared `EmulatorDef`, dropping `deny_unknown_fields` to make + /// that promise possible also stopped every *other* backend + /// rejecting a key it will never act on — a misspelled or + /// unsupported field under a HotSwap or rocjitsu-dbt emulator would + /// parse, and the session would come up with the defaults it was + /// meant to change, silently. + /// + /// So a backend without a passthrough calls this from + /// [`EmulatorBackend::validate_profile`], which the CLI and the + /// daemon both run before a profile is written — the last point at + /// which the user is still looking at the document the key came + /// from. + /// + /// # Errors + /// + /// Names the offending keys and the backend that has no use for + /// them. + pub fn reject_extra(&self, backend: &str) -> std::result::Result<(), String> { + if self.extra.is_empty() { + return Ok(()); + } + let keys = self + .extra + .keys() + .map(|key| format!("{key:?}")) + .collect::>() + .join(", "); + Err(format!( + "emulator field(s) {keys} are not {backend} settings and {backend} has \ + no configuration to pass them to; remove them, or correct the spelling \ + of the mirage field that was meant" + )) + } + + /// The agent every GPU in this system instantiates, when this + /// definition is resolved. + /// + /// `None` while either reference is still a name. Resolving one means + /// reading the on-disk stores, which is [`Self::resolve_refs`]'s job + /// and not something an accessor should do behind a caller's back — + /// an accessor that silently hit the filesystem would also silently + /// pick up an edit made after bring-up, which is the whole class of + /// bug a session snapshot exists to prevent. + #[must_use] + pub fn agent(&self) -> Option<&crate::agent::AgentDef> { + match &self.topology { + MaybeRef::Owned(topology) => match &topology.agent { + MaybeRef::Owned(agent) => Some(agent), + MaybeRef::Ref(_) => None, + }, + MaybeRef::Ref(_) => None, + } + } + + /// Follow every by-name reference in this definition, in place, so + /// that afterwards [`Self::agent`] answers. + /// + /// Done once, when a session is created, so the rest of that + /// session's life reads documents nothing can edit underneath it. + /// + /// # Errors + /// + /// Returns an error when a topology or agent reference names a + /// document that does not exist or does not parse. + pub fn resolve_refs(&mut self) -> Result<()> { + let mut topology = match &self.topology { + MaybeRef::Owned(topology) => topology.clone(), + MaybeRef::Ref(name) => crate::store::topology_get(name)?, + }; + if let MaybeRef::Ref(name) = &topology.agent { + topology.agent = MaybeRef::Owned(crate::agent::store::get(name)?); + } + self.topology = MaybeRef::Owned(topology); + Ok(()) + } +} + +/// Written as `extra` with the typed fields laid over it. +/// +/// Not derived: `#[serde(flatten)]` streams both halves, so an `extra` +/// holding a typed field's name — `extra` is public, and the profile +/// parser is the only thing that currently keeps one out — emits that +/// key twice. `state::write_json` streams straight to the file, so the +/// result is a profile on disk that fails to parse with `duplicate +/// field`, and the user's only copy of it is unreadable. Building a +/// `Value` first makes the typed field win instead. +impl Serialize for EmulatorDef { + fn serialize( + &self, + serializer: S, + ) -> std::result::Result { + let mut value = serde_json::Value::Object(self.extra.clone()); + for (key, typed) in [ + ("emulator", serde_json::to_value(&self.emulator)), + ("plugins", serde_json::to_value(&self.plugins)), + ("exec_mode", serde_json::to_value(&self.exec_mode)), + ("options", serde_json::to_value(&self.options)), + ("topology", serde_json::to_value(&self.topology)), + ] { + value[key] = typed.map_err(serde::ser::Error::custom)?; + } + value.serialize(serializer) + } +} + /// Whether the host's hardware/environment can actually run an /// emulator. This is distinct from [`EmulatorBackend::installed`]: /// an emulator can be installed yet unsupported (e.g. HotSwap installed @@ -231,6 +357,56 @@ pub trait EmulatorBackend: Sync + Send + std::fmt::Debug { /// runtime is present and responsive. fn health(&self, ctx: &SessionContext) -> SessionHealth; + /// Whether this backend stands up a synthetic GPU whose ISA the + /// workload's own ROCm runtime has to recognise. + /// + /// `false` — the default — for a backend that retargets code onto the + /// host's real GPU. Such a backend still has an agent, and that agent + /// still names a device, but ROCr is never asked to enumerate it, so + /// the ISA says nothing about whether the session will work. A new + /// backend therefore gets silence without having to know that the + /// preflight in [`crate::rocr`] exists. + fn presents_emulated_device(&self) -> bool { + false + } + + /// Reconcile `profile` with configuration only this backend can read, + /// so that the profile describes the machine the session will really + /// stand up. + /// + /// Called once, on an already-resolved profile, before a session + /// freezes it. The case it exists for is a drop-in emulator config + /// file: the file describes a device of its own, and without this the + /// profile's agent would still name the device the profile was + /// written around — a device that is not the one about to exist. + /// Rather than reporting the real device separately and leaving two + /// answers in play, a backend corrects the agent here and everything + /// downstream keeps reading the one document. + /// + /// A backend that cannot tell what the file describes must clear the + /// agent's `gfx_target_version` rather than leave a stale value: no + /// answer is a verdict the preflight handles, and a confident wrong + /// one is not. + /// + /// `session_dir` is the session's scratch directory, already created. + /// A backend that reads a file here should snapshot it there and point + /// the profile at the snapshot, so the device it reports and the + /// configuration it later injects come from one reading of the file + /// rather than two. + /// + /// # Errors + /// + /// Returns an error only when the profile is unusable — not when the + /// backend simply has nothing to correct, which is the default. + fn reconcile_profile( + &self, + profile: &mut ProfileDef, + session_dir: &std::path::Path, + ) -> Result<()> { + let _ = (profile, session_dir); + Ok(()) + } + /// Compute the env vars / `LD_PRELOAD` / files to inject into a /// workload run under this emulator. Returns an error when the /// emulator is selected but its runtime library or assets are diff --git a/emulation/mirage/core/src/hardware.rs b/emulation/mirage/core/src/hardware.rs index 712b60e99867..0ab1ff20080e 100644 --- a/emulation/mirage/core/src/hardware.rs +++ b/emulation/mirage/core/src/hardware.rs @@ -31,6 +31,42 @@ pub fn gpu_gfx_versions() -> Vec { detect_from(Path::new(KFD_NODES)) } +/// A GPU's gfx architecture, as KFD reports it. +/// +/// A newtype over the packed `gfx_target_version` rather than the bare +/// `u32` the wire formats use, because zero is not a target: KFD gives a +/// CPU-only node `gfx_target_version 0`, and so does a device a document +/// simply did not fill in. Holding a [`NonZeroU32`] makes "there is no +/// GPU here" a thing a caller has to handle rather than a value it can +/// forget to compare against, and makes the two cases indistinguishable +/// on purpose — neither names a GPU. +/// +/// [`Display`](std::fmt::Display) renders the conventional name, so the +/// packed encoding never has to be spelled out at a call site. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct GfxTarget(std::num::NonZeroU32); + +impl GfxTarget { + /// The target `version` names, or `None` when it names none. + #[must_use] + pub fn new(version: u32) -> Option { + std::num::NonZeroU32::new(version).map(Self) + } + + /// The packed `gfx_target_version`, for the wire formats and the + /// per-target tables that are keyed on it. + #[must_use] + pub fn version(self) -> u32 { + self.0.get() + } +} + +impl std::fmt::Display for GfxTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&gfx_name(self.0.get())) + } +} + /// Render a packed `gfx_target_version` as a conventional `gfxNNN` /// architecture string. The encoding is decimal `MMMmmpp` (major, /// minor, step), so `90402` → `gfx942` and `90500` → `gfx950`. Step @@ -84,6 +120,22 @@ mod tests { use super::*; + /// Zero is not a target, and the two ways of arriving at it are the + /// same answer: a CPU-only KFD node and a device a document left + /// unfilled both name no GPU. + #[test] + fn a_zero_target_version_is_no_target() { + assert_eq!(GfxTarget::new(0), None); + assert_eq!( + GfxTarget::new(120500).map(|t| t.to_string()), + Some("gfx1250".to_string()) + ); + assert_eq!(GfxTarget::new(90402).unwrap().version(), 90402); + // The hex step the gfx convention uses, so the newtype's own + // rendering cannot drift from `gfx_name`. + assert_eq!(GfxTarget::new(90010).unwrap().to_string(), "gfx90a"); + } + #[test] fn parses_gfx_target_version_line() { let props = "cpu_cores_count 0\ngfx_target_version 90402\nsimd_count 304\n"; diff --git a/emulation/mirage/core/src/lib.rs b/emulation/mirage/core/src/lib.rs index 3d927a9ad486..bf006aa6a6ec 100644 --- a/emulation/mirage/core/src/lib.rs +++ b/emulation/mirage/core/src/lib.rs @@ -38,6 +38,7 @@ pub mod profile; pub mod proto; pub mod reclaim; pub mod registry; +pub mod rocr; pub mod session; pub mod state; pub mod store; diff --git a/emulation/mirage/core/src/profile.rs b/emulation/mirage/core/src/profile.rs index 43ba2b4cbfde..b24d76686f32 100644 --- a/emulation/mirage/core/src/profile.rs +++ b/emulation/mirage/core/src/profile.rs @@ -1,25 +1,17 @@ //! Profile: a reusable bundle of emulator configuration. //! -//! # Why every document type here rejects unknown fields +//! Extra profile fields are normalized into the emulator configuration and +//! passed through to the backend. Agent fields are retained recursively. +//! Mirage-owned container and system-topology controls still reject unknown +//! fields because they cannot be delegated to the emulator. //! -//! These are files people write and edit by hand, and mirage acts on -//! them: a profile decides how many GPUs the emulated machine has, which -//! image its nodes run in, and what is mounted into them. A key serde -//! does not recognise is, overwhelmingly, a key that was *meant* to do -//! something — `"descriptoin"`, `"read-only"`, a field from a newer -//! mirage — and dropping it silently means the machine that comes up is -//! quietly not the one the file describes. `deny_unknown_fields` turns -//! that into a parse error naming the key, at the moment the document is -//! read, which is the only moment the user is still looking at it. -//! -//! The cost is forward compatibility in one direction: a document written -//! by a newer mirage that added a field is rejected by an older one -//! rather than degraded. That is the trade we want. An older mirage -//! cannot honour a field it has never heard of, so its choice is between -//! saying so and running something else instead; and adding an *optional* -//! field is still fully backward compatible, because older documents that -//! omit it keep parsing. Only removals and renames need care, and those -//! need care anyway. +//! Delegating is not the same as accepting, and this layer cannot tell +//! the two apart: [`EmulatorDef::extra`] is one map on a definition +//! shared by every backend, but only `rocjitsu` merges it into a +//! configuration. A backend with nowhere to forward these keys refuses +//! them from its own `validate_profile`, which the CLI and the daemon +//! both run before a profile is written — see +//! [`EmulatorDef::reject_extra`]. use serde::{Deserialize, Serialize}; @@ -350,8 +342,7 @@ pub struct ContainerizedDef { /// A profile is a named, on-disk emulator preset that can be referenced /// by sessions. Profiles live in `$XDG_CONFIG_HOME/mirage/profile/`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ProfileDef { /// The profile's name (also used as the filename, `.json`). pub name: String, @@ -369,6 +360,73 @@ pub struct ProfileDef { pub containerize: Option, } +impl<'de> Deserialize<'de> for ProfileDef { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct Fields { + name: String, + #[serde(default)] + description: Option, + emulator: EmulatorDef, + #[serde(default)] + containerize: Option, + #[serde(flatten)] + extra: serde_json::Map, + } + let value = serde_json::Value::deserialize(deserializer)?; + let name = value["name"].as_str().unwrap_or("").to_string(); + // Only the fields a document may legally leave out are worth a + // warning. `name`, `emulator`, `emulator.emulator` and + // `emulator.topology` are required, so the parse below fails on + // them a line later and names them itself — warning first would + // just report the same omission twice, once under the wrong + // profile name for a document whose `name` is not a string. + for path in [ + "/emulator/plugins", + "/emulator/exec_mode", + "/emulator/options", + ] { + if value.pointer(path).is_none() { + tracing::warn!("profile {name:?} is missing field {path}"); + } + } + let mut fields: Fields = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + for (key, value) in fields.extra { + if matches!( + key.as_str(), + "plugins" | "exec_mode" | "options" | "topology" + ) { + return Err(serde::de::Error::custom(format!( + "profile field {key:?} belongs inside emulator" + ))); + } + if fields.emulator.extra.contains_key(&key) { + return Err(serde::de::Error::custom(format!( + "RocJITsu field {key:?} is specified both on the profile and its emulator" + ))); + } + // Say so. A root key mirage does not recognise is forwarded + // to the emulator, and the emulator drops what *it* does not + // recognise without a word — so a misspelled mirage field + // (`containerise`, `descriptoin`) otherwise disappears + // between the two with nothing anywhere to explain why the + // profile did not do what it says. This is the last layer + // that still knows which file the key came from. + tracing::warn!( + "profile {name:?} field {key:?} is not a mirage field; \ + forwarding it to the emulator, which ignores what it does not know" + ); + fields.emulator.extra.insert(key, value); + } + Ok(Self { + name: fields.name, + description: fields.description, + emulator: fields.emulator, + containerize: fields.containerize, + }) + } +} + impl ProfileDef { /// Ask this profile's emulator backend whether it can run it. /// @@ -393,6 +451,33 @@ impl ProfileDef { )), } } + + /// The agent every GPU in this profile instantiates, when the profile + /// is resolved. See [`crate::emulator::EmulatorDef::agent`]. + #[must_use] + pub fn agent(&self) -> Option<&crate::agent::AgentDef> { + self.emulator.agent() + } + + /// The gfx target a session on this profile presents to the + /// workload's ROCm runtime, or `None` when there is nothing for that + /// runtime to recognise. + /// + /// Two things have to be true for there to be an answer: the agent + /// has to name a GPU, and the backend has to be one that stands a + /// synthetic GPU up rather than retargeting onto the host's real one + /// (see + /// [`EmulatorBackend::presents_emulated_device`](crate::emulator::EmulatorBackend::presents_emulated_device)). + /// `None` on every other path, including a profile whose references + /// have not been followed and a backend this build was not compiled + /// with — the callers of this are diagnostics, and a diagnostic that + /// guesses is worse than one that stays quiet. + #[must_use] + pub fn emulated_gfx_target(&self) -> Option { + let backend = crate::emulator::get_emulator_backend(&self.emulator.emulator)?; + backend.presents_emulated_device().then_some(())?; + self.agent()?.gfx_target() + } } #[cfg(test)] @@ -420,6 +505,30 @@ mod tests { use super::*; + #[test] + fn profile_extras_round_trip_through_emulator() { + let profile: ProfileDef = serde_json::from_value(serde_json::json!({ + "name": "extended", "max_ticks": 123, + "emulator": {"emulator": "rocjitsu", "topology": "t", "vm": {"future": [1, 2]}} + })) + .unwrap(); + let json = serde_json::to_value(&profile).unwrap(); + assert_eq!(json["emulator"]["max_ticks"], 123); + assert_eq!(json["emulator"]["vm"]["future"], serde_json::json!([1, 2])); + assert_eq!(serde_json::from_value::(json).unwrap(), profile); + for extra in [ + serde_json::json!({"exec_mode": "clocked"}), + serde_json::json!({"vm": {}}), + ] { + let mut invalid = serde_json::to_value(&profile).unwrap(); + invalid + .as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + assert!(serde_json::from_value::(invalid).is_err()); + } + } + #[test] fn mount_parse_host_only() { let m = FileMount::parse("/data").unwrap(); diff --git a/emulation/mirage/core/src/proto.rs b/emulation/mirage/core/src/proto.rs index ec2da8bedb1e..8a6254703e8b 100644 --- a/emulation/mirage/core/src/proto.rs +++ b/emulation/mirage/core/src/proto.rs @@ -140,6 +140,35 @@ pub enum Response { pub struct SessionDescription { /// The session this describes. pub session: SessionId, + /// The profile this session was brought up on, with every by-name + /// reference already followed. + /// + /// Sent rather than the individual facts a client wants from it, so + /// that a client asking what device the session emulates reads the + /// agent that defines it instead of a second copy travelling + /// alongside. `mirage exec` cannot resolve this for itself: it takes + /// no `--profile`, and re-reading the stored one would answer with + /// whatever has been edited into it since — for a session that is + /// supposed to keep emulating the device it came up on, the stored + /// document is the wrong source. + /// + /// Optional on the wire so a newer `mirage exec` can still attach to + /// an older run process: it simply arrives absent, and the one thing + /// derived from it — [`SessionDescription::emulated_gfx_target`] — + /// answers `None`, which is the silence a diagnostic falls back to + /// anyway. + /// + /// The other direction is the asymmetric one, and worth knowing + /// before a field is added to a profile: [`ProfileDef`] rejects + /// unknown fields, deliberately, because a mistyped key in a + /// hand-written profile that silently defaulted was its own class of + /// bug. An *older* `mirage exec` attaching to a newer run therefore + /// refuses a profile carrying a field it has never heard of, rather + /// than ignoring it. Both halves of a session are the same binary in + /// every ordinary setup, so this buys the on-disk strictness at a + /// cost paid only by a mixed-version pair. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profile: Option, /// Number of nodes in the session's topology. pub node_count: u32, /// Processes per node in the job this session was created to run, @@ -170,6 +199,21 @@ pub struct SessionDescription { pub head_port: u16, } +impl SessionDescription { + /// The gfx target this session presents to the workload's ROCm + /// runtime, or `None` when there is nothing for that runtime to + /// recognise. + /// + /// Derived from [`SessionDescription::profile`] rather than carried + /// beside it, so there is one answer and it is the agent's. `None` + /// for a description from an older run process, which sent no + /// profile. + #[must_use] + pub fn emulated_gfx_target(&self) -> Option { + self.profile.as_ref()?.emulated_gfx_target() + } +} + /// Where each rank's processes really run, for a containerised session. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ContainerTargets { diff --git a/emulation/mirage/core/src/registry.rs b/emulation/mirage/core/src/registry.rs index 56197a3af4f4..1957f8a9e80c 100644 --- a/emulation/mirage/core/src/registry.rs +++ b/emulation/mirage/core/src/registry.rs @@ -125,6 +125,7 @@ pub fn default_emulator(specs: &[EmulatorInfo]) -> Option<&EmulatorInfo> { /// supplied system topology. pub fn make_def(spec: &EmulatorInfo, topology: TopologyDef) -> EmulatorDef { EmulatorDef { + extra: Default::default(), emulator: EmulatorKind::from(spec.name.clone()), plugins: Default::default(), exec_mode: ExecMode::default(), diff --git a/emulation/mirage/core/src/rocr.rs b/emulation/mirage/core/src/rocr.rs new file mode 100644 index 000000000000..14609d035e06 --- /dev/null +++ b/emulation/mirage/core/src/rocr.rs @@ -0,0 +1,1468 @@ +//! The ROCm runtime a workload will load, and the GPU targets it knows +//! how to enumerate. +//! +//! An emulated device is only half of a working session. mirage (through +//! rocjitsu) synthesises a KFD node for the GPU a session emulates, and +//! then something has to *recognise* it: `libhsa-runtime64.so`, the ROCm +//! runtime the workload itself loads, reads that node's +//! `gfx_target_version`, looks the ISA up in the table it was compiled +//! with, and — when the ISA is not in that table — skips the agent +//! entirely. +//! +//! Skips it silently. The emulated device is presented correctly, the +//! emulator daemon logs the same lines it logs for a target that works, +//! the workload runs, sees a CPU agent and no GPU, and exits 0. Nothing +//! anywhere says the target was the problem, and the obvious reading of +//! that evidence — that the emulator is broken for this GPU — is wrong +//! and expensive: it cost about a day of investigation, and was very +//! nearly filed as an emulator bug (issue #11361). +//! +//! So mirage asks the question before each workload starts. Which ROCm +//! runtime does this command's dynamic loader resolve, and can its image +//! rule out the ISA the session emulates? +//! +//! "The session", throughout, and never "the profile". The two usually +//! name the same GPU and a profile is the usual way to choose it, but a +//! drop-in `--config` describes a device of its own, and what a session +//! emulates is read from the configuration it was brought up on. The +//! terminology matters at the one point it is visible: the warning below +//! tells a user what to change, and naming the profile would send +//! somebody to edit an input that is not deciding it. +//! +//! # Why the answer is read out of the library's own image +//! +//! There is no side-effect-free API for it. `hsa_isa_from_name` would +//! answer exactly this question but needs `hsa_init()` first, which would +//! initialise ROCr in mirage's own process rather than inspect the +//! library and environment selected for the workload. What the binary +//! does guarantee is one-way: every ISA in the registry leaves its +//! target name in the image, and other ROCr subsystems — legacy +//! code-object conversion among them — leave target names there too. So +//! presence proves nothing, and only absence from a recognisably +//! complete ROCr image is a verdict. +//! +//! That is evidence rather than an interface, so this module fails +//! towards silence: a missing loader answer, an unreadable runtime, a +//! target mention, or too few target references all yield no verdict, +//! and even a verdict warns rather than refuses, because this diagnostic +//! must never block a run that would have worked. + +use std::collections::BTreeSet; +use std::ffi::{OsStr, OsString}; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +use goblin::Object; +use goblin::elf::dynamic::{DT_AUDIT, DT_DEPAUDIT}; +use nix::unistd::{AccessFlags, eaccess}; +use rustix::fs::getxattr; +use rustix::io::Errno; + +/// The ROCm runtime SONAME the dynamic loader actually resolves. Unlike +/// the unversioned development symlink, this name exists in a +/// runtime-only install and is what a workload's `DT_NEEDED` requests. +/// +/// Private: it is what this module recognises a runtime by, not +/// something anything outside it has to agree with. +const ROCR_SONAME: &str = "libhsa-runtime64.so.1"; + +/// How many plausible ISA names must be found in a library's image +/// before the absence of one more is worth reporting. +/// +/// The floor is what separates "this runtime does not support the +/// target" from "this scan did not find ROCr's target-bearing image". A +/// stripped, packed, or simply unexpected build yields a handful of +/// fragments at most; a real ROCm runtime references dozens of targets +/// across its ISA registry and code-object conversion support — ROCm +/// 7.0.2.2 references forty-five. Ten is far below a real image and far +/// above an accident. +const MIN_PLAUSIBLE_TARGET_REFERENCES: usize = 10; + +/// The largest workload executable this probe will read. +/// +/// Both images are read whole, so both need a ceiling, and over it there +/// is no verdict — the answer everything else this probe cannot do +/// gives. An executable this size is not one whose `DT_RUNPATH` is worth +/// a diagnostic. +const MAX_EXECUTABLE_BYTES: u64 = 256 * 1024 * 1024; + +/// The largest runtime image this probe will scan. Looser than +/// [`MAX_EXECUTABLE_BYTES`] because a debug ROCr legitimately is large. +const MAX_RUNTIME_BYTES: u64 = 512 * 1024 * 1024; + +/// The shortest run of characters after `gfx` that can be a whole +/// target name. +/// +/// `gfx942`, `gfx90a` and `gfx1250` are three and four; `gfx9`, `gfx11` +/// and `gfx1f` are the architecture-family prefixes and wildcard +/// patterns ROCr also carries, and are not targets anybody emulates. +/// Dropping them keeps the count in +/// [`MIN_PLAUSIBLE_TARGET_REFERENCES`] honest. +const MIN_TARGET_DIGITS: usize = 3; + +/// What the ROCm runtime on this host has to say about one GPU target. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TargetSupport { + /// The resolved runtime is recognisably ROCr and contains no + /// reference to this target. The workload will see no GPU. + Unsupported(Box), + /// No verdict: the command's loader did not identify one runtime, + /// loader-affecting state makes the answer ambiguous, the runtime + /// could not be read, its image is not recognisably ROCr, or it + /// mentions the target without proving that the ISA registry + /// contains it. Nothing is said to the user on this path. + Unknown, +} + +/// A target the ROCm runtime a workload will load does not know about, +/// with everything needed to say so usefully. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UnsupportedTarget { + /// The ISA the session emulates, e.g. `gfx1250`. + pub target: String, + /// The ROCm runtime mirage found, which is the one the workload is + /// expected to load. + pub runtime: PathBuf, + /// The ROCm version recorded beside that runtime, when there is one. + pub version: Option, + /// How many distinct target names the runtime image references. + /// Quoting it makes the heuristic checkable without claiming that + /// every reference is an ISA-registry entry. + pub references: usize, +} + +impl UnsupportedTarget { + /// The sentence mirage prints, in the shape the rest of its notices + /// use: what is wrong, what it will look like, and the way out. + /// + /// The symptom is spelled out rather than left implied. "ROCr does + /// not support gfx1250" is only actionable to somebody who already + /// knows what ROCr does with an agent it cannot name, and the module + /// documentation says why that is the whole point. + #[must_use] + pub fn explain(&self) -> String { + let version = self + .version + .as_deref() + .map_or_else(String::new, |v| format!(" (ROCm {v})")); + format!( + "the ROCm runtime this workload will load does not support {target}, which is \ + the GPU this session emulates. {runtime}{version} contains references to \ + {references} GPU targets but none to {target}, so it will skip the emulated device: the \ + workload will see no GPU at all and is likely to still exit 0. Run it under \ + a ROCm that knows {target} — `--image ` is the usual \ + way — or emulate a GPU this ROCm supports.", + target = self.target, + runtime = self.runtime.display(), + references = self.references, + ) + } +} + +/// Whether the ROCm runtime resolved for one host process rules out +/// `target`. +/// +/// `env` and `inherit_env` are the exact environment delta and +/// inheritance policy the process spawner will use. The command is +/// resolved through that effective `PATH` and inspected statically, +/// without loading or executing any object. A verdict is produced only +/// for a trusted system executable that directly needs ROCr and resolves +/// it through an unambiguous modern `DT_RUNPATH`. +/// +/// Legacy `DT_RPATH`, transitive or `dlopen` dependencies, cache-only +/// resolution, hardware-capability alternatives, mutable executables, +/// privileged executables, a system-wide preload, and loader overrides +/// all produce [`TargetSupport::Unknown`]. These deliberate false +/// negatives keep warnings trustworthy. Extending coverage requires a +/// bounded, loader-equivalent static resolver and can be done later +/// without weakening the rule that preflight never executes workload +/// code or emits a guess. +#[must_use] +pub fn check_target_for_process( + target: &str, + command: &str, + workdir: Option<&Path>, + env: &[(OsString, OsString)], + inherit_env: bool, +) -> TargetSupport { + let effective = merged_environment(std::env::vars_os(), env, inherit_env); + if global_preload_is_ambiguous(Path::new(GLOBAL_PRELOAD)) + || loader_state_is_ambiguous(&effective) + { + return TargetSupport::Unknown; + } + let Some((runtime, image)) = locate_for_process(command, workdir, &effective) else { + return TargetSupport::Unknown; + }; + // The image the loader trace already read, size-capped and checked + // against the executable's ELF class: judging those bytes rather than + // a second read of the file means reading a runtime of up to + // `MAX_RUNTIME_BYTES` once per workload, not twice. + verdict(target, &runtime, rocm_version_beside(&runtime), &image) +} + +fn merged_environment( + inherited: impl IntoIterator, + explicit: &[(OsString, OsString)], + inherit_env: bool, +) -> std::collections::BTreeMap { + let mut effective = if inherit_env { + inherited.into_iter().collect() + } else { + std::collections::BTreeMap::new() + }; + effective.extend(explicit.iter().cloned()); + effective +} + +/// The file glibc preloads into every dynamically linked process on the +/// host, whatever its environment. +const GLOBAL_PRELOAD: &str = "/etc/ld.so.preload"; + +/// Whether a system-wide preload could be interposing ROCr. +/// +/// `LD_PRELOAD` is the environment's half of this; `/etc/ld.so.preload` +/// is the host's, and no workload environment mentions it. A library +/// listed there is loaded before the workload's own dependencies and can +/// interpose the HSA entry points the target table is evidence about, so +/// the file's own image stops being the answer. +/// +/// Only an outright absent file is harmless. Present and empty says +/// nothing is preloaded; unreadable says this probe could not find out, +/// which is not the same as finding out that there is nothing. +fn global_preload_is_ambiguous(path: &Path) -> bool { + match std::fs::read(path) { + Ok(bytes) => bytes.iter().any(|byte| !byte.is_ascii_whitespace()), + Err(e) => e.kind() != std::io::ErrorKind::NotFound, + } +} + +/// State that can replace ROCr or change the ISA ROCr evaluates without +/// changing the loader's ordinary SONAME answer. +fn loader_state_is_ambiguous(env: &std::collections::BTreeMap) -> bool { + env.iter().any(|(key, value)| { + let Some(key) = key.to_str() else { + return false; + }; + // A search-list variable is ambiguous as soon as it is set at + // all, empty or not. An empty *element* of a search list means + // the current directory to glibc, so the meaning of an empty + // whole value is a question about one loader's parsing rather + // than about the workload — and this module answers questions it + // is sure of or stays quiet. + if loader_resolution_env_var(key) { + return true; + } + // The rest name something to load or a version to report, so an + // empty value asks for nothing. + !value.is_empty() && ambiguous_probe_env_var(key) + }) +} + +/// Whether changing `key` can change this process probe's answer. +/// +/// Used by the multi-rank caller to deduplicate only specs that resolve +/// the same command and runtime. Keep the exact loader inputs, the +/// variables that force an unknown verdict, and `PATH`, which selects +/// the command whose dependencies are traced. +#[must_use] +pub fn affects_process_probe(key: &str) -> bool { + key == "PATH" || loader_resolution_env_var(key) || ambiguous_probe_env_var(key) +} + +fn ambiguous_probe_env_var(key: &str) -> bool { + matches!(key, "LD_PRELOAD" | "LD_AUDIT" | "HSA_OVERRIDE_GFX_VERSION") + || key.starts_with("HSA_OVERRIDE_GFX_VERSION_") +} + +fn loader_resolution_env_var(key: &str) -> bool { + matches!( + key, + "LD_LIBRARY_PATH" + | "LD_ORIGIN_PATH" + | "LD_HWCAP_MASK" + | "LD_ASSUME_KERNEL" + | "GLIBC_TUNABLES" + ) +} + +fn locate_for_process( + command: &str, + workdir: Option<&Path>, + effective: &std::collections::BTreeMap, +) -> Option<(PathBuf, Vec)> { + let workdir = absolute_workdir(workdir)?; + let executable = resolve_command(command, Some(&workdir), effective)?; + let executable = trusted_system_file(&executable)?; + unprivileged_executable(&executable)?; + let metadata = executable.metadata().ok()?; + (metadata.len() <= MAX_EXECUTABLE_BYTES).then_some(())?; + let image = std::fs::read(&executable).ok()?; + let (runtime, runtime_image) = direct_runpath_runtime(&executable, &image, true)?; + Some((trusted_system_file(&runtime)?, runtime_image)) +} + +/// The ROCr the loader resolves for `executable` through its `RUNPATH`, +/// with the image read while checking it. +fn direct_runpath_runtime( + executable: &Path, + image: &[u8], + require_trusted_paths: bool, +) -> Option<(PathBuf, Vec)> { + let Object::Elf(elf) = Object::parse(image).ok()? else { + return None; + }; + // An executable's own audit tags install callbacks without any + // LD_AUDIT environment variable; la_objsearch may replace the + // runtime before RUNPATH resolution. + if !elf.rpaths.is_empty() + || !elf.libraries.contains(&ROCR_SONAME) + || elf.dynamic.as_ref().is_some_and(|dynamic| { + dynamic + .dyns + .iter() + .any(|entry| matches!(entry.d_tag, DT_AUDIT | DT_DEPAUDIT)) + }) + { + return None; + } + + let origin = executable.parent()?.display().to_string(); + for entry in &elf.runpaths { + for directory in entry.split(':') { + if directory.is_empty() { + return None; + } + let directory = expand_origin(directory, &origin); + if directory.contains('$') { + return None; + } + let mut directory = PathBuf::from(directory); + if !directory.is_absolute() { + return None; + } + // A `RUNPATH` entry that is not there, or is not a directory, + // is not ambiguous: the loader finds nothing in it and moves on + // to the next entry, and so does this. Anything else about it — + // unreadable, untrusted — is a directory the loader *will* + // search and this cannot vouch for, which is a verdict declined + // rather than an entry skipped. + match directory.metadata() { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(_) => return None, + Ok(metadata) if !metadata.is_dir() => continue, + Ok(_) => {} + } + if require_trusted_paths { + directory = trusted_system_directory(&directory)?; + } + if alternative_runtime_exists(&directory, require_trusted_paths)? { + return None; + } + let mut candidate = directory.join(ROCR_SONAME); + if candidate.is_file() { + if require_trusted_paths { + candidate = trusted_system_file(&candidate)?; + } + (candidate.metadata().ok()?.len() <= MAX_RUNTIME_BYTES).then_some(())?; + let candidate_image = std::fs::read(&candidate).ok()?; + let Object::Elf(candidate_elf) = Object::parse(&candidate_image).ok()? else { + return None; + }; + if candidate_elf.header.e_machine != elf.header.e_machine + || candidate_elf.is_64 != elf.is_64 + { + return None; + } + return Some((std::fs::canonicalize(candidate).ok()?, candidate_image)); + } + } + } + None +} + +fn expand_origin(directory: &str, origin: &str) -> String { + let directory = directory.replace("${ORIGIN}", origin); + let mut expanded = String::with_capacity(directory.len()); + let mut rest = directory.as_str(); + while let Some(at) = rest.find("$ORIGIN") { + expanded.push_str(&rest[..at]); + let after = &rest[at + "$ORIGIN".len()..]; + if after + .as_bytes() + .first() + .is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_') + { + expanded.push_str(origin); + } else { + expanded.push_str("$ORIGIN"); + } + rest = after; + } + expanded.push_str(rest); + expanded +} + +/// Whether some other ROCr under `directory` could be selected ahead of +/// the one directly in it. +/// +/// Two families of them, because glibc has had two. Modern glibc +/// augments a RUNPATH directory with `/glibc-hwcaps/`. +/// Before 2.37 it also searched hardware-capability subdirectories +/// directly — `/tls`, `/x86_64`, `/haswell` and the +/// platform's own name — so a runtime in one of those is the one a +/// workload on such a host loads while this probe reads the base copy. +/// +/// Any immediate subdirectory holding a ROCr is an alternative. Legacy +/// *combinations* need one more rule: glibc builds a bounded power set +/// from the capability names for this architecture. Those component +/// names are fixed by glibc's ABI, so mirage follows every existing +/// permutation of them, without walking any other directory. +/// +/// What it is *not* is a recursive filesystem search. That would invent +/// candidates the loader cannot select and, on an ordinary ROCm install, +/// bury the real runtime under asset trees like `hipblaslt/library`. +fn alternative_runtime_exists(directory: &Path, require_trusted_paths: bool) -> Option { + let listing = match std::fs::read_dir(directory) { + Ok(listing) => listing, + // Gone between the check above and here: nothing in a directory + // that is not there, and the caller moves on as the loader does. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Some(false), + Err(_) => return None, + }; + for entry in listing { + let path = entry.ok()?.path(); + if !path.is_dir() { + continue; + } + let found = if path.file_name() == Some(OsStr::new("glibc-hwcaps")) { + hwcap_runtime_exists(&path, require_trusted_paths)? + } else { + // Only whether it is there. An alternative this probe cannot + // rule out means no verdict either way, so nothing about it + // is worth a trust check the selected runtime already gets. + path.join(ROCR_SONAME).is_file() + }; + if found { + return Some(true); + } + } + legacy_combination_runtime_exists(directory, require_trusted_paths) +} + +/// The component names glibc can combine in its legacy hwcap search. +/// +/// This mechanism disappeared in glibc 2.37, so the list is the final +/// ABI for each architecture rather than a table that grows with modern +/// `glibc-hwcaps`. On an architecture not listed here, silence is safer +/// than claiming the base runtime is the one the loader selects. +fn legacy_hwcap_components() -> Option<&'static [&'static str]> { + #[cfg(target_arch = "x86_64")] + { + // Synthetic TLS, AT_PLATFORM values, and HWCAP_IMPORTANT names. + Some(&["tls", "x86_64", "haswell", "xeon_phi", "avx512_1"]) + } + #[cfg(target_arch = "x86")] + { + Some(&["tls", "i586", "i686", "sse2"]) + } + #[cfg(target_arch = "aarch64")] + { + Some(&["tls", "aarch64", "atomics"]) + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64")))] + { + None + } +} + +/// Whether a bounded legacy hwcap combination holds a ROCr. +/// +/// glibc 2.36 and earlier form a power set from the architecture's +/// components. Following only those names catches nested candidates such +/// as `haswell/tls` without mistaking `hipblaslt/library` for a loader +/// path. Permutations deliberately over-approximate glibc's preferred +/// ordering: an extra `Unknown` is safe, while missing an order can emit +/// a false `Unsupported`. +fn legacy_combination_runtime_exists( + directory: &Path, + require_trusted_paths: bool, +) -> Option { + let components = legacy_hwcap_components()?; + let mut frontier = vec![(directory.to_path_buf(), 0_u32)]; + for _ in 0..components.len() { + let mut next = Vec::new(); + for (parent, used) in frontier { + for (index, component) in components.iter().enumerate() { + let bit = 1_u32 << index; + if used & bit != 0 { + continue; + } + let path = parent.join(component); + match path.metadata() { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(_) => return None, + Ok(metadata) if !metadata.is_dir() => continue, + Ok(_) => {} + } + let (path, found) = searchable_directory(&path, require_trusted_paths)?; + if found { + return Some(true); + } + next.push((path, used | bit)); + } + } + frontier = next; + } + Some(false) +} + +/// Whether a modern `/glibc-hwcaps/` holds a ROCr. +fn hwcap_runtime_exists(hwcaps: &Path, require_trusted_paths: bool) -> Option { + let (hwcaps, _) = searchable_directory(hwcaps, require_trusted_paths)?; + let listing = match std::fs::read_dir(hwcaps) { + Ok(listing) => listing, + // The same race, and the same answer: no alternative here. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Some(false), + Err(_) => return None, + }; + for entry in listing { + let path = entry.ok()?.path(); + if !path.is_dir() { + continue; + } + if searchable_directory(&path, require_trusted_paths)?.1 { + return Some(true); + } + } + Some(false) +} + +/// One directory the loader could search, as this probe is able to see +/// it: the path to descend from, and whether a ROCr sits in it. +/// +/// `None` declines the verdict, which is what a directory or a runtime +/// this probe cannot vouch for has to do — that one the loader *will* +/// search, so being unable to judge it is being unable to judge the +/// answer. The returned path is the canonical one the trust check +/// produced, so a caller that descends further descends from the +/// directory it actually checked. +fn searchable_directory(directory: &Path, require_trusted_paths: bool) -> Option<(PathBuf, bool)> { + let directory = if require_trusted_paths { + trusted_system_directory(directory)? + } else { + directory.to_path_buf() + }; + let runtime = directory.join(ROCR_SONAME); + if !runtime.is_file() { + return Some((directory, false)); + } + if require_trusted_paths { + trusted_system_file(&runtime)?; + } + Some((directory, true)) +} + +fn trusted_system_file(path: &Path) -> Option { + trusted_system_path(path, true) +} + +fn unprivileged_executable(path: &Path) -> Option<()> { + let metadata = path.metadata().ok()?; + (metadata.mode() & 0o6000 == 0).then_some(())?; + let mut capabilities = [0_u8; 64]; + match getxattr(path, "security.capability", &mut capabilities[..]) { + Ok(0) | Err(Errno::NODATA) => Some(()), + Ok(_) | Err(_) => None, + } +} + +fn trusted_system_directory(path: &Path) -> Option { + trusted_system_path(path, false) +} + +fn trusted_system_path(path: &Path, require_file: bool) -> Option { + trusted_path_chain(path, require_file)?; + let path = std::fs::canonicalize(path).ok()?; + trusted_path_chain(&path, require_file)?; + Some(path) +} + +fn trusted_path_chain(path: &Path, require_file: bool) -> Option<()> { + for (index, component) in path.ancestors().enumerate() { + let metadata = component.metadata().ok()?; + if metadata.uid() != 0 || metadata.mode() & 0o022 != 0 { + return None; + } + if index == 0 { + let expected_kind = if require_file { + metadata.is_file() + } else { + metadata.is_dir() + }; + expected_kind.then_some(())?; + } + } + Some(()) +} + +fn resolve_command( + command: &str, + workdir: Option<&Path>, + env: &std::collections::BTreeMap, +) -> Option { + let base = absolute_workdir(workdir)?; + if command.contains('/') { + let path = PathBuf::from(command); + let path = if path.is_absolute() { + path + } else { + base.join(path) + }; + return is_executable_file(&path).then_some(path); + } + let path = env.get(OsStr::new("PATH"))?; + std::env::split_paths(path).find_map(|dir| { + let dir = if dir.is_absolute() { + dir.to_path_buf() + } else { + base.join(dir) + }; + let candidate = dir.join(command); + is_executable_file(&candidate).then_some(candidate) + }) +} + +fn absolute_workdir(workdir: Option<&Path>) -> Option { + match workdir { + Some(path) if path.is_absolute() => Some(path.to_path_buf()), + Some(path) => Some(std::env::current_dir().ok()?.join(path)), + None => std::env::current_dir().ok(), + } +} + +fn is_executable_file(path: &Path) -> bool { + path.metadata().is_ok_and(|metadata| metadata.is_file()) + && eaccess(path, AccessFlags::X_OK).is_ok() +} + +/// Decide what `image` says about `target`, given where it came from. +/// +/// Split from [`check_target_for_process`] so the rule can be tested +/// against a fabricated library: the question is about the ROCm on the +/// host, which a test cannot choose, and a test that could only assert +/// about the machine it happened to run on would assert nothing on most +/// of them. +fn verdict(target: &str, runtime: &Path, version: Option, image: &[u8]) -> TargetSupport { + let known = target_names(image); + if known.len() < MIN_PLAUSIBLE_TARGET_REFERENCES { + // Whatever this file is, a recognisable ROCr image is not what + // was read, and a "your ROCm does not support this GPU" drawn + // from a failed scan would be a confident lie. + return TargetSupport::Unknown; + } + if mentions(image, target) { + // The concrete reason presence is inconclusive: ROCr carries + // names its ISA registry does not, notably gfx600/601/602 for + // legacy code-object conversion. + return TargetSupport::Unknown; + } + TargetSupport::Unsupported(Box::new(UnsupportedTarget { + target: target.to_string(), + runtime: runtime.to_path_buf(), + version, + references: known.len(), + })) +} + +/// Every plausible gfx target name in `image`. +/// +/// Used for the count in the report and for the sanity floor. This is +/// the strict half of the scan: a name is only counted when the whole +/// word it sits in is `gfx` followed by nothing but lowercase hex +/// digits, so the fragments a binary scan inevitably turns up cannot +/// inflate the image it is judged against. +fn target_names(image: &[u8]) -> BTreeSet { + const TAG: &[u8] = b"gfx"; + let mut out = BTreeSet::new(); + let mut at = 0; + while at + TAG.len() <= image.len() { + if &image[at..at + TAG.len()] != TAG { + at += 1; + continue; + } + // The whole word, `_` included: stopping at `_` would read + // `gfx942_helper` as a gfx942 reference, and enough identifiers + // shaped like that would clear the floor below and condemn a + // target the runtime supports. + let mut end = at + TAG.len(); + while end < image.len() && is_word_byte(image[end]) { + end += 1; + } + // `gfx` inside a longer word is somebody else's identifier, not + // a target name. + let starts_a_word = at == 0 || !is_word_byte(image[at - 1]); + if starts_a_word && let Some(name) = plausible_target(&image[at + TAG.len()..end]) { + out.insert(name); + } + at = end.max(at + 1); + } + out +} + +/// The target name `digits` completes, if it can be one. +fn plausible_target(digits: &[u8]) -> Option { + if digits.len() < MIN_TARGET_DIGITS || !digits[0].is_ascii_digit() { + return None; + } + if !digits + .iter() + .all(|b| b.is_ascii_digit() || b.is_ascii_lowercase() && *b <= b'f') + { + return None; + } + let digits = std::str::from_utf8(digits).ok()?; + Some(format!("gfx{digits}")) +} + +/// Whether `image` names `target` anywhere at all. +/// +/// Deliberately looser than [`target_names`]. Any mention makes the +/// result inconclusive, because a binary string does not say whether it +/// belongs to the ISA registry. The only thing insisted on is that the +/// match is not the prefix of a longer target: `gfx1200` must not answer +/// for `gfx120`. The word boundaries [`target_names`] insists on are +/// deliberately not insisted on here — `gfx942_helper` counts as a +/// mention of `gfx942` — because accepting more of them only produces +/// more silence. +fn mentions(image: &[u8], target: &str) -> bool { + let needle = target.as_bytes(); + if needle.is_empty() { + return false; + } + image.windows(needle.len()).enumerate().any(|(at, window)| { + window == needle + && image + .get(at + needle.len()) + .is_none_or(|next| !next.is_ascii_alphanumeric()) + }) +} + +/// Whether `byte` can be part of an identifier, and so cannot precede +/// the start of one. +fn is_word_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} + +/// The ROCm version recorded beside the runtime at `lib`, if any. +/// +/// A ROCm install writes its version to `/.info/version`, and +/// `lib` is `/lib/libhsa-runtime64.so.1`. Best effort: the version +/// is the most recognisable thing to say about a ROCm install and the +/// least essential, so a layout that does not have it costs the report a +/// parenthesis and nothing else. +fn rocm_version_beside(lib: &Path) -> Option { + let root = lib.parent()?.parent()?; + let version = std::fs::read_to_string(root.join(".info").join("version")).ok()?; + let version = version.trim(); + (!version.is_empty()).then(|| version.to_string()) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use std::os::unix::fs::PermissionsExt; + + use super::*; + + /// A stand-in for a ROCm runtime's image: the targets it was built + /// for, laid out the way an ELF string table lays them out. + /// + /// Interleaved with the architecture-family prefixes and the + /// wildcard patterns a real `libhsa-runtime64.so` carries beside the + /// real names, because those are what a naive scan miscounts as + /// targets and the floor in [`MIN_PLAUSIBLE_TARGET_REFERENCES`] + /// exists to be judged against real names only. + fn rocr_image(targets: &[&str]) -> Vec { + let mut out: Vec = b"\x7fELF\0\0\0\0some other rodata\0".to_vec(); + for noise in ["gfx9", "gfx10", "gfx11", "gfx12", "gfx9f", "gfx1f"] { + out.extend_from_slice(noise.as_bytes()); + out.push(0); + } + for target in targets { + out.extend_from_slice(format!("amdgcn-amd-amdhsa--{target}").as_bytes()); + out.push(0); + out.extend_from_slice(target.as_bytes()); + out.push(0); + } + out + } + + /// A slice of what a ROCm 7.0 runtime names — every target the seven + /// builtin profiles ask for, plus enough of the rest to clear + /// [`MIN_PLAUSIBLE_TARGET_REFERENCES`]. The real list is longer + /// (forty-five on the host the bug was found on); the point of the + /// fixture is which names are in it and which one is not. + const ROCM_7: &[&str] = &[ + "gfx900", "gfx902", "gfx904", "gfx906", "gfx908", "gfx90a", "gfx90c", "gfx942", "gfx950", + "gfx1010", "gfx1030", "gfx1100", "gfx1101", "gfx1151", "gfx1200", "gfx1201", + ]; + + fn check(target: &str, image: &[u8]) -> TargetSupport { + verdict( + target, + Path::new("/opt/rocm/lib/libhsa-runtime64.so.1"), + Some("7.0.2.2".to_string()), + image, + ) + } + + /// The regression, in the shape it actually happened: seven profiles + /// on a ROCm 7.0 host, and only the absent target can be condemned. + /// A present string stays inconclusive because it may be conversion + /// metadata rather than an ISA-registry entry. + #[test] + fn the_one_target_the_runtime_does_not_name_is_the_one_reported() { + let image = rocr_image(ROCM_7); + for mentioned in [ + "gfx90a", "gfx942", "gfx950", "gfx1100", "gfx1151", "gfx1201", + ] { + assert_eq!( + check(mentioned, &image), + TargetSupport::Unknown, + "{mentioned} is mentioned, but a raw string cannot prove registry support" + ); + } + + let TargetSupport::Unsupported(problem) = check("gfx1250", &image) else { + panic!("gfx1250 is absent from the image and must be reported"); + }; + assert_eq!(problem.target, "gfx1250"); + assert_eq!(problem.references, ROCM_7.len()); + } + + /// ROCr carries these full target names for legacy code-object + /// conversion but does not register the corresponding ISAs. They + /// are the concrete counterexample to treating a binary-string hit + /// as support. + #[test] + fn legacy_conversion_names_do_not_count_as_registry_support() { + let mut image = rocr_image(ROCM_7); + for legacy in ["gfx600", "gfx601", "gfx602"] { + image.extend_from_slice(format!("amdgcn-amd-amdhsa--{legacy}\0").as_bytes()); + assert_eq!( + check(legacy, &image), + TargetSupport::Unknown, + "{legacy} is conversion metadata, not proof of ISA support" + ); + } + } + + /// The message is the whole of the fix, so it is asserted on rather + /// than assumed. It has to name the target, the file the verdict was + /// read out of, the ROCm version, and what the failure will look + /// like — because what it looks like is success. + #[test] + fn the_report_names_the_target_the_runtime_and_the_symptom() { + let TargetSupport::Unsupported(problem) = check("gfx1250", &rocr_image(ROCM_7)) else { + panic!("gfx1250 must be reported"); + }; + let message = problem.explain(); + + assert!(message.contains("gfx1250"), "{message}"); + assert!(message.contains("libhsa-runtime64.so.1"), "{message}"); + assert!(message.contains("ROCm 7.0.2.2"), "{message}"); + // The symptom, spelled out: a session that starts, a command that + // runs and an exit status of 0 is what a working run looks like + // too, and that is why this took a day to diagnose. + assert!(message.contains("no GPU"), "{message}"); + assert!(message.contains("exit 0"), "{message}"); + // And a way out. + assert!(message.contains("--image"), "{message}"); + // The target is the session's, which for a drop-in `--config` is + // not the profile's: naming the profile would point a user at + // the input that is not deciding this. + assert!(message.contains("this session emulates"), "{message}"); + assert!(!message.contains("profile"), "{message}"); + } + + /// A runtime with no version file beside it still gets a report, + /// just without the parenthesis. + #[test] + fn a_missing_rocm_version_costs_the_report_nothing_else() { + let TargetSupport::Unsupported(problem) = verdict( + "gfx1250", + Path::new("/somewhere/libhsa-runtime64.so.1"), + None, + &rocr_image(ROCM_7), + ) else { + panic!("gfx1250 must be reported"); + }; + let message = problem.explain(); + assert!( + message.contains("/somewhere/libhsa-runtime64.so.1"), + "{message}" + ); + assert!(!message.contains("ROCm )"), "{message}"); + assert!(!message.contains("()"), "{message}"); + } + + /// Anything that is not recognisably a ROCr target-bearing image + /// says nothing. + /// + /// This is the half that keeps a heuristic honest. The verdict is + /// read out of a binary rather than asked of an API, so the scan can + /// simply fail — a stripped build, a layout nobody anticipated, a + /// file that is not ROCr at all — and a "your ROCm does not support + /// this GPU" drawn from a failed scan would send a user to fix + /// something that was never wrong. + #[test] + fn an_unrecognisable_image_yields_no_verdict() { + for image in [ + &b""[..], + &b"\x7fELF not really a runtime"[..], + // A couple of real names is not a recognisable image: ROCr + // carries dozens. + &rocr_image(&["gfx942", "gfx950"])[..], + // Neither are the family prefixes and wildcards on their own, + // which is exactly what a scan that counted everything + // `gfx`-shaped would mistake for one. + &rocr_image(&[])[..], + ] { + assert_eq!( + check("gfx1250", image), + TargetSupport::Unknown, + "an unrecognisable image must not produce a verdict" + ); + } + } + + /// The prefixes and wildcards ROCr carries beside the real names are + /// not targets, and counting them would let a stripped library clear + /// the floor that exists to catch it. + #[test] + fn family_prefixes_and_wildcards_are_not_counted_as_targets() { + let names = target_names(&rocr_image(&["gfx942", "gfx1250"])); + assert_eq!( + names, + ["gfx1250".to_string(), "gfx942".to_string()] + .into_iter() + .collect::>() + ); + } + + /// A name is only a name when it is a whole word: `libgfx1250stuff` + /// and `gfx942_helper` are somebody's symbols, not supported targets. + #[test] + fn a_gfx_inside_a_longer_identifier_is_not_a_target() { + assert!(target_names(b"\0libgfx1250_handler\0").is_empty()); + assert_eq!(target_names(b"\0-gfx1250\0").len(), 1); + assert!(target_names(b"\0gfx942_helper\0").is_empty()); + assert!(target_names(b"\0gfx942helper\0").is_empty()); + } + + /// Identifiers that merely begin with a target name cannot pass for + /// a target table. + /// + /// The floor in [`MIN_PLAUSIBLE_TARGET_REFERENCES`] is the whole + /// defence against condemning a target on the strength of a scan + /// that never found ROCr's names. Ending a token at `_` would let + /// ten identifiers like `gfx900_helper` clear it, and an arbitrary + /// image would then report every target as unsupported. + #[test] + fn suffixed_identifiers_cannot_clear_the_confidence_floor() { + let mut image: Vec = b"\x7fELF\0".to_vec(); + for n in 0..MIN_PLAUSIBLE_TARGET_REFERENCES + 5 { + image.extend_from_slice(format!("gfx{}_helper\0", 900 + n).as_bytes()); + } + + assert!(target_names(&image).is_empty()); + assert_eq!(check("gfx1250", &image), TargetSupport::Unknown); + } + + /// The membership test must not let a longer target answer for a + /// shorter one. A `gfx1200` mention says nothing about `gfx120`. + #[test] + fn a_longer_target_does_not_answer_for_a_shorter_one() { + let image = rocr_image(ROCM_7); + assert!(mentions(&image, "gfx1200")); + assert!(!mentions(&image, "gfx120")); + assert!(!mentions(&image, "")); + } + + /// PATH lookup has to follow executable candidates, as `execvp` + /// does. Stopping at an earlier non-executable file could inspect a + /// different program from the one the workload will run. + #[test] + fn command_resolution_skips_non_executable_path_entries() { + let tmp = tempfile::tempdir().unwrap(); + let blocked_dir = tmp.path().join("blocked"); + let runnable_dir = tmp.path().join("runnable"); + std::fs::create_dir_all(&blocked_dir).unwrap(); + std::fs::create_dir_all(&runnable_dir).unwrap(); + let blocked = blocked_dir.join("workload"); + let runnable = runnable_dir.join("workload"); + std::fs::write(&blocked, b"not executable").unwrap(); + std::fs::write(&runnable, b"executable").unwrap(); + // No execute bit is set, so both ordinary users and UID 0 reject + // the candidate. Root treats any execute bit as sufficient for + // X_OK, unlike the owner/group selection an ordinary user gets. + std::fs::set_permissions(&blocked, std::fs::Permissions::from_mode(0o400)).unwrap(); + std::fs::set_permissions(&runnable, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let env = [( + OsString::from("PATH"), + std::env::join_paths([blocked_dir, runnable_dir]).unwrap(), + )] + .into_iter() + .collect(); + assert_eq!(resolve_command("workload", None, &env), Some(runnable)); + assert_eq!( + resolve_command(blocked.to_str().unwrap(), None, &env), + None, + "a command containing a slash must fail rather than fall through PATH" + ); + } + + #[test] + fn only_immutable_system_files_can_produce_a_warning() { + assert!(trusted_system_file(Path::new("/bin/true")).is_some()); + + let tmp = tempfile::tempdir().unwrap(); + let executable = tmp.path().join("workload"); + std::fs::copy("/bin/true", &executable).unwrap(); + assert_eq!(trusted_system_file(&executable), None); + + std::fs::set_permissions(&executable, std::fs::Permissions::from_mode(0o4755)).unwrap(); + assert_eq!(unprivileged_executable(&executable), None); + } + + #[test] + fn dependency_analysis_does_not_execute_transitive_audit_code() { + use std::process::Command; + + if Command::new("cc").arg("--version").output().is_err() { + return; + } + let tmp = tempfile::tempdir().unwrap(); + let marker = tmp.path().join("audit-ran"); + let audit_source = tmp.path().join("audit.c"); + let audit = tmp.path().join("libaudit.so"); + std::fs::write( + &audit_source, + format!( + "#include \n\ + __attribute__((constructor)) static void run(void) {{ \ + FILE *f = fopen(\"{}\", \"w\"); if (f) fclose(f); }}\n", + marker.display() + ), + ) + .unwrap(); + let built_audit = Command::new("cc") + .args(["-shared", "-fPIC"]) + .arg(&audit_source) + .arg("-o") + .arg(&audit) + .status() + .unwrap(); + if !built_audit.success() { + return; + } + + let runtime_source = tmp.path().join("runtime.c"); + let runtime = tmp.path().join(ROCR_SONAME); + std::fs::write(&runtime_source, "int hsa_stub(void) { return 0; }\n").unwrap(); + let built_runtime = Command::new("cc") + .args(["-shared", "-fPIC"]) + .arg(&runtime_source) + .arg(format!("-Wl,-soname,{ROCR_SONAME}")) + .arg(format!("-Wl,--audit,{}", audit.display())) + .arg("-o") + .arg(&runtime) + .status() + .unwrap(); + if !built_runtime.success() { + return; + } + + let workload_source = tmp.path().join("workload.c"); + let workload = tmp.path().join("workload"); + std::fs::write( + &workload_source, + "extern int hsa_stub(void); int main(void) { return hsa_stub(); }\n", + ) + .unwrap(); + let built_workload = Command::new("cc") + .arg(&workload_source) + .arg("-L") + .arg(tmp.path()) + .arg(format!("-l:{ROCR_SONAME}")) + .arg("-Wl,--enable-new-dtags") + .arg("-Wl,-rpath,$ORIGIN") + .arg("-o") + .arg(&workload) + .status() + .unwrap(); + if !built_workload.success() { + return; + } + + let workload_image = std::fs::read(&workload).unwrap(); + assert_eq!( + direct_runpath_runtime(&workload, &workload_image, false).map(|(path, _)| path), + None, + "the linker's propagated DT_DEPAUDIT must suppress the verdict" + ); + assert!( + !marker.exists(), + "static dependency analysis must not load an audit module" + ); + + let built_runtime = Command::new("cc") + .args(["-shared", "-fPIC"]) + .arg(&runtime_source) + .arg(format!("-Wl,-soname,{ROCR_SONAME}")) + .arg("-o") + .arg(&runtime) + .status() + .unwrap(); + assert!(built_runtime.success(), "linking the clean runtime"); + let built_workload = Command::new("cc") + .arg(&workload_source) + .arg("-L") + .arg(tmp.path()) + .arg(format!("-l:{ROCR_SONAME}")) + .arg("-Wl,--enable-new-dtags") + .arg("-Wl,-rpath,$ORIGIN") + .arg("-o") + .arg(&workload) + .status() + .unwrap(); + assert!(built_workload.success(), "linking the clean workload"); + let workload_image = std::fs::read(&workload).unwrap(); + assert_eq!( + direct_runpath_runtime(&workload, &workload_image, false).map(|(path, _)| path), + Some(runtime.canonicalize().unwrap()) + ); + + let unrelated = tmp.path().join("hipblaslt").join("library"); + std::fs::create_dir_all(&unrelated).unwrap(); + std::os::unix::fs::symlink(&runtime, unrelated.join(ROCR_SONAME)).unwrap(); + assert_eq!( + direct_runpath_runtime(&workload, &workload_image, false).map(|(path, _)| path), + Some(runtime.canonicalize().unwrap()), + "a library below an unrelated asset directory is not a loader alternative" + ); + + let hwcap = tmp.path().join("glibc-hwcaps").join("x86-64-v3"); + std::fs::create_dir_all(&hwcap).unwrap(); + std::os::unix::fs::symlink(&runtime, hwcap.join(ROCR_SONAME)).unwrap(); + assert_eq!( + direct_runpath_runtime(&workload, &workload_image, false).map(|(path, _)| path), + None, + "a symlinked hardware-capability alternative makes the selected runtime ambiguous" + ); + std::fs::remove_dir_all(tmp.path().join("glibc-hwcaps")).unwrap(); + + // The other half of the same question, on the glibc most hosts + // are still running: before 2.37 the hardware-capability + // directories sat directly under the RUNPATH entry, so a runtime + // in `tls` is the one such a loader picks while this probe reads + // the base copy beside it. + let legacy = tmp.path().join("tls"); + std::fs::create_dir_all(&legacy).unwrap(); + std::os::unix::fs::symlink(&runtime, legacy.join(ROCR_SONAME)).unwrap(); + assert_eq!( + direct_runpath_runtime(&workload, &workload_image, false).map(|(path, _)| path), + None, + "a pre-2.37 glibc searches /tls, so the selected runtime is ambiguous" + ); + std::fs::remove_dir_all(&legacy).unwrap(); + + #[cfg(target_arch = "x86_64")] + for combination in [["tls", "haswell"], ["haswell", "tls"]] { + let nested = tmp.path().join(combination[0]).join(combination[1]); + std::fs::create_dir_all(&nested).unwrap(); + std::os::unix::fs::symlink(&runtime, nested.join(ROCR_SONAME)).unwrap(); + assert_eq!( + direct_runpath_runtime(&workload, &workload_image, false).map(|(path, _)| path), + None, + "the bounded legacy combination {combination:?} must be treated as ambiguous" + ); + std::fs::remove_dir_all(tmp.path().join(combination[0])).unwrap(); + } + + assert_eq!( + direct_runpath_runtime(&workload, &workload_image, false).map(|(path, _)| path), + Some(runtime.canonicalize().unwrap()), + "and the verdict comes back once every loader alternative is gone" + ); + + // A `RUNPATH` entry that does not exist is not a reason to stop + // looking. The loader finds nothing there and tries the next + // entry; treating it as ambiguous would let one stale directory + // silence the diagnostic for the whole list. + let stale = tmp.path().join("gone"); + let multi_entry = tmp.path().join("workload-multi"); + let built = Command::new("cc") + .arg(&workload_source) + .arg("-L") + .arg(tmp.path()) + .arg(format!("-l:{ROCR_SONAME}")) + .arg("-Wl,--enable-new-dtags") + .arg(format!("-Wl,-rpath,{}:$ORIGIN", stale.display())) + .arg("-o") + .arg(&multi_entry) + .status() + .unwrap(); + assert!( + built.success(), + "linking a workload with a stale first entry" + ); + assert!(!stale.exists()); + let multi_image = std::fs::read(&multi_entry).unwrap(); + // Proven rather than assumed: a linker that dropped the missing + // entry would leave this test passing on a single-entry RUNPATH, + // which is not the case it exists for. + let Object::Elf(elf) = Object::parse(&multi_image).unwrap() else { + panic!("the linker produced an ELF"); + }; + assert!( + elf.runpaths.iter().any(|entry| { + let mut entries = entry.split(':'); + entries.next() == Some(stale.to_str().unwrap()) && entries.next().is_some() + }), + "the fixture needs the missing directory first and another entry after it: {:?}", + elf.runpaths + ); + assert_eq!( + direct_runpath_runtime(&multi_entry, &multi_image, false).map(|(path, _)| path), + Some(runtime.canonicalize().unwrap()), + "a missing RUNPATH entry must be skipped, not treated as ambiguous" + ); + + // Nor is an entry that names a file rather than a directory: the + // loader's open of `/libhsa-runtime64.so.1` fails and it + // moves on, so this must too rather than decline the verdict. + std::fs::write(&stale, b"not a directory").unwrap(); + let (resolved, resolved_image) = direct_runpath_runtime(&multi_entry, &multi_image, false) + .expect("a non-directory RUNPATH entry must be skipped, not treated as ambiguous"); + assert_eq!(resolved, runtime.canonicalize().unwrap()); + // The verdict judges these bytes rather than reading the runtime a + // second time, so they have to be the runtime's own. + assert_eq!(resolved_image, std::fs::read(&runtime).unwrap()); + std::fs::remove_file(&stale).unwrap(); + + for tag in ["--audit", "--depaudit"] { + let audited_workload = tmp + .path() + .join(format!("workload-{}", tag.trim_start_matches("--"))); + let built_workload = Command::new("cc") + .arg(&workload_source) + .arg("-L") + .arg(tmp.path()) + .arg(format!("-l:{ROCR_SONAME}")) + .arg("-Wl,--enable-new-dtags") + .arg("-Wl,-rpath,$ORIGIN") + .arg(format!("-Wl,{tag},{}", audit.display())) + .arg("-o") + .arg(&audited_workload) + .status() + .unwrap(); + assert!(built_workload.success(), "linking an executable with {tag}"); + assert_eq!( + direct_runpath_runtime( + &audited_workload, + &std::fs::read(&audited_workload).unwrap(), + false + ) + .map(|(path, _)| path), + None, + "{tag} can redirect the runtime lookup and must suppress a verdict" + ); + } + } + + #[test] + fn only_complete_origin_tokens_are_expanded() { + assert_eq!(expand_origin("$ORIGIN/lib", "/opt/app"), "/opt/app/lib"); + assert_eq!( + expand_origin("${ORIGIN}_extra", "/opt/app"), + "/opt/app_extra" + ); + assert_eq!( + expand_origin("$ORIGIN_extra:/opt/rocm/lib", "/opt/app"), + "$ORIGIN_extra:/opt/rocm/lib" + ); + } + + #[test] + fn a_relative_workdir_is_resolved_before_the_loader_changes_directory() { + let current = std::env::current_dir().unwrap(); + assert_eq!( + absolute_workdir(Some(Path::new("relative/work"))), + Some(current.join("relative/work")) + ); + } + + /// `--clear-env-vars` means ambient loader state cannot leak into + /// the diagnostic any more than it can leak into the workload. + #[test] + fn a_cleared_environment_drops_ambient_loader_state() { + let inherited = [ + (OsString::from("PATH"), OsString::from("/ambient/bin")), + ( + OsString::from("LD_LIBRARY_PATH"), + OsString::from("/ambient/rocm/lib"), + ), + ]; + let explicit = [(OsString::from("PATH"), OsString::from("/clean/bin"))]; + + let inherited_env = merged_environment(inherited.clone(), &explicit, true); + assert_eq!( + inherited_env.get(OsStr::new("LD_LIBRARY_PATH")), + Some(&OsString::from("/ambient/rocm/lib")) + ); + + let clean_env = merged_environment(inherited, &explicit, false); + assert_eq!(clean_env.get(OsStr::new("LD_LIBRARY_PATH")), None); + assert_eq!( + clean_env.get(OsStr::new("PATH")), + Some(&OsString::from("/clean/bin")) + ); + } + + /// The loader search lists, and the variables that name an object to + /// load or a version to report. + /// + /// One transcription of each for the whole module. The tests below + /// assert different things about the same two lists, and three copies + /// of them would start drifting from each other before they drifted + /// from [`loader_resolution_env_var`] and [`ambiguous_probe_env_var`]. + const SEARCH_LIST_VARS: &[&str] = &[ + "LD_LIBRARY_PATH", + "LD_ORIGIN_PATH", + "LD_HWCAP_MASK", + "LD_ASSUME_KERNEL", + "GLIBC_TUNABLES", + ]; + const NAMED_OBJECT_VARS: &[&str] = &[ + "LD_PRELOAD", + "LD_AUDIT", + "HSA_OVERRIDE_GFX_VERSION", + "HSA_OVERRIDE_GFX_VERSION_0", + "HSA_OVERRIDE_GFX_VERSION_17", + ]; + + /// Preloads can replace the runtime's symbols, HSA overrides change + /// which ISA is looked up, and loader-selection variables can + /// select user-controlled transitive dependencies. None permits a + /// safe verdict from a loader trace. + #[test] + fn preload_and_hsa_overrides_make_the_answer_ambiguous() { + for &key in NAMED_OBJECT_VARS { + let env = [(OsString::from(key), OsString::from("set"))] + .into_iter() + .collect(); + assert!( + loader_state_is_ambiguous(&env), + "{key} must suppress a loader verdict" + ); + } + let unrelated = [(OsString::from("HSA_ENABLE_SDMA"), OsString::from("0"))] + .into_iter() + .collect(); + assert!(!loader_state_is_ambiguous(&unrelated)); + } + + /// `LD_PRELOAD` is the environment's half of the preload question + /// and `/etc/ld.so.preload` is the host's, which no workload + /// environment mentions and which glibc applies to every + /// dynamically linked process on the machine. + /// + /// Absent is the ordinary case and the only harmless one. Present + /// and blank preloads nothing; unreadable is not the same fact as + /// "there is nothing there", and this module does not round the + /// second into the first. + #[test] + fn a_system_wide_preload_makes_the_answer_ambiguous() { + let tmp = tempfile::tempdir().unwrap(); + let preload = tmp.path().join("ld.so.preload"); + + assert!(!global_preload_is_ambiguous(&preload)); + + for blank in [&b""[..], b"\n", b" \t\n \n"] { + std::fs::write(&preload, blank).unwrap(); + assert!(!global_preload_is_ambiguous(&preload)); + } + + std::fs::write(&preload, b"/usr/lib/libinterpose.so\n").unwrap(); + assert!(global_preload_is_ambiguous(&preload)); + + // A directory stands in for the file that is there and cannot be + // read: the error is not `NotFound`, so nothing was ruled out. + assert!(global_preload_is_ambiguous(tmp.path())); + } + + /// Setting a loader search list to nothing is still setting it. + /// + /// glibc gives an empty *element* of a search list the meaning + /// "the current directory", so what an empty whole value means is a + /// question about one loader's parsing of one variable. This module + /// does not answer those: the workload and the probe have to resolve + /// the same file, and an explicitly emptied search list is not + /// evidence that they will. The variables that name something to + /// load or a version to report are different — empty asks for + /// nothing — and must stay harmless, or a workload environment that + /// merely mentions `LD_PRELOAD` would silence the diagnostic. + #[test] + fn an_emptied_search_list_is_ambiguous_but_an_empty_preload_is_not() { + for &key in SEARCH_LIST_VARS { + let env = [(OsString::from(key), OsString::new())] + .into_iter() + .collect(); + assert!( + loader_state_is_ambiguous(&env), + "an emptied {key} is still a loader instruction" + ); + } + + for &key in NAMED_OBJECT_VARS { + let env = [(OsString::from(key), OsString::new())] + .into_iter() + .collect(); + assert!( + !loader_state_is_ambiguous(&env), + "an empty {key} loads nothing and overrides nothing" + ); + } + } + + #[test] + fn every_loader_input_is_deduplicated_and_redirects_are_ambiguous() { + for &key in std::iter::once(&"PATH") + .chain(SEARCH_LIST_VARS) + .chain(NAMED_OBJECT_VARS) + { + assert!( + affects_process_probe(key), + "{key} can change the probe answer" + ); + } + assert!( + !affects_process_probe("BASH_ENV"), + "unrelated workload setup must not split deduplication" + ); + + for &key in SEARCH_LIST_VARS { + let effective = [(OsString::from(key), OsString::from("set"))] + .into_iter() + .collect(); + assert!( + loader_state_is_ambiguous(&effective), + "{key} can select a transitive object whose audit tags have not been inspected" + ); + } + } +} diff --git a/emulation/mirage/core/src/state.rs b/emulation/mirage/core/src/state.rs index 6cd444edc15c..48932d70fa0d 100644 --- a/emulation/mirage/core/src/state.rs +++ b/emulation/mirage/core/src/state.rs @@ -162,19 +162,23 @@ mod tests { #[test] fn a_failed_write_names_the_document_and_not_the_scratch_file() { - // Saving a profile into a directory the user cannot write used to - // fail with `io error on /…/cdna4.tmp.4711` — a path that had - // never existed as far as they were concerned, does not exist by - // the time they look, and names neither the document they were - // saving nor anything they could fix. - use std::os::unix::fs::PermissionsExt; - + // Saving a profile that cannot be written used to fail with `io + // error on /…/cdna4.tmp.4711` — a path that had never existed as + // far as the user was concerned, does not exist by the time they + // look, and names neither the document they were saving nor + // anything they could fix. + // + // The failure is forced by putting a non-empty *directory* where + // the document goes, so `rename` returns `EISDIR`. The obvious + // way to write this is a `chmod 500` parent, but that tests the + // kernel's permission check rather than mirage: root ignores the + // mode bits, the write succeeds, and the test fails on a machine + // where nothing is wrong. CI runs as root in a container. let dir = tempfile::tempdir().unwrap(); - let readonly = dir.path().join("readonly"); - fs::create_dir(&readonly).unwrap(); - fs::set_permissions(&readonly, fs::Permissions::from_mode(0o500)).unwrap(); + let dest = dir.path().join("cdna4.json"); + fs::create_dir(&dest).unwrap(); + fs::write(dest.join("occupied"), "x").unwrap(); - let dest = readonly.join("cdna4.json"); let err = write_json( &dest, &Sample { @@ -186,14 +190,35 @@ mod tests { .full_message(); assert!(err.contains("cdna4.json"), "{err}"); assert!(!err.contains(".tmp."), "the scratch name is ours: {err}"); - assert!(err.contains(&readonly.display().to_string()), "{err}"); + assert!(err.contains(&dir.path().display().to_string()), "{err}"); assert!(err.contains("writable"), "what to check: {err}"); - assert!(err.contains("Permission denied"), "the reason: {err}"); // And a write that did not land leaves nothing behind in a // directory the user reads and edits by hand. - assert_eq!(fs::read_dir(&readonly).unwrap().count(), 0); + let leftovers: Vec<_> = fs::read_dir(dir.path()) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|n| n.contains(".tmp.")) + .collect(); + assert!( + leftovers.is_empty(), + "scratch files left behind: {leftovers:?}" + ); + } - fs::set_permissions(&readonly, fs::Permissions::from_mode(0o700)).unwrap(); + #[test] + fn an_unwritable_directory_is_reported_as_one() { + // The wording for the commonest cause, asserted against a + // synthesized `EACCES` rather than a real one: a test that has to + // make the kernel refuse a write cannot run as root, and this is + // the whole of what such a test would add over the one above. + let err = write_failed( + Path::new("/etc/mirage/cdna4.json"), + &std::io::Error::from_raw_os_error(13), + ) + .full_message(); + assert!(err.contains("Permission denied"), "the reason: {err}"); + assert!(err.contains("/etc/mirage"), "the directory to check: {err}"); + assert!(err.contains("writable"), "what to check: {err}"); } } diff --git a/emulation/mirage/core/src/store.rs b/emulation/mirage/core/src/store.rs index c2b667fceebf..256f6b7ad263 100644 --- a/emulation/mirage/core/src/store.rs +++ b/emulation/mirage/core/src/store.rs @@ -14,6 +14,16 @@ //! //! A *write* asks two more questions, below. //! +//! # Not every document is a file +//! +//! Mirage's builtin profiles are generated rather than stored: each is +//! derived from a rocjitsu config the binary carries, on demand, and +//! nothing is written to `/profile/` ([`BuiltinProfiles`]). So a +//! profile that resolves is not necessarily a profile with a file, and +//! every profile file that *does* exist is the user's own. Builtin +//! agents and topologies are still seeded onto disk, because those are +//! documents a user edits and refers to by name. +//! //! # Writes never destroy the user's work //! //! These files are the user's, and this module is the one door every @@ -37,7 +47,9 @@ //! every missing builtin on the next command — the file would come //! straight back and the "deleted" would have been a lie. Deleting one //! the user *has* changed is allowed, and is how a customised builtin -//! is reset to the shipped version. +//! is reset to the shipped version. A *generated* builtin is refused +//! for the neighbouring reason: there is no file to remove, and the +//! name would still resolve afterwards. //! * A profile is checked before it lands, not when a session later tries //! to use it: its name, the topology and agent references inside it, //! and whether its emulator backend will accept it at all. A topology @@ -140,16 +152,23 @@ impl DocKind { pub enum Stored { /// The name was free, and now holds this document. Created, - /// The name held an untouched builtin, which this replaced. Nothing - /// the user wrote was lost — the replaced copy is still compiled into - /// the binary — but the shipped definition is no longer on disk, and - /// deleting this document is what brings it back. + /// The name resolved to a builtin, and now resolves to this. Nothing + /// the user wrote was lost — the replaced definition is still + /// compiled into the binary — but it is no longer what the name + /// means, and deleting this document is what brings it back. + /// + /// Covers both ways a builtin is taken over: an agent or topology + /// mirage seeded and had left untouched on disk, and a profile + /// mirage generates and never wrote at all. ReplacedBuiltin, } /// The documents a linked-in provider seeds into a fresh config /// directory, and re-seeds whenever one goes missing. /// +/// Agents and topologies only. Builtin profiles are generated rather +/// than written and travel in [`BuiltinProfiles`] instead. +/// /// `mirage_builtin` submits exactly one of these; the core cannot depend /// on it (the dependency runs the other way), so the list arrives at link /// time the same way emulator backends do. A build without the provider @@ -163,6 +182,59 @@ pub struct BuiltinDocuments { inventory::collect!(BuiltinDocuments); +/// The profiles a linked-in provider generates, rather than seeds. +/// +/// Mirage's builtin profiles are not documents at all until somebody +/// asks for one: each is derived from a rocjitsu config the binary +/// carries, on demand, and nothing is ever written to +/// `/profile/`. So they cannot travel in [`BuiltinDocuments`], +/// which is specifically the set mirage *writes* and therefore the set +/// [`is_pristine_builtin`] answers about — a generated profile has no +/// file, so no file of that name can be mirage's own untouched seed, and +/// every one that exists is the user's. +/// +/// [`profile_get`] and [`profile_list`] fall back to this registry, so a +/// builtin profile is visible on a fresh machine — and on one whose +/// config directory mirage cannot write, which used to read as a machine +/// with no profiles. +/// +/// Registered at link time exactly as [`BuiltinDocuments`] is, and for +/// the same reason: the core cannot depend on `mirage_builtin`. A build +/// without the provider simply has no builtin profiles. +#[derive(Debug)] +pub struct BuiltinProfiles { + /// Every profile the provider generates, as its name and its + /// definition. + pub profiles: fn() -> Vec<(String, ProfileDef)>, +} + +inventory::collect!(BuiltinProfiles); + +/// Every builtin profile, by canonical name. +fn builtin_profiles() -> &'static BTreeMap { + static INDEX: OnceLock> = OnceLock::new(); + INDEX.get_or_init(|| { + let mut out = BTreeMap::new(); + for provider in inventory::iter:: { + for (name, profile) in (provider.profiles)() { + out.insert(DocKind::Profile.canonical(&name), profile); + } + } + out + }) +} + +/// The builtin profile called `name`, if mirage generates one. +/// +/// Public because "is this name a builtin?" is a question the CLI has to +/// ask for itself: a profile with no file is not necessarily a profile +/// that does not exist, and `mirage profile delete` and the +/// shipped-version-restored report both turn on the difference. +#[must_use] +pub fn builtin_profile(name: &str) -> Option<&'static ProfileDef> { + builtin_profiles().get(&DocKind::Profile.canonical(name)) +} + /// The content mirage ships for `name`, if it ships one at all. /// /// Compared as parsed JSON rather than as bytes: what matters is whether @@ -386,8 +458,11 @@ pub fn referrers_to(target: DocKind, name: &str) -> Vec { if matches!(target, DocKind::Agent | DocKind::Topology) { for profile in profile_list().unwrap_or_default() { - let Ok(def) = crate::state::read_json::(&DocKind::Profile.path(&profile)) - else { + // Through `profile_get` rather than off disk: a builtin + // profile has no file and pins an agent all the same, so + // reading the directory would miss exactly the referrers a + // user is least able to repoint. + let Ok(def) = profile_get(&profile) else { continue; }; // A profile reaches an agent through the topology it carries @@ -636,26 +711,52 @@ pub fn dangling_ref(referrer: Referrer<'_>, missing: DocKind, name: &str) -> Mir /// List every profile name, sorted. /// +/// Both the ones on disk and the builtins mirage generates +/// ([`BuiltinProfiles`]), which have no file. A builtin whose name a +/// file has taken is listed once, under the name the file is addressed +/// by: the file is the profile that will be used, and listing the same +/// name twice would say there were two. +/// /// # Errors /// /// Returns an error if the profile directory exists but cannot be read. pub fn profile_list() -> Result> { - list_json_stems(&crate::paths::profile_root()) + let mut names = list_json_stems(&crate::paths::profile_root())?; + let on_disk: std::collections::BTreeSet = names + .iter() + .map(|name| DocKind::Profile.canonical(name)) + .collect(); + names.extend( + builtin_profiles() + .keys() + .filter(|name| !on_disk.contains(*name)) + .cloned(), + ); + names.sort(); + Ok(names) } /// Read one profile. /// +/// The file wins where there is one, so writing a profile under a +/// builtin's name shadows the builtin — and deleting that file brings +/// the builtin back, with nothing to re-seed and nothing to go stale. +/// /// # Errors /// -/// Returns [`MirageError::ProfileNotFound`] if there is no such profile, -/// or a parse error if it is malformed. +/// Returns [`MirageError::ProfileNotFound`] if there is no such profile +/// on disk and mirage generates none of that name, or a parse error if +/// the file is malformed. pub fn profile_get(name: &str) -> Result { validate_name(DocKind::Profile, name)?; let path = crate::paths::profile_path(name); - if !path.exists() { - return Err(MirageError::not_found(DocKind::Profile, name)); + if path.exists() { + return crate::state::read_json(&path); } - crate::state::read_json(&path) + if let Some(builtin) = builtin_profile(name) { + return Ok(builtin.clone()); + } + Err(MirageError::not_found(DocKind::Profile, name)) } /// Write a new profile. @@ -679,20 +780,61 @@ pub fn profile_put(profile: &ProfileDef) -> Result { profile .validate() .map_err(|e| MirageError::other(format!("profile {}: {e}", shown(&profile.name))))?; - put_document(DocKind::Profile, &profile.name, profile) + let stored = put_document(DocKind::Profile, &profile.name, profile)?; + // A builtin profile has no file, so `put_document` sees a free name + // and reports `Created`. It is not free: the name resolved to + // mirage's own definition a moment ago and resolves to this one now, + // which is the fact `ReplacedBuiltin` exists to have said out loud. + // Nothing was destroyed — the builtin is generated, so deleting this + // file brings it straight back. + Ok(match stored { + Stored::Created if builtin_profile(&profile.name).is_some() => Stored::ReplacedBuiltin, + stored => stored, + }) +} + +/// Say why a fileless builtin profile cannot be deleted. +/// +/// Its one caller used to be [`profile_delete`], and callers that only +/// wanted to *know* whether a name was a generated builtin reached the +/// delete to find out — which answers the question by removing whatever +/// file has appeared in the meantime. The sentence lives here so asking +/// costs nothing. +#[must_use] +pub fn generated_profile_refusal(name: &str) -> MirageError { + MirageError::other(format!( + "profile {name:?} is a builtin: mirage generates it from a RocJITsu \ + config it ships rather than storing it, so there is no file to \ + delete and it would still be there afterwards. To use your own in \ + its place, write one under that name (`mirage profile create \ + {name} ...`) \u{2014} it shadows the builtin, and deleting it later \ + restores the builtin." + )) } /// Delete a profile. /// +/// Deleting a file that shadows a builtin is allowed and is how a +/// customised builtin is reset: the generated profile is what the name +/// resolves to again afterwards. Deleting the builtin *itself* is +/// refused — mirage generates it, so there is no file to remove and it +/// would still be there. That is [`guard_delete`]'s rule reached by the +/// other road: one refuses a delete that a re-seed would undo, this one +/// a delete with nothing to undo. Both are still here, because whether a +/// provider seeds its profiles or generates them is the provider's +/// business and not this module's. +/// /// # Errors /// /// Returns [`MirageError::ProfileNotFound`] if there is no such profile, -/// or a rejection if it is an untouched builtin (which mirage would -/// simply write back — see `guard_delete`). +/// or a rejection if the name is a builtin mirage generates. pub fn profile_delete(name: &str) -> Result<()> { validate_name(DocKind::Profile, name)?; let path = crate::paths::profile_path(name); if !path.exists() { + if builtin_profile(name).is_some() { + return Err(generated_profile_refusal(name)); + } return Err(MirageError::not_found(DocKind::Profile, name)); } guard_delete(DocKind::Profile, name)?; @@ -888,6 +1030,7 @@ mod tests { name: name.to_string(), description: None, emulator: EmulatorDef { + extra: Default::default(), emulator: TEST_EMULATOR.to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -916,6 +1059,142 @@ mod tests { BuiltinDocuments { documents: seeded_documents } } + /// A [`BuiltinProfiles`] provider standing in for the generated half + /// of `mirage_builtin`: a profile that resolves with no file behind + /// it. Its topology is inline and names an agent, because a + /// generated profile refers to documents like any other and + /// [`referrers_to`] has to see it. + const GENERATED_PROFILE: &str = "generated"; + + fn generated_profile() -> ProfileDef { + profile_referring_to( + GENERATED_PROFILE, + MaybeRef::Owned(TopologyDef { + num_nodes: 1, + gpus_per_node: 1, + agent: MaybeRef::Ref("generated-gpu".to_string()), + }), + ) + } + + fn generated_profiles() -> Vec<(String, ProfileDef)> { + vec![(GENERATED_PROFILE.to_string(), generated_profile())] + } + + inventory::submit! { + BuiltinProfiles { profiles: generated_profiles } + } + + /// Every profile on disk: [`profile_list`] minus the builtin this + /// module generates. + /// + /// That builtin is in every list on every machine, so the tests + /// about *storing* profiles would each have to name it, and would + /// then be asserting the registry rather than the store. The two + /// tests that are about the union say so explicitly instead. + fn stored_profiles() -> Vec { + profile_list() + .unwrap() + .into_iter() + .filter(|name| name != GENERATED_PROFILE) + .collect() + } + + /// A generated builtin is a profile on a machine with no profile + /// directory at all — which is the point of generating it: a config + /// directory mirage cannot write used to read as a machine with no + /// profiles, and `--profile mi350x` then blamed a missing profile + /// that mirage itself would have written. + #[test] + fn a_generated_profile_resolves_without_a_file() { + let _g = crate::paths::test_env_lock(); + let dir = tempfile::tempdir().unwrap(); + crate::paths::set_test_root(dir.path()); + + assert_eq!(profile_list().unwrap(), vec![GENERATED_PROFILE]); + assert_eq!(profile_get(GENERATED_PROFILE).unwrap(), generated_profile()); + assert_eq!( + builtin_profile("GENERATED").cloned(), + Some(generated_profile()), + "builtins are addressed case-insensitively, like every profile" + ); + assert!(!crate::paths::profile_path(GENERATED_PROFILE).exists()); + assert!(!crate::paths::profile_root().exists()); + + crate::paths::clear_test_root(); + } + + /// Writing over a generated builtin's name shadows it and says so; + /// deleting the file hands the name straight back. + #[test] + fn a_file_shadows_a_generated_profile_and_deleting_it_gives_the_name_back() { + let _g = crate::paths::test_env_lock(); + let dir = tempfile::tempdir().unwrap(); + crate::paths::set_test_root(dir.path()); + + let mut mine = generated_profile(); + mine.description = Some("mine now".to_string()); + // `Created` would be true of the filesystem and false of what the + // name meant a moment ago, which is the half the user needs. + assert_eq!(profile_put(&mine).unwrap(), Stored::ReplacedBuiltin); + assert_eq!(profile_get(GENERATED_PROFILE).unwrap(), mine); + // Shadowed, not duplicated: one name, one profile. + assert_eq!(profile_list().unwrap(), vec![GENERATED_PROFILE]); + // And it is the user's now, so a second write is refused like any + // other document of theirs. + let err = profile_put(&mine).unwrap_err().to_string(); + assert!(err.contains("already exists"), "{err}"); + + profile_delete(GENERATED_PROFILE).unwrap(); + assert_eq!(profile_get(GENERATED_PROFILE).unwrap(), generated_profile()); + + crate::paths::clear_test_root(); + } + + /// The builtin itself cannot be deleted — there is no file, and the + /// name would still resolve, so "deleted" would be a lie. Same rule + /// as a seeded builtin, reached the other way. + #[test] + fn deleting_a_generated_profile_is_refused_rather_than_faked() { + let _g = crate::paths::test_env_lock(); + let dir = tempfile::tempdir().unwrap(); + crate::paths::set_test_root(dir.path()); + + let err = profile_delete(GENERATED_PROFILE).unwrap_err().to_string(); + assert!(err.contains("is a builtin"), "{err}"); + assert!(err.contains(GENERATED_PROFILE), "{err}"); + // And it is not the same complaint as a name nobody ships. + assert!(matches!( + profile_delete("ghost"), + Err(MirageError::ProfileNotFound { .. }) + )); + + crate::paths::clear_test_root(); + } + + /// A generated profile refers to an agent like any other, so + /// deleting that agent has to name it. Reading the profile directory + /// would miss it — it has no file — and it is exactly the referrer a + /// user cannot repoint by hand. + #[test] + fn a_generated_profile_is_a_referrer_to_the_agent_it_pins() { + let _g = crate::paths::test_env_lock(); + let dir = tempfile::tempdir().unwrap(); + crate::paths::set_test_root(dir.path()); + + assert_eq!( + referrers_to(DocKind::Agent, "GENERATED-GPU"), + vec![Reference { + kind: DocKind::Profile, + name: GENERATED_PROFILE.to_string(), + }], + "an agent reference is matched case-insensitively, file or no file" + ); + assert!(referrers_to(DocKind::Agent, "some-other-gpu").is_empty()); + + crate::paths::clear_test_root(); + } + /// Write the shipped copy of [`SEEDED_PROFILE`] to disk the way /// `mirage_builtin` does on startup — around the store, so the store's /// own guards do not get a say. @@ -933,14 +1212,14 @@ mod tests { let dir = tempfile::tempdir().unwrap(); crate::paths::set_test_root(dir.path()); - assert!(profile_list().unwrap().is_empty()); + assert!(stored_profiles().is_empty()); assert_eq!(profile_put(&profile("a")).unwrap(), Stored::Created); profile_put(&profile("b")).unwrap(); - assert_eq!(profile_list().unwrap(), vec!["a", "b"]); + assert_eq!(stored_profiles(), vec!["a", "b"]); assert_eq!(profile_get("a").unwrap().name, "a"); profile_delete("a").unwrap(); - assert_eq!(profile_list().unwrap(), vec!["b"]); + assert_eq!(stored_profiles(), vec!["b"]); assert!(matches!( profile_get("a"), Err(MirageError::ProfileNotFound { .. }) @@ -955,7 +1234,7 @@ mod tests { crate::paths::set_test_root(dir.path()); profile_put(&profile("mixedcase")).unwrap(); - assert_eq!(profile_list().unwrap(), vec!["mixedcase"]); + assert_eq!(stored_profiles(), vec!["mixedcase"]); assert_eq!(profile_get("MIXEDCASE").unwrap().name, "mixedcase"); crate::paths::clear_test_root(); } @@ -973,7 +1252,7 @@ mod tests { let err = profile_put(&profile("MyProfile")).unwrap_err().to_string(); assert!(err.contains("\"MyProfile\""), "{err}"); assert!(err.contains("\"myprofile\""), "{err}"); - assert!(profile_list().unwrap().is_empty(), "nothing may be stored"); + assert!(stored_profiles().is_empty(), "nothing may be stored"); let err = agent_put("MI350X", &AgentDef::default()) .unwrap_err() @@ -1117,7 +1396,7 @@ mod tests { let inline = profile_referring_to("trav", MaybeRef::Owned(escaping.clone())); let err = profile_put(&inline).unwrap_err().to_string(); assert!(err.contains("../../outside/evil"), "{err}"); - assert!(profile_list().unwrap().is_empty()); + assert!(stored_profiles().is_empty()); let by_name = profile_referring_to("trav", MaybeRef::Ref("../../etc/passwd".to_string())); assert!(profile_put(&by_name).is_err()); @@ -1145,23 +1424,20 @@ mod tests { let err = profile_put(&p).unwrap_err().to_string(); assert!(err.contains("no-such-emulator"), "{err}"); assert!(err.contains("mirage emulators"), "{err}"); - assert!(profile_list().unwrap().is_empty()); + assert!(stored_profiles().is_empty()); crate::paths::clear_test_root(); } #[test] fn an_unknown_field_is_reported_rather_than_dropped() { - // A typo'd key used to be discarded in silence, so the emulated - // machine was quietly not the one the file described. - let err = serde_json::from_str::( + let profile = serde_json::from_str::( r#"{"name":"p","descriptoin":"typo", "emulator":{"emulator":"test","plugins":{}, "exec_mode":"Functional","options":{}, "topology":"t"}}"#, ) - .unwrap_err() - .to_string(); - assert!(err.contains("descriptoin"), "{err}"); + .unwrap(); + assert_eq!(profile.emulator.extra["descriptoin"], "typo"); let err = serde_json::from_str::( r#"{"num_nodes":2,"gpus_pernode":8,"agent":"MI350X"}"#, @@ -1170,12 +1446,14 @@ mod tests { .to_string(); assert!(err.contains("gpus_pernode"), "{err}"); - let err = serde_json::from_str::( + let agent = serde_json::from_str::( r#"{"vm":{"arch":"cdna4","gpu":{"num_xdcs":8}},"topology":{"root":{"name":"soc","type":"soc"}}}"#, ) - .unwrap_err() - .to_string(); - assert!(err.contains("num_xdcs"), "{err}"); + .unwrap(); + assert_eq!( + serde_json::to_value(agent).unwrap()["vm"]["gpu"]["num_xdcs"], + 8 + ); } fn topology() -> TopologyDef { @@ -1201,7 +1479,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); crate::paths::set_test_root(dir.path()); // A fresh machine has written nothing yet; that is not an error. - assert!(profile_list().unwrap().is_empty()); + assert!(stored_profiles().is_empty()); assert!(agent_list().unwrap().is_empty()); crate::paths::clear_test_root(); } diff --git a/emulation/mirage/ctl/src/lib.rs b/emulation/mirage/ctl/src/lib.rs index f47f07ce36d3..649ed2862e31 100644 --- a/emulation/mirage/ctl/src/lib.rs +++ b/emulation/mirage/ctl/src/lib.rs @@ -292,27 +292,30 @@ fn emulators_cmd(long: bool, json: bool) { println!("\n* = default emulator for new profiles"); } -/// Best-effort: materialise all builtin state on disk — agents, -/// topologies, and profiles — writing only what's missing. Not fatal; -/// the user can always force a full rewrite with `mirage state -/// builtins`. +/// Best-effort: materialise the builtin state that lives on disk — +/// agents and topologies — writing only what's missing. Not fatal; the +/// user can always force a full rewrite with `mirage state builtins`. +/// +/// Not profiles. Mirage generates those from the rocjitsu configs it +/// ships, on demand, and writes none — so there is nothing here to +/// unpack and nothing that can fail to be unpacked. See +/// [`mirage_builtin::profiles`]. /// /// Shared by the CLI ([`dispatch`]) and the daemon so both surfaces /// auto-unpack the builtins the first time they run, instead of /// requiring the user to invoke `mirage state builtins` by hand. /// /// A failure here is said out loud rather than only `tracing::warn!`ed, -/// because it is never local to the builtins. Everything mirage knows -/// about profiles, agents and topologies is files in one directory, so a -/// directory it cannot write reads as a machine with no configuration at -/// all: `profile list` prints nothing and exits 0, and `run` then blames -/// a missing `mi350x` — which exists, and would have been written here. -/// `config_dir_hint` turns that into the sentence the user needs. +/// because it is never local to the builtins. What mirage knows about +/// agents and topologies is files in one directory, so a directory it +/// cannot write reads as a machine with no hardware at all: `agent list` +/// prints nothing and exits 0, and `run` then blames a topology whose +/// agent reference dangles. `config_dir_hint` turns that into the +/// sentence the user needs. pub fn ensure_builtins_present() { let outcome = [ mirage_builtin::ensure_agents(false).map(|_| ()), mirage_builtin::ensure_topologies(false).map(|_| ()), - mirage_builtin::ensure_profiles(false).map(|_| ()), ] .into_iter() .find_map(Result::err); @@ -708,7 +711,7 @@ pub struct ExecArgsCli { #[derive(Subcommand, Debug)] pub enum StateCmd { - /// (Re)write the builtin agents, topologies and profiles under + /// (Re)write the builtin agents and topologies under /// ``. /// /// Every command writes back any builtin that has gone missing, so @@ -719,6 +722,10 @@ pub enum StateCmd { /// replacing it would destroy the only copy. Delete it first if you /// want the shipped version back — that is what a delete of an /// edited builtin is for. + /// + /// Builtin profiles are not written at all: mirage generates them + /// from the RocJITsu configs it ships, so they are always current and + /// there is nothing to refresh. Builtins, /// Remove mirage's runtime directory and reclaim orphaned containers. /// @@ -732,8 +739,9 @@ pub enum StateCmd { force: bool, /// Also remove the mirage config directory: every agent, /// topology and profile, including the ones you wrote. Mirage - /// writes its own builtin agents, topologies and profiles back - /// the next time it runs; nothing else comes back. + /// writes its own builtin agents and topologies back the next + /// time it runs, and its builtin profiles never left; nothing + /// else comes back. #[arg(long)] all: bool, }, @@ -1111,6 +1119,13 @@ fn long_profiles(names: &[String]) -> Vec { "name": p.name, "emulator": p.emulator.emulator, "description": p.description, + // Mirage generates its builtin profiles rather than + // writing them, so a script can no longer tell one from + // a profile the user wrote by looking for the file — + // there is no file. It is also what says whether `mirage + // profile delete` will work on this name. + "builtin": mirage_core::store::builtin_profile(n).is_some() + && !mirage_core::paths::profile_path(n).exists(), }), // The text form prints "(unreadable)" for one of these. A // reader of the JSON needs the same fact, and needs it as a @@ -1142,9 +1157,9 @@ fn report_replaced_builtin(kind: DocKind, name: &str, stored: Stored) { } let kind = kind.as_str(); eprintln!( - "mirage: {name} was a builtin {kind} mirage ships, and this replaced it. \ - Nothing you wrote was lost; `mirage {kind} delete {name}` restores the \ - shipped version." + "mirage: {name} was a builtin {kind}, and this took its name. Nothing you \ + wrote was lost; `mirage {kind} delete {name}` gives the name back to the \ + builtin." ); } @@ -1261,6 +1276,27 @@ fn report_change(json: bool, kind: DocKind, name: &str, verb: &str) -> anyhow::R Ok(()) } +/// Say why a name with no file behind it does not name a document. +/// +/// No file is not always no document: a builtin profile is generated at +/// run time and never written, so `mirage profile list` shows it and a +/// flat "no such profile" answers a name the user has just read out of +/// that list. +/// +/// Telling the two apart used to be done by calling the store's +/// *delete*, whose refusal is the sentence wanted here — which meant +/// asking a question by performing the act. Between the existence check +/// above and that call, another process creating the profile has its +/// file removed: before the confirmation prompt, with `--force` never +/// consulted, and the delete then reports the profile as missing. This +/// only ever needs to look a name up. +fn missing_document(kind: DocKind, name: &str) -> mirage_core::error::MirageError { + if kind == DocKind::Profile && mirage_core::store::builtin_profile(name).is_some() { + return mirage_core::store::generated_profile_refusal(name); + } + mirage_core::error::MirageError::not_found(kind, name) +} + /// Delete one stored document, having asked first unless told not to. /// /// One function for all three resource verbs because everything here is @@ -1299,7 +1335,7 @@ fn delete_document( // bad name it is rather than probed for on disk. mirage_core::store::validate_name(kind, name)?; if !kind.path(name).exists() { - return Err(mirage_core::error::MirageError::not_found(kind, name).into()); + return Err(missing_document(kind, name).into()); } if !force && !confirm(&format!("delete {word} {name}?"))? { if json { @@ -1360,6 +1396,14 @@ fn delete_document( /// config directory mirage cannot write is exactly the case where the /// old claim was worst, and it is the case where this one says no. fn restore_shipped(kind: DocKind, name: &str) -> bool { + // A builtin profile is generated rather than seeded, so removing the + // file that shadowed it *is* the restore — there is nothing to write + // and nothing that can fail to be written. Reading it back off disk + // like the other two kinds would report `false` for the one case + // that is always true. + if kind == DocKind::Profile { + return mirage_core::store::builtin_profile(name).is_some(); + } if mirage_core::store::shipped(kind, name).is_none() { return false; } @@ -2971,12 +3015,20 @@ async fn state_cmd(cmd: StateCmd, json: bool) -> anyhow::Result { /// `mirage state builtins`: refresh every builtin document on disk. /// -/// All three kinds, always. This used to be three `?`-chained calls, so -/// the first kind holding an edited builtin abandoned the other two — and -/// then said "Every other builtin was refreshed", which was false, and -/// under `--json` threw away the report of everything that *had* been -/// refreshed. Repairing an edited agent, topology and profile therefore -/// took three runs to even discover. +/// Both kinds that have a file, always. This used to be three +/// `?`-chained calls, so the first kind holding an edited builtin +/// abandoned the others — and then said "Every other builtin was +/// refreshed", which was false, and under `--json` threw away the report +/// of everything that *had* been refreshed. Repairing an edited agent +/// and topology therefore took two runs to even discover. +/// +/// Builtin profiles are not refreshed because they are not on disk: +/// mirage generates each one from the rocjitsu config it ships, every +/// time it is asked for, so there is no stale copy for this command to +/// rewrite. A profile *file* is the user's own, and refreshing it is +/// exactly what this command must not do. It is named instead, because a +/// file under a builtin's name silently outranks the generated one +/// everywhere else. /// /// Exit 0 when nothing failed, edited builtins included. `--help` /// presents "a builtin you have edited is left alone and named" as the @@ -2990,7 +3042,6 @@ fn builtins_cmd(json: bool) -> anyhow::Result { let kinds = [ (DocKind::Agent, mirage_builtin::ensure_agents(true)?), (DocKind::Topology, mirage_builtin::ensure_topologies(true)?), - (DocKind::Profile, mirage_builtin::ensure_profiles(true)?), ]; if json { @@ -3052,9 +3103,57 @@ fn builtins_cmd(json: bool) -> anyhow::Result { (`mirage delete `) and run `mirage state builtins` again." ); } + + // A profile file under a builtin's name is the definition that name + // resolves to, for good — the store looks at disk before it + // generates, and this command cannot refresh the file because it + // cannot tell a profile the user wrote from one an older mirage + // seeded before it stopped seeding them. Which leaves the file + // invisible: `profile show mi350x` answers with a definition that no + // longer matches what mirage ships, `profile list --long` calls it + // not-a-builtin, and this command, whose whole job is to say what is + // stale, has never mentioned profiles at all. So name them. Nothing + // here writes or removes anything. + let shadowed = shadowed_builtin_profiles(); + if !shadowed.is_empty() { + let n = Plural(shadowed.len()); + eprintln!( + "mirage: {} profile file{} share{} a name with a builtin profile, and {} \ + what that name resolves to:", + shadowed.len(), + n.pick("", "s"), + n.pick("s", ""), + n.pick("is", "are"), + ); + for name in &shadowed { + eprintln!( + " profile {name} -> {}", + mirage_core::paths::profile_path(name).display() + ); + } + eprintln!( + "mirage: mirage generates its builtin profiles rather than storing them, so \ + this command has no copy of yours to refresh and leaves the file as it is. \ + To hand the name back to the builtin, delete it \ + (`mirage profile delete `)." + ); + } Ok(ExitCode::from(0)) } +/// Name every builtin profile that a file on disk stands in for. +/// +/// The store looks at disk before it generates, so such a file is the +/// definition the name resolves to — in `profile show`, in `run`, and in +/// `profile list`, which reports it as not being a builtin at all. +fn shadowed_builtin_profiles() -> Vec<&'static str> { + mirage_builtin::profiles() + .into_iter() + .map(|(name, _)| name) + .filter(|name| mirage_core::paths::profile_path(name).exists()) + .collect() +} + /// Stop every live run and remove mirage's on-disk state. async fn purge(all: bool, json: bool) -> anyhow::Result { // Ask every live run to stop, by signalling nothing: a run owns its @@ -3330,6 +3429,7 @@ mod tests { name: "mi450x".to_string(), description: None, emulator: EmulatorDef { + extra: Default::default(), emulator: EmulatorKind::from("rocjitsu"), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -4078,19 +4178,22 @@ exit 0 #[test] fn purge_fails_when_the_runtime_directory_survives() { - use std::os::unix::fs::PermissionsExt as _; - let _lock = mirage_core::paths::test_env_lock(); let root = tempfile::tempdir().unwrap(); mirage_core::paths::set_test_root(root.path()); let runtime = mirage_core::paths::mirage_runtime_dir(); - // A directory whose contents cannot be unlinked: `chmod 500` on - // the parent of a file is enough, and is what a stuck mount or a - // root-owned leftover looks like. - let stuck = runtime.join("stuck"); - std::fs::create_dir_all(&stuck).unwrap(); - std::fs::write(stuck.join("held"), "x").unwrap(); - std::fs::set_permissions(&stuck, std::fs::Permissions::from_mode(0o500)).unwrap(); + // Something at the runtime path that `remove_dir_all` refuses and + // leaves behind: a plain file, which fails with `ENOTDIR`. What a + // stuck mount or a root-owned leftover produces is the same thing + // as far as purge is concerned — a path that is still there after + // the removal, and must not be reported as purged. + // + // Written with `chmod 500` on a subdirectory this test passed + // everywhere but CI, which runs as root in a container: root + // ignores the mode bits, the tree comes away, and the test then + // fails on a machine where nothing is wrong. + std::fs::create_dir_all(runtime.parent().unwrap()).unwrap(); + std::fs::write(&runtime, "x").unwrap(); // Blocking rather than `#[tokio::test]`: the lock guard above must // not be held across an await. @@ -4100,11 +4203,13 @@ exit 0 .unwrap() .block_on(purge(false, false)); - std::fs::set_permissions(&stuck, std::fs::Permissions::from_mode(0o700)).unwrap(); let survived = runtime.exists(); mirage_core::paths::clear_test_root(); - assert!(survived, "the test needs a directory purge cannot remove"); + assert!( + survived, + "the test needs a runtime path purge cannot remove" + ); // `ExitCode` has no `PartialEq`; its `Debug` is the only thing to // compare, and comparing it against a known value rather than a // literal string keeps the test independent of how std renders it. @@ -4225,18 +4330,17 @@ exit 0 #[test] fn state_builtins_refreshes_every_kind_and_survives_an_edited_one() { - // The three kinds were `?`-chained, so the first one holding an - // edited builtin abandoned the other two — and then claimed - // "Every other builtin was refreshed", which was false. Repairing - // an edited agent, topology and profile took three runs to even - // discover. + // The kinds were `?`-chained, so the first one holding an edited + // builtin abandoned the rest — and then claimed "Every other + // builtin was refreshed", which was false. Repairing an edited + // agent and topology took two runs to even discover. let _lock = mirage_core::paths::test_env_lock(); let root = tempfile::tempdir().unwrap(); mirage_core::paths::set_test_root(root.path()); ensure_builtins_present(); // Edit one of each kind, in the order the command visits them, so - // that stopping at the first would hide the other two. + // that stopping at the first would hide the second. let mut agent = mirage_core::store::agent_get("mi300x").unwrap(); agent.vm.gpu.num_xcds = 1; mirage_core::state::write_json(&mirage_core::paths::agent_path("mi300x"), &agent).unwrap(); @@ -4244,6 +4348,9 @@ exit 0 topology.num_nodes = 3; mirage_core::state::write_json(&mirage_core::paths::topology_path("MI350X-2x8"), &topology) .unwrap(); + // And a profile written over a builtin's name. This one is not a + // builtin at all any more — mirage generates its own and writes + // nothing — so the command must not so much as look at it. let mut profile = mirage_core::store::profile_get("mi450x").unwrap(); profile.description = Some("mine".to_string()); mirage_core::state::write_json(&mirage_core::paths::profile_path("mi450x"), &profile) @@ -4251,9 +4358,9 @@ exit 0 let code = builtins_cmd(false).unwrap(); - // All three survived, which is only possible if all three kinds - // ran, and the run is not a failure: nothing was lost and every - // other document was refreshed. + // All three survived, which is only possible if both kinds ran, + // and the run is not a failure: nothing was lost and every other + // document was refreshed. assert_eq!( mirage_core::store::agent_get("mi300x") .unwrap() @@ -4274,6 +4381,17 @@ exit 0 .description, Some("mine".to_string()) ); + // Deleting it hands the name back to the generated profile, + // without `state builtins` having to put anything there. + mirage_core::store::profile_delete("mi450x").unwrap(); + assert!( + !mirage_core::paths::profile_path("mi450x").exists(), + "a builtin profile is never written" + ); + assert_eq!( + mirage_core::store::profile_get("mi450x").unwrap(), + mirage_builtin::profile_named("mi450x").unwrap() + ); mirage_core::paths::clear_test_root(); assert_eq!( format!("{code:?}"), @@ -4479,6 +4597,75 @@ exit 0 ); } + /// A profile file under a builtin's name is reported, not refreshed. + /// + /// This is the document an older mirage seeded, before it stopped + /// seeding profiles and started generating them. It shadows the + /// generated one permanently, and nothing said so: `state builtins` + /// reported only agents and topologies, so the one command whose job + /// is naming stale copies was silent about the only copy it cannot + /// refresh. + #[test] + fn a_profile_file_shadowing_a_builtin_is_named() { + let _lock = mirage_core::paths::test_env_lock(); + let root = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(root.path()); + ensure_builtins_present(); + + let path = mirage_core::paths::profile_path("mi350x"); + mirage_core::state::write_json( + &path, + &serde_json::json!({ + "name": "mi350x", + "emulator": { + "emulator": "rocjitsu", + "exec_mode": "Functional", + "options": {}, + "plugins": {}, + "topology": {"agent": "MI350X", "gpus_per_node": 1, "num_nodes": 1} + } + }), + ) + .unwrap(); + + assert_eq!(shadowed_builtin_profiles(), vec!["mi350x"]); + assert_eq!(builtins_cmd(false).unwrap(), ExitCode::from(0)); + assert!(path.is_file(), "and reporting it does not disturb it"); + let still = mirage_core::state::read_json::(&path).unwrap(); + assert_eq!( + still["emulator"]["topology"]["agent"], "MI350X", + "nor rewrite it" + ); + } + + /// Finding out what a name is does not delete anything. + /// + /// A generated builtin has no file, so classifying one used to be + /// done by calling the store's delete and reading its refusal. The + /// state this sets up is the one a concurrent `profile create` + /// leaves behind a moment after the existence check has already + /// answered false — and that delete removed it, before the + /// confirmation prompt and without consulting `--force`, then + /// reported the profile as missing. + #[test] + fn classifying_a_generated_builtin_never_removes_a_file() { + let _lock = mirage_core::paths::test_env_lock(); + let root = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(root.path()); + ensure_builtins_present(); + + let path = mirage_core::paths::profile_path("mi350x"); + mirage_core::state::write_json( + &path, + &serde_json::json!({"name": "mi350x", "emulator": {"emulator": "rocjitsu"}}), + ) + .unwrap(); + + let e = missing_document(DocKind::Profile, "mi350x").to_string(); + assert!(e.contains("mi350x") && e.contains("builtin"), "{e}"); + assert!(path.is_file(), "somebody else's profile is still theirs"); + } + /// The shipped version is on disk before mirage says it is back. /// /// Deleting an edited builtin removes the user's copy and the diff --git a/emulation/mirage/ctl/src/run.rs b/emulation/mirage/ctl/src/run.rs index 4beefb25fbcb..5b1b49430153 100644 --- a/emulation/mirage/ctl/src/run.rs +++ b/emulation/mirage/ctl/src/run.rs @@ -522,13 +522,23 @@ pub async fn exec_cmd(a: ExecArgsCli) -> anyhow::Result { // `mirage exec` spawns its ranks as its own children in its own // terminal, so the streams to ask about are this process's; see - // [`mirage_supervisor::CallerStreams`]. - let specs = mirage_supervisor::build_specs( - &desc, - &def, - &id, - mirage_supervisor::CallerStreams::probe(), - )?; + // [`mirage_supervisor::CallerStreams`]. Asked here rather than + // inside the blocking task so the one ambient read stays at the + // edge, where that type puts it. + let caller = mirage_supervisor::CallerStreams::probe(); + // On a blocking thread, for the reason `Session::start_exec` puts + // its own `build_specs` on one: it blocks. The containerised + // `--workdir` probe is a provider round trip, and the ROCr preflight + // reads and parses the workload's executable and the runtime it + // resolves. Left on a runtime worker, either one stalls the + // `select!` below it, and this task is the one holding the lease and + // the signal relays. + let (def, id, specs) = tokio::task::spawn_blocking(move || { + let specs = mirage_supervisor::build_specs(&desc, &def, &id, caller)?; + anyhow::Ok((def, id, specs)) + }) + .await + .map_err(|e| anyhow::anyhow!("building the exec's process specs: {e}"))??; // The same headroom, in the process that will actually hold the // pipes: an exec spawns its ranks as its own children. Asked after // `build_specs` because that is what settles how many there are — diff --git a/emulation/mirage/docs/architecture.md b/emulation/mirage/docs/architecture.md index 7e30381b0efc..d7b0897ecebe 100644 --- a/emulation/mirage/docs/architecture.md +++ b/emulation/mirage/docs/architecture.md @@ -30,7 +30,7 @@ itself has none. | `mirage_core` | Shared types (`ProfileDef`, `SessionDef`, `ExecDef`, …), XDG path resolution for configuration, atomic file I/O and the config store built on it, the `run`↔`exec` wire protocol, and the emulator/registry traits. | | `mirage_supervisor` | The engine a `mirage run` is built out of: `Run`, `Session`, `Exec`, the shared spec builder, process spawn/supervise/reap, and the socket a run serves. | | `mirage_container` | Container provider abstraction (podman/docker) for containerised sessions. | -| `mirage_builtin` | Embedded builtin agents, topologies, and profiles, plus their unpackers. | +| `mirage_builtin` | The RocJITsu configs embedded as builtin agents and the unpacker that seeds them, Mirage's system topologies, and the builtin profiles it generates from those agents rather than storing. | | `mirage_rocjitsu` | The `rocjitsu` (and `rocjitsu-dbt`) backend. | | `mirage_hotswap` | The `hotswap` load-time ISA-rewriting backend. | | `rocjitsu_sys` | FFI bindings to `librocjitsu.so`, plus safe RAII wrappers over them. | diff --git a/emulation/mirage/docs/building.md b/emulation/mirage/docs/building.md index 3aa71280e856..b4d80d681a9a 100644 --- a/emulation/mirage/docs/building.md +++ b/emulation/mirage/docs/building.md @@ -22,6 +22,115 @@ Rust is the only toolchain the mirage build itself needs. Everything below that is about rocjitsu, which is a separate C++ project mirage merely loads at runtime. +### Upgrading Existing Agents + +The complete builtin `vm` and component `topology` are read from the +RocJITsu configs and validated at build time. Every GPU RocJITsu has a +config for becomes one builtin agent: `mi300x`, `mi350x` and `mi450x` +from `gfx942_cdna3.json`, `gfx950_mi355x.json` and `gfx1250_mi455x.json` +(including their 320, 288 and 256 CU topologies), and `gfx90a_mi210_kmd`, +`gfx1100_w7900`, `gfx1151` and `gfx1201_r9700` under their config's own +name. Where RocJITsu ships several configs for one GPU — a `_kmd` +variant, an `_Ngpu` one — Mirage takes the plainest, because the GPU +count is a Mirage topology setting (`--gpus-per-node`) that would +override a baked-in one anyway. Mirage no longer maintains separate +hardware values or uniform 256-CU layouts. + +Builtin **profiles** are not read from disk or written to it. Each is +derived from the agent it pins, whenever it is asked for, so there is +nothing to seed and nothing to go stale on upgrade. A profile file is +always one you wrote, and it shadows the builtin of that name until you +delete it. + +RocJITsu refuses a device that has SDMA engines and no queues on them, +and it refuses it while loading the config — so a session dies at daemon +start rather than at profile validation. Mirage never wrote +`vm.gpu.device.num_sdma_queues_per_engine` before this release, so every +agent an older Mirage left on disk is one RocJITsu now rejects, `mi350x` +included — which is what `--profile` defaults to. The shipped MI300X and +MI350X presets use 8 queues per engine, and MI450X uses 2. + +So on startup Mirage replaces an agent file it can prove it wrote itself +with the one it ships now. Proof means the stored document is either the +shipped one with just that field missing, or byte-for-meaning one of the +documents a previous release wrote, kept verbatim in +`builtin/legacy/*.json` because they cannot be derived from anything — +the code that produced them is gone. Recognition is exact, and the answer +is always the whole shipped document rather than a patched copy of the +old one: the previous `mi350x` has five SDMA engines and four CUs per +shader array against today's two and nine, so pasting today's queue count +into it would leave a machine neither Mirage nor RocJITsu ships. + +Anything else is left exactly as it is. An agent you edited is yours to +keep, and so is one from a release too old to be in that table — the +serialized shape moved more than once before this, and a document from +one of those is simply not recognised. Adding a release to the table is a +matter of dumping its agents and appending the file. An explicit `0` is +likewise an answer, and Mirage does not argue with it — RocJITsu will. + +An unrecognised file is not left silent. `mirage state builtins` names it +along with every other builtin document that differs from the shipped +one, with its path and how to take the shipped version instead +(`mirage agent delete `, then run `mirage state builtins` again). +Profile validation refuses the incomplete pair with the reason, at +`profile create`, `profile import` and `run` — while the document can +still be edited, rather than at daemon start. + +Agents with older or customized layouts are not overwritten. Missing fields +are reported against the selected RocJITsu preset during profile validation +and config synthesis. Preserve any customizations before deleting an old +builtin agent; the next invocation recreates a missing builtin from the +current preset. + +### Profile Compatibility + +Additional JSON fields on an agent are preserved recursively and passed to +RocJITsu. Additional profile fields may be placed inside `emulator` or at the +profile root; root extras are normalized into `emulator` when saved. For example: + +```json +{ + "name": "custom", + "emulator": { + "emulator": "rocjitsu", + "plugins": {}, + "exec_mode": "Functional", + "options": {}, + "topology": "MI350X-1x1", + "max_ticks": 200000, + "vm": { "gpu": { "device": { "capability2": 1 } } } + } +} +``` + +Objects are merged recursively over the agent configuration; scalar values +and arrays replace the corresponding values. Mirage's per-node GPU count +and explicitly selected plugins remain authoritative. Duplicate root and +emulator overrides are rejected. Mirage-only container and system-topology +controls still reject unknown fields *within* them. Replacement stops at +the shape of a field Mirage has a name for: `vm` still has to describe a +device after merging, so a scalar over `vm.gpu.device` or a string over +`num_sdma_engines` is refused when the profile is written. + +Passthrough is not validation, and RocJITsu is not a backstop: it parses +the synthesised config with `skip_unexpected_fields_in_json` set, so a key +it does not recognise is dropped without a word (see +`config_common.h`). A misspelled key therefore survives Mirage and +vanishes in RocJITsu, and the machine that comes up is quietly not the one +the file describes. What Mirage can still tell you is that the key was not +one of *its own*: a profile-root field Mirage does not recognise is +forwarded to the emulator with a warning naming it, which is the signal to +read for `containerise`, `descriptoin` and the like. Inside `emulator` and +inside an agent there is no such signal — check those against the preset. + +Missing profile fields produce warnings on stderr. Omitted `plugins`, +`exec_mode` and `options` use their defaults; required structural fields +still cause a parse error. Missing agent fields stay absent in the generated +JSON so RocJITsu's schema defaults apply. Warnings identify fields present +in the matching preset, including component config entries; they do not +replace RocJITsu validation. Explicit zeroes are preserved, not mistaken +for missing values. `--config` continues to use the supplied file verbatim. + ## Prerequisites | Tool | Version | Needed for | Notes | @@ -49,9 +158,10 @@ cargo build --release # optimized This builds the `mirage` binary at `target/debug/mirage` (or `target/release/mirage`). The build embeds: -- the **builtin agents, topologies and profiles** — Rust constructors in - the `builtin` crate, so the data is validated by the compiler rather - than by a runtime parse, and +- the **builtin agents**, one per GPU in `rocjitsu/configs/`, read from + that directory and validated by the `builtin` build script — plus + Mirage's system topologies, and the builtin profiles generated from + those agents at run time, and - the **third-party dependency manifest** that `mirage about` prints, distilled from `cargo metadata` by the root `build.rs`. @@ -241,6 +351,16 @@ alive, every one of those tests is bounded by a process it started: a suite that crashes cannot leave a session behind for the next one to trip over. +The release lane of [RocJITsu CI](../../../.github/workflows/rocjitsu-corpus-tests.yml) +runs `cargo test --locked --workspace --no-fail-fast -- --test-threads=4 --nocapture` +against that job's freshly built RocJITsu libraries, with missing-emulator +skips disabled. Changes to either project trigger the workflow. This +includes daemon startup for every builtin agent — the list is taken from +`mirage agent list`, so a GPU added to RocJITsu is covered without +editing a test — legacy-agent upgrades, the container and lifecycle +suites, and the software-emulator matrix. +Hardware-dependent DBT cases remain capability-gated. + ## Linting The workspace lint policy lives in [`Cargo.toml`](../Cargo.toml) under @@ -285,6 +405,33 @@ test suite. - **`no mirage run is serving session `** — the run that owned the session has exited. A session exists exactly as long as its `mirage run` does; start one in another terminal and `mirage exec` into that. +- **The workload sees no GPU, and exits 0 anyway** — `rocminfo` reports + only the CPU agent, the session came up normally and nothing failed. + Almost always this is a ROCm that predates the GPU the session + emulates: `libhsa-runtime64.so` enumerates agents by looking their ISA + up in the table it was compiled with, and one it does not recognise is + skipped without a word. Before each host workload starts, mirage + statically checks the command's unambiguous direct `DT_RUNPATH` + resolution and warns — + + > `mirage: the ROCm runtime this workload will load does not support + > gfx1250, which is the GPU this session emulates. …` + + — naming the resolved runtime and the ROCm version beside it. Run the + workload under a newer ROCm (`mirage run --image ` + is the usual way), or emulate a GPU this ROCm supports: a session's + target comes from its profile, or from the device a drop-in `--config` + describes. A `--config` with a DBT guest enabled names no target, because + the session presents the host's devices and the guest together. The + check is deliberately conservative — it warns only for a + trusted system executable that links ROCr directly through a modern + absolute or `$ORIGIN`-based `RUNPATH`, with no loader override or + hardware-capability alternative in the way. Everything else stays + silent: user-built executables, anything that loads ROCr later with + `dlopen` (Python included), transitive dependencies, legacy `RPATH`, + cache-only resolution, preloads including a system-wide + `/etc/ld.so.preload`, HSA overrides, and set-ID or file-capability + executables. Its silence therefore does not rule this cause out. - **A backend reported as not installed** — run `mirage emulators -l` first. It prints every path that was searched for that backend's library and the environment variables that would resolve it, which is diff --git a/emulation/mirage/docs/cli.md b/emulation/mirage/docs/cli.md index 5bc11c27cf3f..54f7b8768f69 100644 --- a/emulation/mirage/docs/cli.md +++ b/emulation/mirage/docs/cli.md @@ -686,8 +686,38 @@ run fails to find an emulator — see [`building.md`](building.md). ## `mirage profile` -Profiles are reusable emulator presets stored in -`$XDG_CONFIG_HOME/mirage/profile/.json`. +Profiles are reusable emulator presets. The ones you write are stored in +`$XDG_CONFIG_HOME/mirage/profile/.json`; the builtin ones are not +stored at all. + +Mirage ships one builtin profile per GPU RocJITsu has a config for, and +**generates it on demand rather than writing it to disk** — there is no +file for `mi350x`, and `mirage profile show mi350x` works anyway. Each is +named after the agent it pins, targets `rocjitsu`, and describes one GPU +on one node; `--num-nodes` and `--gpus-per-node` widen that per run. + +Writing your own profile under a builtin's name still works: the file +shadows the builtin, and deleting the file hands the name back to the +builtin, immediately and with nothing to re-seed. Deleting a builtin that +has no file of its own is refused — there is nothing to remove, and the +name would still resolve afterwards. + +```sh +$ mirage profile list -l +NAME EMULATOR DESCRIPTION +gfx1100_w7900 rocjitsu AMD Radeon Pro W7900 (rdna3), from RocJITsu's configs/gfx1100_w7900.json +gfx1151 rocjitsu AMD Radeon Graphics (rdna3_5), from RocJITsu's configs/gfx1151.json +gfx1201_r9700 rocjitsu AMD Radeon AI PRO R9700 (rdna4), from RocJITsu's configs/gfx1201_r9700.json +gfx90a_mi210_kmd rocjitsu AMD Instinct MI210 (cdna2), from RocJITsu's configs/gfx90a_mi210_kmd.json +mi300x rocjitsu AMD Instinct MI300X (cdna3), from RocJITsu's configs/gfx942_cdna3.json +mi350x rocjitsu AMD Instinct MI350X (cdna4), from RocJITsu's configs/gfx950_mi355x.json +mi450x rocjitsu AMD Instinct MI455X (cdna5), from RocJITsu's configs/gfx1250_mi455x.json +``` + +Under `--json`, `profile list --long` carries a `builtin` field, because +a script can no longer tell a builtin from a profile you wrote by looking +for the file. It is also what says whether `mirage profile delete` will +work on that name. ```text mirage profile list [-l|--long] @@ -717,13 +747,17 @@ mirage profile delete [-f|--force] there. These files are the only copy, and a profile someone tuned is not recoverable once it has been written over — so a name that is taken is an error naming the path, and `mirage profile delete ` is how - you say you meant it. An untouched builtin is the one exception: mirage - wrote it, so replacing it destroys nothing. + you say you meant it. A builtin is the one exception: mirage generates + it and holds no file, so taking its name destroys nothing. Mirage says + so on stderr when it happens, and `mirage profile delete ` gives + the builtin the name back. ## `mirage topology` Topologies describe a rack/node/GPU layout and reference an agent. Builtin -topologies are written on first run. +topologies are written on first run. They cover the MI300X and MI350X +layouts; for any other agent, `--gpus-per-node` and `--num-nodes` on +`mirage run` widen its profile without a stored topology. ```text mirage topology list @@ -735,8 +769,15 @@ mirage topology delete [-f|--force] ## `mirage agent` -Agents are hardware GPU definitions (e.g. `MI300X`, `MI350X`, `MI450X`). -Builtin agents are written on first run. +Agents are hardware GPU definitions. Mirage ships one per GPU RocJITsu +has a config for — `mi300x`, `mi350x`, `mi450x`, `gfx90a_mi210_kmd`, +`gfx1100_w7900`, `gfx1151`, `gfx1201_r9700` — each being the `vm` and +`topology` of the matching `rocjitsu/configs/*.json`, and each written to +disk on first run. Where RocJITsu ships several configs for one GPU (a +`_kmd` variant, an `_Ngpu` one), mirage takes the plainest: the GPU count +is a mirage topology setting (`--gpus-per-node`) and would be overwritten +anyway. `mirage agent list` is therefore a list of machines, not of +files. Names are case-insensitive. ```text mirage agent list @@ -780,38 +821,82 @@ on: works, because that is a stdin with somebody on the other end of it; only `y` and `yes`, in any case, are yes. -## Configuration documents reject what they do not understand +## What a document may say that mirage does not understand + +A key mirage has no name for is one of two things: a typo, or a RocJITsu +setting newer than this mirage. Refusing every one of them would put +mirage between a profile and the emulator it configures; accepting every +one silently would let `emulaotr` load, report success, and run with +settings the file plainly did not ask for. So the answer depends on where +the key is, and there are three of them. This applies to every document +mirage reads: `import` on all three kinds, and the profiles read by `run` +and `exec`. + +**Rejected outright** — the parts of a document that are mirage's own, +where an unknown key can only be a mistake because there is nobody to +forward it to. `containerize` and everything under it, including its file +mounts and port mappings; and a topology document's own fields +(`num_nodes`, `gpus_per_node`, `agent`), as distinct from the SoC tree +inside an agent, which is forwarded. Also rejected: `plugins`, +`exec_mode`, `options` and `topology` written at the profile *root*, which +are refused with "belongs inside emulator"; the same key given both at the +root and under `emulator`; and any extra under `emulator` for a backend +that has no passthrough, which is every backend but RocJITsu. -Profiles, agents and topologies are parsed strictly. A key the schema does -not know is a parse error naming it — +```text +error: json error on t.json: unknown field `nodes`, expected one of `num_nodes`, `gpus_per_node`, `agent` at line 1 column 57 +error: json error on p.json: profile field "exec_mode" belongs inside emulator +``` + +**Retained and forwarded** — ordinary RocJITsu profile and agent extras, +recursively, nested device and topology objects included. These reach the +synthesised RocJITsu configuration exactly as written, which is how a +profile reaches a setting this mirage has never heard of. An extra at the +profile *root* is forwarded with a warning naming it, because that is +where a misspelt mirage key (`descriptoin`, `containerise`) lands; inside +`emulator` and inside an agent there is no warning, because a key mirage +does not know is the expected case there. + +**Retained but shape-checked** — `vm` and everything under it. Forwarding +is not licence to change the type of a field mirage does have a name for, +and passthrough merging replaces nodes rather than deepening them, so +`"device": 7` would hand RocJITsu an integer where a device belongs and +`"num_sdma_engines": "4"` a string where a count belongs. Those are +refused when the profile is written rather than at daemon start: ```text -error: json error on : unknown field `emulaotr`, expected one of `name`, `description`, `emulator`, `containerize` at line 1 column 42 +error: profile "p": rocjitsu cannot use this profile: profile's RocJITsu configuration has a vm that does not describe a device: invalid type: string "4", expected u32. Fields mirage has no name for are passed through as written, but the ones it does have to keep their shape. ``` -— rather than a field quietly dropped on the way in. The two outcomes are -easy to confuse and are not remotely equivalent: a profile with a typo'd -key used to load, report success, and then not be what the file plainly -said it was, which surfaces much later as an emulation that ran with the -wrong settings. Failing at parse time makes the file and the behaviour the -same thing again. It applies to every document mirage reads — `import` on -all three kinds, and the profiles read by `run` and `exec`. +`emulator.topology` is a mirage field throughout, and a malformed one is +refused outright. + +Note that RocJITsu itself drops configuration keys it does not recognise, +without complaint — so a forwarded key that this mirage passes on is not +thereby a key that took effect. See the Profile Compatibility section of +[building.md](building.md). ## `mirage state` ```text -mirage state builtins # (re)write builtin agents/topologies/profiles +mirage state builtins # (re)write builtin agents/topologies mirage state purge [-f|--force] [--all] ``` -mirage writes any *missing* builtin agent, topology and profile on every -command, so the shipped set is always there. `state builtins` additionally +mirage writes any *missing* builtin agent and topology on every command, +so the shipped set is always there. `state builtins` additionally refreshes the ones that exist, which is what you want after upgrading — but it refreshes only the ones you have not touched. A builtin whose file differs from the shipped version is *your* document that happens to share a name, and rewriting it would discard your edits with no way back, so that one is left alone and everything else is still refreshed. +Builtin **profiles** are not in this at all. Mirage generates them from +the RocJITsu configs it ships and writes none, so they are current by +construction and there is nothing here to refresh — a profile file is +always one you wrote, and refreshing it is exactly what this command must +not do. + It reports both halves, one line per document, so the two outcomes are told apart at a glance rather than inferred from what is missing: @@ -826,12 +911,11 @@ mirage: 1 builtin document differs from the one mirage ships and was left alone: mirage: rewriting one would discard your edits, and everything else was refreshed. To take the shipped version of one after all, delete it (`mirage delete `) and run `mirage state builtins` again. ``` -The list of edited documents covers **all three kinds at once**, on -stderr, after the report of what did land — so a script reading the list -of what changed does not have to filter it out, and repairing an edited -agent, an edited topology and an edited profile takes one run rather -than three. Take the shipped version of one by deleting the file and -running the command again. +The list of edited documents covers **both kinds at once**, on stderr, +after the report of what did land — so a script reading the list of what +changed does not have to filter it out, and repairing an edited agent and +an edited topology takes one run rather than two. Take the shipped +version of one by deleting the file and running the command again. **Having edited a builtin is not a failure**, and `state builtins` exits 0 on it. Every document that could be refreshed was, nothing was lost, @@ -850,12 +934,19 @@ Deleting follows from the same fact, in both directions: reset one: it really does remove your version, and the shipped document takes its place. That is the round trip — edit to customise, delete to revert. +* A **builtin profile** is refused for the neighbouring reason: mirage + generates it and never wrote a file, so there is nothing to remove and + the name would still resolve. The round trip is the same shape — + `mirage profile create ` to take the name, delete to give + it back. `state purge` is the blunt tool: * It does everything `mirage cleanup` does, below, and then removes the runtime directory. The config directory (profiles, topologies, agents) - is left alone unless `--all` is given. + is left alone unless `--all` is given. `--all` takes the profiles you + wrote with it; the builtin profiles are generated rather than stored + and are still there afterwards. * It refuses while any `mirage run` is live, and tells you which. Killing someone else's foreground command from a state-cleanup subcommand would be a surprise; stop it with Ctrl-C in its own terminal @@ -1156,6 +1247,10 @@ the invocation — pass `--force`, or correct the flag. ```sh $ mirage --json profile list [ + "gfx1100_w7900", + "gfx1151", + "gfx1201_r9700", + "gfx90a_mi210_kmd", "mi300x", "mi350x", "mi450x" diff --git a/emulation/mirage/docs/demo-debugging-at-scale.md b/emulation/mirage/docs/demo-debugging-at-scale.md index 84022173fa2e..1e5c649c99d7 100644 --- a/emulation/mirage/docs/demo-debugging-at-scale.md +++ b/emulation/mirage/docs/demo-debugging-at-scale.md @@ -196,7 +196,7 @@ flowchart TB root --> ctl["mirage_ctl
CLI verbs · run · exec"] root --> core["mirage_core
types · XDG paths · proto · traits"] ctl --> sup["mirage_supervisor
session/exec/process engine"] - ctl --> builtin["mirage_builtin
embedded agents/topologies/profiles"] + ctl --> builtin["mirage_builtin
embedded agents/topologies
generated profiles"] sup --> cont["mirage_container
podman/docker provider"] root -.link-only.-> rj["mirage_rocjitsu (+ dbt)"] root -.link-only.-> hot["mirage_hotswap"] diff --git a/emulation/mirage/docs/state-layout.md b/emulation/mirage/docs/state-layout.md index a159f4ed3b92..399d17c5dc80 100644 --- a/emulation/mirage/docs/state-layout.md +++ b/emulation/mirage/docs/state-layout.md @@ -161,6 +161,20 @@ daemon binds a socket in the same place: └── exec//.pid # containerised ranks only (see above) ``` +The socket is also the one thing here the workload has to be told *who* +is on the other end of. rocjitsu's interposer asks the daemon for +permission to be read out of — pageable transfers are serviced by the +daemon reaching into the workload with `process_vm_readv`, which Yama +refuses to a non-descendant without that grant — and `SO_PEERCRED` alone +cannot say whether the peer that answered is the right process, since the +socket path is predictable. So mirage exports `$ROCJITSU_DAEMON_PID` +naming the process hosting the daemon, which is the `mirage run` process +itself: the daemon is started in-process (`rj_daemon_start`), not forked +the way the `rocjitsu` CLI does it. Without that variable the interposer +trusts whoever answered and says so on the workload's stderr at every +launch. It is set only in daemon mode; `--in-process` connects to no +socket. + Mirage never reads these back to answer a control-plane query. The session creates the directory before bring-up and removes it wholesale during teardown, so it cannot outlive its session. diff --git a/emulation/mirage/hotswap/Cargo.toml b/emulation/mirage/hotswap/Cargo.toml index 897364057873..ac88d2d0d61e 100644 --- a/emulation/mirage/hotswap/Cargo.toml +++ b/emulation/mirage/hotswap/Cargo.toml @@ -10,5 +10,11 @@ path = "src/lib.rs" mirage_core = { workspace = true } inventory = { workspace = true } +# Tests only: this backend takes no configuration of its own, but it has +# to prove it refuses a profile that hands it some, and a profile is +# JSON. +[dev-dependencies] +serde_json = { workspace = true } + [lints] workspace = true diff --git a/emulation/mirage/hotswap/src/lib.rs b/emulation/mirage/hotswap/src/lib.rs index 4784b1c78eba..c682a405f0dd 100644 --- a/emulation/mirage/hotswap/src/lib.rs +++ b/emulation/mirage/hotswap/src/lib.rs @@ -106,7 +106,14 @@ impl EmulatorBackend for Hotswap { fn shutdown(&self, _ctx: &SessionContext) {} - fn validate_profile(&self, _def: &ProfileDef) -> std::result::Result<(), String> { + fn validate_profile(&self, def: &ProfileDef) -> std::result::Result<(), String> { + // HotSwap runs the workload on a real GPU through its own + // intercept and takes no synthesised configuration, so an extra + // field under `emulator` has nothing here to reach. Say so + // rather than accept it: `EmulatorDef::extra` exists for + // rocjitsu's config passthrough, and the profile parser cannot + // tell which backend a key was meant for. + def.emulator.reject_extra("HotSwap")?; // HotSwap is not bundled or built by mirage; it must be // installed separately. Surface actionable guidance now, at // profile-creation time, rather than only when a session is @@ -616,6 +623,32 @@ mod tests { assert_eq!(status.location.is_found(), status.location.path().is_some()); } + /// `EmulatorDef::extra` is rocjitsu's config passthrough, and the + /// profile parser cannot tell which backend a stray key was meant + /// for. HotSwap has no configuration to forward it to, so it says + /// so at import rather than starting a session with the default the + /// key was written to change. + #[test] + fn an_unknown_field_under_the_emulator_is_refused_at_import() { + let profile: ProfileDef = serde_json::from_value(serde_json::json!({ + "name": "p", + "emulator": { + "emulator": "hotswap", + "topology": "t", + "vm": {"gpu": {"num_xcds": 8}}, + "exec_mode_": "clocked" + } + })) + .unwrap(); + let message = Hotswap.validate_profile(&profile).unwrap_err(); + assert!(message.contains("\"vm\""), "{message}"); + assert!(message.contains("\"exec_mode_\""), "{message}"); + assert!(message.contains("HotSwap"), "{message}"); + // And the check comes first, so it is the answer on a machine + // with no HotSwap install as much as on one with it. + assert!(!message.contains("HOTSWAP_HOME"), "{message}"); + } + #[test] fn search_targets_the_intercept_lib() { let s = lib_search(); @@ -700,6 +733,7 @@ mod tests { name: "hotswap-test".to_string(), description: None, emulator: mirage_core::emulator::EmulatorDef { + extra: Default::default(), emulator: "hotswap".to_string(), plugins: Default::default(), exec_mode: mirage_core::emulator::ExecMode::Functional, diff --git a/emulation/mirage/rocjitsu/Cargo.toml b/emulation/mirage/rocjitsu/Cargo.toml index d3ba16b9dc54..f15752af099a 100644 --- a/emulation/mirage/rocjitsu/Cargo.toml +++ b/emulation/mirage/rocjitsu/Cargo.toml @@ -8,6 +8,7 @@ path = "src/lib.rs" [dependencies] mirage_core = { workspace = true } +mirage_builtin = { workspace = true } rocjitsu_sys = { workspace = true } inventory = { workspace = true } chrono = { workspace = true } @@ -20,7 +21,6 @@ thiserror = { workspace = true } serde_json = { workspace = true } [dev-dependencies] -mirage_builtin = { workspace = true } tempfile = { workspace = true } [lints] diff --git a/emulation/mirage/rocjitsu/src/dbt.rs b/emulation/mirage/rocjitsu/src/dbt.rs index d40ac6623785..09295788f3ee 100644 --- a/emulation/mirage/rocjitsu/src/dbt.rs +++ b/emulation/mirage/rocjitsu/src/dbt.rs @@ -28,8 +28,7 @@ use std::collections::BTreeMap; use std::path::PathBuf; -use mirage_core::agent::AgentDef; -use mirage_core::common::{MaybeRef, SimpleValue}; +use mirage_core::common::SimpleValue; use mirage_core::config::OptionDef; use mirage_core::discovery::{self, LibSearch, RuntimeLocation}; use mirage_core::emulator::{ @@ -42,7 +41,6 @@ use mirage_core::hardware::{gfx_name, gpu_gfx_versions}; use mirage_core::plugin::PluginsDef; use mirage_core::profile::{FileMount, ProfileDef}; use mirage_core::session::{SessionContext, SessionHealth}; -use mirage_core::topology::TopologyDef; /// The canonical name the DBT backend registers under (the value stored /// in [`mirage_core::emulator::EmulatorDef::emulator`]). @@ -163,6 +161,13 @@ impl RocjitsuDbt { def: &ProfileDef, env_lookup: &impl EnvLookup, ) -> std::result::Result<(), String> { + // The DBT backend reads only its typed topology and options; it + // synthesises no rocjitsu simulation config, so it has nowhere + // to forward `EmulatorDef::extra` — the passthrough belongs to + // the regular `rocjitsu` backend alone. An unrecognised field + // under this emulator is a mistake, and this is the last layer + // that can still say which document it is in. + def.emulator.reject_extra("rocjitsu-dbt")?; // Resolving the guest gfx version exercises the same topology + // agent reference resolution the run path needs, so any error // here is exactly what would otherwise surface at run time. @@ -347,18 +352,17 @@ fn supported_isa_list() -> String { } /// Resolve the guest GPU's `gfx_target_version` from `def` by following -/// its topology + agent references. Mirrors the resolution -/// [`crate::kmd_config`] performs. +/// its topology + agent references. +/// +/// Through [`EmulatorDef::resolve_refs`] rather than a walk of its own: +/// the agent is where the target is written down, and a second copy of +/// the route to it is a second thing to keep in step with the stores. fn guest_gfx_version(def: &EmulatorDef) -> Result { - let topology: TopologyDef = match &def.topology { - MaybeRef::Owned(t) => t.clone(), - MaybeRef::Ref(name) => mirage_core::topology::store::get(name)?, - }; - let agent: AgentDef = match &topology.agent { - MaybeRef::Owned(a) => a.clone(), - MaybeRef::Ref(name) => mirage_core::agent::store::get(name)?, - }; - Ok(agent.vm.gpu.device.gfx_target_version) + let mut def = def.clone(); + def.resolve_refs()?; + Ok(def + .agent() + .map_or(0, |agent| agent.vm.gpu.device.gfx_target_version)) } /// How the ISA resolvers read environment overrides. @@ -531,8 +535,10 @@ mod tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use super::*; - use mirage_core::common::SimpleValue; + use mirage_core::agent::AgentDef; + use mirage_core::common::{MaybeRef, SimpleValue}; use mirage_core::emulator::ExecMode; + use mirage_core::topology::TopologyDef; #[test] fn the_installed_flag_and_the_located_library_are_one_answer() { @@ -548,6 +554,7 @@ mod tests { fn def_with(topology: MaybeRef) -> EmulatorDef { EmulatorDef { + extra: Default::default(), emulator: NAME.to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -572,6 +579,40 @@ mod tests { assert_eq!(backend.description().name, NAME); } + /// A backend that retargets onto the host's real GPU reports no + /// emulated target, however specific its agent is. + /// + /// DBT translates the guest's code to run on a physical card, so ROCr + /// is never asked to enumerate the agent's device — and a warning + /// that its ISA is unsupported would be about a GPU nothing is + /// standing up. The agent still names one, which is exactly why the + /// question is asked of the backend rather than of the agent alone. + #[test] + fn a_retargeting_backend_emulates_no_device() { + let mut agent = AgentDef::default(); + agent.vm.gpu.device.gfx_target_version = 90500; + let profile = ProfileDef { + name: "dbt".to_string(), + description: None, + emulator: def_with(MaybeRef::Owned(TopologyDef { + num_nodes: 1, + gpus_per_node: 1, + agent: MaybeRef::Owned(agent), + })), + containerize: None, + }; + + assert!( + profile.agent().and_then(AgentDef::gfx_target).is_some(), + "the fixture's agent must name a device for the test to mean anything" + ); + assert_eq!( + profile.emulated_gfx_target(), + None, + "DBT runs on a real GPU, so its agent's ISA is not a target to check" + ); + } + /// An environment containing exactly the given pairs. /// /// These tests used to call `std::env::set_var`, which Rust 2024 @@ -692,6 +733,31 @@ mod tests { assert!(err.contains("not a translatable"), "unexpected: {err}"); } + /// The DBT backend synthesises no rocjitsu config, so it has + /// nowhere to forward a passthrough field — and the shared + /// `EmulatorDef` no longer rejects one on its behalf. + #[test] + fn an_unknown_field_under_the_emulator_is_refused_at_import() { + let profile: ProfileDef = serde_json::from_value(serde_json::json!({ + "name": "p", + "emulator": { + "emulator": "rocjitsu-dbt", + "topology": "t", + "max_ticks": 5 + } + })) + .unwrap(); + let message = RocjitsuDbt + .validate_profile_with(&profile, &empty_env()) + .unwrap_err(); + assert!(message.contains("\"max_ticks\""), "{message}"); + assert!(message.contains("rocjitsu-dbt"), "{message}"); + // Refused before the topology reference is followed, so the + // stray key is what the user is told about — not the "t" that + // this profile was never going to resolve either. + assert!(!message.contains("\"t\""), "{message}"); + } + #[test] fn validate_accepts_translatable_guest() { let _g = mirage_core::paths::test_env_lock(); diff --git a/emulation/mirage/rocjitsu/src/lib.rs b/emulation/mirage/rocjitsu/src/lib.rs index 06f4eb753511..0e0733a86876 100644 --- a/emulation/mirage/rocjitsu/src/lib.rs +++ b/emulation/mirage/rocjitsu/src/lib.rs @@ -15,7 +15,7 @@ use std::path::PathBuf; -use mirage_core::agent::AgentDef; +use mirage_core::agent::{AgentDef, KfdDeviceInfo}; use mirage_core::common::{MaybeRef, SimpleMap, SimpleValue}; use mirage_core::config::OptionDef; use mirage_core::discovery::{LibSearch, RuntimeLocation}; @@ -40,8 +40,8 @@ pub mod dbt; /// model and stay on a single loopback socket, while ROCr must use SDMA /// copies and skip scratch reclaim. rocprofiler-register is disabled /// since the simulated GPU does not back it. Applied as defaults in -/// [`Rocjitsu::injection_def`]; the per-exec environment overrides any -/// of them. +/// [`Rocjitsu::injection_def_with`]; the per-exec environment overrides +/// any of them. const RCCL_ENV_DEFAULTS: &[(&str, &str)] = &[ ("HSA_ENABLE_SDMA", "1"), ("ROCPROFILER_REGISTER_ENABLED", "0"), @@ -178,12 +178,153 @@ impl EmulatorBackend for Rocjitsu { } fn injection_def(&self, ctx: &SessionContext) -> Result { + self.injection_def_with(ctx, kmd_preload()) + } + + /// rocjitsu synthesises the KFD node the workload enumerates, so the + /// agent's ISA is exactly what its ROCm runtime has to recognise. + fn presents_emulated_device(&self) -> bool { + true + } + + /// Point the profile's agent at the device a drop-in `--config` + /// describes, because that file — not the profile's own agent — is + /// what the interposer will stand up. + /// + /// A config mirage cannot read leaves the agent with no target rather + /// than the profile's original one: the session is about to emulate + /// whatever that file says, and naming the device it replaced would be + /// a confident wrong answer where silence is available. rocjitsu + /// itself reads the file through FlatBuffers, which accepts JSON this + /// does not, so failing to parse it here says nothing about whether + /// the run will work — only that mirage cannot describe it. + /// + /// The file is read once. Its snapshot in `session_dir` becomes the + /// profile's `config`, so [`kmd_config`] injects the bytes the device + /// was read from instead of re-reading a file the user may have + /// rewritten since. + fn reconcile_profile( + &self, + profile: &mut ProfileDef, + session_dir: &std::path::Path, + ) -> Result<()> { + let Some(SimpleValue::String(path)) = profile.emulator.options.get("config") else { + return Ok(()); + }; + let original = absolute_supplied_config(PathBuf::from(path)); + // A file that cannot be read or snapshotted is left for the + // injection to report: it fails on the same file with the error the + // user needs, and there is no device to name in the meantime. + let device = match snapshot_supplied_config(&original, session_dir) { + Ok((snapshot, bytes)) => { + profile.emulator.options.insert( + "config".to_string(), + SimpleValue::String(snapshot.display().to_string()), + ); + device_of_bytes(&bytes) + } + Err(_) => None, + }; + // A default device is one with no `gfx_target_version`, which is + // the "mirage cannot name this" answer the doc above promises. + let device = device.unwrap_or_default(); + // Both already owned by the contract on the trait method: the + // caller resolves the profile first. Matching rather than + // asserting keeps a misuse a no-op instead of a panic, and the + // preflight's answer for an unresolved profile is silence either + // way. + if let MaybeRef::Owned(topology) = &mut profile.emulator.topology + && let MaybeRef::Owned(agent) = &mut topology.agent + { + agent.vm.gpu.device = device; + } + Ok(()) + } + + fn daemon_capability(&self) -> Result<()> { + // Answered once per process; see `located_daemon_capability`. + located_daemon_capability() + .as_ref() + .map_or_else(|e| Err(MirageError::Other(e.to_string())), |()| Ok(())) + } + + fn start_daemon(&self, ctx: &SessionContext) -> Result>> { + // One rocjitsu daemon per session. If the KMD library cannot be + // located there is nothing to host the emulated device with; + // return `None` rather than erroring, since the per-exec + // `injection_def` already fails loudly in that case. + let Some(lib) = kmd_preload() else { + tracing::warn!( + "rocjitsu: KMD library ({LIB_NAME}) not found; \ + not starting daemon" + ); + return Ok(None); + }; + // The configuration `injection_def` already materialised for this + // session, not a second resolution of the profile: an edit + // between the two would leave the daemon emulating one device + // while `SessionDescription` reports another and the preflight + // warns about a third. See [`session_config`]. + let config = session_config(&ctx.runtime_dir)?; + // The daemon binds its socket under the same runtime directory the + // workload's interposer probes (`$ROCJITSU_RUNTIME_DIR`), which is + // exactly what `injection_def` exports — so the workload connects + // to *this* daemon with no extra wiring. Both live in the + // session's scratch directory and go away with it. + let runtime_dir = write_config_discovery(&ctx.runtime_dir, &config)?; + let daemon = rocjitsu_sys::daemon::Daemon::start(&lib, &config, &runtime_dir) + .map_err(|e| MirageError::Other(format!("rocjitsu daemon: {e}")))?; + Ok(Some(Box::new(RocjitsuDaemon(daemon)))) + } +} + +impl Rocjitsu { + /// [`EmulatorBackend::injection_def`] against an explicit KMD + /// interposer, as [`kmd_preload`] located it. + /// + /// Threaded in for the reason `hotswap` and `rocjitsu-dbt` thread + /// theirs: whether rocjitsu is installed is a fact about the host, + /// Rust 2024 makes `set_var` `unsafe` and this workspace forbids + /// `unsafe`, so a test that could not supply the library could only + /// assert about the machine it happened to run on. With it as a + /// parameter, the injection this backend really hands the supervisor + /// is what the tests check. + /// + /// # Errors + /// + /// Returns an error when the profile does not describe a machine + /// rocjitsu can stand up, when `preload` is `None`, or when the + /// session's config or discovery file cannot be written. + pub fn injection_def_with( + &self, + ctx: &SessionContext, + preload: Option, + ) -> Result { let def = ctx.emulator(); let config = kmd_config(def, &ctx.runtime_dir)?; + // A node container sees the session's scratch directory, not + // arbitrary host files a supplied config names. This checks both + // halves: the config itself when `kmd_config` had to leave it + // where it lay, and a `dbt_guest.simulator_config` that a + // successfully copied config still points at outside the + // session. Checking only `config` misses the second one. + if ctx.profile.containerize.is_some() + && let Some(unreachable) = container_unreachable_config_path(&config, &ctx.runtime_dir)? + { + return Err(MirageError::Other(format!( + "rocjitsu: the drop-in config {} cannot be used in a \ + containerised session because it depends on {}, which is \ + outside the session and is not mounted into the container. \ + Run it without `--image`, or use a self-contained config \ + with no external `dbt_guest.simulator_config`.", + config.display(), + unreachable.display(), + ))); + } // Refuse to run unemulated: if the KMD interposer can't be // located there is nothing to emulate the workload, so fail // loudly rather than silently running on real hardware. - let ld_preload = kmd_preload().ok_or_else(|| { + let ld_preload = preload.ok_or_else(|| { // The search itself says where it looked, so this cannot // drift from it the way the hand-written list did. let detail = runtime_location() @@ -217,12 +358,10 @@ impl EmulatorBackend for Rocjitsu { // the supervisor existed, a per-node `mirage host` process inside // each container re-resolved the whole injection instead.) // - // The runtime directory is the session's, whoever wrote the - // config: in drop-in `--config` mode `config` is a file of the - // user's, and deriving the runtime directory from *its* location - // would leave the discovery file and the daemon socket beside it, - // outside the session, uncleaned, and shared with any other run - // pointed at the same config. + // The runtime directory is the session's whatever config it is + // handed: derived from it, two runs pointed at the same config + // would share a discovery file and a daemon socket. Whether the + // config itself is the session's is [`kmd_config`]'s question. let runtime_dir = write_config_discovery(&ctx.runtime_dir, &config)?; let mut env = std::collections::BTreeMap::new(); @@ -231,6 +370,51 @@ impl EmulatorBackend for Rocjitsu { runtime_dir.display().to_string(), ); + // Name the process hosting the daemon, so the interposer can tell + // that the peer answering its socket is the daemon mirage started + // for it and not merely *a* process of this user that got there + // first. + // + // The interposer asks the daemon for permission to be read out of + // (`PR_SET_PTRACER`) — pageable transfers the runtime registered + // through KFD SVM are serviced by the daemon reaching into the + // workload with process_vm_readv, and Yama refuses that to a + // non-descendant without the grant. `SO_PEERCRED` answers who is + // connected, which is weaker than it looks: the socket path is + // predictable, so any process of this user could be listening. So + // the interposer takes the launcher's word for which PID it + // should be talking to, from `$ROCJITSU_DAEMON_PID`, and where no + // launcher left one it trusts the peer anyway and warns — on the + // workload's stderr, at every single launch: + // + // [rj warn] daemon: authorizing pid N to read this address + // space; no launcher named a daemon for this client, ... + // + // Mirage *is* the launcher and has always known the answer: it + // hosts the daemon in-process (`start_daemon` calls + // `rj_daemon_start` on this very process, rather than forking a + // separate one the way the `rocjitsu` CLI does), so the PID the + // interposer should see is ours. Saying so silences the warning + // by making the claim true, rather than by hiding it — the grant + // stops resting on "whoever answered". + // + // Only in daemon mode. `--in-process` emulates inside the + // workload, connects to no socket, and would be told about a + // daemon that does not exist. + // + // A containerised workload is unaffected either way: its PID + // namespace does not contain this process, so `SO_PEERCRED` + // reports no usable peer there and the interposer never consults + // this variable. It is exported regardless, because a container + // that *does* share the namespace is then told the truth rather + // than nothing. + if ctx.daemon { + env.insert( + "ROCJITSU_DAEMON_PID".to_string(), + std::process::id().to_string(), + ); + } + // Default runtime tuning the emulated workload needs to behave // under rocjitsu. These mirror the environment the upstream // rocjitsu RCCL collective tests run with (see @@ -286,37 +470,6 @@ impl EmulatorBackend for Rocjitsu { host_gpus: false, }) } - - fn daemon_capability(&self) -> Result<()> { - // Answered once per process; see `located_daemon_capability`. - located_daemon_capability() - .as_ref() - .map_or_else(|e| Err(MirageError::Other(e.to_string())), |()| Ok(())) - } - - fn start_daemon(&self, ctx: &SessionContext) -> Result>> { - // One rocjitsu daemon per session. If the KMD library cannot be - // located there is nothing to host the emulated device with; - // return `None` rather than erroring, since the per-exec - // `injection_def` already fails loudly in that case. - let Some(lib) = kmd_preload() else { - tracing::warn!( - "rocjitsu: KMD library ({LIB_NAME}) not found; \ - not starting daemon" - ); - return Ok(None); - }; - let config = kmd_config(ctx.emulator(), &ctx.runtime_dir)?; - // The daemon binds its socket under the same runtime directory the - // workload's interposer probes (`$ROCJITSU_RUNTIME_DIR`), which is - // exactly what `injection_def` exports — so the workload connects - // to *this* daemon with no extra wiring. Both live in the - // session's scratch directory and go away with it. - let runtime_dir = write_config_discovery(&ctx.runtime_dir, &config)?; - let daemon = rocjitsu_sys::daemon::Daemon::start(&lib, &config, &runtime_dir) - .map_err(|e| MirageError::Other(format!("rocjitsu daemon: {e}")))?; - Ok(Some(Box::new(RocjitsuDaemon(daemon)))) - } } /// Describe the rocjitsu emulator backend for the registry. Owned by @@ -435,17 +588,28 @@ pub fn enabled_plugin_libs(preload: &std::path::Path, plugins: &PluginsDef) -> V .collect() } -/// Name of the synthesised rocjitsu `SimulationConfig` written into a -/// session's scratch directory. +/// Name of the rocjitsu `SimulationConfig` a session runs on, written +/// into its scratch directory — synthesised from the profile, or copied +/// there from a drop-in `--config` [`kmd_config`] was able to pin. pub const RJ_CONFIG_NAME: &str = "rj_config.json"; -/// Path of the synthesised `SimulationConfig` inside a session's scratch -/// directory. +/// Path of the `SimulationConfig` inside a session's scratch directory. +/// +/// Written by [`kmd_config`] during bring-up. Ask [`session_config`] for +/// the document a live session actually emulates rather than assuming it +/// is this one: a drop-in `--config` that cannot be pinned is left where +/// it lies, and then there is no file here at all. #[must_use] pub fn rj_config_path(runtime_dir: &std::path::Path) -> PathBuf { runtime_dir.join(RJ_CONFIG_NAME) } +/// Name of the KMD interposer's discovery file, read from +/// `$ROCJITSU_RUNTIME_DIR`. Its contents are the path of the +/// `SimulationConfig` the session runs on — the interposer's answer to +/// which config that is, and therefore everything else's. +pub const CONFIG_PATH_NAME: &str = "config_path"; + /// Adapter making a [`rocjitsu_sys::daemon::Daemon`] usable as the /// emulator-agnostic handle mirage's supervisor holds. #[derive(Debug)] @@ -462,23 +626,26 @@ impl EmulatorDaemon for RocjitsuDaemon { /// runtime directory (to export as `ROCJITSU_RUNTIME_DIR`). /// /// The interposer resolves its `SimulationConfig` by reading a -/// `config_path` file from its per-user runtime directory; the file's -/// contents are the path to the config JSON. +/// [`CONFIG_PATH_NAME`] file from its per-user runtime directory; the +/// file's contents are the path to the config JSON. /// /// The runtime directory is always [`RUNTIME_SUBDIR`] under /// `session_dir` — the session's own scratch directory — and never /// derived from where `config` happens to live. The daemon socket lands /// there too, so both are owned by the session, disappear with it, and -/// stay distinct between two runs. That matters for the drop-in -/// `--config` mode in particular, where `config` is a file of the -/// user's that mirage has no business writing next to and that two -/// concurrent runs may well share. +/// stay distinct between two runs. +/// +/// `config` is a session-owned path in every mode but one: [`kmd_config`] +/// copies a drop-in `--config` in rather than naming the user's file, +/// because the interposer reopens whatever is written here for as long as +/// the session lives. The exception is a config mirage cannot copy +/// without changing what it means, which it names where it lies. pub fn write_config_discovery( session_dir: &std::path::Path, config: &std::path::Path, ) -> Result { let runtime_dir = session_dir.join(RUNTIME_SUBDIR); - let config_path_file = runtime_dir.join("config_path"); + let config_path_file = runtime_dir.join(CONFIG_PATH_NAME); mirage_core::state::write_bytes( &config_path_file, format!("{}\n", config.display()).as_bytes(), @@ -748,13 +915,52 @@ fn spread_thread_allocations_over_gpus( #[derive(Debug)] enum SimConfig { /// A config file of the user's own, named by the drop-in `--config` - /// option and used verbatim. Already on disk; mirage only reads it. + /// option. Already on disk; mirage reads it and copies it into the + /// session when it can preserve the meaning of its references — + /// byte for byte but for the reference [`pin_external_references`] + /// pins — and never writes to the original. + /// + /// Always absolute: [`absolute_supplied_config`] settles that at the + /// boundary, because everything downstream reads this path from + /// somewhere else — including the pin, which anchors the config's + /// siblings to its directory. Supplied(PathBuf), /// Config JSON synthesised from the profile's topology + agent, /// still to be written into a session's scratch directory. Synthesised(Vec), } +/// A supplied config path, made absolute against the working directory +/// mirage resolved the profile in. +/// +/// This is the boundary where a profile option becomes a path this +/// backend acts on, and it is the last moment the option's own spelling +/// still means what its author meant. `mirage run --config` canonicalises +/// what it is given, but that is the CLI's doing and not every profile +/// arrives that way: one written by hand, by another tool, or by an +/// older mirage carries whatever was typed. Left relative it would be +/// re-read against whatever directory the next reader happens to be in — +/// and the session's copy of it lives somewhere else entirely, so the +/// references pinned to it would follow the copy rather than the +/// original. +/// +/// Absolute, not canonical: rocjitsu resolves a relative +/// `dbt_guest.simulator_config` against the parent of the path it is +/// handed, without following symlinks, so resolving them here would move +/// the anchor a symlinked config's siblings hang from. +fn absolute_supplied_config(path: PathBuf) -> PathBuf { + if path.is_absolute() { + return path; + } + match std::env::current_dir() { + Ok(working_directory) => working_directory.join(path), + // Nothing to make it absolute against. The path is still the one + // the profile asked for, and the existence check below is about + // to report it if it does not resolve. + Err(_) => path, + } +} + /// Resolve the rocjitsu `SimulationConfig` `def` asks for, writing /// nothing. /// @@ -785,8 +991,10 @@ fn resolve_sim_config(def: &EmulatorDef) -> Result { // applies, because upstream `rocjitsu --cpu-thread-budget` takes it // alongside `--config`; it goes to a session copy, never to the file. if let Some(SimpleValue::String(path)) = def.options.get("config") { - let cfg = PathBuf::from(path); + let cfg = absolute_supplied_config(PathBuf::from(path)); if !cfg.exists() { + // Named as the user spelled it, which is the spelling they + // can compare against what they wrote. return Err(MirageError::Other(format!( "rocjitsu config not found: {path}" ))); @@ -803,9 +1011,18 @@ fn resolve_sim_config(def: &EmulatorDef) -> Result { MaybeRef::Owned(t) => t.clone(), MaybeRef::Ref(name) => mirage_core::topology::store::get(name)?, }; - let agent: AgentDef = match &topology.agent { - MaybeRef::Owned(a) => a.clone(), - MaybeRef::Ref(name) => mirage_core::agent::store::get(name)?, + let (agent, expected_config): (AgentDef, Option) = match &topology.agent { + MaybeRef::Owned(a) => { + let agent = a.clone(); + let expected = mirage_builtin::agents::source_config_for_agent(&agent); + (agent, expected) + } + MaybeRef::Ref(name) => { + let agent = mirage_core::agent::store::get(name)?; + let expected = mirage_builtin::agents::source_config(name) + .or_else(|| mirage_builtin::agents::source_config_for_agent(&agent)); + (agent, expected) + } }; let exec_mode = match def.exec_mode { ExecMode::Functional => "functional", @@ -828,19 +1045,52 @@ fn resolve_sim_config(def: &EmulatorDef) -> Result { // (deriving per-GPU identities from the single `device` template). // Each node's host process emulates the GPUs local to that node, so // the per-node `gpus_per_node` is what the config requests. - let thread_allocations = target_thread_allocations( - agent.vm.gpu.device.gfx_target_version, - topology.gpus_per_node, - ); - let mut vm = agent.vm; - vm.gpu.num_gpus = topology.gpus_per_node.max(1); let mut sim = serde_json::json!({ "max_ticks": 100000u64, - "exec_mode": exec_mode, - "vm": vm, - "topology": agent.topology, - "thread_allocations": thread_allocations, }); + merge_config( + &mut sim, + serde_json::to_value(&agent).map_err(|error| MirageError::Other(error.to_string()))?, + ); + merge_config(&mut sim, serde_json::Value::Object(def.extra.clone())); + // `exec_mode` is mirage's, not the document's. An agent is now a + // rocjitsu config with its `vm` and `topology` taken out, and the + // ones mirage ships from are whole configs with `exec_mode`, + // `max_ticks` and `num_threads` at their root — so an agent imported + // from one carries those keys in `AgentDef::extra` and, merged here, + // would quietly outrank `--exec-mode`. Functional and clocked are + // different runs with different results; the profile decides, and + // this is asserted after both merges so nothing can take it back. + // `max_ticks` is seeded above instead, where a document may still + // override it: it is a default mirage has no opinion about, not a + // claim about how the session runs. + if let Some(map) = sim.as_object_mut() { + map.insert("exec_mode".to_string(), exec_mode.into()); + } + if !sim["vm"].is_object() { + return Err(MirageError::Other("rocjitsu vm must be an object".into())); + } + if sim["vm"].get("gpu").is_none() { + sim["vm"]["gpu"] = serde_json::json!({}); + } + let gpu = sim["vm"]["gpu"] + .as_object_mut() + .ok_or_else(|| MirageError::Other("rocjitsu vm.gpu must be an object".into()))?; + gpu.insert("num_gpus".into(), topology.gpus_per_node.max(1).into()); + let gfx_target_version = sim["vm"]["gpu"]["device"]["gfx_target_version"] + .as_u64() + .filter(|value| *value <= u64::from(u32::MAX)) + .map_or(agent.vm.gpu.device.gfx_target_version, |value| value as u32); + // Target allocation tables are defaults. A profile or imported agent + // that supplies one is choosing its own scheduling policy. + if sim.get("thread_allocations").is_none() { + sim["thread_allocations"] = serde_json::to_value(target_thread_allocations( + gfx_target_version, + topology.gpus_per_node, + )) + .map_err(|error| MirageError::Other(format!("rocjitsu thread allocations: {error}")))?; + } + // Multi-partition RCCL collectives currently hang on multi-GPU VMs. // Serial dispatch also avoids the measured small-collective slowdown. // Match the native multi-GPU presets; an explicit option below can override. @@ -876,17 +1126,62 @@ fn resolve_sim_config(def: &EmulatorDef) -> Result { } } } + + // The merges above put the profile's and the agent's unrecognised + // keys into this document as written, which is the point — mirage + // does not know rocjitsu's whole schema and will not stand between a + // profile and a field it wants to set. But a key mirage *does* know + // is one it has already given a shape, and passthrough merging + // replaces nodes rather than deepening them: `vm.gpu.device: 7` + // arrives here as the integer 7 where an object belongs, and + // `num_sdma_engines: "4"` as a string where a count belongs, which + // `settle_sdma_pair` then reads as absent and silently answers with + // zero. Re-parsing `vm` says so now, while the profile is being + // written and the user is there to fix it, rather than at daemon + // start. Unknown keys survive it by construction: every type under + // `VirtualMachineConfig` carries a flattened `extra` map. This is a + // check and nothing else — the parsed value is dropped, so no + // default it supplies is written back into the config. + serde_json::from_value::(sim["vm"].clone()).map_err( + |error| { + MirageError::Other(format!( + "profile's RocJITsu configuration has a vm that does not describe a device: \ + {error}. Fields mirage has no name for are passed through as written, but \ + the ones it does have to keep their shape." + )) + }, + )?; + for path in missing_config_fields(&sim, expected_config.as_ref()) { + tracing::warn!( + "profile's RocJITsu configuration is missing field {path}; RocJITsu schema defaults apply" + ); + } + // After the omission report, so that report still describes the + // document rather than the repair below. + settle_sdma_pair(&mut sim)?; // Carry the profile's plugin selection into the synthesised rocjitsu // config so the interposer (local path) and the per-node daemon both // enable them through the rocjitsu plugin loader. `def.plugins` maps a // plugin name to its argument object — exactly the shape rocjitsu's - // `plugins` config section expects. Only emit the key when a plugin is - // actually selected so a plugin-free profile still produces a clean, - // minimal config (and the near-zero-overhead no-plugin path). - if !def.plugins.is_empty() - && let serde_json::Value::Object(map) = &mut sim - { - map.insert("plugins".to_string(), plugins_to_json(&def.plugins)); + // `plugins` config section expects. + // + // The profile decides, and it decides both ways: an agent imported + // from a whole rocjitsu config carries that config's root `plugins` + // through `AgentDef::extra` and into the merge above, so a selection + // that is only ever *written* when nonempty leaves the agent's own + // plugins standing for a profile that selects none. Container runs + // would then differ from local ones as well, because the libraries + // bind-mounted into a node container come from `def.plugins` alone + // (see `injection_def`) — the interposer would be told to load a + // plugin whose shared object is not in the container. So the key is + // removed when the selection is empty, which also keeps a + // plugin-free profile on rocjitsu's near-zero-overhead path. + if let serde_json::Value::Object(map) = &mut sim { + if def.plugins.is_empty() { + map.remove("plugins"); + } else { + map.insert("plugins".to_string(), plugins_to_json(&def.plugins)); + } } let bytes = serde_json::to_vec_pretty(&sim).map_err(|e| { MirageError::Other(format!("rocjitsu kmd_config: serialize sim config: {e}")) @@ -894,6 +1189,158 @@ fn resolve_sim_config(def: &EmulatorDef) -> Result { Ok(SimConfig::Synthesised(bytes)) } +/// rocjitsu's schema default for `KfdDeviceInfo.num_sdma_engines`. +/// +/// Together with [`SDMA_QUEUE_DEFAULT`] this is the pair a `device` +/// object that names neither field gets, and rocjitsu's config loader +/// rejects it (`config/config_common.h`): the queue count was appended +/// to the table after the engine count, so it could not be given a +/// nonzero default without moving field IDs, which leaves the +/// *defaults themselves* incoherent. +const SDMA_ENGINE_DEFAULT: u64 = 2; + +/// rocjitsu's schema default for +/// `KfdDeviceInfo.num_sdma_queues_per_engine`. See +/// [`SDMA_ENGINE_DEFAULT`]. +const SDMA_QUEUE_DEFAULT: u64 = 0; + +/// Keep the synthesised config's SDMA pair one rocjitsu will load. +/// +/// A `device` object that names neither field is not making a claim +/// about SDMA at all — but leaving both out hands rocjitsu the +/// incoherent default pair above and the run dies during config load, +/// before anything the user can see. Mirage used to serialize every +/// field of the agent whether the document mentioned it or not, so such +/// an agent arrived carrying `num_sdma_engines: 0` and emulated a GPU +/// with no SDMA; that is still the machine it describes, so say so +/// explicitly rather than let a preserved omission change it. +/// +/// A document that names *one* of the two is making a claim, and the +/// claim is incoherent. There is no value mirage could pick that is not +/// an invention about the device, so it is refused here — at profile +/// validation, via [`check_config`], where the document can still be +/// edited — rather than at run time from inside the emulator. +fn settle_sdma_pair(sim: &mut serde_json::Value) -> Result<()> { + // No `device` object: rocjitsu leaves the whole KFD identity + // defaulted and `kfd_device_from_fb` returns before the check. + let Some(device) = sim + .pointer_mut("/vm/gpu/device") + .and_then(serde_json::Value::as_object_mut) + else { + return Ok(()); + }; + let field = |key: &str| device.get(key).and_then(serde_json::Value::as_u64); + let engines = field("num_sdma_engines"); + let queues = field("num_sdma_queues_per_engine"); + if engines.unwrap_or(SDMA_ENGINE_DEFAULT) == 0 || queues.unwrap_or(SDMA_QUEUE_DEFAULT) != 0 { + return Ok(()); + } + if engines.is_none() && queues.is_none() { + device.insert("num_sdma_engines".into(), 0.into()); + tracing::warn!( + "profile's RocJITsu configuration names neither \ + vm.gpu.device.num_sdma_engines nor \ + vm.gpu.device.num_sdma_queues_per_engine; emulating a GPU with no \ + SDMA engines, since RocJITsu's own defaults for the two do not \ + describe a usable device" + ); + return Ok(()); + } + Err(MirageError::Other(format!( + "profile's RocJITsu configuration has vm.gpu.device.num_sdma_engines = {} \ + and vm.gpu.device.num_sdma_queues_per_engine = {}, which RocJITsu refuses \ + to load: engines with nowhere to queue to. Give the agent a nonzero \ + num_sdma_queues_per_engine, or set num_sdma_engines to 0 to emulate a GPU \ + with no SDMA. Values not written in the document are RocJITsu's schema \ + defaults, {SDMA_ENGINE_DEFAULT} and {SDMA_QUEUE_DEFAULT}.", + engines.unwrap_or(SDMA_ENGINE_DEFAULT), + queues.unwrap_or(SDMA_QUEUE_DEFAULT), + ))) +} + +fn merge_config(base: &mut serde_json::Value, overrides: serde_json::Value) { + match (base, overrides) { + (serde_json::Value::Object(base), serde_json::Value::Object(overrides)) => { + for (key, value) in overrides { + merge_config(base.entry(key).or_insert(serde_json::Value::Null), value); + } + } + (base, value) => *base = value, + } +} + +fn missing_config_fields( + config: &serde_json::Value, + expected_config: Option<&serde_json::Value>, +) -> Vec { + let skeleton = serde_json::json!({ + "vm": {"arch": "", "gpu": {"device": {}}}, + "topology": {"root": {}} + }); + let expected = expected_config.unwrap_or(&skeleton); + let mut missing = Vec::new(); + for key in ["vm", "topology"] { + collect_missing_fields( + config.get(key), + &expected[key], + &format!("/{key}"), + &mut missing, + ); + } + missing +} + +fn collect_missing_fields( + actual: Option<&serde_json::Value>, + expected: &serde_json::Value, + path: &str, + missing: &mut Vec, +) { + let Some(actual) = actual else { + missing.push(path.to_string()); + return; + }; + if let Some(object) = expected.as_object() { + for (key, value) in object { + collect_missing_fields(actual.get(key), value, &format!("{path}/{key}"), missing); + } + } else if let (Some(actual), Some(expected)) = (actual.as_array(), expected.as_array()) { + if path.ends_with("/config") { + for entry in expected { + if let Some(key) = entry["key"].as_str() + && !actual.iter().any(|entry| entry["key"] == key) + { + missing.push(format!("{path}/{key}")); + } + } + } else if path.ends_with("/children") { + for (index, child) in actual.iter().enumerate() { + if let Some(reference) = + expected.iter().find(|entry| entry["type"] == child["type"]) + { + collect_missing_fields( + Some(child), + reference, + &format!("{path}/{index}"), + missing, + ); + } + } + // A child the tree does not have at all. Matching runs from + // `actual` above, so without this a component the preset has + // and the document dropped — the whole IOD memory tier, say — + // is the one omission that goes unreported. + for reference in expected { + if let Some(kind) = reference["type"].as_str() + && !actual.iter().any(|child| child["type"] == kind) + { + missing.push(format!("{path}/{kind}")); + } + } + } + } +} + /// Re-emit the supplied `--config` file with `cpu_thread_budget` replaced. /// /// A run's budget belongs to that run, so the override produces a session copy @@ -950,28 +1397,386 @@ fn supplied_config_with_budget(cfg: &std::path::Path, budget: &SimpleValue) -> R /// path. That path is what gets recorded in the rocjitsu `config_path` /// discovery file so the LD_PRELOAD'd interposer loads it. /// -/// One file per session, rewritten on each call so it always reflects -/// the current profile, and removed with the session. A profile that -/// supplies its own config (drop-in `--config`) is returned as-is and -/// nothing is written, unless a run overrides its CPU thread budget — -/// then the session gets a copy carrying the override. +/// One file per session, written by bring-up and removed with the +/// session. +/// +/// A drop-in `--config` is copied here rather than pointed at whenever +/// it can be, which is what makes the device a session emulates fixed +/// for as long as the session lives. `kmd_config` used to hand the +/// interposer the user's own path, and the interposer reopens it: in +/// `--in-process` mode every later workload re-read whatever was on disk +/// *then*, so editing the file after bring-up changed the emulated device +/// while the agent [`Rocjitsu::reconcile_profile`] read it into — and the +/// preflight warning drawn from that agent — still described the old one. +/// Copying costs a few kilobytes in a directory that is already the +/// session's and removes the whole class. +/// +/// In a session the copy already exists: [`Rocjitsu::reconcile_profile`] +/// took it at session creation, read the device from it, and pointed the +/// profile at it, so this hands the interposer those same bytes. It copies +/// here only for a profile that was never reconciled. +/// +/// The copy is byte for byte apart from one thing: a relative +/// `dbt_guest.simulator_config` is made absolute against the original's +/// directory first, because moving the file would otherwise move what +/// that reference means. A config whose references cannot be pinned is +/// not copied at all — see [`pin_external_references`]. +/// +/// A run that overrides its CPU thread budget gets the same session copy, +/// carrying the override — see [`supplied_config_with_budget`]. +/// +/// Nothing is written beside the user's file, then or now. /// /// # Errors /// /// Returns an error when the topology or agent references cannot be /// resolved, the profile asks for more GPUs than rocjitsu will emulate -/// (see [`MAX_GPUS_PER_NODE`]), or the config cannot be written. +/// (see [`MAX_GPUS_PER_NODE`]), or the config cannot be read or written. pub fn kmd_config(def: &EmulatorDef, session_dir: &std::path::Path) -> Result { + let cfg = rj_config_path(session_dir); match resolve_sim_config(def)? { - SimConfig::Supplied(cfg) => Ok(cfg), + // Already the session's own copy: `reconcile_profile` took it at + // session creation and read the device from it. Copying it onto + // itself would change nothing, and every exec would rewrite a file + // the interposer may be reading. + SimConfig::Supplied(path) if path == cfg => Ok(path), + SimConfig::Supplied(path) => snapshot_supplied_config(&path, session_dir).map(|(p, _)| p), SimConfig::Synthesised(bytes) => { - let cfg = rj_config_path(session_dir); mirage_core::state::write_bytes(&cfg, &bytes)?; Ok(cfg) } } } +/// Copy a drop-in `--config` into `session_dir`, as [`kmd_config`] +/// describes, and return where the session's config now is along with +/// the bytes read from the original. +/// +/// The bytes are the ones the interposer will be handed, so they are what +/// [`Rocjitsu::reconcile_profile`] reads the device from: one reading of +/// the user's file serves both. +/// +/// # Errors +/// +/// Returns an error when the original cannot be read or the copy cannot +/// be written. +fn snapshot_supplied_config( + original: &std::path::Path, + session_dir: &std::path::Path, +) -> Result<(PathBuf, Vec)> { + let bytes = std::fs::read(original).map_err(|e| MirageError::io(original.to_path_buf(), e))?; + match pin_external_references(bytes.clone(), original) { + Some(pinned) => { + let cfg = rj_config_path(session_dir); + mirage_core::state::write_bytes(&cfg, &pinned)?; + Ok((cfg, pinned)) + } + // Owning the document is worth less than the document still + // meaning what it says, so the session declines it and the + // interposer is handed the user's own file — as it was before the + // copy existed. What is given up is the narrow guarantee the copy + // buys: an edit to this file can still retarget the session under + // itself. The device is not read from it either — mirage could + // not parse these bytes — so no warning names the old device. + None => { + // Silent otherwise, and it is the one thing that would explain + // a session behaving as it did before the snapshot existed. + tracing::debug!( + config = %original.display(), + "rocjitsu: config not snapshotted into the session; \ + mirage cannot pin the references it names" + ); + Ok((original.to_path_buf(), bytes)) + } + } +} + +/// JSON pointer to the one field in a rocjitsu `SimulationConfig` that +/// names another file relative to the config's own directory. +const SIMULATOR_CONFIG_POINTER: &str = "/dbt_guest/simulator_config"; + +/// A config or one of its dependencies that a node container cannot +/// open at the host path recorded in the config. +/// +/// The supervisor bind-mounts `session_dir` at its own host path because +/// rocjitsu's discovery file and config contents are not rewritten for a +/// container. A config [`kmd_config`] leaves elsewhere is therefore +/// unreachable. A config copied under `session_dir` can still name an +/// external `dbt_guest.simulator_config`; pinning a relative reference +/// makes that dependency absolute but does not make it part of the +/// session or mount it. +/// +/// The dependency exists only when DBT guest mode is enabled with its +/// simulator backend. `simulator_config` is ignored for a disabled or +/// hardware-backed DBT block, so those are not rejected merely for +/// carrying an unused value. +/// +/// A relative reference is resolved exactly as rocjitsu resolves it: +/// beside the config that contains it. Empty means the same config and +/// therefore introduces no dependency. +/// +/// "Inside the session" is decided on the path the loader will reach, not +/// on its spelling: `/../host.json` names the session in its +/// prefix and a file outside it, and so does a symlink out of the session. +fn container_unreachable_config_path( + config: &std::path::Path, + session_dir: &std::path::Path, +) -> Result> { + if !lies_within(config, session_dir) { + return Ok(Some(config.to_path_buf())); + } + let bytes = std::fs::read(config).map_err(|e| MirageError::io(config.to_path_buf(), e))?; + let parsed: serde_json::Value = match serde_json::from_slice(&bytes) { + Ok(parsed) => parsed, + // A dialect only FlatBuffers reads. [`kmd_config`] copies one into + // the session only when it never names `simulator_config`, so it + // has no dependency that could be out of reach. + Err(_) if !mentions_external_reference(&bytes) => return Ok(None), + Err(e) => { + return Err(MirageError::Other(format!( + "rocjitsu: could not inspect materialised config {} for \ + container-visible references: {e}", + config.display() + ))); + } + }; + let dbt = parsed.pointer("/dbt_guest"); + let enabled = dbt + .and_then(|dbt| dbt.get("enabled")) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + let simulator_backend = dbt + .and_then(|dbt| dbt.get("execution_backend")) + .is_some_and(|backend| { + backend.as_str() == Some("simulator") || backend.as_u64() == Some(1) + }); + if !enabled || !simulator_backend { + return Ok(None); + } + let Some(reference) = parsed + .pointer(SIMULATOR_CONFIG_POINTER) + .and_then(serde_json::Value::as_str) + .filter(|path| !path.is_empty()) + else { + return Ok(None); + }; + let reference = std::path::Path::new(reference); + let resolved = if reference.is_absolute() { + reference.to_path_buf() + } else { + config + .parent() + .map_or_else(|| reference.to_path_buf(), |parent| parent.join(reference)) + }; + Ok((!lies_within(&resolved, session_dir)).then_some(resolved)) +} + +/// Whether `path` is under `directory` as a container that mounts +/// `directory` at its own spelling would see it. +/// +/// Both halves have to hold. The mount is made at the directory's +/// spelling, so the path must be spelled inside it once `..` is settled; +/// and the container has only what is under the mount, so it must still +/// be inside once symlinks are followed. A path that does not exist cannot +/// be followed and is judged on its spelling alone — the interposer fails +/// to open it on either side of the mount. +fn lies_within(path: &std::path::Path, directory: &std::path::Path) -> bool { + let spelled_inside = lexically_normal(path).starts_with(lexically_normal(directory)); + let resolved_inside = match ( + std::fs::canonicalize(path), + std::fs::canonicalize(directory), + ) { + (Ok(path), Ok(directory)) => path.starts_with(directory), + _ => true, + }; + spelled_inside && resolved_inside +} + +/// `path` with `.` dropped and each `..` removing the component before it. +fn lexically_normal(path: &std::path::Path) -> PathBuf { + let mut normal = PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::ParentDir => { + normal.pop(); + } + other => normal.push(other), + } + } + normal +} + +/// A drop-in `--config`'s external reference, pinned to where it was +/// written rather than to where the copy lands. +/// +/// rocjitsu resolves a relative `dbt_guest.simulator_config` beside the +/// config file it was handed — `resolve_dbt_host_config_path` joins it +/// onto that file's parent directory — and the file it is handed is +/// whatever the `config_path` discovery file names. Copying the config +/// into the session therefore moves the anchor: the shipped +/// `guest_gfx950_on_simulated_gfx942.json` asks for +/// `gfx942_cdna3_kmd.json` beside itself and would be looked up in the +/// session scratch directory instead. Making the reference absolute +/// against the original's directory is the same path rocjitsu would +/// have resolved, so the composition form keeps working while the bytes +/// the session emulates stay the session's own. +/// +/// An absolute reference and a config without one are returned byte for +/// byte. It is the only such field in the schema — +/// `ProgramConfig::binary_path` is the other path-valued one and nothing +/// loads it — so this is a pin, not a rewriter. +/// +/// `None` means the document cannot be relocated: mirage could not read +/// it, and it mentions the one field whose meaning the move would +/// change. rocjitsu parses its configs with FlatBuffers, which accepts a +/// JSON dialect `serde_json` does not — comments, trailing commas and +/// unquoted keys among it — so a parse failure here is not evidence that +/// rocjitsu will fail too, and it says nothing about whether the +/// document names a sibling. Matching that dialect would mean carrying a +/// second JSON parser in mirage; declining the copy costs one guarantee +/// on configs mirage cannot read, and keeps every config rocjitsu +/// accepts working. [`kmd_config`] is where that is paid. +/// +/// The file it points *at* is still the user's, and still theirs to +/// edit. Following the reference would mean copying a graph; the value +/// here is that a supplied config no longer silently retargets the +/// device, not that every file it can reach becomes immutable. +fn pin_external_references(bytes: Vec, original: &std::path::Path) -> Option> { + let parsed = serde_json::from_slice::(&bytes); + let (Some(directory), Ok(mut config)) = (original.parent(), parsed) else { + return (!mentions_external_reference(&bytes)).then_some(bytes); + }; + let reference = match config + .pointer(SIMULATOR_CONFIG_POINTER) + .and_then(serde_json::Value::as_str) + { + Some(path) if !path.is_empty() && !std::path::Path::new(path).is_absolute() => { + directory.join(path) + } + _ => return Some(bytes), + }; + let Some(slot) = config.pointer_mut(SIMULATOR_CONFIG_POINTER) else { + return Some(bytes); + }; + *slot = serde_json::Value::String(reference.display().to_string()); + // Unreachable in practice — this `Value` came from `from_slice`, so + // it holds nothing serde_json cannot write back — and the original + // bytes are the only honest fallback if it ever is reached. + Some(serde_json::to_vec_pretty(&config).unwrap_or(bytes)) +} + +/// Whether these bytes mention the field whose value moves with the file. +/// +/// Asked of the raw bytes rather than of a parse, because it is only +/// asked when the parse is the thing that failed. A field is set by +/// naming it, and every dialect rocjitsu accepts spells the name the +/// same way, so the literal absence of it is enough to know there is +/// nothing to pin — which is the direction that has to be right. +fn mentions_external_reference(bytes: &[u8]) -> bool { + /// The leaf of [`SIMULATOR_CONFIG_POINTER`]. + const FIELD: &[u8] = b"simulator_config"; + bytes.windows(FIELD.len()).any(|window| window == FIELD) +} + +/// The configuration bring-up materialised for a live session. +/// +/// The counterpart of [`kmd_config`] for everything that runs *after* +/// bring-up: it reads back the answer bring-up recorded rather than +/// resolving the profile a second time, so a topology, agent or +/// `--config` edited while a session is up cannot change what that +/// session emulates. +/// +/// The answer is read out of the `config_path` discovery file, which is +/// the same byte the interposer opens. That is deliberate: the session's +/// config is [`rj_config_path`] in the ordinary case and the user's own +/// file in the one case [`kmd_config`] declines to copy, and a second +/// rule for which of those it is would be a second thing to keep in step +/// with the first. +/// +/// The path is the file's first line, with one trailing `\r` removed and +/// nothing else trimmed. That is how rocjitsu's own reader +/// (`parse_dbt_runtime_config_handoff`) takes it, so a config whose path +/// begins or ends with a space names the same file here as there. +/// +/// Private because it is the second half of a sequence bring-up owns, +/// not an entry point of its own, and reading a session's configuration +/// before bring-up wrote one is a question with no good answer. +/// +/// # Errors +/// +/// Returns an error when the session has no recorded configuration, +/// which means bring-up did not get as far as [`kmd_config`]. +fn session_config(session_dir: &std::path::Path) -> Result { + let discovery = session_dir.join(RUNTIME_SUBDIR).join(CONFIG_PATH_NAME); + let recorded = match std::fs::read_to_string(&discovery) { + Ok(recorded) => recorded, + // "Not there" is the one error with a better answer than itself, + // because it is the one this function exists to explain. A file + // that is there and cannot be read is a real failure and reports + // as one rather than as a session that never came up. + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(MirageError::io(discovery, e)); + } + Err(_) => String::new(), + }; + let recorded = recorded.split('\n').next().unwrap_or_default(); + let recorded = recorded.strip_suffix('\r').unwrap_or(recorded); + if recorded.is_empty() { + return Err(MirageError::Other(format!( + "rocjitsu: no configuration recorded at {}; bring-up writes \ + one before anything reads it", + discovery.display() + ))); + } + Ok(PathBuf::from(recorded)) +} + +/// JSON pointer to the device a rocjitsu `SimulationConfig` describes. +const VM_DEVICE_POINTER: &str = "/vm/gpu/device"; + +/// JSON pointer to whether a rocjitsu `SimulationConfig` adds a DBT guest +/// device. +const DBT_GUEST_ENABLED_POINTER: &str = "/dbt_guest/enabled"; + +/// The device a supplied `SimulationConfig` describes, as the workload's +/// ROCm runtime will see it. +/// +/// `None` when the bytes cannot be parsed, carry no recognisable device, +/// or describe more than one. Every one of those means the same thing to +/// the only caller — mirage does not know what this session will present +/// — and [`Rocjitsu::reconcile_profile`] turns that into an agent with no +/// target rather than a guess. +/// +/// Fields `KfdDeviceInfo` does not model are ignored here: a supplied +/// config is rocjitsu's document, its schema grows independently, and +/// the shipped gfx1250 configs already carry `revision_id` and +/// `num_sdma_queues_per_engine`. Refusing those would silence the +/// warning for the very target it exists for. +/// +/// A config with an enabled DBT guest describes no single device: the +/// interposer appends the guest to the host's topology, so the session +/// presents both, and "the workload will see no GPU" would be false. +fn device_of_bytes(bytes: &[u8]) -> Option { + let config: serde_json::Value = serde_json::from_slice(bytes).ok()?; + if config + .pointer(DBT_GUEST_ENABLED_POINTER) + .and_then(serde_json::Value::as_bool) + == Some(true) + { + return None; + } + let serde_json::Value::Object(mut device) = config.pointer(VM_DEVICE_POINTER)?.clone() else { + return None; + }; + let serde_json::Value::Object(known) = serde_json::to_value(KfdDeviceInfo::default()).ok()? + else { + return None; + }; + device.retain(|field, _| known.contains_key(field)); + serde_json::from_value(serde_json::Value::Object(device)).ok() +} + /// Check that `def` describes a machine rocjitsu can stand up, without /// writing anything. /// @@ -1076,6 +1881,7 @@ mod tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use super::*; + use mirage_core::profile::ContainerizedDef; #[test] fn the_installed_flag_and_the_located_library_are_one_answer() { @@ -1164,6 +1970,7 @@ mod tests { /// GPUs on a default agent, resolvable without touching the stores. fn def_with_gpus(gpus_per_node: u32) -> EmulatorDef { EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -1176,25 +1983,411 @@ mod tests { } } + /// A single-GPU [`EmulatorDef`] whose owned agent emulates + /// `gfx_target_version`. + fn def_for_target(gfx_target_version: u32) -> EmulatorDef { + let mut agent = AgentDef::default(); + agent.vm.gpu.device.gfx_target_version = gfx_target_version; + EmulatorDef { + topology: MaybeRef::Owned(TopologyDef { + num_nodes: 1, + gpus_per_node: 1, + agent: MaybeRef::Owned(agent), + }), + ..def_with_gpus(1) + } + } + #[test] - fn kmd_config_requires_resolvable_topology() { - let _g = mirage_core::paths::test_env_lock(); + fn additional_profile_fields_reach_rocjitsu() { let tmp = tempfile::tempdir().unwrap(); - mirage_core::paths::set_test_root(tmp.path()); - let def = EmulatorDef { - emulator: "rocjitsu".to_string(), - plugins: Default::default(), - exec_mode: ExecMode::Functional, - options: Default::default(), - topology: MaybeRef::Ref("does-not-exist".to_string()), - }; - assert!(check_config(&def).is_err()); - assert!(kmd_config(&def, tmp.path()).is_err()); + let mut json = serde_json::to_value(def_with_gpus(2)).unwrap(); + json["max_ticks"] = serde_json::json!(7654321); + json["topology"]["agent"]["vm"]["gpu"]["device"]["capability2"] = serde_json::json!(9); + let def: EmulatorDef = serde_json::from_value(json).unwrap(); + let config = kmd_config(&def, tmp.path()).unwrap(); + let emitted: serde_json::Value = + serde_json::from_slice(&std::fs::read(config).unwrap()).unwrap(); + assert_eq!(emitted["max_ticks"], 7654321); + assert_eq!(emitted["vm"]["gpu"]["device"]["capability2"], 9); + assert_eq!(emitted["vm"]["gpu"]["num_gpus"], 2); } - /// Validating a profile must not write anything: it happens before a - /// session exists, so whatever it wrote would have no owner and - /// nothing would ever remove it. Validation used to leave a ~5 KB + /// An agent is a rocjitsu config with two keys taken out, so an + /// imported one carries that config's root keys — and one of them + /// says how the session runs. + #[test] + fn an_agents_root_keys_cannot_take_over_the_profiles_exec_mode() { + let tmp = tempfile::tempdir().unwrap(); + let mut json = serde_json::to_value(def_with_gpus(1)).unwrap(); + json["exec_mode"] = serde_json::json!("Clocked"); + json["topology"]["agent"]["exec_mode"] = serde_json::json!("functional"); + json["topology"]["agent"]["max_ticks"] = serde_json::json!(5); + let def: EmulatorDef = serde_json::from_value(json).unwrap(); + assert_eq!(def.exec_mode, ExecMode::Clocked); + let config: serde_json::Value = + serde_json::from_slice(&std::fs::read(kmd_config(&def, tmp.path()).unwrap()).unwrap()) + .unwrap(); + assert_eq!(config["exec_mode"], "clocked"); + // The keys mirage only seeds a default for stay overridable. + assert_eq!(config["max_ticks"], 5); + } + + #[test] + fn missing_config_fields_name_omissions_not_zeroes() { + let original: serde_json::Value = + serde_json::from_str(include_str!("../../../rocjitsu/configs/gfx950_mi355x.json")) + .unwrap(); + let mut config = original.clone(); + assert!(missing_config_fields(&config, Some(&original)).is_empty()); + config["vm"]["gpu"]["device"]["num_sdma_engines"] = 0.into(); + assert!(missing_config_fields(&config, Some(&original)).is_empty()); + config["vm"]["gpu"]["device"] + .as_object_mut() + .unwrap() + .remove("num_sdma_queues_per_engine"); + assert_eq!( + missing_config_fields(&config, Some(&original)), + vec!["/vm/gpu/device/num_sdma_queues_per_engine"] + ); + let mut missing = Vec::new(); + collect_missing_fields( + Some(&serde_json::json!([])), + &serde_json::json!([{"key": "num_wf_slots", "value": "32"}]), + "/topology/root/config", + &mut missing, + ); + assert_eq!(missing, vec!["/topology/root/config/num_wf_slots"]); + + // A component the document dropped entirely is named too. + let mut missing = Vec::new(); + collect_missing_fields( + Some(&serde_json::json!([{"type": "xcd"}])), + &serde_json::json!([{"type": "gpu_memory"}, {"type": "iod"}, {"type": "xcd"}]), + "/topology/root/children", + &mut missing, + ); + assert_eq!( + missing, + vec![ + "/topology/root/children/gpu_memory", + "/topology/root/children/iod" + ] + ); + } + + #[test] + fn missing_config_fields_uses_the_selected_builtin_preset() { + let agent = mirage_builtin::agents::agent("gfx1251_synthetic") + .expect("builtin gfx1251_synthetic agent"); + let expected = mirage_builtin::agents::source_config_for_agent(&agent) + .expect("builtin agent has source config"); + let SimConfig::Synthesised(bytes) = resolve_sim_config(&EmulatorDef { + extra: Default::default(), + emulator: "rocjitsu".to_string(), + plugins: Default::default(), + exec_mode: ExecMode::Functional, + options: Default::default(), + topology: MaybeRef::Owned(TopologyDef { + num_nodes: 1, + gpus_per_node: 1, + agent: MaybeRef::Owned(agent), + }), + }) + .unwrap() else { + panic!("expected generated config"); + }; + let config: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + + assert!(missing_config_fields(&config, Some(&expected)).is_empty()); + } + + #[test] + fn malformed_vm_overrides_return_errors() { + for value in [serde_json::json!(null), serde_json::json!({"gpu": 7})] { + let mut def = def_with_gpus(1); + def.extra.insert("vm".into(), value); + assert!(check_config(&def).is_err()); + } + // The two containers this walks into itself keep their own + // wording, which names the key rather than a parse position. + let mut def = def_with_gpus(1); + def.extra.insert("vm".into(), serde_json::json!({"gpu": 7})); + assert!( + check_config(&def) + .unwrap_err() + .to_string() + .contains("vm.gpu must be an object") + ); + } + + /// A profile extra may not quietly change the shape of a field + /// mirage has a name for. + /// + /// Passthrough merging replaces nodes rather than deepening them, so + /// a profile that writes a scalar over `vm.gpu.device` hands rocjitsu + /// an integer where a device belongs; `num_sdma_engines: "4"` is + /// worse than refused, because `settle_sdma_pair` reads a string as + /// no answer and writes zero engines over it. Both are the user's + /// mistake to see while they are still writing the profile. + /// + /// The review asked for a topology child missing its `type` here + /// too. That document cannot be built: `topology` is a typed field + /// of `EmulatorDef`, so a malformed one is refused when the profile + /// is parsed, long before this. Only `vm` arrives by passthrough. + #[test] + fn wrong_shaped_vm_passthrough_is_refused() { + for value in [ + serde_json::json!({"arch": 7}), + serde_json::json!({"gpu": {"device": 7}}), + serde_json::json!({"gpu": {"memory": 7}}), + serde_json::json!({"gpu": {"device": {"num_sdma_engines": "4"}}}), + serde_json::json!({"gpu": {"device": {"num_sdma_engines": 1.5}}}), + ] { + let mut def = def_with_gpus(1); + def.extra.insert("vm".into(), value.clone()); + let error = check_config(&def) + .expect_err(&format!("{value} should not describe a device")) + .to_string(); + assert!( + error.contains("does not describe a device"), + "{value}: {error}" + ); + } + } + + /// An install upgraded from the previous release can start its + /// builtin profiles. + /// + /// The documents in `builtin/legacy/` are what that release actually + /// wrote, and every one of them has SDMA engines with no queue count + /// — the pair rocjitsu refuses to load. So this is the whole upgrade + /// in one test: seed those documents, run the startup pass every + /// command runs, and ask the backend whether the profile it would + /// build is one it can use. `mi350x` is `--profile`'s own default, + /// so a "no" here is a machine that cannot run anything until its + /// owner works out which file to delete. + #[test] + fn builtin_profiles_work_after_upgrading_from_the_previous_release() { + let _guard = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + for (name, document) in [ + ("mi300x", include_str!("../../builtin/legacy/mi300x.json")), + ("mi350x", include_str!("../../builtin/legacy/mi350x.json")), + ("mi450x", include_str!("../../builtin/legacy/mi450x.json")), + ] { + let seeded: serde_json::Value = serde_json::from_str(document).unwrap(); + // The precondition that makes this an upgrade at all. + assert!( + seeded["vm"]["gpu"]["device"]["num_sdma_engines"] + .as_u64() + .is_some_and(|engines| engines > 0) + && seeded["vm"]["gpu"]["device"] + .get("num_sdma_queues_per_engine") + .is_none(), + "{name}: fixture is not an agent the previous release left unusable" + ); + mirage_core::state::write_json(&mirage_core::paths::agent_path(name), &seeded).unwrap(); + } + + mirage_builtin::ensure_agents(false).unwrap(); + mirage_builtin::ensure_topologies(false).unwrap(); + + for (name, profile) in mirage_builtin::profiles() { + if profile.emulator.emulator != "rocjitsu" { + continue; + } + check_config(&profile.emulator) + .unwrap_or_else(|error| panic!("builtin profile {name} cannot be used: {error}")); + } + + mirage_core::paths::clear_test_root(); + } + + /// The positive control for the check above: a name mirage does not + /// know is still forwarded, at every depth, and the check does not + /// write its own defaults over what the document left out. + #[test] + fn unknown_vm_fields_still_reach_the_config() { + // An agent parsed from a document, so it still knows what that + // document did not say. + let mut def = def_with_agent(serde_json::json!({ + "vm": {"arch": "cdna4", "gpu": {"device": {"num_sdma_engines": 0}}}, + "topology": {"root": {"name": "soc", "type": "soc"}} + })); + def.extra.insert( + "vm".into(), + serde_json::json!({ + "programs": ["a.hsaco"], + "gpu": {"device": {"capability2": 9}} + }), + ); + def.extra.insert("max_ticks".into(), serde_json::json!(42)); + let config = synthesised(&def); + assert_eq!(config["vm"]["programs"], serde_json::json!(["a.hsaco"])); + assert_eq!(config["vm"]["gpu"]["device"]["capability2"], 9); + assert_eq!(config["max_ticks"], 42); + assert!(config["vm"]["gpu"]["device"].get("simd_count").is_none()); + } + + #[test] + fn omitted_agent_fields_keep_rocjitsu_defaults() { + let original = serde_json::json!({ + "vm": {"arch": "cdna4", "gpu": {"device": {"num_sdma_engines": 0}}}, + "topology": {"root": {"name": "soc", "type": "soc", "future": {"enabled": true}}}, + "future_option": [1, 2] + }); + let agent: AgentDef = serde_json::from_value(original.clone()).unwrap(); + assert_eq!(serde_json::to_value(&agent).unwrap(), original); + let mut def = def_with_gpus(1); + if let MaybeRef::Owned(topology) = &mut def.topology { + topology.agent = MaybeRef::Owned(agent); + } + let tmp = tempfile::tempdir().unwrap(); + let path = kmd_config(&def, tmp.path()).unwrap(); + let config: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + assert_eq!( + config["vm"]["gpu"]["device"], + original["vm"]["gpu"]["device"] + ); + assert_eq!(config["future_option"], original["future_option"]); + } + + /// A def whose agent is the parsed `document`, so the agent keeps + /// the record of what the document left out — which is the whole + /// point of these cases and is *not* what a struct literal gives. + fn def_with_agent(document: serde_json::Value) -> EmulatorDef { + let mut def = def_with_gpus(1); + if let MaybeRef::Owned(topology) = &mut def.topology { + topology.agent = MaybeRef::Owned(serde_json::from_value(document).unwrap()); + } + def + } + + fn synthesised(def: &EmulatorDef) -> serde_json::Value { + let tmp = tempfile::tempdir().unwrap(); + let path = kmd_config(def, tmp.path()).unwrap(); + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() + } + + /// RocJITsu's two SDMA schema defaults do not describe a device it + /// will load — 2 engines, 0 queues each — so an agent that mentions + /// neither field cannot simply be forwarded with both omitted. + #[test] + fn an_agent_silent_about_sdma_still_produces_a_loadable_config() { + let def = def_with_agent(serde_json::json!({ + "vm": {"arch": "cdna4", "gpu": {"device": {"drm_render_minor": 128}}}, + "topology": {"root": {"name": "soc", "type": "soc"}} + })); + let device = &synthesised(&def)["vm"]["gpu"]["device"]; + // No SDMA, which is the machine mirage emulated for this + // document before omissions were preserved. + assert_eq!(device["num_sdma_engines"], 0); + // And nothing else the document left out was filled in. + assert!(device.get("num_sdma_queues_per_engine").is_none()); + assert!(device.get("simd_count").is_none()); + assert_eq!(device["drm_render_minor"], 128); + } + + /// Half a claim is refused rather than completed: mirage has no + /// honest queue count to invent, and no engine count either. + #[test] + fn a_half_specified_sdma_pair_is_refused_at_validation() { + for device in [ + serde_json::json!({"num_sdma_engines": 4}), + serde_json::json!({"num_sdma_queues_per_engine": 0}), + serde_json::json!({"num_sdma_engines": 4, "num_sdma_queues_per_engine": 0}), + ] { + let def = def_with_agent(serde_json::json!({ + "vm": {"arch": "cdna4", "gpu": {"device": device}}, + "topology": {"root": {"name": "soc", "type": "soc"}} + })); + let message = check_config(&def).unwrap_err().to_string(); + assert!(message.contains("num_sdma_queues_per_engine"), "{message}"); + assert!(message.contains("num_sdma_engines"), "{message}"); + } + } + + /// A coherent pair, and an explicit `num_sdma_engines: 0`, are the + /// document's to make and are passed through untouched. + #[test] + fn a_coherent_sdma_pair_is_left_alone() { + for device in [ + serde_json::json!({"num_sdma_engines": 4, "num_sdma_queues_per_engine": 8}), + serde_json::json!({"num_sdma_engines": 0}), + serde_json::json!({"num_sdma_queues_per_engine": 8}), + ] { + let def = def_with_agent(serde_json::json!({ + "vm": {"arch": "cdna4", "gpu": {"device": device.clone()}}, + "topology": {"root": {"name": "soc", "type": "soc"}} + })); + assert_eq!(synthesised(&def)["vm"]["gpu"]["device"], device); + } + } + + /// An agent imported from a whole rocjitsu config carries that + /// config's root `plugins`. The profile is what decides which + /// plugins a session runs, in both directions. + #[test] + fn a_profiles_plugin_selection_outranks_an_agents() { + let agent = serde_json::json!({ + "vm": {"arch": "cdna4", "gpu": {"device": {"num_sdma_engines": 0}}}, + "topology": {"root": {"name": "soc", "type": "soc"}}, + "plugins": {"race": {}} + }); + let empty = def_with_agent(agent.clone()); + assert!(synthesised(&empty).get("plugins").is_none()); + + let mut selected = def_with_agent(agent); + selected.plugins = PluginsDef::from([("logging".to_string(), SimpleMap::new())]); + assert_eq!( + synthesised(&selected)["plugins"], + serde_json::json!({"logging": {}}) + ); + + // The config and the libraries bind-mounted into a node + // container are then the same selection, so a containerised run + // enables what a local one does. `def.plugins` is the only + // input to the mount list (see `injection_def`), which is what + // the assertions above have just made authoritative. + let tmp = tempfile::tempdir().unwrap(); + let preload = tmp.path().join(LIB_NAME); + std::fs::write(&preload, b"").unwrap(); + std::fs::write(tmp.path().join("librocjitsu_plugin_race.so"), b"").unwrap(); + std::fs::write(tmp.path().join("librocjitsu_plugin_logging.so"), b"").unwrap(); + assert!(enabled_plugin_libs(&preload, &empty.plugins).is_empty()); + assert_eq!( + enabled_plugin_libs(&preload, &selected.plugins) + .iter() + .filter_map(|path| path.file_name()?.to_str()) + .collect::>(), + vec!["librocjitsu_plugin_logging.so"] + ); + } + + #[test] + fn kmd_config_requires_resolvable_topology() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + let def = EmulatorDef { + extra: Default::default(), + emulator: "rocjitsu".to_string(), + plugins: Default::default(), + exec_mode: ExecMode::Functional, + options: Default::default(), + topology: MaybeRef::Ref("does-not-exist".to_string()), + }; + assert!(check_config(&def).is_err()); + assert!(kmd_config(&def, tmp.path()).is_err()); + } + + /// Validating a profile must not write anything: it happens before a + /// session exists, so whatever it wrote would have no owner and + /// nothing would ever remove it. Validation used to leave a ~5 KB /// `sim_.json` in a fixed, shared, world-writable directory /// under the system temp directory, one per distinct profile, /// forever — so that is where this looks. @@ -1266,7 +2459,9 @@ mod tests { fn generated_config_spreads_target_budget_over_gpus() { let mut def = def_with_gpus(2); if let MaybeRef::Owned(topology) = &mut def.topology { - topology.agent = MaybeRef::Owned(mirage_builtin::agents::mi350x()); + topology.agent = MaybeRef::Owned( + mirage_builtin::agents::agent("mi350x").expect("builtin mi350x agent"), + ); } let SimConfig::Synthesised(bytes) = resolve_sim_config(&def).unwrap() else { panic!("expected generated config"); @@ -1287,6 +2482,56 @@ mod tests { assert_eq!(json["num_threads"], 1); } + #[test] + fn supplied_thread_allocations_survive_validation_and_emission() { + for source in ["agent", "profile"] { + let custom = serde_json::json!([{ + "num_threads": 1, + "cpu_dispatch_threads": 1, + "async_helper_threads": 0 + }]); + let mut agent = mirage_builtin::agents::agent("mi350x").expect("builtin mi350x agent"); + let mut def = def_with_gpus(1); + if source == "agent" { + agent + .extra + .insert("thread_allocations".into(), custom.clone()); + } else { + def.extra + .insert("thread_allocations".into(), custom.clone()); + } + if let MaybeRef::Owned(topology) = &mut def.topology { + topology.agent = MaybeRef::Owned(agent); + } + + check_config(&def).unwrap(); + + let tmp = tempfile::tempdir().unwrap(); + let config = kmd_config(&def, tmp.path()).unwrap(); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(config).unwrap()).unwrap(); + assert_eq!(json["thread_allocations"], custom, "{source}"); + } + } + + #[test] + fn single_gpu_builtin_keeps_native_thread_policy_unpinned() { + let mut def = def_with_gpus(1); + if let MaybeRef::Owned(topology) = &mut def.topology { + topology.agent = MaybeRef::Owned( + mirage_builtin::agents::agent("mi350x").expect("builtin mi350x agent"), + ); + } + let SimConfig::Synthesised(bytes) = resolve_sim_config(&def).unwrap() else { + panic!("expected generated config"); + }; + let json: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let allocations = json["thread_allocations"].as_array().unwrap(); + + assert!(json.get("num_threads").is_none()); + assert!(allocations.iter().any(|choice| choice["num_threads"] == 8)); + } + #[test] fn unknown_target_has_no_implicit_thread_policy() { let SimConfig::Synthesised(bytes) = resolve_sim_config(&def_with_gpus(1)).unwrap() else { @@ -1439,10 +2684,862 @@ mod tests { assert!(resolve_sim_config(&def).is_err()); } - /// A drop-in `--config` is used verbatim, but its runtime directory - /// belongs to the session: the discovery file (and the daemon socket - /// beside it) must never land next to the user's config file, which - /// mirage does not own and cannot clean up. + /// The session context a backend is handed for `emulator`, with a + /// scratch directory of its own named `session` under `root`. + /// + /// The name is a parameter because a session's configuration is + /// materialised once and then belongs to that session: two cases + /// sharing one scratch directory would be one session being brought + /// up twice, which is not a thing that happens. + fn ctx_for(emulator: EmulatorDef, root: &std::path::Path, session: &str) -> SessionContext { + SessionContext { + id: mirage_core::session::SessionId::new(session).unwrap(), + profile: ProfileDef { + name: "isa-test".to_string(), + description: None, + emulator, + containerize: None, + }, + runtime_dir: root.join(session), + daemon: false, + } + } + + /// A resolved profile around `emulator`, as a session would hold it. + /// + /// `ctx_for` builds the same profile for the bring-up tests; this is + /// the half the derivation needs, without a session directory. + fn profile_for(emulator: EmulatorDef) -> ProfileDef { + ProfileDef { + name: "isa-test".to_string(), + description: None, + emulator, + containerize: None, + } + } + + fn containerise(ctx: &mut SessionContext) { + ctx.profile.containerize = Some(ContainerizedDef { + provider: None, + image: "rocm/dev-ubuntu-24.04:latest".to_string(), + mounts: Vec::new(), + ports: Vec::new(), + devices: Vec::new(), + groups: Vec::new(), + hacks: Vec::new(), + }); + } + + /// The interposer this backend would preload, stood in for by a file + /// that merely exists. + /// + /// Whether rocjitsu is installed is a fact about the host, so the + /// library is supplied rather than searched for — see + /// [`Rocjitsu::injection_def_with`]. Nothing here loads it: a + /// non-containerised injection only records its path. + fn stand_in_interposer(root: &std::path::Path) -> Option { + let lib = root.join(LIB_NAME); + std::fs::write(&lib, b"").unwrap(); + Some(lib) + } + + /// The target a session emulates is the one its agent names. + /// + /// This is the fact the whole of issue #11361 turns on: a ROCm that + /// does not know the target skips the emulated agent, so the + /// workload sees no GPU and exits 0 with nothing said. The warning + /// mirage prints is only as good as this answer, and the answer is + /// the agent's own — nothing computes a second copy of it. + #[test] + fn the_emulated_target_is_the_agents_gfx_target() { + // gfx1250 is the MI450X target, and the one a ROCm 7.0 host has + // never heard of; gfx942 is one the same host does support, so a + // passing test cannot be one that answers `gfx1250` to + // everything. + for (version, isa) in [(120500, "gfx1250"), (90402, "gfx942")] { + let profile = profile_for(def_for_target(version)); + assert_eq!( + profile.emulated_gfx_target().map(|t| t.to_string()), + Some(isa.to_string()) + ); + } + } + + /// A drop-in `--config` names a device of its own, and that is the + /// device the interposer will stand up — so `reconcile_profile` makes + /// it the agent's device before anything reads the agent. + /// + /// Leaving the profile's own agent in place would be wrong in both + /// directions here: silent about a config whose target this ROCm + /// cannot see, and warning about a profile agent that is not going + /// to exist. + #[test] + fn a_supplied_config_becomes_the_agents_device() { + let tmp = tempfile::tempdir().unwrap(); + let config = tmp.path().join("mine.json"); + std::fs::write( + &config, + br#"{"vm": {"gpu": {"device": {"gfx_target_version": 90500}}}}"#, + ) + .unwrap(); + + // The profile's own agent is a gfx1250 the `--config` overrides. + let mut def = def_for_target(120500); + def.options.insert( + "config".to_string(), + SimpleValue::String(config.display().to_string()), + ); + + let mut profile = profile_for(def); + Rocjitsu + .reconcile_profile(&mut profile, &tmp.path().join("session")) + .unwrap(); + assert_eq!( + profile.emulated_gfx_target().map(|t| t.to_string()), + Some("gfx950".to_string()) + ); + } + + /// A `--config` mirage cannot read leaves no target at all, rather + /// than the agent's own. + /// + /// rocjitsu reads these through FlatBuffers, which accepts JSON + /// `serde_json` does not, so a file this cannot parse may still run + /// perfectly well — it just describes a device mirage cannot name. + /// Keeping the profile's original agent would name the device the + /// config replaced, which is the one thing worse than saying nothing. + #[test] + fn a_config_that_cannot_be_read_leaves_no_target() { + let tmp = tempfile::tempdir().unwrap(); + let commented = tmp.path().join("commented.json"); + std::fs::write( + &commented, + br#"{ + // the device this session stands up + "vm": {"gpu": {"device": {"gfx_target_version": 90500}}} +}"#, + ) + .unwrap(); + + for path in [commented, tmp.path().join("absent.json")] { + let session_dir = tmp.path().join(path.file_stem().unwrap()); + let mut def = def_for_target(120500); + def.options.insert( + "config".to_string(), + SimpleValue::String(path.display().to_string()), + ); + let mut profile = profile_for(def); + Rocjitsu + .reconcile_profile(&mut profile, &session_dir) + .unwrap(); + assert_eq!( + profile.emulated_gfx_target(), + None, + "a config mirage cannot read must not leave the agent's own target in place" + ); + } + } + + /// The configs rocjitsu ships are read for their target even though + /// they carry device fields mirage does not model. + /// + /// Both gfx1250 configs set `revision_id` and + /// `num_sdma_queues_per_engine`. Decoded strictly they failed, the + /// agent was left with no target, and a gfx1250 session on a ROCm + /// that cannot see gfx1250 — the case issue #11361 is about — said + /// nothing. gfx942 is here so a passing test cannot be one that + /// answers gfx1250 to everything. + #[test] + fn shipped_configs_name_their_target() { + let configs = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../rocjitsu/configs"); + let tmp = tempfile::tempdir().unwrap(); + for (name, isa) in [ + ("gfx1250_mi455x.json", "gfx1250"), + ("gfx1250_mi455x_kmd_4gpu.json", "gfx1250"), + ("gfx942_cdna3_kmd.json", "gfx942"), + ] { + let mut def = def_for_target(90500); + def.options.insert( + "config".to_string(), + SimpleValue::String(configs.join(name).display().to_string()), + ); + let mut profile = profile_for(def); + Rocjitsu + .reconcile_profile(&mut profile, &tmp.path().join(name)) + .unwrap(); + assert_eq!( + profile + .emulated_gfx_target() + .map(|t| t.to_string()) + .as_deref(), + Some(isa), + "{name}" + ); + } + + // Stored agents preserve rocjitsu fields mirage does not model so + // profiles can pass through settings from newer rocjitsu schemas. + let device: KfdDeviceInfo = + serde_json::from_value(serde_json::json!({"revision_id": 1})).unwrap(); + assert_eq!(device.extra["revision_id"], 1); + } + + /// A config with an enabled DBT guest presents the host's devices and + /// the guest together, so there is no one target to warn about. + #[test] + fn a_dbt_guest_config_names_no_target() { + let tmp = tempfile::tempdir().unwrap(); + let config = tmp.path().join("guest.json"); + std::fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "vm": {"gpu": {"device": {"gfx_target_version": 90402}}}, + "dbt_guest": {"enabled": true, "guest_device": {"gfx_target_version": 90500}}, + })) + .unwrap(), + ) + .unwrap(); + + let mut def = def_for_target(120500); + def.options.insert( + "config".to_string(), + SimpleValue::String(config.display().to_string()), + ); + let mut profile = profile_for(def); + Rocjitsu + .reconcile_profile(&mut profile, &tmp.path().join("session")) + .unwrap(); + assert_eq!(profile.emulated_gfx_target(), None); + } + + /// The device a session reports and the config it injects are one + /// reading of the user's file. + /// + /// `reconcile_profile` used to read the file at session creation and + /// `kmd_config` to copy it again at injection, so a rewrite between + /// the two left the session reporting gfx942 while injecting gfx1250. + /// The snapshot is taken once now, and the profile points at it. + #[test] + fn the_reported_and_injected_device_are_one_reading() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + let config = tmp.path().join("cfg.json"); + let write_target = |version: u32| { + std::fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "vm": {"gpu": {"device": {"gfx_target_version": version}}} + })) + .unwrap(), + ) + .unwrap(); + }; + write_target(90402); + + let mut def = def_with_gpus(1); + def.options.insert( + "config".to_string(), + SimpleValue::String(config.display().to_string()), + ); + let mut ctx = ctx_for(def, tmp.path(), "one-reading"); + Rocjitsu + .reconcile_profile(&mut ctx.profile, &ctx.runtime_dir) + .unwrap(); + + write_target(120500); + Rocjitsu + .injection_def_with(&ctx, stand_in_interposer(tmp.path())) + .unwrap(); + + let injected = std::fs::read(session_config(&ctx.runtime_dir).unwrap()).unwrap(); + let injected = device_of_bytes(&injected).and_then(|d| d.gfx_target()); + assert_eq!(injected, ctx.profile.emulated_gfx_target()); + assert_eq!(injected.map(|t| t.to_string()).as_deref(), Some("gfx942")); + } + + /// A device with no ISA is not a device to check a runtime against, + /// and nothing to say is said as nothing: the injection is still the + /// one bring-up wants, just with no target to warn about. + /// + /// A profile that cannot be resolved at all is a different answer. + /// It fails here, as it always did, because that is the error the + /// user needs — inventing an ISA for it would only put a wrong + /// warning in front of the right failure. + #[test] + fn a_device_without_a_gfx_target_names_no_isa() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + // A default agent's `gfx_target_version` is 0. + assert_eq!(profile_for(def_with_gpus(1)).emulated_gfx_target(), None); + + // An unresolvable topology reference. + let mut def = def_with_gpus(1); + def.topology = MaybeRef::Ref("does-not-exist".to_string()); + let ctx = ctx_for(def, tmp.path(), "unresolvable"); + assert!( + Rocjitsu + .injection_def_with(&ctx, stand_in_interposer(tmp.path())) + .is_err() + ); + + // A `--config` that is not there. + let mut def = def_with_gpus(1); + def.options.insert( + "config".to_string(), + SimpleValue::String("/no/such/config.json".to_string()), + ); + let ctx = ctx_for(def, tmp.path(), "absent-config"); + assert!( + Rocjitsu + .injection_def_with(&ctx, stand_in_interposer(tmp.path())) + .is_err() + ); + } + + /// A drop-in `--config` is copied into the session, so the device a + /// live session emulates cannot be edited out from under it. + /// + /// The interposer does not read the config once: it reopens whatever + /// the discovery file names, and in `--in-process` mode every later + /// workload opens it again. Naming the user's own file therefore + /// left a live session tracking a document mirage does not own — an + /// edit after bring-up retargeted the emulated device while the ISA + /// captured at bring-up, and the preflight warning drawn from it, + /// still described the old one. + #[test] + fn a_supplied_config_is_snapshotted_into_the_session() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + // In a directory of the user's, so "nothing beside it" is a + // thing this test can actually look at. + let user_dir = tmp.path().join("mine"); + std::fs::create_dir_all(&user_dir).unwrap(); + let config = user_dir.join("cfg.json"); + std::fs::write( + &config, + br#"{"vm": {"gpu": {"device": {"gfx_target_version": 90500}}}}"#, + ) + .unwrap(); + + let mut def = def_with_gpus(1); + def.options.insert( + "config".to_string(), + SimpleValue::String(config.display().to_string()), + ); + let ctx = ctx_for(def, tmp.path(), "snapshot"); + let injection = Rocjitsu + .injection_def_with(&ctx, stand_in_interposer(tmp.path())) + .unwrap(); + + // What the interposer will open is the session's copy. + let runtime_dir = PathBuf::from( + injection + .env + .get("ROCJITSU_RUNTIME_DIR") + .expect("the injection names a runtime directory"), + ); + let named = PathBuf::from( + std::fs::read_to_string(runtime_dir.join(CONFIG_PATH_NAME)) + .unwrap() + .trim(), + ); + assert_eq!(named, rj_config_path(&ctx.runtime_dir)); + assert_eq!(session_config(&ctx.runtime_dir).unwrap(), named); + + // So rewriting the user's file retargets nothing. + std::fs::write( + &config, + br#"{"vm": {"gpu": {"device": {"gfx_target_version": 120500}}}}"#, + ) + .unwrap(); + assert_eq!( + device_of_bytes(&std::fs::read(&named).unwrap()).and_then(|d| d.gfx_target()), + mirage_core::hardware::GfxTarget::new(90500), + "a live session must emulate the device it was brought up on" + ); + + // And the copy went into the session, not next to the original. + assert_eq!( + std::fs::read_dir(&user_dir).unwrap().count(), + 1, + "nothing may be written beside the user's own config file" + ); + } + + /// Run `f` with the process working directory at `directory`. + /// + /// The working directory is process-wide, so callers hold + /// [`mirage_core::paths::test_env_lock`] for the same reason the + /// path override's callers do. Restoring it is a `Drop` so a failed + /// assertion cannot leave the rest of the binary somewhere else, and + /// the restore is deliberately silent: panicking while unwinding + /// another panic aborts the process and loses the real failure. + fn in_working_directory(directory: &std::path::Path, f: impl FnOnce() -> R) -> R { + struct Restore(PathBuf); + impl Drop for Restore { + fn drop(&mut self) { + let _ = std::env::set_current_dir(&self.0); + } + } + + let _restore = Restore(std::env::current_dir().expect("a working directory")); + std::env::set_current_dir(directory).expect("entering the fixture directory"); + f() + } + + /// A profile's `config` option can be relative, and it has to be + /// absolute before the session pins anything to it. + /// + /// `mirage run --config` canonicalises what it is handed, but that + /// is the CLI's doing: a saved profile reaches the backend spelled + /// however it was written. Left relative, the pin below produces a + /// relative sibling reference, and rocjitsu resolves that against + /// the parent of the config it is handed — the session directory — + /// where the sibling has never existed. + #[test] + fn a_relative_profile_config_still_finds_its_sibling() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + // The shipped composition form: a guest config naming the host + // config beside it. + let configs = tmp.path().join("configs"); + std::fs::create_dir_all(&configs).unwrap(); + std::fs::write( + configs.join("guest.json"), + br#"{"dbt_guest": {"simulator_config": "gfx942_cdna3_kmd.json"}}"#, + ) + .unwrap(); + std::fs::write( + configs.join("gfx942_cdna3_kmd.json"), + br#"{"vm": {"gpu": {"device": {"gfx_target_version": 90402}}}}"#, + ) + .unwrap(); + + let mut def = def_with_gpus(1); + def.options.insert( + "config".to_string(), + SimpleValue::String("configs/guest.json".to_string()), + ); + let ctx = ctx_for(def, tmp.path(), "relative-config"); + let preload = stand_in_interposer(tmp.path()); + + in_working_directory(tmp.path(), || Rocjitsu.injection_def_with(&ctx, preload)) + .expect("a relative config that exists must bring up"); + + let handed = session_config(&ctx.runtime_dir).unwrap(); + let snapshot: serde_json::Value = + serde_json::from_slice(&std::fs::read(&handed).unwrap()).unwrap(); + let reference = snapshot + .pointer(SIMULATOR_CONFIG_POINTER) + .and_then(serde_json::Value::as_str) + .expect("the snapshot keeps the external reference"); + + assert!( + std::path::Path::new(reference).is_absolute(), + "a reference that leaves the original's directory must be absolute: {reference}" + ); + + // And the property that matters, resolved the way + // `resolve_dbt_host_config_path` resolves it. + let parent = handed.parent().expect("the session copy has a directory"); + let resolved = parent.join(reference); + assert!( + resolved.is_file(), + "rocjitsu resolves `{reference}` against {}, and must find the host config there", + parent.display() + ); + + assert_eq!( + device_of_bytes(&std::fs::read(&handed).unwrap()).and_then(|d| d.gfx_target()), + None, + "the guest config names no `vm.gpu.device` of its own" + ); + } + + /// Copying a drop-in `--config` moves it, so the one reference that + /// resolves *beside* it has to be pinned before it travels. + /// + /// `resolve_dbt_host_config_path` joins a relative + /// `dbt_guest.simulator_config` onto the parent of the config the + /// interposer was handed, and after bring-up that is the session's + /// copy — so the shipped `guest_gfx950_on_simulated_gfx942.json` + /// would look for `gfx942_cdna3_kmd.json` in the session scratch + /// directory. Absolute references and configs without one travel + /// byte for byte. + #[test] + fn a_relative_external_reference_survives_the_copy() { + let tmp = tempfile::tempdir().unwrap(); + let user_dir = tmp.path().join("configs"); + std::fs::create_dir_all(&user_dir).unwrap(); + let original = user_dir.join("guest.json"); + + let relative = br#"{"dbt_guest": {"simulator_config": "gfx942_cdna3_kmd.json"}}"#; + let pinned = pin_external_references(relative.to_vec(), &original).unwrap(); + let pinned: serde_json::Value = serde_json::from_slice(&pinned).unwrap(); + assert_eq!( + pinned + .pointer(SIMULATOR_CONFIG_POINTER) + .and_then(serde_json::Value::as_str), + Some( + user_dir + .join("gfx942_cdna3_kmd.json") + .display() + .to_string() + .as_str() + ), + "the reference must still name the file beside the original" + ); + + // Nothing to pin, nothing touched — which is what keeps the + // ordinary copy a copy. + for untouched in [ + &br#"{"dbt_guest": {"simulator_config": "/opt/rocm/share/kmd.json"}}"#[..], + br#"{"dbt_guest": {"enabled": true}}"#, + br#"{"vm": {"gpu": {"device": {"gfx_target_version": 90500}}}}"#, + b"not json at all", + ] { + assert_eq!( + pin_external_references(untouched.to_vec(), &original), + Some(untouched.to_vec()) + ); + } + + // And a config mirage cannot read, which names a sibling it + // therefore cannot pin, is refused rather than moved. + for unreadable in [ + // The dialect rocjitsu accepts and `serde_json` does not. + &br#"{"dbt_guest": { + // the host this guest runs on + "simulator_config": "gfx942_cdna3_kmd.json"}}"#[..], + br#"{"dbt_guest": {"simulator_config": "gfx942_cdna3_kmd.json",}}"#, + br#"{dbt_guest: {simulator_config: "gfx942_cdna3_kmd.json"}}"#, + ] { + assert_eq!( + pin_external_references(unreadable.to_vec(), &original), + None, + "a sibling reference mirage cannot pin must not be relocated" + ); + } + } + + /// A drop-in `--config` mirage cannot parse is left where it lies, so + /// the sibling it names still resolves. + /// + /// rocjitsu reads its configs with FlatBuffers, whose JSON accepts + /// comments, trailing commas and unquoted keys. `serde_json` accepts + /// none of those, and a config it cannot read is one the pin cannot + /// be applied to — so copying it into the session would re-anchor + /// `dbt_guest.simulator_config` to the session directory, where the + /// host config has never been. The run then fails, or picks up a + /// different simulator, for a diagnostic's benefit. + #[test] + fn a_config_that_cannot_be_pinned_is_not_relocated() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + // The shipped composition form, with one comment in it. + let configs = tmp.path().join("configs"); + std::fs::create_dir_all(&configs).unwrap(); + let original = configs.join("guest.json"); + std::fs::write( + &original, + br#"{ + // gfx950 guest on a simulated gfx942 host + "dbt_guest": {"enabled": true, "simulator_config": "gfx942_cdna3_kmd.json"} +}"#, + ) + .unwrap(); + std::fs::write( + configs.join("gfx942_cdna3_kmd.json"), + br#"{"vm": {"gpu": {"device": {"gfx_target_version": 90402}}}}"#, + ) + .unwrap(); + + let mut def = def_with_gpus(1); + def.options.insert( + "config".to_string(), + SimpleValue::String(original.display().to_string()), + ); + let ctx = ctx_for(def, tmp.path(), "unpinnable-config"); + let injection = Rocjitsu + .injection_def_with(&ctx, stand_in_interposer(tmp.path())) + .expect("a config rocjitsu can read must bring up"); + + // The interposer is pointed at the original, so the relative + // reference resolves the way it did before the session existed. + let handed = session_config(&ctx.runtime_dir).unwrap(); + assert_eq!(handed, original); + assert!( + !rj_config_path(&ctx.runtime_dir).exists(), + "a config that cannot be pinned must not be snapshotted" + ); + let resolved = handed + .parent() + .unwrap() + .join("gfx942_cdna3_kmd.json") + .is_file(); + assert!(resolved, "the sibling must still resolve beside the config"); + + // And the daemon is handed the same file the interposer is, which + // is the whole reason this reads the discovery file. + let runtime_dir = PathBuf::from( + injection + .env + .get("ROCJITSU_RUNTIME_DIR") + .expect("the injection names a runtime directory"), + ); + assert_eq!( + std::fs::read_to_string(runtime_dir.join(CONFIG_PATH_NAME)) + .unwrap() + .trim(), + original.display().to_string() + ); + + // Containerised, the same config is refused rather than left for + // an interposer that cannot open it: a node container mounts the + // session's scratch directory and nothing else. + let mut containerised = ctx_for( + ctx.profile.emulator.clone(), + tmp.path(), + "unpinnable-in-container", + ); + containerise(&mut containerised); + let err = Rocjitsu + .injection_def_with(&containerised, stand_in_interposer(tmp.path())) + .expect_err("a config the container cannot reach must be refused") + .to_string(); + assert!(err.contains("containerised session"), "{err}"); + assert!(err.contains("--image"), "{err}"); + } + + /// Copying the top-level config into the session does not make files + /// it names visible in a node container. + /// + /// A relative `dbt_guest.simulator_config` is pinned to an absolute + /// host path before the copy, and an already-absolute one stays as + /// written. Both still point outside the session scratch directory, + /// which is the only path mounted at the same spelling inside the + /// container. Reject both rather than starting an interposer that + /// cannot read the simulator configuration. + #[test] + fn a_containerised_config_cannot_name_host_only_files() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + let user_dir = tmp.path().join("configs"); + std::fs::create_dir_all(&user_dir).unwrap(); + let simulator = user_dir.join("gfx942_cdna3_kmd.json"); + std::fs::write(&simulator, b"{}").unwrap(); + + for (session, reference) in [ + ( + "relative-external-in-container", + simulator + .file_name() + .unwrap() + .to_string_lossy() + .into_owned(), + ), + ( + "absolute-external-in-container", + simulator.display().to_string(), + ), + ] { + let config = user_dir.join(format!("{session}.json")); + let value = serde_json::json!({ + "dbt_guest": { + "enabled": true, + "execution_backend": "simulator", + "simulator_config": reference, + } + }); + std::fs::write(&config, serde_json::to_vec(&value).unwrap()).unwrap(); + + let mut def = def_with_gpus(1); + def.options.insert( + "config".to_string(), + SimpleValue::String(config.display().to_string()), + ); + let mut ctx = ctx_for(def, tmp.path(), session); + containerise(&mut ctx); + + let err = Rocjitsu + .injection_def_with(&ctx, stand_in_interposer(tmp.path())) + .expect_err("an external simulator config is not mounted") + .to_string(); + assert!(err.contains(&simulator.display().to_string()), "{err}"); + assert!(err.contains("self-contained config"), "{err}"); + } + } + + /// `simulator_config` is not a dependency unless the DBT guest + /// simulator backend is active. + #[test] + fn unused_simulator_config_does_not_refuse_a_container() { + let tmp = tempfile::tempdir().unwrap(); + let config = tmp.path().join(RJ_CONFIG_NAME); + let external = tmp.path().parent().unwrap().join("host.json"); + + for dbt in [ + serde_json::json!({ + "enabled": false, + "execution_backend": "simulator", + "simulator_config": external, + }), + serde_json::json!({ + "enabled": true, + "execution_backend": "hardware", + "simulator_config": external, + }), + ] { + std::fs::write( + &config, + serde_json::to_vec(&serde_json::json!({"dbt_guest": dbt})).unwrap(), + ) + .unwrap(); + assert_eq!( + container_unreachable_config_path(&config, tmp.path()).unwrap(), + None + ); + } + } + + /// A reference that climbs out of the session, or a symlink that leads + /// out of it, is outside the mount whatever its spelling starts with. + #[test] + fn a_reference_that_leaves_the_session_is_unreachable() { + let tmp = tempfile::tempdir().unwrap(); + let session = tmp.path().join("session"); + std::fs::create_dir_all(&session).unwrap(); + let host = tmp.path().join("host.json"); + std::fs::write(&host, b"{}").unwrap(); + let link = session.join("linked.json"); + std::os::unix::fs::symlink(&host, &link).unwrap(); + + let config = session.join(RJ_CONFIG_NAME); + for reference in ["../host.json".to_string(), link.display().to_string()] { + std::fs::write( + &config, + serde_json::to_vec(&serde_json::json!({"dbt_guest": { + "enabled": true, + "execution_backend": "simulator", + "simulator_config": reference, + }})) + .unwrap(), + ) + .unwrap(); + assert!( + container_unreachable_config_path(&config, &session) + .unwrap() + .is_some(), + "{reference} resolves outside the session" + ); + } + + // The mount is made at the session's own spelling. Reached through + // an alias, a file named by the real path is the same file on the + // host and absent in the container. + let alias = tmp.path().join("alias"); + std::os::unix::fs::symlink(&session, &alias).unwrap(); + let inside = session.join("sim.json"); + std::fs::write(&inside, b"{}").unwrap(); + std::fs::write( + &config, + serde_json::to_vec(&serde_json::json!({"dbt_guest": { + "enabled": true, + "execution_backend": "simulator", + "simulator_config": inside, + }})) + .unwrap(), + ) + .unwrap(); + assert_eq!( + container_unreachable_config_path(&alias.join(RJ_CONFIG_NAME), &alias).unwrap(), + Some(inside), + "a path spelled outside the mount is not in the container" + ); + } + + /// A self-contained config in a dialect only FlatBuffers reads is not + /// refused for a container: it names no file that could be missing. + #[test] + fn a_self_contained_flatbuffers_config_reaches_a_container() { + let tmp = tempfile::tempdir().unwrap(); + let config = tmp.path().join(RJ_CONFIG_NAME); + std::fs::write( + &config, + b"{\n // FlatBuffers accepts comments and trailing commas\n max_ticks: 1,\n}\n", + ) + .unwrap(); + assert_eq!( + container_unreachable_config_path(&config, tmp.path()).unwrap(), + None + ); + } + + /// Nothing may read a session's configuration before bring-up has + /// recorded one, and the message says so rather than resolving the + /// profile a second time to paper over it. + #[test] + fn a_session_without_a_materialised_config_is_refused() { + let tmp = tempfile::tempdir().unwrap(); + let err = session_config(tmp.path()).unwrap_err().to_string(); + assert!(err.contains(CONFIG_PATH_NAME), "{err}"); + + // A discovery file with nothing in it is the same answer: the + // path is what is being asked for, and there isn't one. + write_config_discovery(tmp.path(), std::path::Path::new("")).unwrap(); + let err = session_config(tmp.path()).unwrap_err().to_string(); + assert!(err.contains(CONFIG_PATH_NAME), "{err}"); + } + + /// The recorded path is read the way rocjitsu reads it: the first + /// line, less one trailing `\r`, with spaces kept. Trimming them named + /// a different file from the one the interposer opens. + #[test] + fn a_recorded_config_path_keeps_its_spaces() { + let tmp = tempfile::tempdir().unwrap(); + let spaced = tmp.path().join(" cfg .json"); + write_config_discovery(tmp.path(), &spaced).unwrap(); + assert_eq!(session_config(tmp.path()).unwrap(), spaced); + + let discovery = tmp.path().join(RUNTIME_SUBDIR).join(CONFIG_PATH_NAME); + std::fs::write(&discovery, format!("{}\r\n12\n", spaced.display())).unwrap(); + assert_eq!(session_config(tmp.path()).unwrap(), spaced); + } + + /// Without the interposer there is nothing to emulate the workload, + /// so the injection fails rather than quietly describing a run on + /// real hardware — and it says which library and where to get it. + #[test] + fn a_missing_interposer_refuses_the_injection() { + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + let ctx = ctx_for(def_for_target(120500), tmp.path(), "no-interposer"); + let err = Rocjitsu.injection_def_with(&ctx, None).unwrap_err(); + + let msg = err.to_string(); + assert!(msg.contains(LIB_NAME), "{msg}"); + assert!(msg.contains("docs/building.md"), "{msg}"); + } + + /// Whatever config it is handed, the runtime directory + /// `write_config_discovery` picks belongs to the session: the + /// discovery file (and the daemon socket beside it) must never land + /// next to a config file mirage does not own and cannot clean up. #[test] fn discovery_file_lands_in_the_session_not_beside_the_config() { let tmp = tempfile::tempdir().unwrap(); @@ -1456,7 +3553,7 @@ mod tests { assert_eq!(runtime_dir, session.join(RUNTIME_SUBDIR)); assert_eq!( - std::fs::read_to_string(runtime_dir.join("config_path")).unwrap(), + std::fs::read_to_string(runtime_dir.join(CONFIG_PATH_NAME)).unwrap(), format!("{}\n", config.display()) ); assert_eq!( diff --git a/emulation/mirage/rocjitsu/tests/daemon.rs b/emulation/mirage/rocjitsu/tests/daemon.rs index 2534d2c08151..363df317f6e5 100644 --- a/emulation/mirage/rocjitsu/tests/daemon.rs +++ b/emulation/mirage/rocjitsu/tests/daemon.rs @@ -79,6 +79,7 @@ fn daemon_serves_handshake() { .next() .expect("at least one builtin agent"); let def = EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -164,6 +165,7 @@ fn dropping_the_handle_stops_the_daemon() { let agent_report = mirage_builtin::ensure_agents(false).unwrap(); let agent_name = agent_report.iter().map(|(n, _)| n.clone()).next().unwrap(); let def = EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -208,6 +210,7 @@ fn daemon_serves_multiple_clients() { let agent_report = mirage_builtin::ensure_agents(false).unwrap(); let agent_name = agent_report.iter().map(|(n, _)| n.clone()).next().unwrap(); let def = EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, diff --git a/emulation/mirage/rocjitsu/tests/dbt.rs b/emulation/mirage/rocjitsu/tests/dbt.rs index b01056257a69..85ea598303ad 100644 --- a/emulation/mirage/rocjitsu/tests/dbt.rs +++ b/emulation/mirage/rocjitsu/tests/dbt.rs @@ -36,6 +36,7 @@ fn def_for_guest(gfx_target_version: u32) -> EmulatorDef { let mut agent = mirage_core::agent::AgentDef::default(); agent.vm.gpu.device.gfx_target_version = gfx_target_version; EmulatorDef { + extra: Default::default(), emulator: dbt::NAME.to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, diff --git a/emulation/mirage/rocjitsu/tests/smoke.rs b/emulation/mirage/rocjitsu/tests/smoke.rs index ffa32c57d391..0bfc695c5820 100644 --- a/emulation/mirage/rocjitsu/tests/smoke.rs +++ b/emulation/mirage/rocjitsu/tests/smoke.rs @@ -27,6 +27,7 @@ fn sim_config_round_trip() { .next() .expect("at least one builtin agent"); let def = EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -62,6 +63,7 @@ fn gpus_per_node_drives_num_gpus() { .expect("at least one builtin agent"); let def = EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -104,6 +106,7 @@ fn injection_emits_rccl_env_defaults() { .expect("at least one builtin agent"); let emulator = EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -151,6 +154,93 @@ fn injection_emits_rccl_env_defaults() { } } +/// The injection names the process hosting the daemon, and only in +/// daemon mode. +/// +/// Without it the interposer has nothing to check the peer answering its +/// socket against, so it trusts whoever answered and says so on the +/// workload's stderr — at every launch, in front of the workload's own +/// output: +/// +/// ```text +/// [rj warn] daemon: authorizing pid N to read this address space; no +/// launcher named a daemon for this client, ... +/// ``` +/// +/// The value is load-bearing rather than cosmetic in both directions. A +/// *wrong* PID is worse than none: the interposer refuses the peer +/// outright (`RefusedPeerMismatch`) and the pageable transfers that need +/// `process_vm_readv` stop working. So this asserts the exact identity — +/// this process, which is the one `start_daemon` calls +/// `rj_daemon_start` on — and not merely that the variable is set. +#[test] +fn injection_names_the_daemon_hosting_process_in_daemon_mode() { + use mirage_core::emulator::get_emulator_backend; + use mirage_core::profile::ProfileDef; + use mirage_core::session::{SessionContext, SessionId}; + + let _g = mirage_core::paths::test_env_lock(); + let tmp = tempfile::tempdir().unwrap(); + mirage_core::paths::set_test_root(tmp.path()); + + if kmd_preload().is_none() { + return; // rocjitsu not installed on this host + } + + let agent_report = mirage_builtin::ensure_agents(false).unwrap(); + let agent_name = agent_report + .iter() + .map(|(n, _)| n.clone()) + .next() + .expect("at least one builtin agent"); + + let backend = get_emulator_backend("rocjitsu").expect("rocjitsu backend registered"); + let injection_for = |daemon: bool| { + let emulator = EmulatorDef { + extra: Default::default(), + emulator: "rocjitsu".to_string(), + plugins: Default::default(), + exec_mode: ExecMode::Functional, + options: Default::default(), + topology: MaybeRef::Owned(mirage_core::topology::TopologyDef { + num_nodes: 1, + gpus_per_node: 1, + agent: MaybeRef::Ref(agent_name.clone()), + }), + }; + let runtime_dir = tmp.path().join(if daemon { "daemon" } else { "local" }); + std::fs::create_dir_all(&runtime_dir).unwrap(); + let ctx = SessionContext { + id: SessionId::new("daemon-pid-test").unwrap(), + profile: ProfileDef { + name: "daemon-pid-test".to_string(), + description: None, + emulator, + containerize: None, + }, + runtime_dir, + daemon, + }; + backend + .injection_def(&ctx) + .expect("injection should succeed with a discoverable KMD library") + }; + + assert_eq!( + injection_for(true).env.get("ROCJITSU_DAEMON_PID"), + Some(&std::process::id().to_string()), + "daemon mode must name this process, the one that hosts the daemon" + ); + + // `--in-process` connects to no socket. Naming a daemon there would + // describe one that was never started. + assert_eq!( + injection_for(false).env.get("ROCJITSU_DAEMON_PID"), + None, + "in-process mode has no daemon to name" + ); +} + /// Drop-in `--config ` mode: the user's config file is used /// verbatim, but the rocjitsu runtime directory it implies belongs to /// the *session*, not to whatever directory the user keeps their config @@ -180,6 +270,7 @@ fn dropin_config_keeps_its_runtime_dir_inside_the_session() { std::fs::write(&user_config, b"{}\n").unwrap(); let mut emulator = EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -221,10 +312,21 @@ fn dropin_config_keeps_its_runtime_dir_inside_the_session() { runtime_dir.display(), session_dir.display() ); - // It is still wired up: the discovery file names the user's config. + // It is still wired up, and what it names is the session's own copy + // of the user's config rather than the user's file: the interposer + // reopens this path for the life of the session, so it has to be a + // document the session owns and an edit cannot retarget. + assert_eq!( + std::fs::read_to_string(runtime_dir.join(mirage_rocjitsu::CONFIG_PATH_NAME)).unwrap(), + format!( + "{}\n", + mirage_rocjitsu::rj_config_path(&session_dir).display() + ) + ); assert_eq!( - std::fs::read_to_string(runtime_dir.join("config_path")).unwrap(), - format!("{}\n", user_config.display()) + std::fs::read(mirage_rocjitsu::rj_config_path(&session_dir)).unwrap(), + std::fs::read(&user_config).unwrap(), + "the copy must be the user's bytes, not a re-synthesised config" ); // And the user's directory is exactly as they left it. let left_behind: Vec<_> = std::fs::read_dir(&user_dir) diff --git a/emulation/mirage/supervisor/src/process.rs b/emulation/mirage/supervisor/src/process.rs index dff995028b51..460f8aea2a70 100644 --- a/emulation/mirage/supervisor/src/process.rs +++ b/emulation/mirage/supervisor/src/process.rs @@ -620,7 +620,7 @@ pub fn spawn(spec: &SpawnSpec, output: mpsc::Sender) -> Result Vec<(std::ffi::OsString, std::ffi::OsString)> { +pub(crate) fn resolved_env(spec: &SpawnSpec) -> Vec<(std::ffi::OsString, std::ffi::OsString)> { let mut env: Vec<(std::ffi::OsString, std::ffi::OsString)> = Vec::new(); if !spec.inherit_env { for key in INHERITED_ENV { diff --git a/emulation/mirage/supervisor/src/run.rs b/emulation/mirage/supervisor/src/run.rs index 1e47b90bb560..c627ef67f583 100644 --- a/emulation/mirage/supervisor/src/run.rs +++ b/emulation/mirage/supervisor/src/run.rs @@ -738,11 +738,21 @@ mod tests { /// Nothing brings a session on it up by accident, which is what /// [`stalled_run`] needs — and when one *is* brought up, bring-up /// fails at its first step with a message worth asserting on. + /// The MI350X agent the stub describes, inline rather than by name: + /// bring-up follows agent references, and these test roots have no + /// agent store. + fn agent_mi350x() -> mirage_core::agent::AgentDef { + let mut agent = mirage_core::agent::AgentDef::default(); + agent.vm.gpu.device.gfx_target_version = 90500; + agent + } + fn stub_profile() -> ProfileDef { ProfileDef { name: "p".to_string(), description: None, emulator: EmulatorDef { + extra: Default::default(), emulator: "stub".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -750,7 +760,7 @@ mod tests { topology: MaybeRef::Owned(TopologyDef { num_nodes: 1, gpus_per_node: 1, - agent: MaybeRef::Ref("MI350X".to_string()), + agent: MaybeRef::Owned(agent_mi350x()), }), }, containerize: None, diff --git a/emulation/mirage/supervisor/src/session.rs b/emulation/mirage/supervisor/src/session.rs index 76390bc8108a..f289aa6d12ed 100644 --- a/emulation/mirage/supervisor/src/session.rs +++ b/emulation/mirage/supervisor/src/session.rs @@ -255,6 +255,19 @@ impl Session { let runtime_dir = mirage_core::paths::session_runtime_dir(&def.id); std::fs::create_dir_all(&runtime_dir) .map_err(|e| MirageError::io(runtime_dir.clone(), e))?; + // Every by-name reference followed once, here, and the backend + // given its chance to correct the agent — so the profile this + // session holds describes the machine it is about to stand up, + // and keeps describing it for as long as the session lives. An + // edit to the stored topology, agent or drop-in config after this + // point reaches nothing: the documents have already been read. + let mut profile = profile; + profile.emulator.resolve_refs()?; + if let Some(backend) = + mirage_core::emulator::get_emulator_backend(&profile.emulator.emulator) + { + backend.reconcile_profile(&mut profile, &runtime_dir)?; + } let node_count = resolve_node_count(&profile)?; let ctx = SessionContext { id: def.id.clone(), @@ -459,7 +472,8 @@ impl Session { alive } - /// Record the emulator injection to apply to every workload. + /// Record the emulator injection and the session metadata derived + /// while that exact injection was materialised. pub fn set_injection(&self, injection: InjectionDef) { self.lock().injection = injection; } @@ -841,10 +855,11 @@ impl Session { /// remapped onto the in-container mounts here, once, rather than at /// every call site. /// - /// Every call describes the same session: the shape is the one fixed - /// at creation, and the rendezvous port is picked once and kept, so a - /// `mirage exec` in another terminal joins the job rather than - /// starting a differently-shaped one beside it. + /// Every call describes the same session: the shape is fixed at + /// creation, the profile is resolved once with it, and the rendezvous + /// port is picked once and kept. + /// A `mirage exec` in another terminal therefore joins the job rather + /// than starting a differently-shaped one beside it. /// /// # Errors /// @@ -914,9 +929,13 @@ impl Session { } None => (injection.env.clone(), injection.ld_preload.clone()), }; - Ok(SessionDescription { session: self.def.id.clone(), + // The snapshot taken at creation, references already + // followed: a client that wants to know what device this + // session emulates asks the agent in here, rather than being + // handed a second copy of the answer. + profile: Some(self.ctx.profile.clone()), node_count, nproc_per_node: job_nproc, workdir: self.def.workdir.clone(), @@ -1884,11 +1903,24 @@ mod tests { session } + /// The MI350X agent these stubs describe, inline rather than by name. + /// + /// Bring-up follows agent references now, and a test root has no + /// agent store unless the test writes one — which is the subject of + /// exactly one test here, not a fixture cost every other one should + /// pay. + fn agent_mi350x() -> mirage_core::agent::AgentDef { + let mut agent = mirage_core::agent::AgentDef::default(); + agent.vm.gpu.device.gfx_target_version = 90500; + agent + } + fn profile(num_nodes: u32, gpus_per_node: u32) -> ProfileDef { ProfileDef { name: "p".to_string(), description: None, emulator: EmulatorDef { + extra: Default::default(), emulator: "rocjitsu".to_string(), plugins: Default::default(), exec_mode: ExecMode::Functional, @@ -1896,7 +1928,7 @@ mod tests { topology: MaybeRef::Owned(TopologyDef { num_nodes, gpus_per_node, - agent: MaybeRef::Ref("MI350X".to_string()), + agent: MaybeRef::Owned(agent_mi350x()), }), }, containerize: None, @@ -1990,7 +2022,7 @@ mod tests { let mut topology = TopologyDef { num_nodes: 1, gpus_per_node: 1, - agent: MaybeRef::Ref("MI350X".to_string()), + agent: MaybeRef::Owned(agent_mi350x()), }; mirage_core::topology::store::put("shared", &topology).unwrap(); @@ -2020,6 +2052,126 @@ mod tests { ); } + /// The device a live session reports is the one its profile named at + /// bring-up, not whatever the agent store says later. + /// + /// The same guarantee as the topology test above, for the field the + /// ROCr preflight reads: a `mirage exec` half an hour into a session + /// has to be warned about the GPU that session is emulating. Bring-up + /// follows the agent reference once and keeps the document, so an + /// edit afterwards reaches nothing. + #[test] + fn a_live_sessions_device_is_the_one_resolved_at_bring_up() { + let dir = tempfile::tempdir().unwrap(); + let _guard = mirage_core::paths::test_env_lock(); + mirage_core::paths::set_test_root(dir.path()); + + let mut agent = mirage_core::agent::AgentDef::default(); + agent.vm.gpu.device.gfx_target_version = 90500; + mirage_core::agent::store::put("shared-agent", &agent).unwrap(); + + let mut p = profile(1, 1); + p.emulator.topology = MaybeRef::Owned(TopologyDef { + num_nodes: 1, + gpus_per_node: 1, + agent: MaybeRef::Ref("shared-agent".to_string()), + }); + let def = make_def( + SessionId::new("device").unwrap(), + MaybeRef::Owned(p.clone()), + "/".to_string(), + false, + ); + let session = Session::new(def, p).unwrap(); + session.set_phase(true, state::READY, None); + + let described = |session: &Arc| -> Option { + let profile = session.describe().unwrap().profile?; + Some(profile.agent()?.gfx_target()?.to_string()) + }; + let before = described(&session); + + // The user retargets the shared agent for their next run. + agent.vm.gpu.device.gfx_target_version = 120500; + mirage_core::agent::store::put("shared-agent", &agent).unwrap(); + let after = described(&session); + + mirage_core::paths::clear_test_root(); + + assert_eq!(before.as_deref(), Some("gfx950")); + assert_eq!( + after, before, + "editing an agent retargeted a session that was already running" + ); + } + + /// A drop-in `--config` rewritten after session creation changes + /// neither what the session reports nor what it injects. + /// + /// Session creation reads the config's device into the agent, and + /// injection hands the interposer a config. When those were two + /// readings of the user's file, a rewrite between them left the + /// session reporting gfx942 while injecting gfx1250, and the preflight + /// warned about a device that was not the one running. + #[test] + fn a_supplied_configs_reported_device_is_the_one_injected() { + let dir = tempfile::tempdir().unwrap(); + let _guard = mirage_core::paths::test_env_lock(); + mirage_core::paths::set_test_root(dir.path()); + + let config = dir.path().join("cfg.json"); + let write_target = |version: u32| { + std::fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "vm": {"gpu": {"device": {"gfx_target_version": version}}} + })) + .unwrap(), + ) + .unwrap(); + }; + write_target(90402); + + let mut p = profile(1, 1); + p.emulator.options.insert( + "config".to_string(), + mirage_core::common::SimpleValue::String(config.display().to_string()), + ); + let def = make_def( + SessionId::new("supplied").unwrap(), + MaybeRef::Owned(p.clone()), + "/".to_string(), + false, + ); + let session = Session::new(def, p).unwrap(); + session.set_phase(true, state::READY, None); + + write_target(120500); + let interposer = dir.path().join("librocjitsu.so"); + std::fs::write(&interposer, b"").unwrap(); + let injection = mirage_rocjitsu::Rocjitsu + .injection_def_with(&session.ctx, Some(interposer)) + .unwrap(); + mirage_core::paths::clear_test_root(); + + let discovery = PathBuf::from(&injection.env["ROCJITSU_RUNTIME_DIR"]) + .join(mirage_rocjitsu::CONFIG_PATH_NAME); + let injected: serde_json::Value = serde_json::from_slice( + &std::fs::read(std::fs::read_to_string(discovery).unwrap().trim()).unwrap(), + ) + .unwrap(); + let reported = session + .describe() + .unwrap() + .profile + .and_then(|profile| Some(profile.agent()?.vm.gpu.device.gfx_target_version)); + assert_eq!( + injected.pointer("/vm/gpu/device/gfx_target_version"), + Some(&serde_json::json!(90402)) + ); + assert_eq!(reported, Some(90402)); + } + #[tokio::test] async fn teardown_waits_for_a_bring_up_that_is_still_creating_things() { // Ctrl-C during bring-up. The bring-up task is not one anybody diff --git a/emulation/mirage/supervisor/src/spec.rs b/emulation/mirage/supervisor/src/spec.rs index dce6468ae97e..918caa782dca 100644 --- a/emulation/mirage/supervisor/src/spec.rs +++ b/emulation/mirage/supervisor/src/spec.rs @@ -18,11 +18,11 @@ //! //! The mapping is very nearly a function of its inputs, and everything //! it decides is testable without a container runtime or an emulator. -//! The one thing it asks the world about is a containerised -//! `--workdir`, which is a question only the image can answer and is -//! asked only when the caller passed one. One more thing is read from -//! the process rather than passed in — which runtime directory this -//! mirage resolved — and both callers necessarily agree on it. +//! One thing it asks the world about is a containerised `--workdir`, +//! which is a question only the image can answer and is asked only when +//! the caller passed one. One more thing is read from the process rather +//! than passed in — which runtime directory this mirage resolved — and +//! both callers necessarily agree on it. //! //! Whether the caller's streams are a terminal used to be read here too, //! and it was the one input that broke the promise above. A test runs @@ -32,6 +32,25 @@ //! most needed pinning was the branch no test could enter. It is a //! parameter now, [`CallerStreams`], which the caller probes once with //! [`CallerStreams::probe`] and a test states outright. +//! +//! The ROCr preflight is the exception that stayed, and the promise +//! above is narrower for it. Once the concrete specs exist, +//! [`build_specs`] asks whether the ROCm runtime each host process will +//! load knows the ISA the session emulates — reading this process's +//! environment, the workload's executable and the runtime that +//! executable resolves — and prints a warning on stderr when it does +//! not. It is here because here is the only place both `mirage run` and +//! every `mirage exec` have a concrete command, workdir and environment +//! to ask about; an earlier version that lived in the run checked the +//! run and silently skipped every exec. +//! +//! What the `CallerStreams` lesson costs is paid rather than ignored. +//! The decision — which specs to ask about, which sessions to skip +//! without asking, how ranks that share loader inputs collapse to one +//! question, and what the notice says — is `unsupported_target_notices`, +//! a pure function of the description, the specs and a verdict its +//! caller supplies, which the tests drive with a stub. Ambient here is +//! the answer, never the policy. use std::collections::BTreeMap; @@ -465,9 +484,102 @@ pub fn build_specs( }); } } + warn_if_the_runtime_cannot_see_the_device(desc, &specs); Ok(specs) } +/// Warn about an emulated ISA only after the workload's concrete +/// [`SpawnSpec`] exists. +/// +/// This placement is what makes the answer belong to the process that +/// will actually run: `mirage run` and every later `mirage exec` both +/// pass here, and the spec already contains their distinct command, +/// workdir, environment layering and `--clear-env-vars` policy. +fn warn_if_the_runtime_cannot_see_the_device(desc: &SessionDescription, specs: &[SpawnSpec]) { + let notices = + unsupported_target_notices(desc.emulated_gfx_target(), desc, specs, |isa, spec| { + let probe = rocr_probe_spec(desc, spec); + let env = crate::process::resolved_env(&probe); + mirage_core::rocr::check_target_for_process( + isa, + &probe.command, + probe.workdir.as_deref().map(std::path::Path::new), + &env, + probe.inherit_env, + ) + }); + for notice in notices { + eprintln!("mirage: {notice}"); + } +} + +/// A workload spec made safe to inspect with the ELF loader trace. +/// +/// rocjitsu's own KMD interposer is known not to select ROCr, but +/// running the trace with it would initialise the interposer outside +/// the session it belongs to. Remove that one known preload only when it +/// is the whole value. A user preload stays in a merged value, and an +/// inherited preload remains in the inherited environment; either +/// makes the answer [`mirage_core::rocr::TargetSupport::Unknown`]. +fn rocr_probe_spec(desc: &SessionDescription, spec: &SpawnSpec) -> SpawnSpec { + let mut probe = spec.clone(); + if let Some(injected) = &desc.ld_preload + && probe.env.get("LD_PRELOAD") == Some(injected) + { + probe.env.remove("LD_PRELOAD"); + } + probe +} + +/// Which notices this session's specs earn, given the target it +/// emulates. +/// +/// `target` is a parameter rather than a field this reads off `desc` for +/// the reason `support` is a parameter: the module doc's rule is that +/// ambient state is the *answer* here and never the decision. Deriving it +/// inside would also make every test of this policy depend on which +/// backends happened to be linked into the test binary, since the +/// derivation asks the registry whether the backend presents a device at +/// all. +fn unsupported_target_notices( + target: Option, + desc: &SessionDescription, + specs: &[SpawnSpec], + mut support: impl FnMut(&str, &SpawnSpec) -> mirage_core::rocr::TargetSupport, +) -> Vec { + if desc.containers.is_some() { + return Vec::new(); + } + let Some(target) = target else { + return Vec::new(); + }; + let isa = target.to_string(); + + let mut checked = std::collections::BTreeSet::new(); + let mut notices = std::collections::BTreeSet::new(); + for spec in specs.iter().filter(|spec| spec.container.is_none()) { + let loader_env: Vec<(String, String)> = spec + .env + .iter() + .filter(|(key, _)| mirage_core::rocr::affects_process_probe(key)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let fingerprint = ( + spec.command.clone(), + spec.workdir.clone(), + spec.inherit_env, + loader_env, + ); + if !checked.insert(fingerprint) { + continue; + } + if let mirage_core::rocr::TargetSupport::Unsupported(problem) = support(&isa, spec) { + notices.insert(problem.explain()); + } + } + notices.into_iter().collect() +} + /// `n` as an English ordinal: `1st`, `2nd`, `3rd`, `4th`, `21st`. /// /// Written out because the error above is a sentence a person reads, and @@ -728,6 +840,7 @@ mod tests { fn job(node_count: u32, nproc: u32) -> SessionDescription { SessionDescription { session: SessionId::new("s").unwrap(), + profile: None, node_count, nproc_per_node: nproc, workdir: "/work".to_string(), @@ -760,6 +873,127 @@ mod tests { ExecId::new("e-1").unwrap() } + /// The target the `mi450x` builtin emulates, which is the one the + /// original report was filed against. + fn gfx1250() -> Option { + mirage_core::hardware::GfxTarget::new(120500) + } + + fn unsupported(isa: &str) -> mirage_core::rocr::TargetSupport { + mirage_core::rocr::TargetSupport::Unsupported(Box::new( + mirage_core::rocr::UnsupportedTarget { + target: isa.to_string(), + runtime: std::path::PathBuf::from("/opt/rocm/lib/libhsa-runtime64.so.1"), + version: Some("7.0.2.2".to_string()), + references: 45, + }, + )) + } + + /// Both `mirage run` and `mirage exec` call `build_specs`; keeping + /// the diagnostic on that shared path means an attached command is + /// checked against its own loader environment too. + #[test] + fn an_unsupported_emulated_target_produces_a_notice_for_the_concrete_process() { + let description = desc(1); + let mut def = exec_def(1, None); + def.clear_env = true; + let specs = build_specs(&description, &def, &id(), CAPTURED).unwrap(); + + let notices = unsupported_target_notices(gfx1250(), &description, &specs, |isa, spec| { + assert_eq!(isa, "gfx1250"); + assert!( + !spec.inherit_env, + "the check must receive the workload's clear-env policy" + ); + unsupported(isa) + }); + + assert_eq!(notices.len(), 1); + assert!(notices[0].contains("gfx1250"), "{notices:?}"); + assert!(notices[0].contains("no GPU"), "{notices:?}"); + } + + #[test] + fn no_target_and_containerised_sessions_are_not_probed() { + let description = desc(1); + let def = exec_def(1, None); + let specs = build_specs(&description, &def, &id(), CAPTURED).unwrap(); + let called = std::cell::Cell::new(false); + let check = |_: &str, _: &SpawnSpec| { + called.set(true); + unsupported("gfx1250") + }; + assert!(unsupported_target_notices(None, &description, &specs, check).is_empty()); + + let mut containerised = description; + containerised.containers = Some(ContainerTargets { + provider: "provider".to_string(), + names: vec!["node".to_string()], + scratch: std::path::PathBuf::from("/tmp"), + }); + assert!( + unsupported_target_notices(gfx1250(), &containerised, &specs, check).is_empty(), + "a containerised session runs the image's ROCm, not the host's" + ); + assert!(!called.get(), "neither exempt session may reach the probe"); + } + + #[test] + fn identical_rank_loader_inputs_are_checked_once() { + let description = job(2, 2); + let def = exec_def(2, None); + let specs = build_specs(&description, &def, &id(), CAPTURED).unwrap(); + let calls = std::cell::Cell::new(0); + + let notices = unsupported_target_notices(gfx1250(), &description, &specs, |isa, _| { + calls.set(calls.get() + 1); + unsupported(isa) + }); + + assert_eq!( + calls.get(), + 1, + "rank-only environment must not repeat the loader trace" + ); + assert_eq!(notices.len(), 1, "the warning must not repeat per rank"); + } + + #[test] + fn only_the_emulator_owned_preload_is_removed_from_the_loader_probe() { + let description = desc(1); + let def = exec_def(1, None); + let spec = build_specs(&description, &def, &id(), CAPTURED) + .unwrap() + .remove(0); + assert_eq!( + spec.env.get("LD_PRELOAD").map(String::as_str), + Some("/lib/interpose.so") + ); + assert!( + !rocr_probe_spec(&description, &spec) + .env + .contains_key("LD_PRELOAD"), + "the loader trace must not initialise the emulator outside its session" + ); + + let mut def = exec_def(1, None); + def.exec + .env + .insert("LD_PRELOAD".to_string(), "/user/mine.so".to_string()); + let spec = build_specs(&description, &def, &id(), CAPTURED) + .unwrap() + .remove(0); + assert_eq!( + rocr_probe_spec(&description, &spec) + .env + .get("LD_PRELOAD") + .map(String::as_str), + Some("/lib/interpose.so:/user/mine.so"), + "a user preload must make the verdict unknown, not be stripped" + ); + } + #[test] fn a_single_process_job_gets_the_terminal_whole() { // The interactive case: `mirage run -- bash` on a one-node diff --git a/emulation/mirage/supervisor/tests/run.rs b/emulation/mirage/supervisor/tests/run.rs index c5ef14f5b092..07bb737642dc 100644 --- a/emulation/mirage/supervisor/tests/run.rs +++ b/emulation/mirage/supervisor/tests/run.rs @@ -280,6 +280,7 @@ fn profile(nodes: u32) -> ProfileDef { name: "stub".to_string(), description: None, emulator: EmulatorDef { + extra: Default::default(), emulator: "stub".to_string(), plugins: PluginsDef::default(), exec_mode: mirage_core::emulator::ExecMode::default(), diff --git a/emulation/mirage/tests/container_e2e.rs b/emulation/mirage/tests/container_e2e.rs index 6c1516fb4bc3..132b6a759e42 100644 --- a/emulation/mirage/tests/container_e2e.rs +++ b/emulation/mirage/tests/container_e2e.rs @@ -1930,7 +1930,7 @@ impl Drop for RealContainer { #[test] fn a_real_engine_creates_and_removes_the_container_with_the_run() { - let env = Env::new(); + let env = BaseEnv::with_host_xdg(); if skip_without_emulator() { return; } @@ -1948,7 +1948,7 @@ fn a_real_engine_creates_and_removes_the_container_with_the_run() { // playing both sides. This is the same claim against a real engine: // mirage asks for the container, a real `podman`/`docker` creates it, // and a Ctrl-C in the run's terminal has to be enough to remove it. - env.base.ok(&[ + env.ok(&[ "profile", "create", "real", @@ -1962,7 +1962,7 @@ fn a_real_engine_creates_and_removes_the_container_with_the_run() { ]); let tag = marker("real-container"); - let mut run = env.base.spawn_run( + let mut run = env.spawn_run( &["--profile", "real"], &["/bin/sh", "-c", &tagged_sleep(&tag)], ); diff --git a/emulation/mirage/tests/e2e.rs b/emulation/mirage/tests/e2e.rs index dc2f6b94bf20..1904c8479f96 100644 --- a/emulation/mirage/tests/e2e.rs +++ b/emulation/mirage/tests/e2e.rs @@ -258,6 +258,177 @@ fn the_builtin_agents_are_unpacked_on_first_use() { assert!(shown.is_object(), "{shown}"); } +#[test] +fn profile_extra_fields_reach_the_daemon_config() { + if skip_without_emulator() { + return; + } + let env = Env::new(); + let mut profile: serde_json::Value = + serde_json::from_str(&env.ok(&["profile", "show", "mi350x"])).unwrap(); + profile["name"] = "extended".into(); + profile["max_ticks"] = 7654321.into(); + profile["emulator"]["vm"] = serde_json::json!({"gpu": {"device": {"capability2": 9}}}); + let source = env.root().join("extended.json"); + std::fs::write(&source, serde_json::to_vec(&profile).unwrap()).unwrap(); + env.ok(&["profile", "import", source.to_str().unwrap()]); + let mut run = env.spawn_run(&["--profile", "extended"], &["/bin/sh", "-c", "sleep 300"]); + let id = run.await_ready(Duration::from_secs(90)); + let config: serde_json::Value = serde_json::from_slice( + &std::fs::read(env.session_scratch(&id).join("rj_config.json")).unwrap(), + ) + .unwrap(); + assert_eq!(config["max_ticks"], 7654321); + assert_eq!(config["vm"]["gpu"]["device"]["capability2"], 9); + assert_eq!( + config["vm"]["gpu"]["device"]["num_sdma_queues_per_engine"], + 8 + ); + run.signal(Signal::SIGINT); + let output = run.wait(Duration::from_secs(60)); + assert!(!String::from_utf8_lossy(&output.stderr).contains("missing field")); +} + +#[test] +fn incomplete_profiles_warn_without_losing_extra_fields() { + if skip_without_emulator() { + return; + } + let env = Env::new(); + let mut profile: serde_json::Value = + serde_json::from_str(&env.ok(&["profile", "show", "mi350x"])).unwrap(); + let mut agent: serde_json::Value = + serde_json::from_str(&env.ok(&["agent", "show", "mi350x"])).unwrap(); + agent["vm"]["gpu"]["device"] + .as_object_mut() + .unwrap() + .remove("mem_clk_max"); + agent["vm"]["gpu"]["device"]["future_field"] = serde_json::json!({"values": [1, true]}); + profile["name"] = "incomplete".into(); + profile["emulator"]["topology"]["agent"] = agent; + profile["emulator"] + .as_object_mut() + .unwrap() + .remove("plugins"); + let source = env.root().join("incomplete.json"); + std::fs::write(&source, serde_json::to_vec(&profile).unwrap()).unwrap(); + let output = env.run(&["profile", "import", source.to_str().unwrap()]); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stderr}"); + assert!( + stderr.contains("missing field /emulator/plugins"), + "{stderr}" + ); + assert!( + stderr.contains("missing field /vm/gpu/device/mem_clk_max"), + "{stderr}" + ); + let stored: serde_json::Value = + serde_json::from_str(&env.ok(&["profile", "show", "incomplete"])).unwrap(); + let device = &stored["emulator"]["topology"]["agent"]["vm"]["gpu"]["device"]; + assert!(device.get("mem_clk_max").is_none()); + assert_eq!( + device["future_field"], + serde_json::json!({"values": [1, true]}) + ); +} + +/// The agents mirage ships, as mirage itself reports them. +/// +/// Every one has a builtin profile of the same name pinning it, so this +/// doubles as the list of builtin profiles. +fn builtin_agents() -> Vec { + let env = Env::new(); + serde_json::from_str(&env.ok(&["--json", "agent", "list"])).unwrap() +} + +#[test] +fn every_builtin_agent_starts_the_rocjitsu_daemon() { + if skip_without_emulator() { + return; + } + // Every builtin, asked for rather than listed here: mirage takes its + // agents from the RocJITsu configs it ships, so a GPU added there + // becomes a builtin without anybody editing this file — and the + // point of this test is that a builtin starts. A hardcoded three + // would have gone on passing while four new ones went unexercised. + for agent in builtin_agents() { + let env = Env::new(); + let output = env.run(&["run", "--profile", &agent, "--", "/bin/true"]); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{agent}: {stderr}"); + assert!(!stderr.contains("missing field"), "{agent}: {stderr}"); + assert!(env.live_runs().is_empty(), "{agent} left a live run"); + } +} + +/// A machine upgraded from the previous release can still run. +/// +/// The documents below are what that release wrote — not the current +/// ones with a field removed, which is a shape only an install that +/// upgraded mid-release would ever have. They differ from what mirage +/// ships by more than the queue count (the previous `mi350x` has five +/// SDMA engines and four CUs per shader array against today's two and +/// nine), and none of them carries the queue count rocjitsu now demands, +/// so before the startup pass learned to recognise them this seeded an +/// install whose *default* profile could not start a session. +#[test] +fn an_install_upgraded_from_the_previous_release_still_runs() { + if skip_without_emulator() { + return; + } + for (agent, document) in [ + ("mi300x", include_str!("../builtin/legacy/mi300x.json")), + ("mi350x", include_str!("../builtin/legacy/mi350x.json")), + ("mi450x", include_str!("../builtin/legacy/mi450x.json")), + ] { + let env = Env::new(); + let path = env.root().join(format!("config/mirage/agent/{agent}.json")); + // Nothing has run yet, so this is the directory an upgrade finds + // already populated rather than one mirage has just made. + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, document).unwrap(); + + let output = env.run(&["run", "--profile", agent, "--", "/bin/true"]); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{agent}: {stderr}"); + + // Not patched into a machine that is neither: the whole document + // is now the one a fresh install gets. + let migrated: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let shipped: serde_json::Value = + serde_json::from_str(&env.ok(&["agent", "show", agent])).unwrap(); + assert_eq!(migrated, shipped, "{agent}"); + } +} + +#[test] +fn builtin_agents_without_sdma_queue_counts_are_upgraded() { + if skip_without_emulator() { + return; + } + for agent in builtin_agents() { + let env = Env::new(); + let mut old: serde_json::Value = + serde_json::from_str(&env.ok(&["agent", "show", &agent])).unwrap(); + old["vm"]["gpu"]["device"] + .as_object_mut() + .unwrap() + .remove("num_sdma_queues_per_engine"); + let path = env.root().join(format!("config/mirage/agent/{agent}.json")); + std::fs::write(&path, serde_json::to_vec_pretty(&old).unwrap()).unwrap(); + env.ok(&["run", "--profile", &agent, "--", "/bin/true"]); + let upgraded: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + assert!( + upgraded["vm"]["gpu"]["device"]["num_sdma_queues_per_engine"] + .as_u64() + .is_some_and(|queues| queues > 0) + ); + } +} + #[test] fn run_streams_output_and_propagates_the_exit_code() { let env = Env::new(); @@ -841,6 +1012,47 @@ fn the_rank_environment_is_injected() { assert_eq!(out.trim(), "0/0/1/0"); } +/// The workload is told which process hosts the emulator daemon. +/// +/// The interposer inside the workload asks that daemon for permission to +/// be read out of, and checks the peer answering its socket against +/// `$ROCJITSU_DAEMON_PID`. With nothing there it trusts whoever answered +/// and warns on the workload's own stderr, at every launch — the +/// variable has to survive the whole way into the workload's +/// environment, not merely be computed. +/// +/// Which PID is correct is pinned in `mirage_rocjitsu`'s own tests, +/// where "the process hosting the daemon" is checkable directly. This +/// asserts the two facts that only a real run can: that it arrives, and +/// that `--in-process` — which starts no daemon and connects to no +/// socket — does not claim one. +#[test] +fn the_workload_is_told_which_process_hosts_the_daemon() { + let env = Env::new(); + if skip_without_emulator() { + return; + } + env.create_profile("p"); + let read_var = ["--", "/bin/sh", "-c", "echo ${ROCJITSU_DAEMON_PID:-unset}"]; + + let mut daemon = vec!["run", "--profile", "p"]; + daemon.extend_from_slice(&read_var); + let named = env.ok(&daemon); + let named = named.trim(); + assert!( + named.parse::().is_ok_and(|pid| pid > 0), + "a daemon-mode run must name the daemon's process, got {named:?}" + ); + + let mut local = vec!["run", "--in-process", "--profile", "p"]; + local.extend_from_slice(&read_var); + assert_eq!( + env.ok(&local).trim(), + "unset", + "--in-process starts no daemon, so there is none to name" + ); +} + #[test] fn a_multi_node_topology_runs_every_node() { let env = Env::new(); @@ -1338,12 +1550,20 @@ fn state_builtins_writes_every_shipped_document_and_names_where() { let env = Env::new(); let text = env.ok(&["state", "builtins"]); - for kind in ["agent", "topology", "profile"] { + // The two kinds that have files. Builtin profiles are generated from + // the RocJITsu configs mirage ships and never written, so this + // command has nothing to say about them — and must not name a path + // for one, which the loop below would then go looking for. + for kind in ["agent", "topology"] { assert!( text.contains(kind), "no {kind} in the builtins report: {text}" ); } + assert!( + !text.contains("profile"), + "builtin profiles are generated, not written: {text}" + ); let json: serde_json::Value = serde_json::from_str(&env.ok(&["--json", "state", "builtins"])).unwrap(); @@ -1371,7 +1591,13 @@ fn state_builtins_writes_every_shipped_document_and_names_where() { // Everything the report claims exists is loadable through the store, // not merely present as bytes. env.ok(&["agent", "show", "mi350x"]); + // And the profile it does not claim is there all the same, without a + // file: that is what generating them means. env.ok(&["profile", "show", "mi350x"]); + assert!( + !env.profile_dir().join("mi350x.json").exists(), + "a builtin profile must not be written to disk" + ); } /// `mirage emulators` is how a user finds out what this build can do. @@ -2983,8 +3209,9 @@ fn an_exec_joins_the_running_job_rather_than_standing_up_a_new_one() { "/bin/sh", "-c", &format!( - "if [ \"$RANK\" = 0 ]; then {report} > {}; fi; sleep 300", - recorded.display() + "if [ \"$RANK\" = 0 ]; then {report} > '{recorded}.tmp' && \ + mv '{recorded}.tmp' '{recorded}'; fi; sleep 300", + recorded = recorded.display() ), ], ); diff --git a/emulation/mirage/tests/harness/mod.rs b/emulation/mirage/tests/harness/mod.rs index 256cb9cc7471..e5b52ef06bc5 100644 --- a/emulation/mirage/tests/harness/mod.rs +++ b/emulation/mirage/tests/harness/mod.rs @@ -35,6 +35,7 @@ pub(crate) struct Env { config: PathBuf, runtime: PathBuf, bin: PathBuf, + preserve_host_xdg: bool, } impl Env { @@ -51,10 +52,17 @@ impl Env { config: dir.path().join("config"), runtime: short_runtime_dir(), bin: PathBuf::from(env!("CARGO_BIN_EXE_mirage")), + preserve_host_xdg: false, dir, } } + pub(crate) fn with_host_xdg() -> Self { + let mut env = Self::new(); + env.preserve_host_xdg = true; + env + } + /// The mirage binary under test. pub(crate) fn bin(&self) -> &Path { &self.bin @@ -102,11 +110,10 @@ impl Env { /// A `mirage` command wired to this environment. pub(crate) fn mirage(&self) -> Command { let mut c = Command::new(&self.bin); - c.env("XDG_CONFIG_HOME", &self.config) - .env("XDG_RUNTIME_DIR", &self.runtime) - .env_remove("MIRAGE_LOG") + c.env_remove("MIRAGE_LOG") .env_remove("MIRAGE_CONFIG") - .env_remove("MIRAGE_RUNTIME"); + .env_remove("MIRAGE_RUNTIME") + .envs(self.child_env()); c } @@ -115,6 +122,18 @@ impl Env { /// Exposed separately from [`Env::mirage`] so a test that builds its /// own command still gets the same isolation. pub(crate) fn child_env(&self) -> Vec<(String, String)> { + if self.preserve_host_xdg { + return vec![ + ( + "MIRAGE_CONFIG".to_string(), + self.config.join("mirage").display().to_string(), + ), + ( + "MIRAGE_RUNTIME".to_string(), + self.runtime.join("mirage").display().to_string(), + ), + ]; + } vec![ ( "XDG_CONFIG_HOME".to_string(), @@ -261,7 +280,11 @@ impl Run { if let Some(child) = self.child.as_mut() && let Ok(Some(status)) = child.try_wait() { - panic!("`mirage run` exited before serving its socket: {status:?}"); + let output = self.wait(Duration::from_secs(1)); + panic!( + "`mirage run` exited before serving its socket: {status:?}\nstderr: {}", + String::from_utf8_lossy(&output.stderr) + ); } if let Ok(entries) = std::fs::read_dir(&self.socket_dir) && let Some(id) = entries diff --git a/emulation/rocjitsu/.gitignore b/emulation/rocjitsu/.gitignore index 20813f4a1616..186f7b9edbbc 100644 --- a/emulation/rocjitsu/.gitignore +++ b/emulation/rocjitsu/.gitignore @@ -23,7 +23,7 @@ # Misc local working artifacts *.swp .vscode -build*/ +/build*/ third_party Testing tmp @@ -43,14 +43,6 @@ __pycache__ .codex .agents -# Website dependencies and generated artifacts -/website/node_modules/ -/website/dist/ -/website/.test-dist/ -/website/.test-data/ -/website/.vite/ -/website/coverage/ -/website/playwright-report/ -/website/test-results/ +# Local website settings /website/**/*.local /website/**/.DS_Store diff --git a/emulation/rocjitsu/CMakeLists.txt b/emulation/rocjitsu/CMakeLists.txt index f5de9a711094..204279dbee2f 100644 --- a/emulation/rocjitsu/CMakeLists.txt +++ b/emulation/rocjitsu/CMakeLists.txt @@ -368,6 +368,9 @@ if(ROCJITSU_ENABLE_VFIO) include(rj_libvfio_user) endif() +# Apply the host CPU policy to rocjitsu targets, after configuring dependencies. +include(rj_host_cpu) + # Libraries add_subdirectory(lib/util) add_subdirectory(lib/simdojo) diff --git a/emulation/rocjitsu/CONTRIBUTING.md b/emulation/rocjitsu/CONTRIBUTING.md index 80a1676f771b..bdfac8601d96 100644 --- a/emulation/rocjitsu/CONTRIBUTING.md +++ b/emulation/rocjitsu/CONTRIBUTING.md @@ -92,12 +92,14 @@ Before writing new infrastructure, check these existing libraries: survive a log-groups-OFF build, and its signal-backtrace path must stay async-signal-safe (`rj_hsa_dbt_hooks.cpp` uses raw `::write`). - **Errors.** Expected failures such as a rejected instruction encoding return - `Result` or `FailureOr` and optionally emit a diagnostic. Exceptions are + `Result` or `FailureOr` and optionally emit a diagnostic. Do not introduce + new exception use unless an existing code path requires it. Exceptions are reserved for unrecoverable initialization, configuration, or execution - failures. Do not throw exceptions in simulation hot paths (event handlers, - instruction execution, cache lookups). Do not add `try`/`catch` blocks unless - you are at a boundary that must translate an error (e.g., the C API layer). - There is no general exception safety guarantee unless explicitly documented. + failures. The project plans to disable exceptions entirely in the future. + Do not throw exceptions in simulation hot paths (event handlers, instruction + execution, cache lookups). Do not add `try`/`catch` blocks unless you are at a + boundary that must translate an error (e.g., the C API layer). There is no + general exception safety guarantee unless explicitly documented. ## ISA codegen workflow diff --git a/emulation/rocjitsu/README.md b/emulation/rocjitsu/README.md index b07b6d89e769..f7df181bb358 100644 --- a/emulation/rocjitsu/README.md +++ b/emulation/rocjitsu/README.md @@ -55,9 +55,19 @@ configs/ GPU topology JSON files schemas/ FlatBuffers schemas tests/ Test suite docs/ Design documents and guides -website/ React dashboard source and web tests +website/ Public web applications + dashboard/ React dashboard source and web tests + handbook/ Markdown documentation and blog site ``` +When adding a guide under `docs/`, also register it in the handbook navigation. +See [Adding a guide](website/handbook/README.md#adding-a-guide) for the steps and +local validation command. + +See [website/handbook/README.md](website/handbook/README.md) to build and preview the +Handbook. The simulation-performance dashboard has its own +[website/dashboard/README.md](website/dashboard/README.md). + ## Building ```bash diff --git a/emulation/rocjitsu/cmake/rj_host_cpu.cmake b/emulation/rocjitsu/cmake/rj_host_cpu.cmake new file mode 100644 index 000000000000..d80936b0f506 --- /dev/null +++ b/emulation/rocjitsu/cmake/rj_host_cpu.cmake @@ -0,0 +1,81 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Use the target platform, including when cross-compiling. Never infer the +# minimum supported CPU from the machine running CMake. +set(_rj_x86_64_linux OFF) +if( + CMAKE_SYSTEM_NAME STREQUAL "Linux" + AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$" + AND CMAKE_SIZEOF_VOID_P EQUAL 8 +) + set(_rj_x86_64_linux ON) +endif() + +set(_rj_host_cpu_default default) +if( + _rj_x86_64_linux + AND CMAKE_C_COMPILER_ID MATCHES "^(GNU|Clang)$" + AND CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang)$" +) + set(_rj_host_cpu_default x86-64-v3) +endif() +set(ROCJITSU_HOST_CPU_BASELINE + "${_rj_host_cpu_default}" + CACHE STRING + "rocjitsu host CPU baseline (default, x86-64, x86-64-v3, x86-64-v4)" +) +set(_rj_host_cpu_values default x86-64 x86-64-v3 x86-64-v4) +set_property( + CACHE ROCJITSU_HOST_CPU_BASELINE + PROPERTY STRINGS ${_rj_host_cpu_values} +) +if(NOT ROCJITSU_HOST_CPU_BASELINE IN_LIST _rj_host_cpu_values) + message( + FATAL_ERROR + "Unsupported ROCJITSU_HOST_CPU_BASELINE='${ROCJITSU_HOST_CPU_BASELINE}'; " + "choose one of: ${_rj_host_cpu_values}" + ) +endif() + +if(NOT ROCJITSU_HOST_CPU_BASELINE STREQUAL "default") + if(NOT _rj_x86_64_linux) + message( + FATAL_ERROR + "An explicit ROCJITSU_HOST_CPU_BASELINE requires a Linux x86-64 target." + ) + endif() + include(CheckCCompilerFlag) + include(CheckCXXCompilerFlag) + set(_rj_host_cpu_flag "-march=${ROCJITSU_HOST_CPU_BASELINE}") + string(MAKE_C_IDENTIFIER "${ROCJITSU_HOST_CPU_BASELINE}" _rj_host_cpu_key) + check_c_compiler_flag( + "${_rj_host_cpu_flag}" + _RJ_C_HOST_CPU_${_rj_host_cpu_key} + ) + check_cxx_compiler_flag( + "${_rj_host_cpu_flag}" + _RJ_CXX_HOST_CPU_${_rj_host_cpu_key} + ) + if( + NOT _RJ_C_HOST_CPU_${_rj_host_cpu_key} + OR NOT _RJ_CXX_HOST_CPU_${_rj_host_cpu_key} + ) + message( + FATAL_ERROR + "The C and C++ compilers must support ${_rj_host_cpu_flag}; " + "use ROCJITSU_HOST_CPU_BASELINE=default to retain toolchain flags." + ) + endif() + # Directory properties cover object libraries and header-only SIMD users, + # including tests, without changing parent projects or fetched dependencies. + add_compile_options("$<$:${_rj_host_cpu_flag}>") + add_link_options("$<$:${_rj_host_cpu_flag}>") +endif() +message(STATUS "rocjitsu host CPU baseline: ${ROCJITSU_HOST_CPU_BASELINE}") + +unset(_rj_x86_64_linux) +unset(_rj_host_cpu_default) +unset(_rj_host_cpu_values) +unset(_rj_host_cpu_flag) +unset(_rj_host_cpu_key) diff --git a/emulation/rocjitsu/configs/gfx1100_w7900.json b/emulation/rocjitsu/configs/gfx1100_w7900.json index bb9b66312f68..dd18a20af0de 100644 --- a/emulation/rocjitsu/configs/gfx1100_w7900.json +++ b/emulation/rocjitsu/configs/gfx1100_w7900.json @@ -5,7 +5,7 @@ {"num_threads": 1, "cpu_dispatch_threads": 4, "async_helper_threads": 0}, {"num_threads": 1, "cpu_dispatch_threads": 8, "async_helper_threads": 0}, {"num_threads": 1, "cpu_dispatch_threads": 16, "async_helper_threads": 0}, - {"num_threads": 1, "cpu_dispatch_threads": 32, "async_helper_threads": 0} + {"num_threads": 1, "cpu_dispatch_threads": 24, "async_helper_threads": 0} ], "max_ticks": 100000, "exec_mode": "functional", @@ -15,6 +15,11 @@ "device": { "gpu_id": 7019, "gfx_target_version": 110000, + "ip_versions": [ + {"hardware_id": 11, "major": 11, "minor": 0, "revision": 0}, + {"hardware_id": 42, "major": 6, "minor": 0, "revision": 0}, + {"hardware_id": 108, "major": 4, "minor": 3, "revision": 0} + ], "vendor_id": 4098, "device_id": 29768, "family_id": 145, diff --git a/emulation/rocjitsu/configs/gfx1151.json b/emulation/rocjitsu/configs/gfx1151.json index 5f32b5b2a359..e17d93dc5c84 100644 --- a/emulation/rocjitsu/configs/gfx1151.json +++ b/emulation/rocjitsu/configs/gfx1151.json @@ -5,7 +5,7 @@ {"num_threads": 1, "cpu_dispatch_threads": 4, "async_helper_threads": 0}, {"num_threads": 1, "cpu_dispatch_threads": 8, "async_helper_threads": 0}, {"num_threads": 1, "cpu_dispatch_threads": 16, "async_helper_threads": 0}, - {"num_threads": 1, "cpu_dispatch_threads": 32, "async_helper_threads": 0} + {"num_threads": 1, "cpu_dispatch_threads": 24, "async_helper_threads": 0} ], "max_ticks": 10000000, "exec_mode": "functional", @@ -15,6 +15,11 @@ "device": { "gpu_id": 5510, "gfx_target_version": 110501, + "ip_versions": [ + {"hardware_id": 11, "major": 11, "minor": 5, "revision": 1}, + {"hardware_id": 42, "major": 6, "minor": 1, "revision": 1}, + {"hardware_id": 108, "major": 7, "minor": 11, "revision": 1} + ], "vendor_id": 4098, "device_id": 5510, "family_id": 145, @@ -74,7 +79,7 @@ "type": "compute_unit", "config": [ { "key": "num_wf_slots", "value": "32" }, - { "key": "sgprs_per_wf", "value": "104" }, + { "key": "sgprs_per_wf", "value": "106" }, { "key": "vgprs_per_wf", "value": "256" }, { "key": "lds_size_kb", "value": "64" } ] diff --git a/emulation/rocjitsu/configs/gfx1201_r9700.json b/emulation/rocjitsu/configs/gfx1201_r9700.json index 17ac695611f8..6f0ae3cbcb03 100644 --- a/emulation/rocjitsu/configs/gfx1201_r9700.json +++ b/emulation/rocjitsu/configs/gfx1201_r9700.json @@ -5,7 +5,7 @@ {"num_threads": 1, "cpu_dispatch_threads": 4, "async_helper_threads": 0}, {"num_threads": 1, "cpu_dispatch_threads": 8, "async_helper_threads": 0}, {"num_threads": 1, "cpu_dispatch_threads": 16, "async_helper_threads": 0}, - {"num_threads": 1, "cpu_dispatch_threads": 32, "async_helper_threads": 0} + {"num_threads": 1, "cpu_dispatch_threads": 24, "async_helper_threads": 0} ], "max_ticks": 100000, "exec_mode": "functional", @@ -15,6 +15,11 @@ "device": { "gpu_id": 8716, "gfx_target_version": 120001, + "ip_versions": [ + {"hardware_id": 11, "major": 12, "minor": 0, "revision": 1}, + {"hardware_id": 42, "major": 7, "minor": 0, "revision": 1}, + {"hardware_id": 108, "major": 6, "minor": 3, "revision": 1} + ], "vendor_id": 4098, "device_id": 30033, "family_id": 152, diff --git a/emulation/rocjitsu/configs/gfx1250_mi455x.json b/emulation/rocjitsu/configs/gfx1250_mi455x.json index c96344857abb..2ac76fe16a1e 100644 --- a/emulation/rocjitsu/configs/gfx1250_mi455x.json +++ b/emulation/rocjitsu/configs/gfx1250_mi455x.json @@ -7,10 +7,8 @@ {"num_threads": 8, "cpu_dispatch_threads": 9, "async_helper_threads": 0}, {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 0}, {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 2}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 4}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 8}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 16} + {"num_threads": 8, "cpu_dispatch_threads": 33, "async_helper_threads": 0}, + {"num_threads": 8, "cpu_dispatch_threads": 33, "async_helper_threads": 8} ], "max_ticks": 100000, "exec_mode": "functional", diff --git a/emulation/rocjitsu/configs/gfx90a_mi210_kmd.json b/emulation/rocjitsu/configs/gfx90a_mi210_kmd.json index 3894fae2267c..928bed35d707 100644 --- a/emulation/rocjitsu/configs/gfx90a_mi210_kmd.json +++ b/emulation/rocjitsu/configs/gfx90a_mi210_kmd.json @@ -5,7 +5,7 @@ {"num_threads": 1, "cpu_dispatch_threads": 4, "async_helper_threads": 0}, {"num_threads": 1, "cpu_dispatch_threads": 8, "async_helper_threads": 0}, {"num_threads": 1, "cpu_dispatch_threads": 16, "async_helper_threads": 0}, - {"num_threads": 1, "cpu_dispatch_threads": 32, "async_helper_threads": 0} + {"num_threads": 1, "cpu_dispatch_threads": 24, "async_helper_threads": 0} ], "max_ticks": 0, "exec_mode": "functional", diff --git a/emulation/rocjitsu/configs/gfx942_cdna3.json b/emulation/rocjitsu/configs/gfx942_cdna3.json index 6b99a7416ba8..457c05a130f6 100644 --- a/emulation/rocjitsu/configs/gfx942_cdna3.json +++ b/emulation/rocjitsu/configs/gfx942_cdna3.json @@ -5,8 +5,7 @@ {"num_threads": 2, "cpu_dispatch_threads": 3, "async_helper_threads": 0}, {"num_threads": 2, "cpu_dispatch_threads": 7, "async_helper_threads": 0}, {"num_threads": 8, "cpu_dispatch_threads": 9, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 0} + {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 0} ], "max_ticks": 100000, "exec_mode": "functional", @@ -16,6 +15,11 @@ "device": { "gpu_id": 50148, "gfx_target_version": 90402, + "ip_versions": [ + {"hardware_id": 11, "major": 9, "minor": 4, "revision": 3}, + {"hardware_id": 42, "major": 4, "minor": 4, "revision": 2}, + {"hardware_id": 108, "major": 7, "minor": 9, "revision": 0} + ], "vendor_id": 4098, "device_id": 29856, "family_id": 146, diff --git a/emulation/rocjitsu/configs/gfx942_cdna3_kmd.json b/emulation/rocjitsu/configs/gfx942_cdna3_kmd.json index 9d4bae95e92d..42071945475b 100644 --- a/emulation/rocjitsu/configs/gfx942_cdna3_kmd.json +++ b/emulation/rocjitsu/configs/gfx942_cdna3_kmd.json @@ -5,8 +5,7 @@ {"num_threads": 2, "cpu_dispatch_threads": 3, "async_helper_threads": 0}, {"num_threads": 2, "cpu_dispatch_threads": 7, "async_helper_threads": 0}, {"num_threads": 8, "cpu_dispatch_threads": 9, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 0} + {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 0} ], "max_ticks": 100000, "exec_mode": "functional", @@ -16,6 +15,11 @@ "device": { "gpu_id": 50148, "gfx_target_version": 90402, + "ip_versions": [ + {"hardware_id": 11, "major": 9, "minor": 4, "revision": 3}, + {"hardware_id": 42, "major": 4, "minor": 4, "revision": 2}, + {"hardware_id": 108, "major": 7, "minor": 9, "revision": 0} + ], "vendor_id": 4098, "device_id": 29857, "family_id": 146, diff --git a/emulation/rocjitsu/configs/gfx950_mi355x.json b/emulation/rocjitsu/configs/gfx950_mi355x.json index dbbf1a011c40..9fc836d04917 100644 --- a/emulation/rocjitsu/configs/gfx950_mi355x.json +++ b/emulation/rocjitsu/configs/gfx950_mi355x.json @@ -6,11 +6,7 @@ {"num_threads": 2, "cpu_dispatch_threads": 7, "async_helper_threads": 0}, {"num_threads": 8, "cpu_dispatch_threads": 9, "async_helper_threads": 0}, {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 2}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 4}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 8}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 16} + {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 8} ], "max_ticks": 100000, "exec_mode": "functional", @@ -20,6 +16,11 @@ "device": { "gpu_id": 38144, "gfx_target_version": 90500, + "ip_versions": [ + {"hardware_id": 11, "major": 9, "minor": 5, "revision": 0}, + {"hardware_id": 42, "major": 4, "minor": 4, "revision": 5}, + {"hardware_id": 108, "major": 7, "minor": 9, "revision": 0} + ], "vendor_id": 4098, "device_id": 30112, "family_id": 160, diff --git a/emulation/rocjitsu/configs/gfx950_mi355x_kmd.json b/emulation/rocjitsu/configs/gfx950_mi355x_kmd.json index 2290002675e1..7d84d0548bf7 100644 --- a/emulation/rocjitsu/configs/gfx950_mi355x_kmd.json +++ b/emulation/rocjitsu/configs/gfx950_mi355x_kmd.json @@ -6,11 +6,7 @@ {"num_threads": 2, "cpu_dispatch_threads": 7, "async_helper_threads": 0}, {"num_threads": 8, "cpu_dispatch_threads": 9, "async_helper_threads": 0}, {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 0}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 2}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 4}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 8}, - {"num_threads": 8, "cpu_dispatch_threads": 25, "async_helper_threads": 16} + {"num_threads": 8, "cpu_dispatch_threads": 17, "async_helper_threads": 8} ], "max_ticks": 0, "exec_mode": "functional", @@ -20,6 +16,11 @@ "device": { "gpu_id": 38144, "gfx_target_version": 90500, + "ip_versions": [ + {"hardware_id": 11, "major": 9, "minor": 5, "revision": 0}, + {"hardware_id": 42, "major": 4, "minor": 4, "revision": 5}, + {"hardware_id": 108, "major": 7, "minor": 9, "revision": 0} + ], "vendor_id": 4098, "device_id": 30112, "family_id": 160, diff --git a/emulation/rocjitsu/configs/gfx950_mi355x_kmd_2gpu.json b/emulation/rocjitsu/configs/gfx950_mi355x_kmd_2gpu.json index aa5a29dfbe51..9014f422bed4 100644 --- a/emulation/rocjitsu/configs/gfx950_mi355x_kmd_2gpu.json +++ b/emulation/rocjitsu/configs/gfx950_mi355x_kmd_2gpu.json @@ -19,6 +19,11 @@ "device": { "gpu_id": 38144, "gfx_target_version": 90500, + "ip_versions": [ + {"hardware_id": 11, "major": 9, "minor": 5, "revision": 0}, + {"hardware_id": 42, "major": 4, "minor": 4, "revision": 5}, + {"hardware_id": 108, "major": 7, "minor": 9, "revision": 0} + ], "vendor_id": 4098, "device_id": 30112, "family_id": 160, diff --git a/emulation/rocjitsu/configs/guest_gfx950_on_gfx1201.json b/emulation/rocjitsu/configs/guest_gfx950_on_gfx1201.json index 4f3099bb0006..c8edf35687f1 100644 --- a/emulation/rocjitsu/configs/guest_gfx950_on_gfx1201.json +++ b/emulation/rocjitsu/configs/guest_gfx950_on_gfx1201.json @@ -7,6 +7,11 @@ "guest_device": { "gpu_id": 38144, "gfx_target_version": 90500, + "ip_versions": [ + {"hardware_id": 11, "major": 9, "minor": 5, "revision": 0}, + {"hardware_id": 42, "major": 4, "minor": 4, "revision": 5}, + {"hardware_id": 108, "major": 7, "minor": 9, "revision": 0} + ], "vendor_id": 4098, "device_id": 30112, "family_id": 160, diff --git a/emulation/rocjitsu/configs/guest_gfx950_on_gfx942.json b/emulation/rocjitsu/configs/guest_gfx950_on_gfx942.json index 2b2f4f364fac..b372c61317f9 100644 --- a/emulation/rocjitsu/configs/guest_gfx950_on_gfx942.json +++ b/emulation/rocjitsu/configs/guest_gfx950_on_gfx942.json @@ -7,6 +7,11 @@ "guest_device": { "gpu_id": 38144, "gfx_target_version": 90500, + "ip_versions": [ + {"hardware_id": 11, "major": 9, "minor": 5, "revision": 0}, + {"hardware_id": 42, "major": 4, "minor": 4, "revision": 5}, + {"hardware_id": 108, "major": 7, "minor": 9, "revision": 0} + ], "vendor_id": 4098, "device_id": 30112, "family_id": 160, diff --git a/emulation/rocjitsu/configs/guest_gfx950_on_simulated_gfx942.json b/emulation/rocjitsu/configs/guest_gfx950_on_simulated_gfx942.json index 3bb5245ecf6b..8be312934513 100644 --- a/emulation/rocjitsu/configs/guest_gfx950_on_simulated_gfx942.json +++ b/emulation/rocjitsu/configs/guest_gfx950_on_simulated_gfx942.json @@ -9,6 +9,11 @@ "guest_device": { "gpu_id": 38144, "gfx_target_version": 90500, + "ip_versions": [ + {"hardware_id": 11, "major": 9, "minor": 5, "revision": 0}, + {"hardware_id": 42, "major": 4, "minor": 4, "revision": 5}, + {"hardware_id": 108, "major": 7, "minor": 9, "revision": 0} + ], "vendor_id": 4098, "device_id": 30112, "family_id": 160, diff --git a/emulation/rocjitsu/docs/architecture.md b/emulation/rocjitsu/docs/architecture.md index 08774651274f..68b86285a3ca 100644 --- a/emulation/rocjitsu/docs/architecture.md +++ b/emulation/rocjitsu/docs/architecture.md @@ -49,8 +49,8 @@ See [simdojo.md](simdojo.md) for the full design. See [vm-design.md](vm-design.md) for the full hardware model design. Models the GPU hardware pipeline: -- **CommandProcessor** — Monitors compute doorbells, fetches AQL and supported - PM4 packets from ring buffers, parses kernel descriptors, and dispatches +- **CommandProcessor** — Monitors compute doorbells, fetches AQL packets, prepares native + PM4 ring submissions for the PM4 processor, parses kernel descriptors, and dispatches workgroups to CUs. It does not own or execute SDMA queues. - **ComputeUnit** — Executes wavefronts. Manages SGPR/VGPR register files, LDS, and scratch memory. Supports functional and cycle-accurate @@ -94,16 +94,33 @@ Models the GPU hardware pipeline: continuation, retirement publication, and terminal state. Legacy KFD and PCI/MES create queues through `GpuQueueRegistry` and the same SDMA binding factory; the command processor is not part of the SDMA path. -- **Packet processors** — `AqlPacketProcessor`, `Pm4PacketProcessor`, and - `SdmaPacketProcessor` implement the same compile-time, one-head-packet +- **Packet processors** — `AqlPacketProcessor` and `SdmaPacketProcessor` implement + the same compile-time, one-head-packet processing contract while retaining protocol-specific request and diagnostic state. An SDMA request carries caller-owned opaque continuation state because - an SDMA packet may block after a partial effect; AQL and PM4 retries are + an SDMA packet may block after a partial effect; AQL retries are restartable and require no continuation. Packet processors do not own queue registration, scheduling, ring lifetime, or completion tracking. Expected guest-packet and descriptor failures are returned as typed per-queue results; they do not escape as simulator exceptions or stop unrelated queues. +- **PM4 packet processing** — `pm4/pm4_packet_processor` processes production + compute rings and DRM indirect-buffer streams. It owns packet fetching, opcode + effects, register updates, nested IB traversal, and cursor publication over + CP-owned queue state. CP supplies architecture, cache flushing, dispatch + admission, retry scheduling, and synchronous fault cancellation callbacks. + Temporary unavailability retains command and cursor-publication state for retry; + cancellation stops resident waves before submission resources are released. + CP retains queue scheduling and dispatch completion. For native rings, CP captures + the VM snapshot, initializes the consumer cursor, builds root submissions from + doorbells, and retries pending cursor publication even while execution is suspended. + The processor traverses submitted packets and IBs and commits/publishes their + retirement through the shared cursor journal. Its `retry` callback must set + `command_retry_pending`; CP clears it when resuming. The `dispatch` callback adds + admitted work to `dispatches.entries`, which CP retires before processing resumes. + The processor sets `publication_faulted` before requesting cancellation on a + terminal publication failure; the cancellation callback marks the queue faulted, + cancels pending work, and notifies failed submissions. - **PCI/VFIO adapters** — PCI configuration, BAR/MMIO, DMA, interrupts, and transport-session lifetime. These adapt accesses into `GpuVm` and the shared block models; they do not contain alternate CP, MES, SDMA, or shader models. @@ -139,18 +156,18 @@ corresponding core model. The firmware-free non-AQL compute ring follows the same boundary. MES maps the queue and selects its `GpuVm` address space, then registers it with the selected command processor through `GpuQueueRegistry`. The registry binding is -only a lifetime and notification adapter. The CP-owned `Pm4QueueController` -holds each queue's ring traversal, cursor publication, retry state, and concrete -`Pm4PacketProcessor`, which implements the exact NOP/SET_UCONFIG_REG subset. The -MMIO model supplies only the supported register-write sink. This is deliberately -not a general PM4 implementation. - -The PM4 controller leases a queue under its management lock, then releases that -lock before VM access or register callbacks. Removal and reconfiguration wait for -an active lease to quiesce, so frontend callbacks cannot stall unrelated queue -registration or notification and queue state cannot be destroyed during service. -The CP invokes PM4 service outside its AQL queue lock, and each PM4 queue turn is -bounded so a large PM4 ring cannot monopolize the CP event. +only a lifetime and notification adapter. `CommandProcessor` owns AQL, native +PM4 and DRM compute queues in `ComputeQueueRecord`. Each record retains ring +configuration, shader registers, command streams, dispatches and cursor publication. +Native register writes require the MMIO model's supported register-write sink; +unsupported writes fault the queue. DRM submissions may ignore unused graphics state. + +Queue service, removal and reconfiguration serialize under `hw_queue_mutex_`. +VM and register callbacks execute under that lock, so callbacks must not acquire +frontend locks whose holders can enter the CP. Each PM4 service turn is bounded. +Graceful removal waits for command streams, shader dispatches and durable +publications; forced removal cancels them. VM-polled native PM4 rings are rejected +until they have an execution poller; explicit and host-polled rings are supported. Each service transaction pins both its `GpuVmAccess` and producer boundary; work announced after a root replacement is deferred to a new snapshot rather than being appended to an older blocked transaction. @@ -181,15 +198,15 @@ Queue lifecycle is shared without forcing unrelated protocols into one execution engine. AQL, PM4, and SDMA use `GpuQueueRegistry`, reusable `QueueBindingFactory` objects, and unique per-registration `QueueBinding` objects. PM4 and SDMA share `ConsumerCursorJournal` for retry-safe consumer publication; SDMA and the -restricted PM4 compute-queue path share `CircularRingReader` for wrap-safe -fetches. AQL, PM4, and SDMA expose the same `PacketProcessResult` envelope +production PM4 compute-ring path share `CircularRingReader` for wrap-safe +fetches. AQL and SDMA expose the same `PacketProcessResult` envelope and core processor operation. The common layer validates only protocol-independent status, retirement, and input-growth invariants. `required_bytes` is the fetch extent needed to process the head packet, while `retirement_bytes` is the cursor advance requested after processing and may include protocol-defined skipped commands. The ring owner validates that retirement extent against its capacity and the -producer-visible cursor. All three remain concrete protocol +producer-visible cursor. The protocols remain concrete protocol implementations because their request, scheduling, retry, and completion semantics differ; SDMA continuation lifetime remains with its ring consumer. diff --git a/emulation/rocjitsu/docs/async-instructions.md b/emulation/rocjitsu/docs/async-instructions.md index 13a8575687a8..2c9aa4bbbd83 100644 --- a/emulation/rocjitsu/docs/async-instructions.md +++ b/emulation/rocjitsu/docs/async-instructions.md @@ -21,45 +21,39 @@ executes its own CUs while helpers can execute eligible MMA instructions. `async_helper_threads` extends the existing allocation tables. Omission or -1 selects the configured policy; zero disables helpers. Explicit counts range from 0 to 128 and take priority over the automatic budget. Automatic selection -uses CPU affinity, keeping engine/dispatch cost at most 32. Helpers are additive; -the shipped single-GPU server presets stop at 48 total threads. +uses CPU affinity and the target's allocation table. Desktop and older CDNA +presets stop at an engine/dispatch cost of 24. gfx950 adds eight helpers for +32 total threads; gfx1250 uses up to 40 engine/dispatch threads and eight +helpers for 48 total. -| Budget / affinity | gfx950 E/D/H | gfx1250 E/D/H | Total execution threads | +| Budget / affinity | gfx950 E/D/H | gfx1250 E/D/H | Total gfx950 / gfx1250 | |---:|---:|---:|---:| -| 1 | 1/1/0 | 1/1/0 | 1 | -| 2 | 1/2/0 | 1/2/0 | 2 | -| 4 | 2/3/0 | 1/4/0 | 4 | -| 8 | 2/7/0 | 2/7/0 | 8 | -| 16 | 8/9/0 | 8/9/0 | 16 | -| 24 | 8/17/0 | 8/17/0 | 24 | -| 32 | 8/25/0 | 8/25/0 | 32 | -| 34 | 8/25/2 | 8/25/2 | 34 | -| 36 | 8/25/4 | 8/25/4 | 36 | -| 40 | 8/25/8 | 8/25/8 | 40 | -| 48 and above | 8/25/16 | 8/25/16 | 48 | +| 1 | 1/1/0 | 1/1/0 | 1 / 1 | +| 2 | 1/2/0 | 1/2/0 | 2 / 2 | +| 4 | 2/3/0 | 1/4/0 | 4 / 4 | +| 8 | 2/7/0 | 2/7/0 | 8 / 8 | +| 16 | 8/9/0 | 8/9/0 | 16 / 16 | +| 24 | 8/17/0 | 8/17/0 | 24 / 24 | +| 32 | 8/17/8 | 8/25/0 | 32 / 32 | +| 40 | 8/17/8 | 8/33/0 | 32 / 40 | +| 48 and above | 8/17/8 | 8/33/8 | 32 / 48 | A budget between entries selects the lower entry; 12 selects 8. Explicit -`cpu_thread_budget` limits the combined allocation. Affinity of 64 still selects -48: the measurements do not justify more than 16 preset helpers. Custom JSON -allocations and explicit knobs remain available for experiments. - -Both targets preserve the synchronous engine/dispatch allocations through 32. -Reserving helpers within that budget regressed Gluon FP16 by displacing CU -workers, even with cached independence admission. Adding helpers above 32 -avoids that tradeoff. `async_helper_threads: 0` disables helpers while retaining -8/25/0 on larger hosts. No workload classification is needed in the selector. - -The 32-thread CPU cap is a conservative resource policy, not a universal -throughput optimum. Extra dispatch workers can outperform extra helpers on -CU-rich workloads such as large-grid Gluon FP16. Helpers offer their largest -gains when few workgroups limit CU parallelism. The preset keeps the existing -CPU allocation and spends additional affinity on this complementary path; -helpers are created lazily, only when eligible MMA reaches the adapter. - -Desktop and MI210 presets retain 1/D/0 through D=32; CDNA3 retains the landed -mixed E/D table with H=0. Two- and four-GPU presets keep **1/1/0** for RCCL. -Mirage embeds these native tables in its RocJITsu backend and exposes the helper -option alongside the engine, dispatch and total-budget options. +`cpu_thread_budget` limits the combined allocation. Larger hosts retain the +highest table entry. Custom JSON allocations and explicit knobs remain +available for workloads that benefit from more workers. + +Both server targets add helpers after reaching their engine/dispatch ceiling. +Lower-budget comparisons on IREE, Gluon and Tensile favor retaining general +CU workers over reserving helper slots. Helpers are created lazily when +eligible MMA reaches the adapter. Disabling helpers retains 8/17/0 on gfx950 +and 8/33/0 on gfx1250 at larger budgets. These defaults limit CPU use while +retaining matrix overlap; they are not universal throughput optima. + +Desktop and MI210 presets stop at 1/24/0; CDNA3 stops at 8/17/0. +Two- and four-GPU presets keep **1/1/0** for RCCL. Mirage embeds these native +tables in its rocjitsu backend and exposes the helper option alongside the +engine, dispatch and total-budget options. ```sh rocjitsu --config configs/gfx950_mi355x.json --thread-budget-table @@ -81,6 +75,12 @@ unrelated completed result. The issuer polls or joins jobs and destroys decoded instructions on their allocator's owning thread. Admission plans retain code snapshots and decisions; their temporary decodes never escape a scan. +Each issuer uses a stable, hashed starting position when selecting a free +helper, preferring reuse across consecutive jobs. This preference does not pin +threads or guarantee cache locality. Other free slots remain available, and +pools larger than 64 helpers retain first-word priority. Selection still falls +back inline after a bounded number of failed reservation attempts. + Both publication and completion use release/acquire synchronization. Helpers inherit the issuer's floating-point environment. A bounded warm wait precedes blocking through standard atomic wait/notify operations. diff --git a/emulation/rocjitsu/docs/blog/index.md b/emulation/rocjitsu/docs/blog/index.md new file mode 100644 index 000000000000..967fa27b4f7f --- /dev/null +++ b/emulation/rocjitsu/docs/blog/index.md @@ -0,0 +1,3 @@ +# Blog + +Technical notes on simulation, GPU execution, debugging, and Rocjitsu development. diff --git a/emulation/rocjitsu/docs/building.md b/emulation/rocjitsu/docs/building.md index f76ed4f7fd72..78e5500a663c 100644 --- a/emulation/rocjitsu/docs/building.md +++ b/emulation/rocjitsu/docs/building.md @@ -24,6 +24,7 @@ cmake --build build | Option | Default | Description | |---|---|---| +| `ROCJITSU_HOST_CPU_BASELINE` | `x86-64-v3` on Linux x86-64 with GCC/Clang; `default` otherwise | Host CPU baseline: `default`, `x86-64`, `x86-64-v3`, or `x86-64-v4` | | `RJ_ENABLE_ASAN` | `OFF` | Enable AddressSanitizer | | `RJ_ENABLE_UBSAN` | `OFF` | Enable UndefinedBehaviorSanitizer | | `RJ_ENABLE_TSAN` | `OFF` | Enable ThreadSanitizer | @@ -33,6 +34,19 @@ cmake --build build | `LTO` | `OFF` | Enable link-time optimization for Release/RelWithDebInfo | | `ROCJITSU_ENABLE_VFIO` | `OFF` | Build Linux VFIO-user support; requires CMake 3.28+ and Linux 6.1+ UAPI headers | +### Host CPU baseline + +By default, Linux x86-64 binaries built with GCC or Clang require x86-64-v3 +CPU and OS support. To build for older x86-64 hosts: + +```bash +cmake -B build -G Ninja -DROCJITSU_HOST_CPU_BASELINE=x86-64 +``` + +Use `ROCJITSU_HOST_CPU_BASELINE=default` to retain your compiler or toolchain's +CPU settings. Explicit x86-64 baselines are supported only for Linux x86-64 +targets. + ### Sanitizer builds ```bash diff --git a/emulation/rocjitsu/docs/concurrent-dispatch.md b/emulation/rocjitsu/docs/concurrent-dispatch.md index 7b8c8837ac03..b82387239335 100644 --- a/emulation/rocjitsu/docs/concurrent-dispatch.md +++ b/emulation/rocjitsu/docs/concurrent-dispatch.md @@ -14,8 +14,10 @@ drain its CUs without holding that mutex. Pending submissions rotate when a worker takes an assignment. An assigned worker finishes its current submission; this is not preemptive scheduling. -Each caller executes only its own submission and cancels unused assignments -when its task index is exhausted. It joins only workers holding that submission. +Each caller executes only its own submission. The first caller or worker to +finish draining the task index cancels unused assignments under the queue mutex, +so workers do not keep claiming empty assignments while other CUs are still +running. The caller joins all workers holding that submission. The last worker notifies before dropping the mutex, protecting the stack-owned submission's lifetime. There is no per-submission queue allocation. @@ -36,15 +38,19 @@ other runtime service threads are outside this allocation. pairs. `resolve_execution_threads()` is a pure function: apply explicit overrides and topology clamps, calculate each entry's total retained thread count, and choose the largest fitting entry. Later entries break ties. It does not fill -unused budget between entries. The default budget is CPU affinity capped at 32; -`cpu_thread_budget` overrides that default. A larger budget can select larger -entries from a custom table. Explicit knob settings take priority over a budget. +unused budget between entries. The default budget is CPU affinity and the +target table sets the allocation ceiling. `cpu_thread_budget` overrides the +budget; explicit knob settings take priority. -The synchronous granules measured below stop at 32. The eight-XCD tables include a 24-thread entry; -otherwise their granules are powers of two. At 32 with async helpers disabled, gfx950 and gfx1250 select -8 engine threads and inclusive dispatch width 25. The [async MMA extension](async-instructions.md) -adds helper allocations and uses E + sum(D - 1) + H for the total budget. Desktop presets select one -engine and dispatch width 32. See [the allocation table](configuration.md#thread-accounting-and-preferred-allocations). +Desktop, MI210 and CDNA3 tables stop at an engine/dispatch cost of 24. +gfx950 adds eight MMA helpers for 32 total threads; gfx1250 uses up to 40 +engine/dispatch threads and eight helpers for 48 total. See +[the allocation table](configuration.md#thread-accounting-and-preferred-allocations) +and [async MMA policy](async-instructions.md#thread-policy). + +The historical study below includes wider synchronous allocations. Subsequent +IREE, Gluon and Tensile measurements favor lower defaults to limit CPU use; +custom tables and explicit overrides remain available for wider workloads. The initial entries were screened in the previous study, then checked on this standalone branch with the synchronous kernels. The final tables below report @@ -73,8 +79,11 @@ below. Multi-GPU presets default to E=1/D=1 for RCCL. Explicit D=0 opts into granules that divide each budget's remaining slots among the per-GPU pools; positive D overrides also remain available. Mirage uses the same defaults and derives the multi-GPU granules from single-GPU tables embedded in the RocJITsu -backend. The hardware agent model contains no host tuning. The ceiling remains 32, with -larger custom entries and explicit knob overrides available. +backend. Those derived granules convert helper slots to dispatch workers, so +gfx950 derives from a total budget of 32 and gfx1250 from 48, while desktop +and CDNA3 use 24. The hardware agent model contains no host tuning. Clocked +engines and legacy table-free dispatch overrides retain their automatic +32-thread cap. Explicit overrides remain available. ### Synchronous server kernels diff --git a/emulation/rocjitsu/docs/configuration.md b/emulation/rocjitsu/docs/configuration.md index 250326deb01d..afe30517ffca 100644 --- a/emulation/rocjitsu/docs/configuration.md +++ b/emulation/rocjitsu/docs/configuration.md @@ -89,8 +89,8 @@ The example above is intentionally minimal. |---|---|---| | `max_ticks` | int | Maximum simulation ticks (0 = unlimited) | | `num_threads` | int | Simdojo engine partitions (one per XCD when partitioned). Omit for the default. | -| `cpu_dispatch_threads` | int | Inclusive functional dispatch width per SoC. Omitted/0 selects a preferred allocation; 1 forces serial dispatch. Clamped to per-CP CU capacity. | -| `cpu_thread_budget` | int | Automatic selection ceiling. Omitted/0 uses CPU affinity, with engine/dispatch cost capped at 32 and additive preset helpers up to 48 total; a positive value overrides the budget. | +| `cpu_dispatch_threads` | int | Inclusive functional dispatch width per SoC. Omitted/0 selects a preferred allocation; 1 forces serial dispatch. Clamped to the shared SoC pool capacity. | +| `cpu_thread_budget` | int | Automatic selection ceiling. Omitted/0 uses CPU affinity and the target allocation table; a positive value overrides the budget. | | `async_helper_threads` | int | Shared MMA helpers per VM. Omitted/-1 selects the table; 0 disables; explicit values are 0–128. | | `thread_allocations` | array | Preferred `num_threads` / `cpu_dispatch_threads` / `async_helper_threads` triples, selected by total execution-thread cost. | | `exec_mode` | string | Execution mode. Use `"clocked"` for clocked execution; `"functional"` is the default/fallback. | @@ -123,14 +123,17 @@ selects the largest effective allocation fitting the budget, after applying explicit knob overrides and topology limits. A budget between table entries uses the lower entry; it does not create workers merely to exhaust the budget. Later entries break ties. The automatic total budget is CPU affinity, with a -minimum of one. Engine/dispatch cost stays capped at 32; helper entries may -use additional affinity. Shipped presets stop at 48 total. A positive -`cpu_thread_budget` overrides the budget, including the engine/dispatch cap. A config without a table uses serial defaults for unspecified -knobs. Clocked mode uses only engines, capped by affinity/budget and XCD count. - -Explicit engine, dispatch and helper values take precedence and may exceed the automatic -ceiling. Engines are clamped to aggregate XCD count, and dispatch width to each -SoC's largest per-CP CU count. No workload inspection is involved. +minimum of one; the target table sets the preferred allocation ceiling. A +positive `cpu_thread_budget` overrides the budget. A config without a table +uses serial defaults for unspecified knobs. Clocked mode uses only engines, +capped automatically at 32, by affinity/budget and by XCD count. + +Explicit engine, dispatch and helper values take precedence and may exceed the +automatic ceiling. Engines are clamped to aggregate XCD count. Each SoC's pool +width is bounded by one plus the sum of CUs-1 across nonempty XCDs; individual +submissions use at most their active CU count. This is the maximum concurrent +capacity; fewer engine partitions may leave workers idle. No workload inspection +is involved. Two separate contracts constrain consumers, and only the first is about the config file. @@ -195,13 +198,15 @@ The single-GPU tables retain these synchronous engine/dispatch pairs (H=0): | 4 | 2/3 | 1/4 | 1/4 | | 8 | 2/7 | 2/7 | 1/8 | | 16 | 8/9 | 8/9 | 1/16 | -| 24 | 8/17 | 8/17 | 1/16 | -| 32 | 8/25 | 8/25 | 1/32 | +| 24 | 8/17 | 8/17 | 1/24 | +| 32 | 8/17 | 8/25 | 1/24 | +| 40 and above | 8/17 | 8/33 | 1/24 | The gfx950/gfx1250 tables also contain the [async MMA triples](async-instructions.md#thread-policy). -Both keep 8/25/0 at 32 threads. Affinity above 32 selects additive helper rows -at totals 34, 36, 40 and 48 (2, 4, 8 and 16 helpers). Larger hosts still select -48. `async_helper_threads: 0` retains the synchronous choices above. +gfx950 adds eight helpers at budget 32, selecting 8/17/8; gfx1250 adds eight +at budget 48, selecting 8/33/8. Larger hosts retain those allocations. +`async_helper_threads: 0` retains the synchronous choices above. Desktop and MI210 tables stop at 1/24/0; +CDNA3 stops at 8/17/0. A budget of 12 selects the eight-thread row. Print allocations for any target without constructing a simulated GPU: @@ -253,7 +258,9 @@ selects them by the agent's GPU target and GPU count. The hardware agent format host scheduling policy. Unknown targets use the serial fallback. For multiple GPUs, it uses the matching native preset when available. Other GPU counts convert each single-GPU granule's budget B to E=1, D=1+floor((B-1)/GPUs) and H=0, -then leave selection to rocjitsu. +then leave selection to rocjitsu. B includes helper slots, which become dispatch +workers in this opt-in path: derived gfx950 tables use a budget of 32 and +gfx1250 uses 48, while desktop and CDNA3 tables use 24. Its multi-GPU configurations default to E=1/D=1/H=0 for the same reasons. A positive engine override can change that pin; `num_threads: 0` retains it. Profile options may override `cpu_thread_budget`, @@ -283,6 +290,28 @@ Components are defined hierarchically under `topology.root`. Range expansion (`xcd[0:8]`) creates multiple instances. Links connect component ports using pattern expressions with loop variables. +gfx1250 `WAVE_HW_ID1.WGP_ID` reads require a shader-array width +(`vm.gpu.device.num_cu_per_sh`) of 1–16. Every shader engine, including groups +with direct CU children, must have `num_shader_arrays_per_engine * num_cu_per_sh` +compute units. If this geometry disagrees with the component hierarchy, the +loader warns once per GPU and leaves the shader-array width unknown. Reads +with unknown or unrepresentable geometry issue an `s_getreg_b32` warning and +write zero. + +Omitted geometry fields inherit defaults selected by `gfx_target_version`. +For a smaller topology, set both `num_cu_per_sh` and +`num_shader_arrays_per_engine` to match its CU groups; for example, two arrays +of two CUs match a four-CU group. The topology need not grow to the generation's +default size to support WGP-ID reads. + +The topology has no shader-array level. Within each shader engine, the first +`num_cu_per_sh` CU children form array 0, the next form array 1, and so on. +The WGP ID is the CU's shader-engine-local index modulo `num_cu_per_sh`. +For gfx1250 shader engines, KFD CU-mask bit order varies XCD fastest, followed +by shader engine, array, and WGP. Within an engine, array `a` and WGP `w` select +CU child `a * num_cu_per_sh + w`, which reports `WGP_ID = w`. The KFD mask loop +does not select direct-CU groups; their WGP IDs still follow CU child order. + ### Memory wait diagnostics With memory wait diagnostics enabled, compute units warn when an instruction reads @@ -311,6 +340,12 @@ The JSON config is validated against FlatBuffers schemas in `schemas/`: - `simulation_config.fbs` — topology and simulation parameters - `checkpoint.fbs` — simulation state checkpointing +Checkpoints preserve the shader-array width used by gfx1250 `WAVE_HW_ID1.WGP_ID` +reads. Older checkpoints lack this geometry, so WGP-ID reads after restoring +them issue an `s_getreg_b32` warning and write zero; the simulator cannot infer a +shader-array-local ID from the shader-engine CU count alone. Saving such a restored +checkpoint again preserves that unknown geometry. + ## Multi-GPU Multi-GPU configs define multiple SoCs with distinct GPU IDs and diff --git a/emulation/rocjitsu/docs/dbi-design.md b/emulation/rocjitsu/docs/dbi-design.md index 0fef7df99538..fa5be6d6582d 100644 --- a/emulation/rocjitsu/docs/dbi-design.md +++ b/emulation/rocjitsu/docs/dbi-design.md @@ -98,6 +98,7 @@ The probe-call envelope, in emit order, is: an in-flight-load drain; the special ### Spill bracket `build_spill_bracket()` produces the prologue (saves) and epilogue (restores) that wrap the call: + - **VGPRs** — a direct `build_scratch_store_dword` in the prologue, `build_scratch_load_dword` in the epilogue. - **AccVGPRs** — the same builders with `acc=true` (CDNA scratch `acc` bit), addressing the accumulator file directly; no bridge. CDNA-only. - **SGPRs** — bridged through one VGPR (`plan.spill_bridge_vgpr`): `v_writelane` then a scratch store in the prologue; a scratch load, load-wait, then `v_readlane` in the epilogue. The single bridge is reused, so each SGPR restore is its own load/wait/readlane. @@ -134,6 +135,7 @@ Three validators sit at the orchestrator boundary, separated so each can evolve Pure predicate over the anchor instruction and its position. Permanent structural checks only — no `InstrumentationPoint` involvement. Reusable by future predicate-based anchor selection (Instrumentor walks blocks and filters candidates) without inheriting milestone noise. Rules enforced: + - `anchor_offset` is dword aligned. - `anchor.size()` is 4 or 8 and fits inside `text_bytes` (subtraction-based bounds check; resists overflow when `anchor_offset` is huge). - `anchor.raw_encoding()` is non-null. @@ -182,6 +184,7 @@ ISA-parameterized helpers for encoding common instructions (`s_branch`, `s_nop`, ### Spill Builders (`code/builders/spill_builders.h`) [DBI-only] Multi-word, generation-specific encoders for the spill bracket, split out from the scalar helpers because their prefixes/opcodes move by ISA and they return variable-length word lists. Not shared with DBT (which emits scratch through its own target-specific path): + - `build_scratch_store_dword` / `build_scratch_load_dword` — per-lane scratch store/load. CDNA3/CDNA4 use the gfx9 FLAT `seg=SCRATCH` encoding (2 words, 13-bit signed offset, `lds`=0); RDNA4 uses the dedicated VSCRATCH encoding (3 words, 24-bit offset, `sve`=0). Both take an `acc` flag that, on CDNA, sets the FLAT `acc` bit to address the AccVGPR file directly (RDNA throws — no acc file). - `build_v_writelane_b32` / `build_v_readlane_b32` — the SGPR↔VGPR lane bridge (VOP3; CDNA prefix `0x34`, RDNA `0x35`). - `build_wait_loads_complete` / `build_wait_stores_complete` — the async-access fences. CDNA uses a unified `s_waitcnt`; RDNA4 splits into `s_wait_loadcnt` (loads on LOADCNT) and `s_wait_storecnt` (stores on STORECNT). `build_wait_all_loads_complete` is the boundary drain used by `emit_probe_call`. @@ -192,10 +195,32 @@ An unmodeled arch throws `UnimplementedInst`. The hard arch gates on spilling ar The plain VALU ops emitted outside the spill bracket, which `instruction_builder.h` (scalar by construction) and `spill_builders.h` (scoped to the bracket) are not the home for. Today the two argument-materialization forms of `v_mov_b32`: `build_v_mov_b32_imm`, the `v_mov_b32 vN, ` pair — a VOP1 word whose `src0` names the literal constant, plus the literal word — and `build_v_mov_b32_src`, the single-word `v_mov_b32 vN, ` used for a register-sourced argument slot (the EXEC temp holding the anchor mask). `build_v_mov_b32_src` rejects the literal `src0` code, since that form needs the trailing literal word only `build_v_mov_b32_imm` emits. Both cover all ten AMDGPU targets, each through its own generation's builder and opcode table — the encodings agree at this opcode, but the generated VOP1 `op` field is 7 bits on cdna5 and rdna4 and 8 bits on the other eight generations, so a shared packer would be right only for opcodes that fit both. +### Scalar Memory Builders (`code/builders/smem_builders.h`) [DBI-only] + +SMEM is scalar but not SOP, so `instruction_builder.h` (SOP-only by its own documentation) is not its home, and it is not part of the spill bracket `spill_builders.h` covers. Today `build_s_load_dwordx2` and `build_wait_scalar_loads_complete`, both covering **all ten AMDGPU targets**. These are pure encoders with no DBI-specific semantics, so they follow `instruction_builder.h` and `vector_builders.h` rather than `spill_builders.h`, whose narrower scope comes from the scratch addressing modes and the CDNA `acc` bit. + +The wrapper exists because three things vary and every one of them is silent when wrong. + +**SBASE is the register index halved** on every generation, and the decoder multiplies it back, so an unhalved base loads through the wrong address. + +**"No SGPR offset" has two shapes.** CDNA1–4 gate the register and the immediate with independent `SOFFSET_EN` and `IMM` bits, so an immediate-only load clears the former and sets the latter. RDNA has neither bit and instead carries an always-present `SOFFSET` field that must name NULL. **The NULL code moved**, from 125 on RDNA1/2 to 124 from RDNA3 on. Leaving it zero names `s0`, so the load silently adds whatever that register holds. Each case takes the code from its own generation's operand table for that reason. + +**The immediate field changes name and width**: `offset` at 21 bits through RDNA3.5, `ioffset` at 24 bits on RDNA4 and CDNA5. Both are **signed**, so `max_smem_byte_offset` bounds a forward offset one bit below the field width (`0x0FFFFF` and `0x7FFFFF`) rather than at the unsigned maximum. + +The scalar-load wait is a separate counter from anything in `spill_builders.h`: the monolithic `s_waitcnt 0` covers it through RDNA3.5, but GFX12 splits the counters and it becomes `s_wait_kmcnt` (CDNA5 has no `s_waitcnt` opcode at all). `build_wait_loads_complete`'s LOADCNT orders VMEM, not SMEM. + +**Three things no simulator test can validate**, all instances of the standing rule that the simulator does not model everything hardware does. The model builds RDNA's offset operand from the immediate alone and ignores `SOFFSET` entirely. It retires scalar loads synchronously, so removing the completion wait does not change a simulated result. And it *zero*-extends the immediate offset (`generated/cdna3/smem.cpp` uses `static_cast(enc->offset)`, unlike the deliberate `<< 20 >> 20` the tree uses where it does model a signed displacement), so an out-of-range offset simulates as a large forward access and runs backwards on hardware. The builder's bound is the only thing rejecting it. + +`tests/patch/smem_builder_test.cpp` asserts SOFFSET against the *encoded word*, since a decode discards it. The offset bound is covered differently: the builder rejects an out-of-range offset, so the test asserts the rejection rather than inspecting a word. The wait has no in-tree observable and is documented rather than tested. + ### Kernel Descriptor Scan (`code/kernel_descriptor_scan.h`) [shared with DBT] Enumerates a code object's kernel descriptors and derives per-kernel allocation facts. `scan_kernel_descriptors(image, text_offset, text_size)` returns each kernel's descriptor file offset, entry, and `private_segment_fixed_size`, with overflow-safe extent checks and a descriptor-bounded-by-owning-section guard (rejects malformed ELFs). `kernel_wavefront_size` and `descriptor_vgpr_granularity_for_wavefront` decode the wave-size-dependent VGPR encoding granule (shared with DBT so the two cannot diverge); the orchestrator multiplies `(GRANULATED_WORKITEM_VGPR_COUNT + 1)` by that granule to get the kernel's VGPR count. The orchestrator currently rejects anything but a single kernel. +The same header decodes the descriptor's kernarg fields, also shared with DBT. `kernarg_segment_ptr_slot` returns the user-SGPR index the kernarg pointer occupies, which is the summed width of whichever of private_segment_buffer (4), dispatch_ptr (2) and queue_ptr (2) the descriptor enables ahead of it. It answers whether or not the pointer is enabled, so a caller *inserting* the pointer can ask for the slot before setting the bit. `kernarg_segment_ptr_sgpr` is the narrower question, empty unless `has_kernarg_segment_ptr`. `kernarg_preload_length` / `kernarg_preload_offset` decode the two halves of the one `kernarg_preload` field. `kernel_descriptor_initial_sgpr_count` adds the dense system SGPRs the hardware initializes after the user block — enabled workgroup IDs, then WORKGROUP_INFO — giving one past the last SGPR the launch ABI writes; DBT repairs that range when it inserts a kernarg pointer, DBI uses it as a floor for framework storage. + +The pointer those first two name is **live only at the kernel entry**. Nothing reserves the pair; the register allocator reclaims it after its last use, which in compiled kernels is typically within the first handful of instructions. A consumer that needs the value at an arbitrary later site has to capture it at entry rather than read the pair there. + ### Register Liveness Analysis [shared with DBT] **Files:** `code/analysis/liveness.h`, `code/analysis/liveness.cpp`, `code/analysis/def_use_chain.h`, `code/analysis/def_use_chain.cpp` @@ -279,11 +304,13 @@ public: ### RegisterRef / RegisterSet [shared with DBT] -**Files:** `isa/register_set.h`, `isa/register_set.cpp` +**File:** `isa/register_set.h` **Used by:** DBT semantic translator, DBI SpillManager and liveness ISA-independent register-file model. `RegisterRef` is `(RegClass, uint16_t index, uint8_t width)` measured in 32-bit lanes. `RegisterSet` is three disjoint bitsets (SGPR / VGPR / ACC_VGPR) sized to the union of CDNA and RDNA hardware bounds (`REGISTER_SET_MAX_*`). For scratch selection across both families, `REGISTER_SET_ALLOCATABLE_SGPRS` gives the conservative `min(CDNA, RDNA)` bound. +`RegisterSet` aliases the header-only `RegisterSetT`, selecting `RegisterSetWordType::Avx2M256` on AVX2 targets and `RegisterSetWordType::StandardUint64` otherwise. The enum selects an internal storage type (`__m256i` or a scalar unsigned integer) without passing vector types as template arguments. The register-set unit tests instantiate both storage types in AVX2 builds so the scalar fallback remains compiled and tested. + `RegisterSet` exposes `expand` / `erase` / `contains` / `none` / `size` / `intersects`, the standard set operators (`|=`, `&=`, `-=`), and a `for_each` visitor that yields tracked single-lane `RegisterRef`s in (SGPR, VGPR, AccVGPR) ascending-index order. --- diff --git a/emulation/rocjitsu/docs/dbi-overview.md b/emulation/rocjitsu/docs/dbi-overview.md index 4d7dabf93fa7..3cb50d856399 100644 --- a/emulation/rocjitsu/docs/dbi-overview.md +++ b/emulation/rocjitsu/docs/dbi-overview.md @@ -220,12 +220,12 @@ not to the code. condition codes, and mode bits, is bit-identical wherever the guest can observe it. - This extends to state that is easy to overlook because it is not a register - the guest names. **Probe memory operations perturb the guest's - outstanding-memory counters**, and guest code performs arithmetic against its - own count of outstanding operations; leaving those disturbed violates this - invariant in a way that manifests as data corruption rather than a fault. - Handling it is the framework's responsibility, not the probe's (§6.6). + This extends to state that is easy to overlook because it is not a register + the guest names. **Probe memory operations perturb the guest's + outstanding-memory counters**, and guest code performs arithmetic against its + own count of outstanding operations; leaving those disturbed violates this + invariant in a way that manifests as data corruption rather than a fault. + Handling it is the framework's responsibility, not the probe's (§6.6). 2. **No silent loss of coverage.** Every instrumentation request resolves to instrumented, or to rejected with a reason. Every dropped record is counted. @@ -252,13 +252,13 @@ not to the code. in the design may depend on kernels running one at a time; that would change the behavior being measured and is very hard to walk back. - This is a constraint on the framework, not a promise about every run. A - sufficiently heavyweight tool can induce serialization as a consequence of - what it asked for; most concretely, a large enough scratch request causes - the runtime to admit one scratch dispatch at a time. The heavier the tool, - the more it perturbs the application; that is inherent, and acceptable. What - is not acceptable is it happening invisibly, so induced serialization is - detected and reported (§7). + This is a constraint on the framework, not a promise about every run. A + sufficiently heavyweight tool can induce serialization as a consequence of + what it asked for; most concretely, a large enough scratch request causes + the runtime to admit one scratch dispatch at a time. The heavier the tool, + the more it perturbs the application; that is inherent, and acceptable. What + is not acceptable is it happening invisibly, so induced serialization is + detected and reported (§7). 6. **The device never waits on the host for an individual record** (§5.5, noting the deliberate exception in lossless mode). @@ -429,21 +429,21 @@ corruption if missed: during its execution, not only at wave launch, so nothing short of completion bounds the last read. - The trap here is that several earlier events look like completion and are not. - The packet being written, the command processor consuming it, and the queue's - read index advancing past it all mean the dispatch was *launched*. A framework - that reclaims on any of them corrupts a kernel still reading its arguments, - and does so intermittently, under load, in a way that will be blamed on the - application. - - Completion is observable only through a signal. A dispatch packet carries at - most one, and applications frequently leave it unset, so the framework cannot - assume there is one to watch and cannot add a second. The dependable - construction is for the framework to introduce **its own** synchronisation - after the dispatch, carrying a signal it owns, and to reclaim when that fires. - Note that this is the same mechanism §7 needs in order to count dispatches in - flight, so it is one facility serving two requirements rather than a cost - incurred here alone. + The trap here is that several earlier events look like completion and are not. + The packet being written, the command processor consuming it, and the queue's + read index advancing past it all mean the dispatch was *launched*. A framework + that reclaims on any of them corrupts a kernel still reading its arguments, + and does so intermittently, under load, in a way that will be blamed on the + application. + + Completion is observable only through a signal. A dispatch packet carries at + most one, and applications frequently leave it unset, so the framework cannot + assume there is one to watch and cannot add a second. The dependable + construction is for the framework to introduce **its own** synchronisation + after the dispatch, carrying a signal it owns, and to reclaim when that fires. + Note that this is the same mechanism §7 needs in order to count dispatches in + flight, so it is one facility serving two requirements rather than a cost + incurred here alone. **Read from the dispatch packet.** A wave can be given a pointer to its own dispatch packet in a preloaded scalar pair. This does not carry the buffer @@ -1070,17 +1070,18 @@ What buys throughput is **producing fewer records**: not conveniences; they are what makes the framework applicable to workloads that cannot be shrunk. - The decision is a table lookup, but the mechanism is not free. Observing - packets requires interposing on submission, which routes every dispatch in the - process through the framework, including those sampling declines to - instrument, and adds latency to each. Applications issuing very many small - dispatches pay this whether or not they are being sampled. It is the price of - admission for everything in §2.1, and it should be counted once, honestly, - rather than assumed away. - - Finer granularity, such as per workgroup, per wave, or every Nth execution of - a site, happens inside the probe and costs cycles there instead. The two - mechanisms should not be conflated. + The decision is a table lookup, but the mechanism is not free. Observing + packets requires interposing on submission, which routes every dispatch in the + process through the framework, including those sampling declines to + instrument, and adds latency to each. Applications issuing very many small + dispatches pay this whether or not they are being sampled. It is the price of + admission for everything in §2.1, and it should be counted once, honestly, + rather than assumed away. + + Finer granularity, such as per workgroup, per wave, or every Nth execution of + a site, happens inside the probe and costs cycles there instead. The two + mechanisms should not be conflated. + - **Filtering in the probe.** The earliest point volume can be reduced. It costs cycles rather than memory, so it survives the memory model. This is the design's main lever and should be a first-class probe capability. diff --git a/emulation/rocjitsu/docs/memory-wait-counter-coverage.md b/emulation/rocjitsu/docs/memory-wait-counter-coverage.md index a603f640414d..9287f5d499c7 100644 --- a/emulation/rocjitsu/docs/memory-wait-counter-coverage.md +++ b/emulation/rocjitsu/docs/memory-wait-counter-coverage.md @@ -32,9 +32,9 @@ both admission and issue accounting (two units for transfers larger than a DWORD but readiness still requires zero. Mixed hardware event types sharing a counter retain separate ordered-class positions: a nonzero wait proves an ordered result complete only when at least that many younger operations in its own class follow it. Generic FLAT on older CDNA is treated -conservatively as unordered. Zero waits reset this ordering state. FLAT keeps both queue -entries even when all lanes use one segment; register dependencies are restricted using -the resolved routing masks. Mixed global/shared FLAT functional execution still has the +conservatively as unordered. Zero waits reset this ordering state. FLAT contributes only +to memory domains used by its resolved requests; register dependencies follow the +resolved routing masks. Mixed global/shared FLAT functional execution still has the existing first-request-lane routing limitation; the checker does not repair memory routing. @@ -56,10 +56,17 @@ partial wait. CDNA5 async load and store completion classes are separate; async barrier arrive orders with async loads. Proven completion also releases the associated replay translation prefix. +FLAT admission is address-dependent, so the checker applies its memory-counter +capacity constraints after routing, before checking result writeback. The instruction +cannot use an unused domain's full counter to prove an older request complete. Address +and other source operands are read before this inference is available. + ## FLAT register readiness -FLAT contributes to both VMEM and DS counters even when every active lane selects -one address space. Its global/scratch and LDS portions can complete independently; +FLAT contributes to VMEM when at least one lane requests global/scratch memory and +to DS when at least one lane requests LDS memory. This conditional-participation +rule is the runtime model across AMD GPU targets; physical validation on every +architecture is not established. Its global/scratch and LDS portions can complete independently; RDNA4 ISA section 5.7.1.3 describes complementary lane masks for these portions. The checker uses the resolved address of each lane, rather than the functional pipeline's first-lane route, to associate a returned VGPR lane with its counter. @@ -93,7 +100,7 @@ wait requirements on untested architectures or under all schedules. |---|---| | Global, scratch, buffer, typed buffer, image loads and returning atomics | Load queue; track returned registers | | Stores and atomics without return | Store queue, or the shared legacy VMEM queue; no destination register | -| Generic FLAT loads, stores and atomics | VMEM and DS queues; returned lanes depend on DS for the resolved shared aperture and VMEM for global/scratch | +| Generic FLAT loads, stores and atomics | Only the VMEM/DS queues used by resolved requests; returned lanes depend on DS for the shared aperture and VMEM for global/scratch | | LDS/GDS, including permutation, swizzle and DS no-op | DS/legacy LGKM queue; returning forms track registers. GDS also contributes to EXP | | Scalar loads, atomics, cache operations, timestamps, barrier-state and wave-ID queries | KM/legacy LGKM queue; returned scalar registers are tracked | | Messages, including message returns | KM/legacy LGKM queue. Return forms contribute two units, with the result pending through the return unit | @@ -108,8 +115,9 @@ wait requirements on untested architectures or under all schedules. | Tensor DMA loads/stores | TENSOR queue | | gfx1250 scalar address operations and multi-group VMEM, including async LDS transfers | X translation queue; protect source dwords against overwrite, plus VMEM EXEC. One X unit per instruction, independently of completion-counter units. Tensor DMA has no X event | -Zero-EXEC vector producers still contribute positions behind pending requests; an empty -queue may skip them. Scalar producers do not depend on EXEC. Prefetches that do not +Zero-EXEC vector producers generally still contribute positions behind pending requests; +an empty queue may skip them. FLAT contributes no VMEM/DS position without a requesting +lane. Scalar producers do not depend on EXEC. Prefetches that do not increment completion counters contribute no entries. If a zero-EXEC instruction occupies an X position but no completion position, it has no completion-to-X mapping. Waiting on that empty completion counter cannot diff --git a/emulation/rocjitsu/docs/mma-admission.md b/emulation/rocjitsu/docs/mma-admission.md index 820fdfd840cb..7ce2223b92ea 100644 --- a/emulation/rocjitsu/docs/mma-admission.md +++ b/emulation/rocjitsu/docs/mma-admission.md @@ -55,10 +55,9 @@ to the CU. ## Configuration -Server presets add 2, 4, 8 or 16 helpers above the existing 32-thread -engine/dispatch allocation, selected automatically when CPU affinity permits. -The largest preset uses 48 total threads. Set `async_helper_threads: 0` to -keep execution synchronous, or `cpu_thread_budget: 32` to limit the total. +Server presets select shared MMA helpers from their execution-thread budget. +See the [thread policy](async-instructions.md#thread-policy) for allocations. +Set `async_helper_threads: 0` to keep execution synchronous. Every selected CDNA4 MFMA and CDNA5 WMMA family requires cached admission, including dense K32 and scaled 16x16x128 WMMA. Lookahead inspects at most eight diff --git a/emulation/rocjitsu/docs/performance-dashboard.md b/emulation/rocjitsu/docs/performance-dashboard.md new file mode 100644 index 000000000000..21b0e49b32c4 --- /dev/null +++ b/emulation/rocjitsu/docs/performance-dashboard.md @@ -0,0 +1,21 @@ +# Performance dashboard + +The Rocjitsu performance dashboard tracks **how quickly the simulator runs benchmark workloads**. It helps developers inspect benchmark history and identify changes in simulator performance. + +!!! info "Simulator performance, not GPU hardware performance" + + These results measure Rocjitsu's simulation runs. They are not benchmarks of physical AMD GPUs and should not be used to compare AMD hardware performance. Target names such as `gfx950` identify the architecture being simulated. + +[**Rocjitsu / Simulation Performance Dashboard** :material-open-in-new:](https://rocm.github.io/rocm-systems/rocjitsu-dashboard/){ .dashboard-launch aria-label="Open Rocjitsu Simulation Performance Dashboard (external site)" } + +## What you can explore + +- Benchmark run history and test durations. +- Results for individual tests and simulated targets. +- Comparisons between a selected run and a baseline to investigate improvements or regressions. + +## How to interpret the results + +For a comparable, completed test, a shorter duration means the simulator finished that workload faster. Interpret changes alongside the test status, workload, simulator configuration, and benchmark environment. Failed or incomplete tests do not establish a performance improvement. + +Use the dashboard to investigate simulator changes over time; use hardware benchmarks when evaluating performance on a physical GPU. diff --git a/emulation/rocjitsu/docs/plugins.md b/emulation/rocjitsu/docs/plugins.md index 9d315b4a153c..f53f5dcba3f8 100644 --- a/emulation/rocjitsu/docs/plugins.md +++ b/emulation/rocjitsu/docs/plugins.md @@ -167,18 +167,22 @@ required exports. The `config_schema` string describes the accepted config keys. Each key maps to an object with a `type` (`string`, `number`, or `boolean`), an -optional `description`, and an optional `default`. Keys without a -`default` are required. Example: +optional `description`, an optional `default`, and an optional boolean +`optional`. A missing key with a `default` receives that value, including when +`optional` is also `true`. A missing key with `optional: true` and no default +remains absent. All other keys are required. Example: ```json { "argname": { "type": "string", "description": "does something important", "default": "defaultvalue" }, - "requiredarg": { "type": "number" } + "requiredarg": { "type": "number" }, + "optionalarg": { "type": "boolean", "optional": true } } ``` -The loader merges defaults, validates types, checks for required keys, -and passes the resolved JSON object to `rocjitsu_plugin_create`. +The loader validates supplied values first, applies defaults to missing keys, +omits missing optional keys, checks for required keys, and passes the resolved +JSON object to `rocjitsu_plugin_create`. ## Plugin output @@ -337,6 +341,34 @@ every high-frequency callback, serializing it with the infrequent callbacks without a per-instruction scan of the plugin list. Plugins that protect their own shared state should retain the parallel default. +Plugins can opt out of individual high-frequency hooks by overriding +`observes_before_execute_instruction()`, +`observes_after_execute_instruction()`, +`observes_async_instruction_issued()`, +`observes_memory_instruction_routing()`, `observes_vgpr_reads()`, +`observes_vgpr_writes()`, `observes_sgpr_reads()`, +`observes_scalar_register_writes()`, `observes_memory_routing()`, or +`observes_tensor_dma_memory_access()`. The group samples these stable flags +when the plugin is added and avoids both fanout and upstream observation work +when no contained plugin consumes a hook. + +For finer-grained selection, `observes_hot_hooks_for_wavefront(wf)` controls +all high-frequency callbacks for one wavefront. After every plugin's +`onAmdgpuWavefrontDispatched()` callback completes, the group samples and caches +that predicate on the wavefront until halt. Reentrant hooks during dispatch or +halt, hooks without a wavefront, and hooks on a resident wave after live plugin- +group replacement use the current group's live predicate instead. The predicate +must therefore be lock-free, thread-safe, and stable for the subscribed portion +of a wavefront's lifetime. Lifecycle, dispatch, workgroup, wavefront, and barrier +callbacks are not filtered by this predicate. + +Live replacement refreshes subscriptions for resident waves, but does not +replay their dispatch callbacks or migrate or clear their wave-local plugin +state. A newly attached stateful plugin therefore has no dispatch-initialized +state for those waves, while a replacement that reuses a plugin slot can see +state left by the previous plugin. Stateful plugin groups should be replaced at +a quiescent boundary unless they explicitly tolerate both cases. + ### Observing memory accesses There are two memory hooks, and they see different things. @@ -366,11 +398,16 @@ return `true`; the group samples this policy when each plugin is added, and its conservative default is `false`. Overriding the hook alone is silent — the plugin simply never sees an access. -Plugins that also need execution state may override the context-preserving -`onAmdgpuMemoryAccessRouted(access, inst, wf)` form. Its default implementation -forwards to the observation-only form, so existing observers keep the same -behavior. The borrowed instruction and wavefront already reflect the selected -route and are valid only during the callback. +The routed-memory callback has a source-compatible overload chain. Existing +plugins may override the observation-only +`onAmdgpuMemoryAccessRouted(access)` form. Plugins that need wave-local state may +instead override `onAmdgpuMemoryAccessRouted(access, wf)`, avoiding a shared +identity lookup. Plugins that also need the decoded instruction may override +the context-preserving `onAmdgpuMemoryAccessRouted(access, inst, wf)` form. +RocJITsu calls the most specific form; its defaults forward through the +wavefront-aware form to the observation-only form. The borrowed instruction and +wavefront already reflect the selected route and are valid only during the +callback. The observation's spans borrow execution-owned storage and are valid only for the duration of the callback. A plugin that keeps one must copy them. @@ -384,7 +421,9 @@ with duplicates preserved. Empty and fully masked transfers produce no callback. Consumers must opt in through `observes_tensor_dma_memory_access()`. A consumer that retains addresses must use `size()` to budget and allocate the event, then call `copy_to()` during the callback; a failed copy means the observation is -malformed and must be rejected. +malformed and must be rejected. Tensor-DMA observers that need wave-local state +may override `onAmdgpuTensorDmaMemoryAccess(access, wf)`; its default forwards to +the observation-only overload. SGPR owner resolution is skipped when no contained plugin observes scalar register reads. Plugins that consume neither `onAmdgpuReadScalarRegister` nor @@ -412,11 +451,13 @@ concurrently. high-frequency and infrequent callbacks. Override `requires_serial_hot_hooks()` when that state cannot be protected within the plugin. -6. Override `observes_sgpr_reads()` to return `false` when the plugin does not - consume `onAmdgpuReadSgpr`, and `observes_memory_routing()` to return `true` - when it does consume `onAmdgpuMemoryAccessRouted`. A consumer of - `onAmdgpuTensorDmaMemoryAccess` must likewise return `true` from - `observes_tensor_dma_memory_access()`. +6. Override the per-hook `observes_*()` methods for the high-frequency hooks the + plugin consumes. Use `observes_hot_hooks_for_wavefront()` when subscription + also depends on the dispatched wavefront; keep that predicate lock-free, + thread-safe, and stable after dispatch. A consumer of + `onAmdgpuMemoryAccessRouted` or `onAmdgpuTensorDmaMemoryAccess` must explicitly + return `true` from the corresponding interest method because those two + conservative defaults are `false`. 7. Enable it by adding `"myname": { ... }` to the `plugins` section of the config file. 8. Return `true` from `supports_async_instructions()` only when the plugin accepts diff --git a/emulation/rocjitsu/docs/race-detector.md b/emulation/rocjitsu/docs/race-detector.md index 119ebc40dc97..6d6f4c8da426 100644 --- a/emulation/rocjitsu/docs/race-detector.md +++ b/emulation/rocjitsu/docs/race-detector.md @@ -8,6 +8,17 @@ Currently detects intra-workgroup races — cases where the value read from a register or LDS is not deterministic due to missing `s_waitcnt` or `s_barrier` instructions. +## Target scope + +The race detector focuses on pre-GFX12 architectures. Its current end-to-end +coverage exercises gfx950 (GFX9/CDNA4) and gfx1151 (GFX11.5/RDNA3.5). GFX12 +and later architectures are not supported. Some GFX12 split-counter behavior +is already modeled and covered by plugin tests, including partial load waits +and address-dependent FLAT counter participation. This partial +coverage does not establish complete counter, scheduling, or writeback support; +issue-time counter-capacity backpressure remains limited to CDNA1 through CDNA4 +and GFX11. + ## Quick start This section is a standalone guide to getting up and running with race @@ -180,12 +191,102 @@ following lifecycle: retired and, from the perspective of all threads in all wavefronts, the operation is complete. -Generic `FLAT_*` instructions have two independent completion obligations: -the vector-memory counter and the LDS counter. A wait on only one domain does -not complete the race-detector event; both domains must be satisfied. The -resolved route still determines whether the event accesses global memory or -LDS. This currently assumes that all active lanes select the same memory space; -mixed LDS/global lanes are tracked separately in #11456. +Generic `FLAT_*` instructions select counter domains from their resolved memory +requests. Global and scratch requests use the vector-memory counter; LDS +requests use the LDS counter. An instruction with requests in both domains uses +both counters. The runtime applies this conditional-participation model across +AMD GPU targets, for loads, stores and atomics. Validation on every physical GPU +architecture is not established. + +For uniform requests, the race detector uses only the selected memory domain. +For example, on gfx950 with global addresses in every requesting lane, this use +of `v8` has a sufficient wait: + +```asm +flat_load_dword v8, v[0:1] +s_waitcnt vmcnt(0) +flat_store_dword v[2:3], v8 +``` + +LDS-only loads instead require `lgkmcnt(0)`. Newer targets use their corresponding +split counters. Routing preserves decoded completion ordering and independent +obligations such as EXPCNT. In particular, CDNA4 FLAT completion stays unordered; +a nonzero wait cannot prove a result ready. + +Functional counter acquisition and both runtime checkers use the selected memory +domains. FLAT capacity constraints are applied after addresses are resolved. +Before reading the address operands, neither domain is guaranteed to need a +counter slot. A full LDS counter followed by a global-only FLAT therefore cannot +prove an older LDS result complete. + +A mixture of global and scratch lanes uses the same VMEM domain. Mixed LDS/global +observations still attach both obligations to the whole race-plugin event, so a +consumer restricted to an already-completed lane group can still get a false +report. `TODO(newling)` regression cases track this remaining #12237 gap. The +core checker's [FLAT register-readiness policy](memory-wait-counter-coverage.md#flat-register-readiness) +tracks the groups separately. Mixed LDS/global functional execution remains +limited by first-request-lane routing, tracked separately in #11456. + +An all-ones wait-count field is the architectural “do not wait” value. CDNA's +four-bit `lgkmcnt(15)` and six-bit `vmcnt(63)` therefore retire no events; +`lgkmcnt(14)` and `vmcnt(62)` are the largest values that can impose an +explicit wait. Hardware also prevents counter overflow by stalling issue. + +The physical LGKMCNT capacity is shared by every event accounted to LGKMCNT, +including LDS, GDS, scalar-memory, and message operations. Sharing that counter +does not mean those event classes complete in order with each other. The +detector therefore retires only the oldest prefix that is provably complete in +an ordered class. For example, a new LGKM-counted instruction issued after 15 +pending local-LDS operations proves that the oldest LDS operation completed, +even when the new instruction is scalar memory or GDS. Mixed-class pressure +that does not identify a completed event remains conservatively pending. VMCNT +is handled the same way for its 63-entry ordered non-FLAT VMEM class. + +The detector's persistent counter accounting is limited to operations routed +through rocJITsu's memory pipelines. A narrow `s_sendmsg*` fallback applies one +token of issue-time LGKMCNT pressure, but message occupancy and returning +message results are not retained as pending events. Timestamp-query operations +such as `s_memtime` and `s_memrealtime` are not accounted for. These operations +therefore remain unsupported rather than being treated as complete message or +timestamp modeling. + +For example, CDNA cannot issue the final scalar load below while all 15 earlier +LGKM tokens remain outstanding: + +```asm +ds_read_b32 v0, v16 +ds_read_b32 v1, v16 +; ... 13 more ordered LDS reads, through v14 ... +s_load_dword s4, s[2:3], 0 +``` + +The scalar load can issue only after the LGKMCNT value drops below its +four-bit capacity. Because all preceding operations are ordered LDS reads, this +proves that the oldest read into `v0` completed. If those pending operations +belonged to different or unordered classes, the capacity stall would not prove +which individual event completed. + +The detector records the target-specific wait-counter family on every event. +It handles both the combined wait fields and the standalone counter forms on +supported targets, for example: + +```asm +s_waitcnt vmcnt(0) lgkmcnt(0) ; combined fields +s_waitcnt_vmcnt null, 0 ; standalone VMCNT form +s_waitcnt_lgkmcnt null, 0 ; standalone LGKMCNT form +``` + +Counter-capacity backpressure is modeled for the VMCNT and LGKMCNT domains on +CDNA1 through CDNA4 and GFX11. CDNA uses a four-bit LGKMCNT and six-bit VMCNT; +GFX11 uses six-bit fields for both. GFX11 vector stores use the separate VSCNT +domain and therefore do not create VMCNT pressure. Generic FLAT contributes to +both modeled domains, so both capacity constraints are applied before operand +reads. + +The same completion-order distinction is used for nonzero partial waits; a +zero wait still completes every event on the selected counter. The detector +also uses these classes to determine whether two asynchronous writes to the +same VGPR are ordered. The plugin keeps track, for all registers and LDS memory bytes, of which memory operations are in flight. When an instruction in the emulator accesses an LDS @@ -249,12 +350,12 @@ regardless of the order the waves execute in: 508–511, the address wave 1 wrote. The detector validates that no live writes overlap: - - *Fast path*: the write count for the 16-byte chunk containing byte 508 is - non-zero (wave 1's write is still live). Falls through to slow path. - - *Slow path*: scans live write events. Finds wave 1's event covering bytes - 508–511. The event is **WAVE_COMPLETE**, not **RETIRED**, and the accessing - wave (0) differs from the owning wave (1). - - **Race reported.** + - *Fast path*: the write count for the 16-byte chunk containing byte 508 is + non-zero (wave 1's write is still live). Falls through to slow path. + - *Slow path*: scans live write events. Finds wave 1's event covering bytes + 508–511. The event is **WAVE_COMPLETE**, not **RETIRED**, and the accessing + wave (0) differs from the owning wave (1). + - **Race reported.** 1. **What `s_barrier` would fix.** If an `s_barrier` had appeared between steps 4 and 5, the detector would flush all **WAVE_COMPLETE** events to @@ -312,11 +413,10 @@ ctest --test-dir $BUILD_DIR -R "RaceTest" (missing `s_waitcnt` and `s_barrier`). It does not detect inter-workgroup races, races between dispatches, or host-device synchronization issues. -- **Limited WAW detection**: VGPR WAW is limited to synchronous instruction - writes that overlap a pending asynchronous load. Scalar-register WAW covers - instruction writes and scalar loads that overlap pending scalar-memory - destinations. WAW between two asynchronous VGPR writers and LDS WAW are not - currently reported. +- **Limited WAW detection**: VGPR WAW covers instruction writes and + asynchronous memory writes that overlap a pending load. Scalar-register WAW + covers instruction writes and scalar loads that overlap pending scalar-memory + destinations. LDS WAW is not currently reported. - **Conservative DPP/SDWA write masks**: WAW precision depends on the execution plugin's instruction-write lane and byte masks. DPP destinations can currently diff --git a/emulation/rocjitsu/docs/rocjitsu-cli.md b/emulation/rocjitsu/docs/rocjitsu-cli.md index c0589ca3eb57..0abdbe4c11fe 100644 --- a/emulation/rocjitsu/docs/rocjitsu-cli.md +++ b/emulation/rocjitsu/docs/rocjitsu-cli.md @@ -172,6 +172,7 @@ data. This is the standard Unix mechanism for cross-process file descriptor sharing (same pattern as QEMU vhost-user and CRIU). Two RPC operations carry memfds: + - `RPC_IOCTL` (ALLOC_MEMORY response) --- allocation backing memfd - `RPC_MMAP` (response) --- doorbell and event page memfds @@ -253,6 +254,7 @@ public: `UnixTransport` wraps a Unix domain socket with SCM_RIGHTS for handle passing. Static factory methods: + - `UnixTransport::listen(endpoint)` --- create a listening socket - `UnixTransport::connect(endpoint)` --- connect to a daemon - `transport->accept()` --- accept a client connection @@ -262,6 +264,7 @@ The endpoint is `$XDG_RUNTIME_DIR/rocjitsu/daemon.sock` by default. ### Platform Portability The transport abstraction enables future Windows support: + - Unix: Unix domain socket + SCM_RIGHTS - Windows: Named pipe + DuplicateHandle diff --git a/emulation/rocjitsu/docs/rocjitsu_dbt_guest.md b/emulation/rocjitsu/docs/rocjitsu_dbt_guest.md index 8bd61f3d5a98..d52096bb999b 100644 --- a/emulation/rocjitsu/docs/rocjitsu_dbt_guest.md +++ b/emulation/rocjitsu/docs/rocjitsu_dbt_guest.md @@ -64,53 +64,53 @@ Underneath: 1. ROCR must internally discover both the real host GPU and the synthetic guest GPU. - The selected host must stay available for queues, allocations, and code - loading. The synthetic guest must also exist so ROCR can build the guest HSA - agent. Public HSA iteration is then shadowed: rocjitsu emits the guest agent - in the selected host's ordinal slot and suppresses the guest's own slot. This - keeps applications that choose the first GPU on the guest path while leaving - the host agent alive for execution. - - If `ROCR_VISIBLE_DEVICES` is set, the launcher preserves its selected order - and expands it with the appended guest ordinal so both sides remain visible - internally before the HSA hook applies public shadowing. Because CLR applies - `HIP_VISIBLE_DEVICES` or `CUDA_VISIBLE_DEVICES` after HSA agent iteration, - the launcher normalizes an existing client selector to equivalent numeric - ordinals after ROCR filtering. This keeps host UUID selection meaningful - after the host's public identity is replaced by the guest. OpenCL's - `GPU_DEVICE_ORDINAL` remains owned by the client runtime and is not used to - choose the DBT host. + The selected host must stay available for queues, allocations, and code + loading. The synthetic guest must also exist so ROCR can build the guest HSA + agent. Public HSA iteration is then shadowed: rocjitsu emits the guest agent + in the selected host's ordinal slot and suppresses the guest's own slot. This + keeps applications that choose the first GPU on the guest path while leaving + the host agent alive for execution. + + If `ROCR_VISIBLE_DEVICES` is set, the launcher preserves its selected order + and expands it with the appended guest ordinal so both sides remain visible + internally before the HSA hook applies public shadowing. Because CLR applies + `HIP_VISIBLE_DEVICES` or `CUDA_VISIBLE_DEVICES` after HSA agent iteration, + the launcher normalizes an existing client selector to equivalent numeric + ordinals after ROCR filtering. This keeps host UUID selection meaningful + after the host's public identity is replaced by the guest. OpenCL's + `GPU_DEVICE_ORDINAL` remains owned by the client runtime and is not used to + choose the DBT host. 2. Guest-facing discovery stays guest-shaped. - Calls that applications use to decide which kernel image to load, such as - agent name and ISA iteration, should report the synthetic guest agent. If we - mapped those calls to the host, the application would choose host kernels - and skip DBT. + Calls that applications use to decide which kernel image to load, such as + agent name and ISA iteration, should report the synthetic guest agent. If we + mapped those calls to the host, the application would choose host kernels + and skip DBT. 3. Execution-facing calls map guest handles to host handles. - Queue creation, code-object load, symbol lookup, memory access, and AMD - extension operations that take an agent are redirected from the guest agent - to the selected host agent. + Queue creation, code-object load, symbol lookup, memory access, and AMD + extension operations that take an agent are redirected from the guest agent + to the selected host agent. 4. KFD emulation stops at guest discovery. - `GuestKfd` should not run queues, allocate real guest VRAM, or process AQL - packets. It does handle enough startup plumbing for ROCR discovery: process - apertures, clock counters, VM acquisition, available-memory queries, memory - policy setup, synthetic guest memory handles, and guest-to-host gpu_id - rewrites for map/unmap requests. If a guest execution ioctl is reached after - the HSA hooks are in place, that is a missed HSA interception and should be - visible in logs. + `GuestKfd` should not run queues, allocate real guest VRAM, or process AQL + packets. It does handle enough startup plumbing for ROCR discovery: process + apertures, clock counters, VM acquisition, available-memory queries, memory + policy setup, synthetic guest memory handles, and guest-to-host gpu_id + rewrites for map/unmap requests. If a guest execution ioctl is reached after + the HSA hooks are in place, that is a missed HSA interception and should be + visible in logs. 5. Code objects are loaded against the host ROCR agent. - ROCR validates code-object ISA against the load agent using ELF header - fields. The hook translates the ELF to host ISA and calls the original - `hsa_executable_load_agent_code_object()` with the host agent, not the guest - agent. Symbol queries using the guest agent are later remapped to the host - agent so application code still works. + ROCR validates code-object ISA against the load agent using ELF header + fields. The hook translates the ELF to host ISA and calls the original + `hsa_executable_load_agent_code_object()` with the host agent, not the guest + agent. Symbol queries using the guest agent are later remapped to the host + agent so application code still works. ## Architecture @@ -208,17 +208,18 @@ rocjitsu goes through the patched table and is covered. `hsa_executable_load_agent_code_object`), so whichever ROCR loads second wraps the first one's wrappers. - Note that `HSA_TOOLS_LIB` does not decide automatic loading. ROCR calls - `Runtime::LoadHotswapTool()` before it reads `HSA_TOOLS_LIB`. Current releases - load the hotswap hook when `HSA_HOTSWAP_ENABLE` is true and a gfx1250 A0 agent - is present; older releases load it unless `HSA_HOTSWAP_DISABLE` is true. - - The supported DBT launch path enforces one hook across both runtime - generations: `rocjitsu` sets `HSA_HOTSWAP_ENABLE=0` and - `HSA_HOTSWAP_DISABLE=1`, then replaces `HSA_TOOLS_LIB` with the DBT hook rather - than appending to it. An inherited parent setting therefore cannot enable the - automatic hotswap hook. Neither library exports `HSA_AMD_TOOL_PRIORITY`, so - nothing pins a slot among tools that do come from `HSA_TOOLS_LIB`. + Note that `HSA_TOOLS_LIB` does not decide automatic loading. ROCR calls + `Runtime::LoadHotswapTool()` before it reads `HSA_TOOLS_LIB`. Current releases + load the hotswap hook when `HSA_HOTSWAP_ENABLE` is true and a gfx1250 A0 agent + is present; older releases load it unless `HSA_HOTSWAP_DISABLE` is true. + + The supported DBT launch path enforces one hook across both runtime + generations: `rocjitsu` sets `HSA_HOTSWAP_ENABLE=0` and + `HSA_HOTSWAP_DISABLE=1`, then replaces `HSA_TOOLS_LIB` with the DBT hook rather + than appending to it. An inherited parent setting therefore cannot enable the + automatic hotswap hook. Neither library exports `HSA_AMD_TOOL_PRIORITY`, so + nothing pins a slot among tools that do come from `HSA_TOOLS_LIB`. + - `HSA_TOOLS_DISABLE_REGISTER=1` is a workaround. The better design is a rocprofiler-register API-table interposer that applies the same shadowing before rocprofiler validates HSA agents. diff --git a/emulation/rocjitsu/docs/simdojo.md b/emulation/rocjitsu/docs/simdojo.md index 2ea3d16681e6..3cfd741594ac 100644 --- a/emulation/rocjitsu/docs/simdojo.md +++ b/emulation/rocjitsu/docs/simdojo.md @@ -16,19 +16,19 @@ event loop via the `Functional` CRTP mixin. | `component.h` | Node, Component, CompositeComponent, Link, Port, QueuedLink | | `component.cpp` | Implementation for Component, CompositeComponent, Link | | `clock_domain.h` | ClockDomain (frequency, period, phase offset) | -| `clocked.h` | Clocked\ Curiously Recurring Template Pattern (CRTP) mixin for clock-driven components | +| `clocked.h` | `Clocked` Curiously Recurring Template Pattern (CRTP) mixin for clock-driven components | | `topology.h` | Topology, Partition, AdjacencyGraph, Partitioner | | `topology.cpp` | Topology wiring, graph construction, multilevel Fiduccia-Mattheyses (FM) partitioning | -| `functional.h` | Functional\ CRTP mixin for untimed components | -| `exec_mode.h` | ExecMode enum (FUNCTIONAL, CLOCKED) and ExecBase\ alias | +| `functional.h` | `Functional` CRTP mixin for untimed components | +| `exec_mode.h` | ExecMode enum (FUNCTIONAL, CLOCKED) and `ExecBase` alias | | `pacing_controller.h` | PacingController - PI-controlled wall-clock pacing with token bucket burst absorption | | `simulation.h` | PartitionContext, SimulationEngine | | `simulation.cpp` | Engine run loop, event dispatch, LBTS computation (multi-threaded) | | `components/memory_interface.h` | MemoryInterface - abstract byte-addressed memory controller | -| `components/cache.h` | Cache\ - set-associative cache template | +| `components/cache.h` | `Cache` - set-associative cache template | | `components/sparse_memory.h` | SparseMemory - page-allocated flat address space | | `components/register_file.h` | RegisterFile - typed register storage with read/write interface | -| `components/vector_reg.h` | VectorReg\ - SIMD-width vector register type | +| `components/vector_reg.h` | `VectorReg` - SIMD-width vector register type | | *(uses `util/debug_print.h`)* | Conditional debug output (`util::debug` namespace) | --- @@ -229,7 +229,10 @@ following dimensions: `RegisterFile` divides the physical VGPR index space into fixed 4 KiB chunks. An untouched chunk is represented by a null entry and reads as zero; its storage is allocated and zero-initialized on the first mutable access. Retiring -a wave releases wholly covered chunks and clears any shared boundary chunk. +a wave makes wholly covered chunks logically absent and clears any shared +boundary chunk. Each lazy register file retains up to 64 KiB of retired chunks, +which are zeroed when reused; excess chunks release their allocations. +`materialized_chunk_count()` excludes these retained spares. The implementation uses only portable C++ allocation and does not require virtual-memory APIs. Mutable handles and handles into materialized registers remain stable until their allocation retires, but the complete register file is diff --git a/emulation/rocjitsu/docs/sphinx/conceptual/gpu-vm-design.md b/emulation/rocjitsu/docs/sphinx/conceptual/gpu-vm-design.md index 7b829935daf7..deeb93b55f57 100644 --- a/emulation/rocjitsu/docs/sphinx/conceptual/gpu-vm-design.md +++ b/emulation/rocjitsu/docs/sphinx/conceptual/gpu-vm-design.md @@ -51,15 +51,28 @@ embedded in the code object, determines the grid and workgroup dimensions, and dispatches workgroups to available CUs across the shader engines within its XCD. -AQL, PM4, and SDMA use a common compile-time packet-processor contract but -retain distinct concrete processors. The command processor owns the AQL and -PM4 processors; the SoC-owned SDMA scheduler owns the SDMA processor. Queue +AQL and SDMA use a common compile-time packet-processor contract but +retain distinct concrete processors. The command processor owns the AQL +processor; the SoC-owned SDMA scheduler owns the SDMA processor. Queue owners, rather than packet processors or PCI/VFIO adapters, retain ring cursors, VM snapshots, retry state (including SDMA's opaque typed continuation), and scheduling policy. They also retain a VM binding lease until detach, preventing address-space teardown while future snapshots can still be created without conflating execution lifetime with frontend references. +PM4 compute rings and DRM indirect-buffer submissions are processed in +`pm4/pm4_packet_processor`. Packet fetching, opcode effects, register updates, +IB traversal, and cursor publication operate on CP-owned queue state. CP supplies +cache flushing, dispatch admission, retry scheduling, and fault cancellation. +For native rings, CP captures the VM snapshot, initializes the consumer cursor, +builds root submissions from doorbells, and retries pending cursor publication +even while execution is suspended. The processor traverses those submissions and +nested IBs and commits/publishes packet retirement through the shared cursor journal. +Retry sets `command_retry_pending` until CP resumes the queue; dispatch admission +appends to `dispatches.entries` until CP retires the work. The processor marks +`publication_faulted` before requesting cancellation on a terminal publication +failure. + SDMA queues use the separate SoC-owned SDMA scheduler, ring consumer, and packet processor; no SDMA packet logic is part of the CP. A completion tracker inside each XCD monitors per-dispatch workgroup diff --git a/emulation/rocjitsu/docs/sphinx/conceptual/json-configuration.md b/emulation/rocjitsu/docs/sphinx/conceptual/json-configuration.md index 24ba4412c76f..741438f574fb 100644 --- a/emulation/rocjitsu/docs/sphinx/conceptual/json-configuration.md +++ b/emulation/rocjitsu/docs/sphinx/conceptual/json-configuration.md @@ -44,7 +44,7 @@ objects for the virtual machine and the topology. | `max_ticks` | int | Maximum simulation ticks. A value of `0` means unlimited. | | `num_threads` | int | Simdojo engine partitions, clamped to the aggregate XCD count. | | `cpu_dispatch_threads` | int | Inclusive dispatch width per SoC; omitted/0 selects from preferred allocations, 1 forces serial. | -| `cpu_thread_budget` | int | Selection ceiling; omitted/0 uses affinity capped at 32, positive values override it. | +| `cpu_thread_budget` | int | Selection ceiling; omitted/0 uses affinity and target allocations. Positive values override the budget. | | `async_helper_threads` | int | Shared MMA helpers: -1 selects the table, 0 disables, 1–128 overrides. | | `thread_allocations` | array | Preferred engine, dispatch and helper allocations for this target. Largest fitting effective allocation wins. | | `exec_mode` | string | Execution mode: `"functional"` or `"clocked"`. | @@ -55,7 +55,8 @@ objects for the virtual machine and the topology. single XCD is never split between engine partitions. In functional mode, `cpu_dispatch_threads` controls the host parallelism used to execute accepted CU work. Its pool is shared by all command processors in a SoC, and each width -is clamped to per-CP CU capacity. The selection counts engines, all retained +is bounded by one plus the sum of CUs-1 across nonempty XCDs. Each submission +still uses at most its active CU count. The selection counts engines, all retained dispatch workers and shared helpers: E + sum(D - 1) + H. Explicit knobs override the selected allocation; configs without a table use serial defaults. Clocked mode always uses serial dispatch. See the source diff --git a/emulation/rocjitsu/docs/sphinx/how-to/configure-topology.md b/emulation/rocjitsu/docs/sphinx/how-to/configure-topology.md index 10e0b09ab6d6..7b1941bc2a49 100644 --- a/emulation/rocjitsu/docs/sphinx/how-to/configure-topology.md +++ b/emulation/rocjitsu/docs/sphinx/how-to/configure-topology.md @@ -66,7 +66,9 @@ keep all XCDs on one engine partition. The host-thread width used to execute accepted CU work in functional mode. A nonzero value is applied per SoC and shared by all command processors in that SoC. Omission or `0` selects a preferred allocation from `thread_allocations`; -`1` forces serial dispatch. The effective width is capped by per-CP CU capacity. +`1` forces serial dispatch. The shared pool covers concurrent XCD submissions; +its width is bounded by one plus the sum of CUs-1 across nonempty XCDs. +Each submission uses at most its active CU count. Clocked mode always uses a width of one. ### `async_helper_threads` @@ -80,8 +82,10 @@ other shipped targets and multi-GPU defaults keep H=0. See ### Execution budget and preferred allocations `cpu_thread_budget` is a ceiling for engines, retained dispatch workers and shared MMA helpers: -E + sum(D - 1) + H. Its default is process affinity capped at 32. -A positive value overrides the ceiling. The selector picks the largest fitting +E + sum(D - 1) + H. Its default is process affinity and the target allocation +table. Desktop/older CDNA presets stop at 24 engine/dispatch threads; gfx950 +adds eight helpers, while gfx1250 uses up to 40 engine/dispatch threads and +eight helpers. A positive budget overrides affinity. The selector picks the largest fitting entry in `thread_allocations`; it leaves unused budget between granules. Explicit E/D/H knobs take priority and may exceed the automatic budget. A config without a table uses serial defaults for unspecified knobs. diff --git a/emulation/rocjitsu/docs/vm-design.md b/emulation/rocjitsu/docs/vm-design.md index fbfced409235..5a3c07f3bd67 100644 --- a/emulation/rocjitsu/docs/vm-design.md +++ b/emulation/rocjitsu/docs/vm-design.md @@ -31,13 +31,12 @@ execution. | `amdgpu/sdma_ring_consumer.h/cpp` | `SdmaRingConsumer`: per-queue ring cursor, fetch, retry, execution, and retirement publication | | `amdgpu/sdma_packet_processor.h/cpp` | SdmaPacketProcessor: SDMA packet decoding and effects over caller-owned typed continuation state | | `amdgpu/packet_processor.h` | Shared compile-time one-packet contract and validated protocol-independent result envelope | -| `amdgpu/aql_packet_processor.h/cpp` | AqlPacketProcessor: fixed-size AQL decode and durable CP admission | -| `amdgpu/pm4_packet_processor.h/cpp` | Pm4PacketProcessor: PM4 framing, validation, and supported packet effects | -| `amdgpu/pm4_ring_consumer.h/cpp` | `Pm4RingConsumer`: per-queue ring traversal, retry, and cursor publication owned by the CP | -| `amdgpu/pm4_queue_controller.h/cpp` | Pm4QueueController: CP-owned PM4 queue, VM snapshot, retry, and cursor-publication state | -| `amdgpu/pm4_queue_binding_factory.h/cpp` | Thin PM4 lifetime/notification adapter from GpuQueueRegistry to the owning CP | +| `amdgpu/aql/aql_packet_processor.h/cpp` | AqlPacketProcessor: fixed-size AQL decode and durable CP admission | +| `amdgpu/pm4/pm4_packet_processor.h/cpp` | Production PM4 fetch, opcode effects, register updates, IB traversal, and cursor publication | +| `amdgpu/pm4/pm4.h` | PM4 opcodes, registers, scratch leases, and persistent submission state | +| `amdgpu/dispatch_entry.h` | ComputeQueueRecord: CP-owned AQL/PM4 queues, nested command streams, shader dispatches and cursor publication | | `amdgpu/gpu_queue_registry.h/cpp` | Frontend-neutral queue admission, lifetime, routing, and generation-safe handles | -| `amdgpu/aql_queue_binding_factory.h/cpp` | Reusable binding adapter from GpuQueueRegistry to the AQL command processor | +| `amdgpu/compute_queue_binding_factory.h/cpp` | Reusable binding adapter from GpuQueueRegistry to the unified compute command processor | --- @@ -85,10 +84,16 @@ VMID/PASID as lifetime identity. An address-space slot generation changes when a slot is destroyed and reused. Its translation epoch changes when a root is replaced or invalidated. A -`GpuVmAccess` captures the handle, epoch, translator, and physical backing under -one lock for the duration of an operation. This prevents a multi-page access -from combining an old root with a replacement backing and provides -`VmCacheNamespace` for virtually indexed clean caches. Access results are +`GpuVmAccess` selects an immutable translator/backing generation under the +registry lock. Ordinary snapshots share its retirement state; each pinned +snapshot has independent retirement state and survives root replacement. +Invalidation and unregistration revoke both kinds. Access methods hold a shared +revocation lease, allowing an in-flight access to finish without combining an +old root with a replacement backing. The captured handle and epoch provide +`VmCacheNamespace` for virtually indexed clean caches. Functional instruction +fetch reuses an ordinary snapshot within a quantum and checks `is_current()` +before reuse; the access method still takes a lease and checks for retirement. +Access results are typed as complete, temporarily unavailable, faulted, or malformed; transport availability is not encoded as a fake mapping in `GpuMemory`. @@ -176,12 +181,12 @@ its published read pointer. These front ends must not duplicate scheduling, ring fetch, packet decoding, retry, or cursor-publication state. The command processor serves compute queues only and has no SDMA implementation or state. -The AQL command processor, restricted PM4 compute-queue path, and SDMA -scheduler use the same compile-time packet-processor interface and +The AQL command processor and SDMA scheduler use the same compile-time +packet-processor interface and `PacketProcessResult` envelope, not a common execution engine. SDMA may suspend after a partial effect, so its ring consumer owns an opaque typed continuation that only `SdmaPacketProcessor` -interprets; AQL and PM4 block only before committing an effect and can restart +interprets; AQL blocks only before committing an effect and can restart the same request. The common `required_bytes` field requests more head-packet input; `retirement_bytes` reports the eventual cursor advance and can include a protocol-defined skipped extent, which the ring owner must bound against its ring @@ -189,13 +194,32 @@ and producer cursor. Ring ownership, VM snapshots, scheduling, consumer publicat and completion remain outside packet processors. Guest-controlled AQL validation failures are reported through the typed packet/admission result and fault only their queue; internal invariant violations remain exceptions. -The firmware-free PM4 path supports a restricted compute-queue packet subset, +The firmware-free PM4 path supports the modeled compute-queue packet subset, rather than general PM4. MES maps its MQD and address space, but submits the queue through `GpuQueueRegistry` to a selected command processor. The binding retains only a CP registration token; the CP-owned queue controller retains the -VM snapshot, ring traversal, packet processor, and retry-safe cursor publication. -The PCI/MMIO layer supplies only the narrow register-write sink. PM4 service runs -outside the AQL queue lock and uses bounded queue turns for event-loop fairness. +VM snapshot and retry-safe cursor publication state. The production packet +processor in `pm4/` performs ring traversal, opcode effects, register updates, +and nested IB traversal over that CP-owned state. CP supplies cache flushing, +dispatch admission, retry scheduling, and synchronous fault cancellation +callbacks. Temporary unavailability retains command and cursor-publication state +for retry; cancellation stops resident waves before submission resources are +released. A snapshot captured before GART publication is unready and must be +released before retrying, so the next turn can capture the published binding. +Once ready, the snapshot stays pinned through execution and retirement publication. +For native rings, CP captures `command_access`, initializes the consumer cursor, +builds root submissions from doorbells, and retries pending publication even when +execution is suspended. The processor traverses packets and nested IBs, captures +access for submitted streams as needed, and commits/publishes packet retirement +through `read_pointer_journal`. The retry callback sets `command_retry_pending`, +which CP clears before resuming. Dispatch admission appends to `dispatches.entries`; +CP retires these entries before packet processing continues. On terminal publication +failure, the processor sets `publication_faulted` before invoking cancellation. +The cancellation callback marks the queue faulted, cancels pending work, and notifies +failed submissions while preserving the publication failure. +The PCI/MMIO layer supplies only the narrow register-write sink. PM4 service +uses bounded queue turns for event-loop fairness and retains CP queue +serialization. Graceful AQL removal is a publication barrier. The CP retains the registration while a consumed ring cursor, dispatch completion, or queue-idle signal still @@ -485,6 +509,7 @@ producer writes an AQL packet into the ring, then rings the doorbell The in-flight record the CP builds from that packet is a `DispatchEntry`, which carries among other things: + - `kernel_entry_pc` - byte address of kernel code in GPU memory - `total_wgs` - workgroups to launch, narrowed to this XCD's share once sharded - `wfs_per_workgroup` - wavefronts per workgroup @@ -525,6 +550,7 @@ stale reads. Supported integer atomics: swap, cmpswap, add, sub, smin/umin, smax/umax, and, or, xor, inc, dec. **Cache management instructions:** + - `s_dcache_inv` / `s_dcache_inv_vol` — invalidate the L1 scalar cache - `s_gl1_inv` — invalidate the L1 vector cache diff --git a/emulation/rocjitsu/docs/vulkan-compute.md b/emulation/rocjitsu/docs/vulkan-compute.md index ac89ec7f8792..6f860098effe 100644 --- a/emulation/rocjitsu/docs/vulkan-compute.md +++ b/emulation/rocjitsu/docs/vulkan-compute.md @@ -26,6 +26,11 @@ from a build with the Vulkan runtime driver and SPIR-V compiler backend enabled. Compare individual test results as well as suite results: an additional skipped case does not establish parity. +The release corpus workflow runs selected Vulkan CTS compute and texel-buffer +cases on both targets. See the +[corpus test guide](../tests/corpus/README.md#vulkan-compute-and-buffer-cts) +for a local smoke command using the same runner and dependencies as CI. + ## Scope and limitations Matched Vulkan workloads have been validated on W7900 (`gfx1100`) and R9700 diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 74ea7194fbbb..b14cfd4da7e0 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -22,6 +22,11 @@ """ import cgen + +from amdisa.codegen.execute.floating_policy import ( + FLUSH_NEAREST_F32_OPS, + ROUNDED_F16_OPS, +) import textwrap import re import os @@ -44,7 +49,12 @@ fieldless_policy, operand_participates, ) -from amdisa.semantics import F32_TO_INTEGER_DTYPES, InstructionSemantics, SemanticsSpec +from amdisa.semantics import ( + F16_INPUT_CONVERSION_DTYPES, + F32_TO_INTEGER_DTYPES, + InstructionSemantics, + SemanticsSpec, +) from amdisa.isa_profile import DppOpcodeRule, FloatDotAccumulation from amdisa.codegen.config import CodegenConfig @@ -394,6 +404,7 @@ class CodeGenerator: 'tbuffer_store': 'vmem_store', 'buffer_load_format_d16': 'vmem_load', 'buffer_store_format_d16': 'vmem_store', + 'lds_direct_load': 'direct', 'ds_read': 'local', 'ds_read2': 'local', 'ds_write': 'local', @@ -962,6 +973,15 @@ def _constructor_operand_type( ) -> str: if self._uses_generic_wmma_accumulator_selector(inst_sem, opnd): return 'OPR_SRC' + if ( + inst_sem + and inst_sem.semantic_class in ('vector_readfirstlane', 'vector_readlane') + and opnd.is_output + and opnd.operand_type == 'OPR_SREG_NOVCC' + ): + # CDNA1-4 metadata marks only these two lane-read instructions NOVCC, + # but their scalar destinations can hold VCC spill reloads. + return 'OPR_SREG' if ( inst_sem and inst_sem.semantic_class in ('vector_readfirstlane', 'vector_readlane') @@ -985,6 +1005,12 @@ def _uses_generic_wmma_accumulator_selector( and opnd.operand_type == 'OPR_SRC_VGPR_OR_INLINE' ) + @staticmethod + def _sdwa_output_policy(sem: InstructionSemantics | None) -> str: + if sem and sem.name in FLUSH_NEAREST_F32_OPS: + return ', amdgpu::sdwa::OutputPolicy::FLUSH_NEAREST' + return '' + @staticmethod def _sdwa_result_format(sem: InstructionSemantics | None) -> str: """Return the C++ result format for numerical SDWA output modifiers. @@ -1018,11 +1044,14 @@ def _sdwa_result_format(sem: InstructionSemantics | None) -> str: return f'amdgpu::sdwa::ResultFormat::{suffix}' @staticmethod - def _apply_sdwa_f16_omod(body: str, instruction: str) -> str: + def _apply_sdwa_f16_omod( + body: str, instruction: str, *, rounded_result: bool = False + ) -> str: """Apply SDWA OMOD at the F16 producer, before result narrowing.""" + helper = 'finish_rounded_f16' if rounded_result else 'round_f16_result' body = body.replace( 'util::f32_to_f16_mode(', - f'amdgpu::sdwa::round_f16_result({instruction}, wf, ', + f'amdgpu::sdwa::{helper}({instruction}, wf, ', ) body = body.replace( 'amdgpu::fp_mode::finish_arithmetic_f16(', @@ -1061,6 +1090,25 @@ def _sdwa_source_modifier_format( }.get(opnd.data_format_name, 'NONE') return f'amdgpu::sdwa::SourceModifierFormat::{suffix}' + @staticmethod + def _dpp_source_sign_bit( + sem: InstructionSemantics, source_index: int, opnd: Operand + ) -> int: + if source_index == 1 and ( + sem.semantic_class in ('vector_cmp_class', 'vector_cmpx_class') + or (sem.semantic_class == 'vector_binop' and sem.operation == 'ldexp') + ): + return 0 + if sem.name == 'V_PK_FMAC_F16': + return 0 + bit = { + 'FMT_NUM_F16': 15, + 'FMT_NUM_BF16': 15, + 'FMT_NUM_F32': 31, + 'FMT_NUM_F64': 63, + }.get(opnd.data_format_name) + return 0 if bit is None else 1 << bit + @staticmethod def _literal_encoding_info( enc: InstEncoding, inst_enc_obj: InstEncoding | None, inst: Instruction @@ -1876,6 +1924,12 @@ def gen_execution_backend(self) -> None: f' &execute_with_backend<{class_name}>,\n' for class_name in self._split_execution_classes ) + # Generated executors follow Instruction's decoded-reuse contract: + # non-memory instructions without DynamicInstState can execute again + # on another wave. Read values and EXEC from the current context, + # restore temporary operand delegates, and assign per-execution flags + # on every call. Persistent per-issue state belongs in DynamicInstState, + # not decoded encoding or operand members. source = textwrap.dedent(f'''\ {CppFile._prologue_comment()} #include "{generated_arch}/execution_backend.h" @@ -2065,7 +2119,7 @@ def opcode_mask(name: str, opcodes: frozenset[int]) -> str: { float result = amdgpu::fp_mode::arithmetic(std::bit_cast(src0), std::bit_cast(src1), - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(*slot.dst, lane)), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(*slot.dst, lane)), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } ''', @@ -2076,7 +2130,7 @@ def opcode_mask(name: str, opcodes: frozenset[int]) -> str: { float result = amdgpu::fp_mode::arithmetic(std::bit_cast(src0), std::bit_cast(src1), - std::bit_cast(src2), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + std::bit_cast(src2), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } ''', @@ -2087,7 +2141,7 @@ def opcode_mask(name: str, opcodes: frozenset[int]) -> str: { float result = amdgpu::fp_mode::arithmetic(std::bit_cast(src0), std::bit_cast(src2), - std::bit_cast(src1), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + std::bit_cast(src1), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } ''', @@ -2096,7 +2150,7 @@ def opcode_mask(name: str, opcodes: frozenset[int]) -> str: ('VopdMulF32',), ''' { - float result = amdgpu::fp_mode::arithmetic(std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + float result = amdgpu::fp_mode::arithmetic(std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } ''', @@ -2115,7 +2169,7 @@ def opcode_mask(name: str, opcodes: frozenset[int]) -> str: ('VopdAddF32',), ''' { - float result = amdgpu::fp_mode::arithmetic(std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + float result = amdgpu::fp_mode::arithmetic(std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } ''', @@ -2203,7 +2257,7 @@ def opcode_mask(name: str, opcodes: frozenset[int]) -> str: { float result = amdgpu::fp_mode::arithmetic(std::bit_cast(src0), std::bit_cast(src1), - std::bit_cast(src2), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + std::bit_cast(src2), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } ''', @@ -4021,9 +4075,17 @@ def gen_encodings(self) -> None: ' std::memcpy(raw_words_.data(), inst, size_);' ' raw_encoding_ = raw_words_.data();' ) + if dpp_opcodes and enc_upper in ('ENC_VOP1', 'ENC_VOP2', 'ENC_VOPC'): + size_line += ( + ' if (has_encoded_dpp())' + ' dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode(' + f'*reinterpret_cast(inst));' + ) if supports_fixed_size_embedding: size_line += ' }' validation_body += ' }' + if profile.vskip_affected_encoding(enc_upper): + size_line += ' flags_ |= VSKIP_AFFECTED;' validation_body += ' return Result::success();' if has_encoding_validation: public_members.append( @@ -4057,44 +4119,45 @@ def gen_encodings(self) -> None: f'{{{size_line}}}' ) class_func_impls.append(cgen.Line(class_ctor_impl)) - if profile.renders_gfx11_image_syntax and enc_upper == 'ENC_MIMG': + if profile.has_gfx11_image_address_extension and enc_upper == 'ENC_MIMG': public_members.append( cgen.Line( 'void capture_nsa_words(const MachineInst *inst, ' 'const Operand *vaddr);' ) ) - public_members.append( - cgen.Line( - 'void append_src_operand(std::string &out, ' - 'uint8_t operand_index) const override {\n' - ' const Operand *operand = src_operands_[operand_index];\n' - ' if (!inst_.nsa || operand != nsa_vaddr_operand_) {\n' - ' Instruction::append_src_operand(out, operand_index);\n' - ' return;\n' - ' }\n' - ' const uint32_t vaddr_words = (operand->size_bits() + 31) / 32;\n' - ' out += "[";\n' - ' uint32_t consumed_words = 0;\n' - ' for (uint32_t index = 0; index < 5 && consumed_words < vaddr_words; ++index) {\n' - ' const uint32_t group_words = gfx11_mimg_nsa_group_width(index, vaddr_words);\n' - ' if (group_words == 0) break;\n' - ' if (index != 0) out += ", ";\n' - ' const uint32_t selector = index == 0\n' - ' ? inst_.vaddr\n' - ' : (raw_words_[2] >> ((index - 1) * 8)) & 0xffu;\n' - ' if (group_words > 1) {\n' - ' out += "v[" + std::to_string(selector) + ":";\n' - ' out += std::to_string(selector + group_words - 1) + "]";\n' - ' } else {\n' - ' out += "v" + std::to_string(selector);\n' - ' }\n' - ' consumed_words += group_words;\n' - ' }\n' - ' out += "]";\n' - '}' + if profile.renders_gfx11_image_syntax: + public_members.append( + cgen.Line( + 'void append_src_operand(std::string &out, ' + 'uint8_t operand_index) const override {\n' + ' const Operand *operand = src_operands_[operand_index];\n' + ' if (!inst_.nsa || operand != nsa_vaddr_operand_) {\n' + ' Instruction::append_src_operand(out, operand_index);\n' + ' return;\n' + ' }\n' + ' const uint32_t vaddr_words = (operand->size_bits() + 31) / 32;\n' + ' out += "[";\n' + ' uint32_t consumed_words = 0;\n' + ' for (uint32_t index = 0; index < 5 && consumed_words < vaddr_words; ++index) {\n' + ' const uint32_t group_words = gfx11_mimg_nsa_group_width(index, vaddr_words);\n' + ' if (group_words == 0) break;\n' + ' if (index != 0) out += ", ";\n' + ' const uint32_t selector = index == 0\n' + ' ? inst_.vaddr\n' + ' : (raw_words_[2] >> ((index - 1) * 8)) & 0xffu;\n' + ' if (group_words > 1) {\n' + ' out += "v[" + std::to_string(selector) + ":";\n' + ' out += std::to_string(selector + group_words - 1) + "]";\n' + ' } else {\n' + ' out += "v" + std::to_string(selector);\n' + ' }\n' + ' consumed_words += group_words;\n' + ' }\n' + ' out += "]";\n' + '}' + ) ) - ) class_func_impls.append( cgen.Line( f'void {inst_enc.fmt_enc_name}::capture_nsa_words(' @@ -4388,7 +4451,7 @@ def gen_encodings(self) -> None: f'std::array raw_words_{{}}' ) ) - elif profile.renders_gfx11_image_syntax and enc_upper == 'ENC_MIMG': + elif profile.has_gfx11_image_address_extension and enc_upper == 'ENC_MIMG': class_members.append( cgen.Statement('std::array raw_words_{}') ) @@ -4418,6 +4481,9 @@ def gen_encodings(self) -> None: if _dpp8_struct: class_members.append(cgen.Statement('uint32_t dpp8_lane_sel_ = 0')) if _enc_upper in ('ENC_VOP1', 'ENC_VOP2', 'ENC_VOPC'): + class_members.append( + cgen.Statement('amdgpu::dpp::SourceModifiers dpp_modifiers_') + ) # SDWA fields (CDNA and RDNA1/2 have hardware SDWA encoding; fields # are present on all ISAs for uniform codegen even if unused). class_members.append( @@ -6303,9 +6369,17 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: is_f32_to_integer = ( cls == 'vector_unary' and dtype in F32_TO_INTEGER_DTYPES ) + is_f16_input_conversion = ( + cls == 'vector_unary' and dtype in F16_INPUT_CONVERSION_DTYPES + ) if ( is_vop3 - and (is_float_op or is_integer_to_f32 or is_f32_to_integer) + and ( + is_float_op + or is_integer_to_f32 + or is_f32_to_integer + or is_f16_input_conversion + ) and not is_true16_mov and cls != 'pseudo_scalar_unary' ): @@ -6435,27 +6509,6 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: f' amdgpu::RegisterAccess(wf).write_lane({dst_ops[0]}, lane, (({selector_read} >> lane) & 1) ? src1_value : src0_value);\n' ' }\n' ) - if ( - inst.name == 'V_CVT_F32_F16' - and is_true16_vop3 - and src_ops - and dst_ops - ): - return ( - ' uint64_t exec = wf.exec();\n' - ' const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_);\n' - ' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {\n' - ' if (!(exec & (1ULL << lane)))\n' - ' continue;\n' - f' uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src({src_ops[0]}, wf, lane, opsel, 0);\n' - ' float src = util::f16_to_f32(static_cast(raw));\n' - ' if (inst_.abs & (1u << 0))\n' - ' src = std::fabs(src);\n' - ' if (inst_.neg & (1u << 0))\n' - ' src = -src;\n' - f' amdgpu::RegisterAccess(wf).write_lane({dst_ops[0]}, lane, std::bit_cast(src));\n' - ' }\n' - ) true16_special_vop3_ops = { 'V_ASHRREV_I16': ( 2, @@ -6538,7 +6591,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: ' src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u,\n' ' inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u,\n' ' wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod,\n' - ' inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));\n' + ' inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));\n' f'{write_result}\n' ' }\n' ) @@ -6657,7 +6710,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: f' src0_bits, src1_bits, accumulator, {abs0}, {abs1}, false,\n' f' {neg0}, {neg1}, false, wf.fp_round_mode_f16_f64(),\n' f' wf.fp_denorm_mode_f16_f64(), omod, {clamp}, wf.fp16_ovfl(),\n' - ' amdgpu::floating_clamp_nan_to_zero(wf));\n' + ' amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));\n' f'{write_result}\n' ' }\n' ) @@ -6726,6 +6779,14 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: return self._sleep_body(sem) return self._trap_control_body(sem) or ' (void)wf;' + if cls == 'set_vskip': + return ( + ' const uint32_t source = amdgpu::RegisterAccess(wf).read_scalar(ssrc0);\n' + ' const uint32_t bit = amdgpu::RegisterAccess(wf).read_scalar(ssrc1) & 31u;\n' + ' const uint32_t vskip = ((source >> bit) & 1u) * Wavefront::VSKIP_BIT;\n' + ' wf.set_mode_raw((wf.mode_raw() & ~Wavefront::VSKIP_BIT) | vskip);' + ) + if cls == 'gpr_idx': if op == 'on': return ( @@ -7234,15 +7295,12 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: if cls == 'vector_readfirstlane': L.append(' uint64_t exec = wf.exec();') - L.append(' uint32_t val = 0;') - L.append(' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {') - L.append(' if (exec & (1ULL << lane)) {') L.append( - f' val = amdgpu::RegisterAccess(wf).read_lane({src_ops[0]}, lane);' + ' uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0;' + ) + L.append( + f' uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane({src_ops[0]}, lane);' ) - L.append(' break;') - L.append(' }') - L.append(' }') L.append(f' amdgpu::RegisterAccess(wf).write_scalar({dst_ops[0]}, val);') return '\n'.join(L) @@ -7290,7 +7348,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: f' uint16_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane({s2_expr}, lane));' ) L.append( - f' uint16_t result = amdgpu::fp_mode::fma_f16(s0, k, s2, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));' + f' uint16_t result = amdgpu::fp_mode::fma_f16(s0, k, s2, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));' ) L.append( f' amdgpu::RegisterAccess(wf).write_lane({dst_ops[0]}, lane, result);' @@ -7330,7 +7388,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: f' float s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane({s2_expr}, lane));' ) L.append( - f' amdgpu::RegisterAccess(wf).write_lane({dst_ops[0]}, lane, std::bit_cast(amdgpu::fp_mode::arithmetic(s0, k, s2, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())));' + f' amdgpu::RegisterAccess(wf).write_lane({dst_ops[0]}, lane, std::bit_cast(amdgpu::fp_mode::arithmetic(s0, k, s2, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode())));' ) L.append(' }') return '\n'.join(L) @@ -7355,7 +7413,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: ) L.append(f' uint16_t k = static_cast({k_expr});') L.append( - ' uint16_t result = amdgpu::fp_mode::fma_f16(s0, s1, k, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));' + ' uint16_t result = amdgpu::fp_mode::fma_f16(s0, s1, k, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));' ) L.append( f' amdgpu::RegisterAccess(wf).write_lane({dst_ops[0]}, lane, result);' @@ -7395,7 +7453,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: ) L.append(f' float k = std::bit_cast({k_expr});') L.append( - f' amdgpu::RegisterAccess(wf).write_lane({dst_ops[0]}, lane, std::bit_cast(amdgpu::fp_mode::arithmetic(s0, s1, k, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())));' + f' amdgpu::RegisterAccess(wf).write_lane({dst_ops[0]}, lane, std::bit_cast(amdgpu::fp_mode::arithmetic(s0, s1, k, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode())));' ) L.append(' }') return '\n'.join(L) @@ -7442,6 +7500,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: opsel=opsel, dtype=dtype, is_vop3=is_vop3, + has_abs=has_abs, fp8_format_select=fp8_format_select, arch_name=self.isa_spec.arch_name, ) @@ -7742,6 +7801,9 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: L.append(f' }}') return '\n'.join(L) + if inst.name.upper() in ('LDS_DIRECT_LOAD', 'DS_DIRECT_LOAD'): + return ' amdgpu::execute_lds_direct_load(*this, wf, inst_.vdst);' + # ── Image pipeline stubs ────────────────────────────────────────── # NOTE for image execution: the image ADDRESS is carried as the # fieldless ``vaddr`` operand (OPR_VGPR), currently emitted as an @@ -7994,6 +8056,8 @@ def _wait_counter_type( 'amdgpu::WaitCounterType::VSCNT)' ) return 'amdgpu::WaitCounterType::VMCNT' + if kind == 'direct': + return 'amdgpu::WaitCounterType::EXPCNT' if kind == 'local': return ( 'amdgpu::WaitCounterType::DSCNT' @@ -8030,6 +8094,8 @@ def _memory_completion_class( if is_flat: kind = kind.replace('flat_', 'vmem_', 1) + if kind == 'direct': + return ordered_local if kind == 'local': if 'gds' in inst_fields: return f'(inst_.gds != 0 ? {ordered_gds} : {ordered_local})' @@ -8540,6 +8606,12 @@ def _append_atomic_fp_policy( self, lines: list[str], sem: InstructionSemantics, *, ds: bool ) -> None: """Carry manual-defined scalar FP policies through deferred memory execution.""" + if ( + not ds + and self.isa_spec.profile.atomic_source_nan_first + and sem.operation in ('fadd', 'pk_add_f16', 'pk_add_bf16') + ): + lines.append(' d->atomic_source_nan_first = true;') if sem.operation not in ('fadd', 'fmin', 'fmax', 'fcmpswap'): return profile = self.isa_spec.profile @@ -11707,7 +11779,7 @@ def gen_insts(self) -> None: ) if ( - profile.renders_gfx11_image_syntax + profile.has_gfx11_image_address_extension and enc.enc_name.upper() == 'ENC_MIMG' ): ctor_body_parts.append('capture_nsa_words(inst, &vaddr);') @@ -11994,6 +12066,20 @@ def gen_insts(self) -> None: if len(_src_input_ops) > 1 else None ) + _dpp_sign_bits = ( + [ + self._dpp_source_sign_bit(sem, i, o) + for i, o in enumerate(_src_input_ops[:2]) + ] + if sem is not None + and _supports_dpp_encoding + and enc.enc_name.upper() + in ('ENC_VOP1', 'ENC_VOP2', 'ENC_VOPC') + else [] + ) + _dpp_modifies_src1 = len(_dpp_sign_bits) > 1 and bool( + _dpp_sign_bits[1] + ) # Tripwire: the permuted sources must be field-bearing, # so a future change to the selection above cannot # silently reintroduce reading a fieldless literal as @@ -12042,7 +12128,9 @@ def gen_insts(self) -> None: _dpp_preamble += ( ' std::optional dpp_src0_;\n' ) - if _has_sdwa_encoding and _src1_name: + if ( + _has_sdwa_encoding or _dpp_modifies_src1 + ) and _src1_name: _dpp_preamble += ( ' std::optional dpp_src1_;\n' ) @@ -12245,6 +12333,13 @@ def _legacy_dpp_write_mask_lines( + _sdwa_src1_block + ' }\n' ) + for index, sign in enumerate(_dpp_sign_bits): + if sign: + source = _src_input_ops[index].name + _dpp_preamble += ( + f' amdgpu::dpp::apply_source_modifiers({source}, dpp_src{index}_, wf,\n' + f' {sign}ull, dpp_modifiers_, {index});\n' + ) if not _semantic_stages_src0: _dpp_preamble += ( f' ScopedOperandDelegate dpp_src0_binding_({_src0_name},\n' @@ -12254,7 +12349,8 @@ def _legacy_dpp_write_mask_lines( ) + ( f' ScopedOperandDelegate dpp_src1_binding_({_src1_name},\n' ' dpp_src1_ ? &*dpp_src1_ : nullptr);\n' - if _has_sdwa_encoding and _src1_name + if (_has_sdwa_encoding or _dpp_modifies_src1) + and _src1_name else '' ) if _supports_dpp_encoding and not _is_compare and _dst_name: @@ -12445,15 +12541,18 @@ def _legacy_dpp_write_mask_lines( ' }\n' ) _result_format = self._sdwa_result_format(sem) + _output_policy = self._sdwa_output_policy(sem) _local_body = body if _result_format == 'amdgpu::sdwa::ResultFormat::F16': _local_body = self._apply_sdwa_f16_omod( - _local_body, '*this' + _local_body, + '*this', + rounded_result=sem.name in ROUNDED_F16_OPS, ) _local_body = re.sub( r'amdgpu::RegisterAccess\(wf\)\.write_lane\(\s*' r'([A-Za-z_][A-Za-z0-9_]*),\s*lane,\s*', - rf'amdgpu::sdwa::write_lane<{_result_format}>' + rf'amdgpu::sdwa::write_lane<{_result_format}{_output_policy}>' r'(*this, wf, \1, lane, ', _local_body, ) @@ -12967,6 +13066,7 @@ def _legacy_dpp_write_mask_lines( ('rocjitsu/isa/arch/amdgpu/shared/gfx11_dot2.h', False), ('rocjitsu/isa/arch/amdgpu/shared/gfx12_dot.h', False), ('rocjitsu/isa/arch/amdgpu/shared/division.h', False), + ('rocjitsu/isa/arch/amdgpu/shared/cube.h', False), ('util/except.h', False), ] _MEM_ENC_NAMES = frozenset( @@ -13022,6 +13122,10 @@ def _legacy_dpp_write_mask_lines( cpp_includes.append( ('rocjitsu/isa/arch/amdgpu/shared/image_resource.h', False) ) + if enc.enc_name.upper() in ('ENC_LDSDIR', 'ENC_VDSDIR'): + cpp_includes.append( + ('rocjitsu/isa/arch/amdgpu/shared/lds_direct.h', False) + ) has_matrix_exec = any( self.semantics and (s := self.semantics.instructions.get(i.name)) @@ -13520,6 +13624,7 @@ def scoped_includes(impls: list[object]) -> list[tuple[str, bool]]: 'rocjitsu/isa/arch/amdgpu/shared/fp_mode.h': 'fp_mode::', 'rocjitsu/isa/arch/amdgpu/shared/gfx11_dot2.h': 'gfx11_dot2_f32', 'rocjitsu/isa/arch/amdgpu/shared/gfx12_dot.h': 'gfx12_dot2_f32', + 'rocjitsu/isa/arch/amdgpu/shared/cube.h': 'cube::', 'rocjitsu/isa/arch/amdgpu/shared/division.h': ( 'div_scale(', 'div_fmas(', @@ -13859,7 +13964,11 @@ def _write_shared_execute_templates(self) -> None: r'(? None: r'\s*\(void\)wf;\s*(?://[^\n]*)?\n?', '\n', prefixed_body ) result_format = self._sdwa_result_format(sem) + output_policy = self._sdwa_output_policy(sem) prefixed_body = _re.sub( r'amdgpu::RegisterAccess\(wf\)\.write_lane\(\s*' r'(inst\.[A-Za-z_][A-Za-z0-9_]*),\s*lane,\s*', - rf'sdwa::write_lane<{result_format}>' r'(inst, wf, \1, lane, ', + rf'sdwa::write_lane<{result_format}{output_policy}>' + r'(inst, wf, \1, lane, ', prefixed_body, ) prefixed_body = _re.sub( @@ -13983,8 +14094,10 @@ def _write_shared_execute_templates(self) -> None: '#include "rocjitsu/isa/arch/amdgpu/shared/pseudo_scalar.h"', '#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h"', '#include "rocjitsu/isa/arch/amdgpu/shared/gfx11_dot2.h"', + '#include "rocjitsu/isa/arch/amdgpu/shared/lds_direct.h"', '#include "rocjitsu/isa/arch/amdgpu/shared/gfx12_dot.h"', '#include "rocjitsu/isa/arch/amdgpu/shared/division.h"', + '#include "rocjitsu/isa/arch/amdgpu/shared/cube.h"', *simd_extra_includes(), '#include "util/data_types.h"', '#include "util/except.h"', @@ -14029,6 +14142,11 @@ def _write_shared_execute_templates(self) -> None: true16_vop3=is_true16_vop3, result_writer='commit_result' if uses_result_writer else None, ) + if mnemonic.rsplit('_', 1)[0].upper() in FLUSH_NEAREST_F32_OPS: + # These mappings ignore guest rounding. Keep output scaling and clamp + # under the saved environment too: OMOD can overflow or touch + # signaling NaNs, and SIMD clamp compares NaN results. + lines.append(' fp_mode::ScopedEnvironment environment(0);') alu_classifiers = { 'v_mul_f32_vop2': 'classify_mul_f32_vop2', 'v_mul_f32_vop3': 'classify_mul_f32_vop3', @@ -14808,6 +14926,12 @@ def gen_operand(self) -> None: ' bool packed_16bit_dst_ = false;\n' ) + if uses_packed_16bit_sources and self.isa_spec.profile.split_execution_sources: + packed_16bit_field += ( + ' std::optional resolved_vgpr_offset_exec(\n' + ' const amdgpu::Wavefront &wf) const;\n' + ) + # WARNING: final, kStaticRegisterAccess, and this friendship form the typed-access # contract. Removing any of these restores vtable dispatch, silently degrading performance. execution_decls = ( @@ -16076,6 +16200,28 @@ def _inert_guard(ret_expr: str = '', cond: str = _inert_cond) -> str: full_execution_backend())); } ''') + if uses_packed_16bit_sources: + # Ownership checks, raw storage, and observation hooks must all + # resolve a packed half to the same underlying physical VGPR. + execution_code = execution_code.replace( + 'detail::resolved_vgpr_offset_for_operand(wf, *this)', + 'resolved_vgpr_offset_exec(wf)', + ) + execution_code += '\n\n' + textwrap.dedent('''\ + std::optional + Operand::resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const { + if (!reads_value()) + return std::nullopt; + auto packed = packed_16bit_vgpr_source( + packed_16bit_source_, size_bits_, opr_type_, encoding_value_); + if (!packed) + packed = packed_16bit_vgpr_dst( + packed_16bit_dst_, size_bits_, opr_type_, encoding_value_); + if (packed) + return packed->reg + (wf.vgpr_msb_for_role(vgpr_msb_role()) << 8); + return detail::resolved_vgpr_offset_for_operand(wf, *this); + } + ''') execution_code = execution_code.replace( 'raw_compute_unit(wf.cu())', 'amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu())', diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py index 03894294305b..26dc38199e49 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py @@ -293,6 +293,7 @@ def _register_handlers() -> None: ), dtype=c.dtype, is_vop3=c.is_vop3, + has_abs=c.has_abs, fp8_format_select=( 'inst_.clamp' if c.cls == 'vector_cvt_pk' @@ -359,6 +360,7 @@ def _register_handlers() -> None: op_sel_hi_2_expr=c.op_sel_hi_2_expr, opsel_exprs=c.opsel_exprs, use_cdna5_helpers=c.arch_name == 'cdna5', + fused_result=c.arch_name.startswith('rdna') or c.arch_name == 'cdna5', ) DISPATCH['mad_mixlo_f16'] = lambda c: gen_mad_mix_lo_hi( c.dst_ops, @@ -367,6 +369,7 @@ def _register_handlers() -> None: op_sel_hi_2_expr=c.op_sel_hi_2_expr, opsel_exprs=c.opsel_exprs, use_cdna5_helpers=c.arch_name == 'cdna5', + fused_result=c.arch_name.startswith('rdna') or c.arch_name == 'cdna5', ) DISPATCH['mad_mixhi_f16'] = lambda c: gen_mad_mix_lo_hi( c.dst_ops, @@ -375,6 +378,7 @@ def _register_handlers() -> None: op_sel_hi_2_expr=c.op_sel_hi_2_expr, opsel_exprs=c.opsel_exprs, use_cdna5_helpers=c.arch_name == 'cdna5', + fused_result=c.arch_name.startswith('rdna') or c.arch_name == 'cdna5', ) DISPATCH['mad_mix_f32_bf16'] = lambda c: gen_mad_mix_bf16( c.dst_ops, diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/cube.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/cube.py new file mode 100644 index 000000000000..03b53de4bbfd --- /dev/null +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/cube.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Shared cube opcode mapping and helper calls for execute emitters.""" + +from __future__ import annotations + +CUBE_OPERATIONS = {'cubeid': 'ID', 'cubesc': 'SC', 'cubetc': 'TC', 'cubema': 'MA'} + + +def cube_expression(op: str, x: str, y: str, z: str) -> str: + """Emit cube selection with the wave's rounding and NaN policy.""" + return ( + f'::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::{CUBE_OPERATIONS[op]}>' + f'({x}, {y}, {z}, wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode())' + ) + + +def cube_omod(expression: str, inst: str = 'inst_') -> str: + """Emit cube scaling; callers disable generic OMOD and apply CLAMP separately.""" + return ( + f'::rocjitsu::amdgpu::cube::apply_omod({expression}, wf.fp_round_mode_f32(), ' + f'wf.cu().arch(), wf.ieee_mode(), {inst}.omod)' + ) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/float_compare.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/float_compare.py new file mode 100644 index 000000000000..639dd0361274 --- /dev/null +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/float_compare.py @@ -0,0 +1,86 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Floating VOPC relations evaluated by shared/comparison.h. + +Scalar, SIMD and arch-local compare bodies all emit the same +``comparison::evaluate`` call on raw source encodings, so source modifiers, +MODE input flushing and NaN ordering stay in one place. +""" + +from amdisa.semantics import FLOAT_COMPARE_RELATIONS, is_float_relation + +__all__ = ['is_float_relation'] + +_NS = 'amdgpu::comparison' + +# VOPC relation mnemonic -> comparison.h relation. F and T/TRU read no source +# and keep their constant bodies. +RELATIONS: dict[str, str] = { + 'lt': 'Lt', + 'eq': 'Eq', + 'le': 'Le', + 'gt': 'Gt', + 'lg': 'Lg', + 'ge': 'Ge', + 'o': 'O', + 'u': 'U', + 'nge': 'Nge', + 'nlg': 'Nlg', + 'ngt': 'Ngt', + 'nle': 'Nle', + 'neq': 'Neq', + 'nlt': 'Nlt', +} + +assert RELATIONS.keys() == FLOAT_COMPARE_RELATIONS + +FORMATS: dict[str, str] = {'f16': 'F16', 'f32': 'F32', 'f64': 'F64'} + +# Name of the per-instruction policy the scalar bodies declare before the lane loop. +POLICY = 'compare_policy' + + +def lane_type(dtype: str) -> str: + """Raw-encoding lane type: F16 occupies the low half of a 32-bit lane.""" + return 'uint64_t' if dtype == 'f64' else 'uint32_t' + + +def policy_expr(dtype: str) -> str: + mode = 'f32' if dtype == 'f32' else 'f16_f64' + return f'{_NS}::Policy::make(wf.fp_denorm_mode_{mode}())' + + +def policy_decl(dtype: str, indent: str = ' ') -> str: + return f'{indent}const auto {POLICY} = {policy_expr(dtype)};' + + +def evaluate_expr( + dtype: str, + op: str, + a: str, + b: str, + policy: str = POLICY, + modifiers: tuple[str, str] | None = None, +) -> str: + """Return the relation on raw encodings ``a`` and ``b``. + + ``modifiers`` is the (ABS, NEG) field pair of a VOP3 form; VOPC forms + receive sources that already carry any DPP modifiers. + """ + args = [a, b, *(modifiers or ()), policy] + return ( + f'{_NS}::evaluate<{_NS}::{FORMATS[dtype]}, {_NS}::{RELATIONS[op]}>' + f'({", ".join(args)})' + ) + + +def simd_functor(dtype: str, op: str, modifiers: tuple[str, str] | None = None) -> str: + """Return a SIMD compare functor; the policy and modifiers are captured once.""" + captures = [f'{POLICY} = {policy_expr(dtype)}'] + if modifiers is not None: + captures += [f'abs_mods = {modifiers[0]}', f'neg_mods = {modifiers[1]}'] + call = evaluate_expr( + dtype, op, 'a', 'b', modifiers=('abs_mods', 'neg_mods') if modifiers else None + ) + return f'[{", ".join(captures)}](auto a, auto b) {{ return {call}; }}' diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/floating_policy.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/floating_policy.py new file mode 100644 index 000000000000..b2a4b40a05e0 --- /dev/null +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/floating_policy.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Instruction policies shared by scalar, SIMD and SDWA output lowering.""" + +# These F32 results cannot preserve output denormals, independently of MODE. +# Their SDWA forms use the same output modifiers as VOP3. SIN can produce +# subnormals and retains its separate MODE policy. +FLUSH_NEAREST_F32_OPS = frozenset( + {'V_LOG_F32', 'V_EXP_F32', 'V_RCP_F32', 'V_RSQ_F32', 'V_SQRT_F32', 'V_COS_F32'} +) + +# Evaluate and round these operations to half before output modifiers. +ROUNDED_F16_OPS = frozenset( + { + 'V_LOG_F16', + 'V_EXP_F16', + 'V_RCP_F16', + 'V_RSQ_F16', + 'V_SQRT_F16', + 'V_SIN_F16', + 'V_COS_F16', + } +) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py index 11834e3be6ef..2070461d7ac8 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py @@ -119,7 +119,7 @@ def gen_pk_binop( f' if (inst_.{neg} & {1 << bit}u) {name}_{half} ^= 0x8000u;' ) L.append( - f' const uint16_t r{half} = amdgpu::fp_mode::packed_binary_f16(amdgpu::fp_mode::PackedBinaryOp::{op.upper()}, a_{half}, b_{half}, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));' + f' const uint16_t r{half} = amdgpu::fp_mode::packed_binary_f16(amdgpu::fp_mode::PackedBinaryOp::{op.upper()}, a_{half}, b_{half}, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode());' ) L.append( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, rlo | (static_cast(rhi) << 16));' @@ -286,7 +286,7 @@ def gen_pk_ternary( f' if (inst_.{neg} & {1 << index}u) {name}_{half} ^= 0x8000u;' ) L.append( - f' const uint16_t r{half} = amdgpu::fp_mode::packed_select3_f16(amdgpu::fp_mode::PackedBinaryOp::{selection}, first_{half}, second_{half}, third_{half}, wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf));' + f' const uint16_t r{half} = amdgpu::fp_mode::packed_select3_f16(amdgpu::fp_mode::PackedBinaryOp::{selection}, first_{half}, second_{half}, third_{half}, wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode());' ) L.append( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, rlo | (static_cast(rhi) << 16));' @@ -306,10 +306,10 @@ def gen_pk_ternary( f' uint16_t {name} = static_cast({selector} ? ({raw} >> 16) : {raw});' ) L.append( - ' uint16_t rlo = amdgpu::fp_mode::fma_f16(a_lo, b_lo, c_lo, false, false, false, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));' + ' uint16_t rlo = amdgpu::fp_mode::fma_f16(a_lo, b_lo, c_lo, false, false, false, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));' ) L.append( - ' uint16_t rhi = amdgpu::fp_mode::fma_f16(a_hi, b_hi, c_hi, false, false, false, inst_.neg_hi & 1u, inst_.neg_hi & 2u, inst_.neg_hi & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));' + ' uint16_t rhi = amdgpu::fp_mode::fma_f16(a_hi, b_hi, c_hi, false, false, false, inst_.neg_hi & 1u, inst_.neg_hi & 2u, inst_.neg_hi & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));' ) L.append( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, static_cast(rlo) | (static_cast(rhi) << 16));' @@ -557,8 +557,8 @@ def gen_pk_fmac_vop2(dst: list[str], src: list[str]) -> str: f' uint32_t raw1 = amdgpu::RegisterAccess(wf).read_lane({s1}, lane);', f' uint32_t rawd = amdgpu::RegisterAccess(wf).read_lane({d}, lane);', ' const uint32_t omod = amdgpu::sdwa::output_modifier(*this, wf);', - ' uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), static_cast(rawd), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));', - ' uint32_t r1 = amdgpu::fp_mode::fma_f16(static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));', + ' uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), static_cast(rawd), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));', + ' uint32_t r1 = amdgpu::fp_mode::fma_f16(static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));', f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, r0 | (r1 << 16));', ' }', ] @@ -585,8 +585,8 @@ def gen_pk_fmac_vop3(dst: list[str], src: list[str]) -> str: f' raw1 = (raw1 & 0xffffu) * 0x10001u;', f' uint32_t rawd = amdgpu::RegisterAccess(wf).read_lane({d}, lane);', ' uint32_t omod = amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), true, inst_.omod);', - ' uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), static_cast(rawd), inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));', - ' uint32_t r1 = amdgpu::fp_mode::fma_f16(static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf));', + ' uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), static_cast(rawd), inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));', + ' uint32_t r1 = amdgpu::fp_mode::fma_f16(static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()));', f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, r0 | (r1 << 16));', ' }', ] @@ -632,7 +632,7 @@ def gen_pk_binop_f32( L.append(' if (inst_.neg_hi & 2) b_hi = -b_hi;') for half, result in (('lo', 'rlo'), ('hi', 'rhi')): L.append( - f' uint32_t {result} = amdgpu::fp_mode::packed_f32(a_{half}, b_{half}, 0.0f, amdgpu::fp_mode::PackedF32Op::{op.upper()}, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf));' + f' uint32_t {result} = amdgpu::fp_mode::packed_f32(a_{half}, b_{half}, 0.0f, amdgpu::fp_mode::PackedF32Op::{op.upper()}, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode());' ) L.append( f' amdgpu::RegisterAccess(wf).write_lane64({d}, lane, static_cast(rlo) | (static_cast(rhi) << 32));' @@ -688,10 +688,10 @@ def gen_pk_ternary_f32( L.append(' if (inst_.neg_hi & 2) b_hi = -b_hi;') L.append(' if (inst_.neg_hi & 4) c_hi = -c_hi;') L.append( - ' uint32_t rlo = amdgpu::fp_mode::packed_f32(a_lo, b_lo, c_lo, amdgpu::fp_mode::PackedF32Op::FMA, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf));' + ' uint32_t rlo = amdgpu::fp_mode::packed_f32(a_lo, b_lo, c_lo, amdgpu::fp_mode::PackedF32Op::FMA, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode());' ) L.append( - ' uint32_t rhi = amdgpu::fp_mode::packed_f32(a_hi, b_hi, c_hi, amdgpu::fp_mode::PackedF32Op::FMA, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf));' + ' uint32_t rhi = amdgpu::fp_mode::packed_f32(a_hi, b_hi, c_hi, amdgpu::fp_mode::PackedF32Op::FMA, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode());' ) L.append( f' amdgpu::RegisterAccess(wf).write_lane64({d}, lane, static_cast(rlo) | (static_cast(rhi) << 32));' @@ -910,6 +910,7 @@ def gen_mad_mix_f32( op_sel_hi_2_expr: str = '', opsel_exprs: tuple[str, str] = ('', ''), use_cdna5_helpers: bool = False, + fused_result: bool = False, ) -> str: """Generate V_MAD_MIX_F32: mixed-precision FMA with op_sel selecting f16/f32 per src.""" d, s0, s1, s2 = dst[0], src[0], src[1], src[2] @@ -967,9 +968,16 @@ def gen_mad_mix_f32( L.append(' if (inst_.neg & 1) a = -a;') L.append(' if (inst_.neg & 2) b = -b;') L.append(' if (inst_.neg & 4) c = -c;') - L.append( - f' float result = {"std::fma(a, b, c)" if use_cdna5_helpers else "a * b + c"};' - ) + if fused_result: + L.extend( + [ + ' float result = amdgpu::fp_mode::arithmetic(', + ' a, b, c, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(),', + ' wf.cu().arch(), wf.ieee_mode());', + ] + ) + else: + L.append(' float result = a * b + c;') L.append(' if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf);') L.append( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, std::bit_cast(result));' @@ -985,6 +993,7 @@ def gen_mad_mix_lo_hi( op_sel_hi_2_expr: str = '', opsel_exprs: tuple[str, str] = ('', ''), use_cdna5_helpers: bool = False, + fused_result: bool = False, ) -> str: """Generate V_MAD_MIXLO_F16 / V_MAD_MIXHI_F16.""" d, s0, s1, s2 = dst[0], src[0], src[1], src[2] @@ -1042,11 +1051,26 @@ def gen_mad_mix_lo_hi( L.append(' if (inst_.neg & 1) a = -a;') L.append(' if (inst_.neg & 2) b = -b;') L.append(' if (inst_.neg & 4) c = -c;') - L.append( - f' float result = {"std::fma(a, b, c)" if use_cdna5_helpers else "a * b + c"};' - ) - L.append(' if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf);') - L.append(f' uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl());') + if fused_result: + L.extend( + [ + ' amdgpu::fp_mode::ScopedEnvironment environment(0);', + ' uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment(', + ' a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(),', + ' amdgpu::floating_clamp_nan_to_zero(wf));', + ] + ) + else: + L.append( + f' float result = {"std::fma(a, b, c)" if use_cdna5_helpers else "a * b + c"};' + ) + L.append( + ' if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf);' + ) + L.append(' uint16_t h = amdgpu::pseudo_scalar::round_f16_result(') + L.append( + ' result, wf.fp_round_mode_f16_f64(), 0, false, wf.fp16_ovfl(), false);' + ) if is_lo: L.append( f' ::rocjitsu::amdgpu::write_vop3_true16_dst({d}, wf, lane, 0u, h);' diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index 252bf2b26b7b..5850c057ae68 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -9,10 +9,15 @@ from __future__ import annotations +from amdisa.codegen.execute.floating_policy import FLUSH_NEAREST_F32_OPS + from dataclasses import dataclass, field, replace from enum import Enum, auto +from amdisa.codegen.execute import float_compare +from amdisa.codegen.execute.cube import CUBE_OPERATIONS, cube_expression, cube_omod from amdisa.codegen.execute.fp8_formats import fp8_helper_name +from amdisa.sema_derive import FLOAT_COMPARE_CALL from amdisa.sema_ast import ( ExecModel, SemaBlock, @@ -130,6 +135,8 @@ class LoweringContext: mode_sensitive_f16_dst: bool = True mode_arithmetic: bool = True dx9_zero_fma: bool = False + flush_nearest_f32: bool = False + arithmetic_flush_output: str | None = None integer_saturation_dtype: str | None = None @@ -204,7 +211,11 @@ def lower_sema_block(block: SemaBlock, ctx: LoweringContext | None = None) -> st # Legacy MAD has a distinct intermediate-precision policy. ctx.mode_arithmetic = not block.instruction_name.startswith('V_MAD') - ctx.dx9_zero_fma = block.instruction_name == 'V_FMA_DX9_ZERO_F32' + ctx.dx9_zero_fma = block.instruction_name in ( + 'V_FMA_DX9_ZERO_F32', + 'V_FMAC_DX9_ZERO_F32', + ) + ctx.flush_nearest_f32 = block.instruction_name in FLUSH_NEAREST_F32_OPS body_lines = _lower_stmt(block.body, ctx) if ctx.exec_model == ExecModel.VECTOR: @@ -553,6 +564,10 @@ def _mode_arithmetic( arguments += [f'wf.fp_round_mode_{mode}()', f'wf.fp_denorm_mode_{mode}()'] if operation == 'FMA' and ctx.dx9_zero_fma: operation = 'FMA_DX9_ZERO' + if width == 32 and operation in ('ADD', 'MUL', 'FMA', 'FMA_DX9_ZERO'): + arguments += ['wf.cu().arch()', 'wf.ieee_mode()'] + if ctx.arithmetic_flush_output is not None: + arguments.append(ctx.arithmetic_flush_output) helper = 'arithmetic_f16' if width == 16 else 'arithmetic' return ( f'amdgpu::fp_mode::{helper}' @@ -637,6 +652,8 @@ def _lower_expr(node: SemaNode, ctx: LoweringContext) -> str: c = _lower_expr(node.children[2], ctx) if (arithmetic := _mode_arithmetic(node, ctx, 'FMA', [a, b, c])) is not None: return arithmetic + if node.ty and node.ty.base == 'F' and node.ty.size == 32: + return f'amdgpu::fp_mode::arithmetic({a}, {b}, {c}, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode())' return f'std::fma({a}, {b}, {c})' if kind == SemaNodeKind.BITNEG: @@ -1361,6 +1378,8 @@ def _lower_arrayderef(node: SemaNode, ctx: LoweringContext) -> str: # generic generated scalar F16 arithmetic is gated out in CodeGenerator. 'cvt_f16_f32': 'util::f32_to_f16_mode(std::bit_cast(static_cast({0})), wf.fp16_ovfl())', 'cvt_f32_f16': 'std::bit_cast(util::f16_to_f32(static_cast({0})))', + # VALU applies MODE policy; keep the existing raw SALU conversion separate. + 'cvt_f32_f16_valu': 'amdgpu::fp_mode::cvt_f32_f16({0}, wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())', 'cvt_f32_bf16': 'std::bit_cast(util::bf16_to_f32(static_cast({0})))', 'cvt_f32_fp8': 'std::bit_cast(util::fp8_e4m3_to_f32(static_cast({0})))', 'cvt_f32_bf8': 'std::bit_cast(util::bf8_e5m2_to_f32(static_cast({0})))', @@ -1381,22 +1400,25 @@ def _lower_arrayderef(node: SemaNode, ctx: LoweringContext) -> str: 'cvt_f32_f64': 'std::bit_cast(static_cast(std::bit_cast(static_cast({0}))))', 'cvt_f16_u16': 'util::f32_to_f16_mode(static_cast(static_cast({0})), wf.fp16_ovfl())', 'cvt_f16_i16': 'util::f32_to_f16_mode(static_cast(static_cast({0} & 0xFFFF)), wf.fp16_ovfl())', - 'cvt_u16_f16': '[&]() -> uint32_t {{ float s = util::f16_to_f32(static_cast({0}));' + # Decoded floats include ABS/NEG; integer conversion truncates toward zero. + 'cvt_u16_f16': '[&]() -> uint32_t {{ float s = {0};' ' if (std::isnan(s) || s < 0.0f) return 0u;' ' if (s >= 65536.0f) return static_cast(UINT16_MAX);' ' return static_cast(static_cast(s)); }}()', - 'cvt_i16_f16': '[&]() -> uint32_t {{ float s = util::f16_to_f32(static_cast({0}));' + 'cvt_i16_f16': '[&]() -> uint32_t {{ float s = {0};' ' if (std::isnan(s)) return 0u;' ' if (s >= 32768.0f) return static_cast(static_cast(INT16_MAX));' ' if (s < -32768.0f) return static_cast(static_cast(INT16_MIN));' ' return static_cast(static_cast(static_cast(s))); }}()', - 'cvt_norm_i16_f16': '[&]() -> uint32_t {{ float s = util::f16_to_f32(static_cast({0}));' + # These arguments are decoded floats, including source modifiers. Conversion + # rounds once with ties to even, independently of MODE.ROUND. + 'cvt_norm_i16_f16': '[&]() -> uint32_t {{ float s = {0};' ' if (std::isnan(s)) return 0u;' - ' float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f);' + ' double scaled = util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0));' ' return static_cast(static_cast(static_cast(scaled))); }}()', - 'cvt_norm_u16_f16': '[&]() -> uint32_t {{ float s = util::f16_to_f32(static_cast({0}));' + 'cvt_norm_u16_f16': '[&]() -> uint32_t {{ float s = {0};' ' if (std::isnan(s)) return 0u;' - ' float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f);' + ' double scaled = util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0));' ' return static_cast(static_cast(scaled)); }}()', 'cvt_off_f32_i4': '[&]() -> float {{ int32_t nibble = static_cast({0} & 0xfu);' ' if (nibble & 0x8) nibble -= 16;' @@ -1800,24 +1822,7 @@ def _effectful_inline_binary_op(name: str, template: str) -> InlineBinaryOp: ' if (w == 0) return 0u; uint32_t mask = (uint32_t{{1}} << w) - 1u;' ' uint32_t extracted = static_cast(static_cast(src) >> off) & mask;' ' uint32_t signbit = uint32_t{{1}} << (w - 1u); return (extracted ^ signbit) - signbit; }}()', - 'cubeid': '[&]() {{ auto x={0}; auto y={1}; auto z={2};' - ' float ax=std::fabs(x), ay=std::fabs(y), az=std::fabs(z);' - ' if (ax >= ay && ax >= az) return x >= 0 ? 0.0f : 1.0f;' - ' if (ay >= ax && ay >= az) return y >= 0 ? 2.0f : 3.0f;' - ' return z >= 0 ? 4.0f : 5.0f; }}()', - 'cubesc': '[&]() {{ auto x={0}; auto y={1}; auto z={2};' - ' float ax=std::fabs(x), ay=std::fabs(y), az=std::fabs(z);' - ' if (ax >= ay && ax >= az) return x >= 0 ? z : -z;' - ' if (ay >= ax && ay >= az) return x;' - ' return z >= 0 ? -x : x; }}()', - 'cubetc': '[&]() {{ auto x={0}; auto y={1}; auto z={2};' - ' float ax=std::fabs(x), ay=std::fabs(y), az=std::fabs(z);' - ' if (ax >= ay && ax >= az) return -y;' - ' if (ay >= ax && ay >= az) return y >= 0 ? -z : z;' - ' return -y; }}()', - 'cubema': '[&]() {{ auto x={0}; auto y={1}; auto z={2};' - ' float ax=std::fabs(x), ay=std::fabs(y), az=std::fabs(z);' - ' return 2.0f * std::fmax(ax, std::fmax(ay, az)); }}()', + **{op: cube_expression(op, '{0}', '{1}', '{2}') for op in CUBE_OPERATIONS}, } @@ -1831,6 +1836,8 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: return _lower_apply_omod(node, ctx) if callee == 'apply_clamp': return _lower_apply_clamp(node, ctx) + if callee.startswith(FLOAT_COMPARE_CALL): + return _lower_float_compare(node, ctx) args = [_lower_expr(c, ctx) for c in node.children[1:]] args_str = ', '.join(args) @@ -1911,8 +1918,13 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: operation_name = pseudo_scalar_operations[operation] mode_suffix = 'f32' if precision == 'f32' else 'f16_f64' fp16_ovfl = ', wf.fp16_ovfl()' if precision == 'f16' else '' + helper = ( + 'amdgpu::transcendental::execute_pseudo_f16' + if precision == 'f16' + else 'amdgpu::pseudo_scalar::execute_f32' + ) return ( - f'amdgpu::pseudo_scalar::execute_{precision}(' + f'{helper}(' f'amdgpu::pseudo_scalar::Operation::{operation_name}, {args[0]}, ' f'(inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, ' f'wf.fp_round_mode_{mode_suffix}(), wf.fp_denorm_mode_{mode_suffix}(), ' @@ -1954,6 +1966,44 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: return f'static_cast(util::f32_to_f16_mode({fp8_decode_fn}(static_cast({arg})), wf.fp16_ovfl()))' return f'static_cast(util::f32_to_f16_mode({bf8_decode_fn}(static_cast({arg})), wf.fp16_ovfl()))' + if ( + len(args) == 1 + and callee in ('rcp', 'rsq', 'sqrt', 'sin', 'cos') + and node.ty == SemaType.F16 + ): + return ( + f'amdgpu::transcendental::map_f16(' + f'{args[0]}, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))' + ) + if len(args) == 1 and callee in ('rcp', 'rsq', 'sqrt') and node.ty == SemaType.F32: + return ( + f'amdgpu::transcendental::{callee}_f32({args[0]}, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))' + ) + if len(args) == 1 and callee in ('sin', 'cos') and node.ty == SemaType.F32: + return ( + f'amdgpu::transcendental::{callee}_f32({args[0]}, ' + 'wf.fp_denorm_mode_f32(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))' + ) + if ( + len(args) == 1 + and callee in ('log', 'log2', 'exp', 'exp2') + and node.ty in (SemaType.F16, SemaType.F32) + ): + function = 'log' if callee.startswith('log') else 'exp' + if node.ty == SemaType.F16: + logarithm = 'true' if function == 'log' else 'false' + return ( + f'amdgpu::transcendental::log_exp_f16<{logarithm}>({args[0]}, ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))' + ) + return ( + f'amdgpu::transcendental::{function}_f32({args[0]}, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))' + ) if len(args) == 1 and callee in _INLINE_UNARY_OPS: return _INLINE_UNARY_OPS[callee].format(args[0]) if len(args) == 2 and callee in _INLINE_BINARY_OPS: @@ -1983,6 +2033,40 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: return f'{callee}({args_str})' +def _lower_float_compare(node: SemaNode, ctx: LoweringContext) -> str: + """Lower a floating VOPC relation to comparison::evaluate on raw encodings. + + CALL children: [ID(name), src0, src1], each a typed cast of a register + read, possibly wrapped in apply_src_mod by enrichment. The relation applies + ABS/NEG to the encoding itself, so the wrappers are reduced to the + instruction's modifier fields. + """ + op = (node.call_name or '').removeprefix(FLOAT_COMPARE_CALL) + has_abs = has_neg = False + dtype = None + reads = [] + for src in node.children[1:3]: + if src.kind == SemaNodeKind.CALL and src.call_name == 'apply_src_mod': + has_neg |= src.children[3].lit_value == '1' + has_abs |= src.children[4].lit_value == '1' + src = src.children[1] + if src.ty and src.ty.base == 'F': + dtype = f'f{src.ty.size}' + while src.kind == SemaNodeKind.CAST: + src = src.children[0] + if dtype is None or src.kind != SemaNodeKind.INSTOPERAND: + raise ValueError(f'unexpected {node.call_name} source: {src}') + lane = float_compare.lane_type(dtype) + reads.append(f'static_cast<{lane}>({_lower_expr(src, ctx)})') + declaration = float_compare.policy_decl(dtype) + if declaration not in ctx.vector_preamble: + ctx.vector_preamble.append(declaration) + modifiers = None + if has_abs or has_neg: + modifiers = ('inst_.abs' if has_abs else '0u', 'inst_.neg' if has_neg else '0u') + return float_compare.evaluate_expr(dtype, op, *reads, modifiers=modifiers) + + def _lower_apply_src_mod(node: SemaNode, ctx: LoweringContext) -> str: """Lower apply_src_mod CALL to inline VOP3 NEG/ABS code. @@ -2042,7 +2126,8 @@ def _lower_apply_omod(node: SemaNode, ctx: LoweringContext) -> str: """ if len(node.children) < 2: return '0' - rhs = _lower_expr(node.children[1], ctx) + if any(_contains_call(node.children[1], op) for op in CUBE_OPERATIONS): + return cube_omod(_lower_expr(node.children[1], ctx)) is_f64 = node.ty and node.ty.size == 64 mode_arithmetic = ctx.mode_arithmetic and _contains_mode_arithmetic(node) wide_result = is_f64 or (node.ty == SemaType.F16 and mode_arithmetic) @@ -2070,14 +2155,54 @@ def _lower_apply_omod(node: SemaNode, ctx: LoweringContext) -> str: 'wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), inst_.omod)' ) else: + # DX9 FMA and these transcendentals disable output denormals regardless of MODE, + # so those bits cannot suppress their output modifiers. + force_output_flush = ctx.dx9_zero_fma or ctx.flush_nearest_f32 + denorm_expr = '0' if force_output_flush else 'wf.fp_denorm_mode_f32()' omod_expr = ( 'amdgpu::fp_mode::effective_omod(wf.cu().arch(), ' - 'wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst_.omod)' + f'{denorm_expr}, wf.ieee_mode(), inst_.omod)' ) + arithmetic_f32 = ( + node.ty == SemaType.F32 + and ctx.mode_arithmetic + and any( + n.kind in (SemaNodeKind.ADD, SemaNodeKind.MUL, SemaNodeKind.FMA) + for n in node.children[1].walk() + ) + ) + rhs = _lower_expr( + node.children[1], + ( + replace(ctx, arithmetic_flush_output=f'({omod_expr} != 0)') + if arithmetic_f32 + else ctx + ), + ) if node.ty in (SemaType.F32, SemaType.F64) and any( child.kind == SemaNodeKind.LDEXP for child in node.children[1].walk() ): return f'amdgpu::div_apply_omod({rhs}, wf.fp_round_mode_{mode}(), {omod_expr})' + if arithmetic_f32: + # The arithmetic helper establishes MODE and restores the host state. + # Only active scaling needs a second environment, after the operation. + return ( + f'[&]() {{ float v = {rhs};' + f' const uint32_t effective_omod = {omod_expr};' + ' if (effective_omod == 0) return v;' + f' {environment}' + ' return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }()' + ) + if node.ty == SemaType.F32: + return ( + f'[&]() {{ {environment}float v = {rhs};' + f' return amdgpu::fp_mode::apply_omod_f32(v, {omod_expr}); }}()' + ) + if node.ty == SemaType.F16 and any( + _contains_call(node.children[1], op) + for op in ('log', 'log2', 'exp', 'exp2', 'rcp', 'rsq', 'sqrt', 'sin', 'cos') + ): + return f'amdgpu::fp_mode::apply_omod_f16({rhs}, {omod_expr}, wf.fp16_ovfl())' return ( f'[&]() {{ {environment}{fp_type} v = {rhs};' f' const uint32_t effective_omod = {omod_expr};' @@ -2111,6 +2236,14 @@ def _lower_apply_clamp(node: SemaNode, ctx: LoweringContext) -> str: if mode_arithmetic else '' ) + if node.ty == SemaType.F32 and any( + child.kind == SemaNodeKind.FMA for child in node.children[1].walk() + ): + return ( + f'[&]() {{ float v = {rhs};' + f' if (inst_.clamp) {{ {environment}' + ' v = amdgpu::clamp_floating_result(v, wf); } return v; }()' + ) return ( f'[&]() {{ {environment}{fp_type} v = {rhs};' ' if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf);' diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py index 6799dd94ca33..32cb7e061e49 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py @@ -28,6 +28,27 @@ from __future__ import annotations +from amdisa.codegen.execute import float_compare +from amdisa.codegen.execute.floating_policy import ( + FLUSH_NEAREST_F32_OPS, + ROUNDED_F16_OPS, +) + +from amdisa.codegen.execute.cube import CUBE_OPERATIONS, cube_expression, cube_omod + + +def _binary_f32_op( + operation: str, *, modifiers: bool = False, reverse: bool = False +) -> str: + capture = '&inst, &wf' if modifiers else '&wf' + omod = ', amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)' if modifiers else '' + operands = 'b, a' if reverse else 'a, b' + return ( + f'[{capture}](auto a, auto b) {{ return ' + f'amdgpu::binary_f32_simd({operands}, wf{omod}); }}' + ) + + # template_name -> (cpp_element_type, cpp_binary_op_functor) # # template_name matches the symbol emitted by _generator.gen_shared_execute: @@ -37,10 +58,10 @@ # try_execute_binary_vop2_simd. Use std::*<> for stateless ops. SIMD_VOP2_BINARY: dict[str, tuple[str, str]] = { # --- float32 (IEEE-754 single-rounded, bit-identical to scalar body) --- - "v_add_f32_vop2": ("float32_t", "std::plus<>{}"), - 'v_sub_f32_vop2': ('float32_t', 'std::minus<>{}'), - 'v_subrev_f32_vop2': ('float32_t', '[](auto a, auto b) { return b - a; }'), - 'v_mul_f32_vop2': ('float32_t', 'std::multiplies<>{}'), + "v_add_f32_vop2": ("float32_t", _binary_f32_op('ADD')), + 'v_sub_f32_vop2': ('float32_t', _binary_f32_op('SUB')), + 'v_subrev_f32_vop2': ('float32_t', _binary_f32_op('SUB', reverse=True)), + 'v_mul_f32_vop2': ('float32_t', _binary_f32_op('MUL')), # Legacy / DX9 zero-multiply: (a==0 || b==0) ? 0 : a*b. The ==0 matches both # ±0 (as the scalar `a == 0.0f` does). Routed via the VOP3 binary fp glue for # the _vop3 twin (which applies abs/neg/omod/clamp around this functor). @@ -243,8 +264,8 @@ ), # v_cvt_pkrtz_f16_f32 (both spellings): pack two f32 -> two f16 with # round-toward-zero narrowing; proven bit-identical to the scalar helper. - # Inputs arrive as raw u32 lanes, bit_cast to f32. The VOP3 twins carry no - # modifiers (verified) so they auto-route to VOP3_BINARY_INT with this functor. + # Inputs arrive as raw u32 lanes, bit_cast to f32. VOP3 source modifiers + # require a scalar fallback before using this raw-word functor. 'v_cvt_pkrtz_f16_f32_vop2': ( 'uint32_t', '[](auto a, auto b) {' @@ -300,11 +321,9 @@ # un_op(simd) -> simd # inside try_execute_unary_vop1_simd. Tin and Tout are both 32-bit lane # types and may differ (e.g. int32->float32 for v_cvt_f32_i32). Eligible -# kernels are those whose host SIMD result is bit-identical to the scalar -# generated body: elementwise bit ops, exact int<->float casts, and the -# correctly-rounded IEEE operations (div, sqrt, and — verified per toolchain -# via the parity test — exp2/log2). NaN/clamp-bearing conversions and the -# inexact transcendentals (sin/cos) are excluded. +# kernels preserve the generated scalar result: elementwise bit operations, +# exact casts, and floating-point helpers with matching guest policies. +# LOG/EXP and RCP/RSQ use the shared hardware mappings. SIN/COS stay scalar. # VOP1 base mnemonics whose VOP3 form applies float abs/neg/omod/clamp modifiers # over an f32 source and result (so the VOP3 twin routes through the f32 unary # modifier glue rather than reusing the plain VOP1 path). @@ -607,7 +626,8 @@ 'v_rcp_f32_vop1': ( 'float32_t', 'float32_t', - '[](auto a) { return util::rcp_f32_simd(a); }', + '[&wf](auto a) { return util::rcp_f32_simd(a, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }', ), 'v_rcp_iflag_f32_vop1': ( 'float32_t', @@ -617,27 +637,28 @@ 'v_rsq_f32_vop1': ( 'float32_t', 'float32_t', - '[](auto a) { return util::rsq_f32_simd(a); }', + '[&wf](auto a) { return util::rsq_f32_simd(a, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }', ), 'v_sqrt_f32_vop1': ( 'float32_t', 'float32_t', - '[](auto a) { return util::sqrt_f32_simd(a); }', + '[&wf](auto a) { return util::sqrt_f32_simd(a, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }', ), - # --- transcendental exp2/log2. util::{exp,log}_f32_simd wrap stdx::exp2/log2 - # with the same FTZ flush / special-case guards as the scalar transcendental - # reference; the underlying vector libm is bit-exact to scalar std::* on the - # supported toolchains (libstdc++ 13 / AVX-512), guarded by - # UtilSimd.Exp2/Log2_*_BitExact. v_sin/v_cos excluded: vector libm ~1 ULP off. --- + # LOG and EXP share their integer scalar mappings. Their complete + # instruction bodies save the host environment for output scaling. 'v_exp_f32_vop1': ( 'float32_t', 'float32_t', - '[](auto a) { return util::exp_f32_simd(a); }', + '[&wf](auto a) { return util::exp_f32_simd(a, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }', ), 'v_log_f32_vop1': ( 'float32_t', 'float32_t', - '[](auto a) { return util::log_f32_simd(a); }', + '[&wf](auto a) { return util::log_f32_simd(a, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }', ), # --- float -> int conversions with NaN->0 and saturating clamp. The float # comparison masks are re-typed to the int lane via simd_mask_as<> before @@ -701,45 +722,53 @@ ' auto f = util::f16_to_f32_simd(a);' ' return util::f32_to_f16_simd(f - util::floor_simd(f)); }', ), - # f16 transcendentals mirror the scalar f32_to_f16(_f32(f16_to_f32(x))) - # by applying the f32-domain util::*_f32_simd helper (FTZ flush + canonical - # qNaN / NaN-input guards) on the f16->f32 intermediate. + # Half transcendentals apply input policy and round before output modifiers. 'v_rcp_f16_vop1': ( 'uint32_t', 'uint32_t', - '[](auto a) {' - ' return util::f32_to_f16_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); }', + '[&wf](auto a) { return util::f32_to_f16_simd(' + 'amdgpu::transcendental::map_f16_simd(' + 'util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }', ), 'v_rsq_f16_vop1': ( 'uint32_t', 'uint32_t', - '[](auto a) {' - ' return util::f32_to_f16_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); }', + '[&wf](auto a) { return util::f32_to_f16_simd(' + 'amdgpu::transcendental::map_f16_simd(' + 'util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }', ), 'v_sqrt_f16_vop1': ( 'uint32_t', 'uint32_t', - '[](auto a) {' - ' return util::f32_to_f16_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); }', + '[&wf](auto a) { return util::f32_to_f16_simd(' + 'amdgpu::transcendental::map_f16_simd(' + 'util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }', ), - # exp/log_f16 inherit the exp2/log2 toolchain guard (UtilSimd.Exp2/Log2_*). + # Half LOG/EXP rounds once before applying output modifiers. 'v_exp_f16_vop1': ( 'uint32_t', 'uint32_t', - '[](auto a) {' - ' return util::f32_to_f16_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); }', + '[&wf](auto a) { return util::f32_to_f16_simd(' + 'amdgpu::transcendental::log_exp_f16_simd(util::f16_to_f32_simd(a), ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }', ), 'v_log_f16_vop1': ( 'uint32_t', 'uint32_t', - '[](auto a) {' - ' return util::f32_to_f16_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); }', + '[&wf](auto a) { return util::f32_to_f16_simd(' + 'amdgpu::transcendental::log_exp_f16_simd(util::f16_to_f32_simd(a), ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }', ), # --- f16 <-> f32 / int16 conversions --- 'v_cvt_f32_f16_vop1': ( 'uint32_t', 'float32_t', - '[](auto a) { return util::f16_to_f32_simd(a); }', + '[&wf](auto a) { return ::rocjitsu::amdgpu::cvt_f32_f16_mode_simd(a, wf); }', ), 'v_cvt_f16_f32_vop1': ( 'float32_t', @@ -867,9 +896,15 @@ # performs the fused operation in native chunks and applies the current # FP16 MODE controls without an intervening f32 rounding. v_fmac_f64 likewise # has a dedicated split-VGPR, MODE-aware path in SIMD_VOP2_FMA_F64. -_FMA_ACC_F32 = '[](auto a, auto b, auto d, auto) { return util::stdx::fma(a, b, d); }' -_FMA_ADDK_F32 = '[](auto a, auto b, auto, auto k) { return util::stdx::fma(a, b, k); }' -_FMA_MULK_F32 = '[](auto a, auto b, auto, auto k) { return util::stdx::fma(a, k, b); }' +_FMA_ACC_F32 = ( + '[&wf](auto a, auto b, auto d, auto) { return amdgpu::fma_f32_simd(a, b, d, wf); }' +) +_FMA_ADDK_F32 = ( + '[&wf](auto a, auto b, auto, auto k) { return amdgpu::fma_f32_simd(a, b, k, wf); }' +) +_FMA_MULK_F32 = ( + '[&wf](auto a, auto b, auto, auto k) { return amdgpu::fma_f32_simd(a, k, b, wf); }' +) _FMA_ACC_F16 = ( '[](auto a, auto b, auto d, auto) {' ' return util::f32_to_f16_simd(util::stdx::fma(' @@ -893,7 +928,11 @@ SIMD_VOP2_TERNARY: dict[str, tuple[str, str, str]] = { # --- f32 dst-accumulate --- 'v_fmac_f32_vop2': ('float32_t', '0u', _FMA_ACC_F32), - 'v_fmac_dx9_zero_f32_vop2': ('float32_t', '0u', _FMA_ACC_F32), + 'v_fmac_dx9_zero_f32_vop2': ( + 'float32_t', + '0u', + '[&wf](auto a, auto b, auto d, auto) { return amdgpu::fma_dx9_zero_f32_simd(a, b, d, wf); }', + ), 'v_mac_f32_vop2': ('float32_t', '0u', _FMA_ACC_F32), # --- f32 inline literal --- 'v_fmaak_f32_vop2': ('float32_t', _FMA_K_READ, _FMA_ADDK_F32), @@ -1092,13 +1131,12 @@ def simd_ternary_literal_operand_name(template_name: str) -> str | None: 'v_div_fmas_f64_vop3', } -# VOP3P fma_mix / mad_mix family. The six ops share one body (`a*b + c` plus -# optional clamp to [0,1]); only the destination shape differs: -# - F32 -> v_fma_mix_f32_vop3p (RDNA3+), v_mad_mix_f32_vop3p (CDNA1-4) +# VOP3P fma_mix / mad_mix family. FMA uses the fused, mode-aware path: +# - F32 -> v_fma_mix_f32_vop3p (RDNA), v_mad_mix_f32_vop3p (CDNA1-4) # - F16_LO -> v_fma_mixlo_f16_vop3p, v_mad_mixlo_f16_vop3p # - F16_HI -> v_fma_mixhi_f16_vop3p, v_mad_mixhi_f16_vop3p # Per-source op_sel_hi gates the f16<->f32 widening shape; op_sel picks the f16 -# half. neg flips the sign bit. No abs, no omod. Functorless / fixed-op. +# half. neg flips the sign bit and neg_hi applies ABS. No OMOD. SIMD_VOP3P_FMA_MIX_F32: set[str] = { 'v_fma_mix_f32_vop3p', 'v_mad_mix_f32_vop3p', @@ -1250,9 +1288,9 @@ def simd_ternary_literal_operand_name(template_name: str) -> str | None: } # pk_fma_f32 — 3-source FMA per half, with independent source-half selection. -# NaN-input payload divergence accepted. +# Scalar and SIMD execution share hardware NaN selection. SIMD_VOP3P_PK_TERNARY_F32: dict[str, str] = { - 'v_pk_fma_f32_vop3p': '[](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }', + 'v_pk_fma_f32_vop3p': '[&wf](auto a, auto b, auto c) { return amdgpu::fma_f32_simd(a, b, c, wf); }', } # v_pk_mov_b32 — each src is a 64-bit SGPR or VGPR pair. op_sel[0] selects the @@ -1419,13 +1457,20 @@ def _vopc_functor(conv: str, rel: str) -> str: return f'[](auto a, auto b) {{ return {body}; }}' +def _float_vopc_entry(suf: str, rel: str) -> tuple[str, str]: + """Float relations evaluate raw encodings through shared/comparison.h.""" + if float_compare.is_float_relation(suf, rel): + return float_compare.lane_type(suf), float_compare.simd_functor(suf, rel) + lane_t, conv = _VOPC_SUFFIX[suf] + return lane_t, _vopc_functor(conv, rel) + + def _build_simd_vopc() -> dict[str, tuple[str, str]]: table: dict[str, tuple[str, str]] = {} # 16-/32-bit lane suffixes only; f64/i64/u64 (64-bit lane) wired separately. for suf in ('f16', 'f32'): - lane_t, conv = _VOPC_SUFFIX[suf] for rel in _VOPC_FLOAT_RELS: - table[f'v_cmp_{rel}_{suf}_vopc'] = (lane_t, _vopc_functor(conv, rel)) + table[f'v_cmp_{rel}_{suf}_vopc'] = _float_vopc_entry(suf, rel) for suf in ('i16', 'u16', 'i32', 'u32'): lane_t, conv = _VOPC_SUFFIX[suf] for rel in _VOPC_INT_RELS: @@ -1436,9 +1481,8 @@ def _build_simd_vopc() -> dict[str, tuple[str, str]]: def _build_simd_vopc64() -> dict[str, tuple[str, str]]: """64-bit-lane VOPC compares (f64/i64/u64), routed through the VOPC64 glue.""" table: dict[str, tuple[str, str]] = {} - lane_t, conv = _VOPC_SUFFIX['f64'] for rel in _VOPC_FLOAT_RELS: - table[f'v_cmp_{rel}_f64_vopc'] = (lane_t, _vopc_functor(conv, rel)) + table[f'v_cmp_{rel}_f64_vopc'] = _float_vopc_entry('f64', rel) for suf in ('i64', 'u64'): lane_t, conv = _VOPC_SUFFIX[suf] for rel in _VOPC_INT_RELS: @@ -1601,11 +1645,10 @@ def _build_simd_vopc64() -> dict[str, tuple[str, str]]: # try_execute_vopc_vop3_int_simd (32-bit lane) or # try_execute_vopc64_vop3_int_simd (64-bit lane). # -# Floating-point VOPC bodies in VOP3 form apply abs (std::fabs) then neg per -# source on the already-widened/converted operand; the float-bucket tables -# below build new functors that take the post-modifier value (no in-functor -# widen), and the corresponding glue applies the modifier outside before -# calling. +# Floating-point VOPC bodies in VOP3 form read the raw encodings through the +# same raw-lane glue; shared/comparison.h applies the captured abs/neg fields, +# MODE input flushing and NaN ordering inside the functor. F16 sources occupy +# 32-bit lanes and may select their half with true16 OPSEL. # # VOP3 fp adds two extra rels vs VOPC: 't' alongside 'tru' (both constant # always-true), so the table keys span the union of the two. @@ -1640,57 +1683,26 @@ def _build_simd_vopc_vop3_int_64() -> dict[str, tuple[str, str]]: SIMD_VOPC_VOP3_INT_64: dict[str, tuple[str, str]] = _build_simd_vopc_vop3_int_64() -def _build_simd_vopc_vop3_f32() -> dict[str, str]: - """VOP3 form of the f32 VOPC relations (16 from VOPC + 't' constant). +def _build_simd_vopc_vop3_float() -> dict[str, tuple[str, str]]: + """VOP3 form of the f16/f32/f64 VOPC relations (16 from VOPC + 't'). - Keyed by ``_vop3``; the functor is the same shape as the VOPC f32 one - (identity operand conversion), and operates on already-modifier-applied - `native` arguments — the VOP3 fp32 glue applies abs/neg outside the - functor. + Keyed by ``_vop3``; values are (raw lane type, functor). Constant relations + ignore their operands. """ - table: dict[str, str] = {} - _, conv = _VOPC_SUFFIX['f32'] - for rel in _VOP3_FLOAT_RELS: - table[f'v_cmp_{rel}_f32_vop3'] = _vopc_functor(conv, rel) - return table - - -SIMD_VOPC_VOP3_F32: dict[str, str] = _build_simd_vopc_vop3_f32() - - -def _build_simd_vopc_vop3_f16() -> dict[str, str]: - """VOP3 form of the f16 VOPC relations (17 — same set as f32). - - The f16 VOP3 glue widens raw lanes via util::f16_to_f32_simd and applies - abs/neg on the f32 outside the functor; the functor itself takes - already-widened `native` arguments, so it is the same functor as - the f32 path (identity operand conversion). - """ - table: dict[str, str] = {} - _, conv = _VOPC_SUFFIX['f32'] - for rel in _VOP3_FLOAT_RELS: - table[f'v_cmp_{rel}_f16_vop3'] = _vopc_functor(conv, rel) - return table - - -SIMD_VOPC_VOP3_F16: dict[str, str] = _build_simd_vopc_vop3_f16() - - -def _build_simd_vopc_vop3_f64() -> dict[str, str]: - """VOP3 form of the f64 VOPC relations (17 — same set as f32/f16). - - Lane type `double`; the glue applies abs/neg outside the functor on the - f64 value, so the functor itself takes already-modified `native` - arguments and reuses the VOPC64 f64 builder (identity operand conversion). - """ - table: dict[str, str] = {} - _, conv = _VOPC_SUFFIX['f64'] - for rel in _VOP3_FLOAT_RELS: - table[f'v_cmp_{rel}_f64_vop3'] = _vopc_functor(conv, rel) + modifiers = ('inst.inst_.abs', 'inst.inst_.neg') + table: dict[str, tuple[str, str]] = {} + for suf in ('f16', 'f32', 'f64'): + lane_t = float_compare.lane_type(suf) + for rel in _VOP3_FLOAT_RELS: + if float_compare.is_float_relation(suf, rel): + functor = float_compare.simd_functor(suf, rel, modifiers) + else: + functor = _vopc_functor('{x}', rel) + table[f'v_cmp_{rel}_{suf}_vop3'] = (lane_t, functor) return table -SIMD_VOPC_VOP3_F64: dict[str, str] = _build_simd_vopc_vop3_f64() +SIMD_VOPC_VOP3_FLOAT: dict[str, tuple[str, str]] = _build_simd_vopc_vop3_float() # --- VOP3 integer/bitwise ternary (3-source) ------------------------------- @@ -1761,7 +1773,7 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]: # them on the (correct) scalar path. Only the int-domain twins # v_cvt_pk_i16_i32 / v_cvt_pk_u16_u32 above are safe to route through this glue. # Normalized f32 pack-converts. src read as raw f32 (bit_cast), scale by K, - # clamp, isnan->0, truncate, pack 16|16. Helpers in util/simd.h. + # clamp, isnan->0, round once to nearest-even, pack 16|16. Helpers in util/simd.h. 'v_cvt_pknorm_i16_f32_vop3': ( 'uint32_t', '[](auto a, auto b) {' @@ -1993,29 +2005,42 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]: } -# VOP3 f16 unary: widen f16->f32, apply abs/neg, op, omod/clamp, narrow back. -# Rounding ops (ceil/floor/trunc/rndne) have no FTZ. sqrt is also no-FTZ -# (transcendental::sqrt_f32 keeps denormals). The four transcendentals -# (rcp/rsq/exp/log) reuse util::*_f32_simd which already wraps the scalar -# transcendental::flush_denorm_f32 carve-outs (FTZ input + matching ±0/Inf -# blends + NaN-passthrough), so the f16 scalar -# f32_to_f16(transcendental::op_f32(f16_to_f32(...))) maps directly. +# VOP3 f16 unary operations widen to f32 for modifiers and narrow the result. +# Integral rounding preserves input denormals. Transcendentals apply half +# denormal, overflow and NaN policies and round to f16 before OMOD. SIMD_VOP3_UNARY_FP16: dict[str, str] = { 'v_ceil_f16_vop3': '[](auto a) { return util::ceil_simd(a); }', 'v_floor_f16_vop3': '[](auto a) { return util::floor_simd(a); }', 'v_trunc_f16_vop3': '[](auto a) { return util::trunc_simd(a); }', 'v_rndne_f16_vop3': '[](auto a) { return util::rndne_simd(a); }', 'v_sqrt_f16_vop3': ( - '[](auto a) {' - ' auto r = util::stdx::sqrt(a);' - ' util::stdx::where(util::stdx::isnan(a), r) = a;' - ' util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN();' - ' return r; }' - ), - 'v_rcp_f16_vop3': '[](auto a) { return util::rcp_f32_simd(a); }', - 'v_rsq_f16_vop3': '[](auto a) { return util::rsq_f32_simd(a); }', - 'v_exp_f16_vop3': '[](auto a) { return util::exp_f32_simd(a); }', - 'v_log_f16_vop3': '[](auto a) { return util::log_f32_simd(a); }', + '[&wf](auto a) { return amdgpu::transcendental::map_f16_simd<' + 'amdgpu::transcendental::HalfOperation::SQRT>(a, ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }' + ), + 'v_rcp_f16_vop3': ( + '[&wf](auto a) { return amdgpu::transcendental::map_f16_simd<' + 'amdgpu::transcendental::HalfOperation::RCP>(a, ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }' + ), + 'v_rsq_f16_vop3': ( + '[&wf](auto a) { return amdgpu::transcendental::map_f16_simd<' + 'amdgpu::transcendental::HalfOperation::RSQ>(a, ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }' + ), + 'v_exp_f16_vop3': ( + '[&wf](auto a) { return amdgpu::transcendental::log_exp_f16_simd(a, ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }' + ), + 'v_log_f16_vop3': ( + '[&wf](auto a) { return amdgpu::transcendental::log_exp_f16_simd(a, ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }' + ), # v_fract_f16: x - floor(x) in the widened f32 domain (the glue widens/narrows # and applies abs/neg/omod/clamp), mirroring the covered v_fract_f16_vop1. 'v_fract_f16_vop3': '[](auto a) { return a - util::floor_simd(a); }', @@ -2061,14 +2086,10 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]: # --- VOP3 floating-point ternary (FMA / MAD family) ------------------------ # -# v_fma_*: util::stdx::fma (fused multiply-add, single-rounded). v_fmac/v_mac: -# same body (the scalar generator emits std::fma for both because of dst- -# accumulate semantics — src2 == vdst). v_mad: non-fused `a * b + c`. NaN-input -# divergence between stdx::fma and std::fma (gcc-13 packed FMA picks a -# different NaN operand to quiet) is accepted, same as the existing VOP2 -# ternary FMA slice — the A/B test skips NaN-input lanes. +# F32 FMA and accumulator forms share scalar NaN policy; finite lanes retain +# native SIMD arithmetic. MAD retains its non-fused multiply/add semantics. SIMD_VOP3_TERNARY_FP32: dict[str, str] = { - 'v_fma_f32_vop3': '[](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }', + 'v_fma_f32_vop3': '[&inst, &wf](auto a, auto b, auto c) { return amdgpu::fma_f32_simd(a, b, c, wf, amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); }', 'v_mad_f32_vop3': '[](auto a, auto b, auto c) { return a * b + c; }', 'v_mad_legacy_f32_vop3': '[](auto a, auto b, auto c) { return a * b + c; }', # min3/max3/med3 (f32): the scalar body composes std::fmax/std::fmin @@ -2079,11 +2100,7 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]: # lanes and uses no ±0 inputs). omod/clamp applied by the glue. # DX9 FMA has zero-product and mandatory flushing rules distinct from FMA. 'v_fma_dx9_zero_f32_vop3': ( - '[&wf](auto a, auto b, auto c) {' - ' return decltype(a)([&](auto i) {' - ' return amdgpu::fp_mode::arithmetic(' - ' static_cast(a[i]), static_cast(b[i]), static_cast(c[i]),' - ' wf.fp_round_mode_f32(), 0); }); }' + '[&wf](auto a, auto b, auto c) { return amdgpu::fma_dx9_zero_f32_simd(a, b, c, wf); }' ), 'v_max3_f32_vop3': '[](auto a, auto b, auto c) { return util::stdx::fmax(util::stdx::fmax(a, b), c); }', 'v_min3_f32_vop3': '[](auto a, auto b, auto c) { return util::stdx::fmin(util::stdx::fmin(a, b), c); }', @@ -2104,13 +2121,15 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]: 'v_minimum3_f32_vop3': '[](auto a, auto b, auto c) { return util::ieee_minimum_simd(util::ieee_minimum_simd(a, b), c); }', 'v_maximumminimum_f32_vop3': '[](auto a, auto b, auto c) { return util::ieee_minimum_simd(util::ieee_maximum_simd(a, b), c); }', 'v_minimummaximum_f32_vop3': '[](auto a, auto b, auto c) { return util::ieee_maximum_simd(util::ieee_minimum_simd(a, b), c); }', - # Cubemap face ops: ternary f32, per-source abs/neg + result omod/clamp via - # the glue; the face-selection is a bit-exact where-cascade (helpers in - # simd_glue.h). cubema is inline (exact ×2 of fmax of abs). - 'v_cubeid_f32_vop3': '[](auto x, auto y, auto z) { return util::cube_id_f32_simd(x, y, z); }', - 'v_cubema_f32_vop3': '[](auto x, auto y, auto z) { return 2.0f * util::stdx::fmax(util::stdx::abs(x), util::stdx::fmax(util::stdx::abs(y), util::stdx::abs(z))); }', - 'v_cubesc_f32_vop3': '[](auto x, auto y, auto z) { return util::cube_sc_f32_simd(x, y, z); }', - 'v_cubetc_f32_vop3': '[](auto x, auto y, auto z) { return util::cube_tc_f32_simd(x, y, z); }', + # Cube applies OMOD itself; false leaves only CLAMP to the operand glue. + **{ + f'v_{op}_f32_vop3': ( + '[&inst, &wf](auto x, auto y, auto z) { return ' + + cube_omod(cube_expression(op, 'x', 'y', 'z'), 'inst.inst_') + + '; }, false' + ) + for op in CUBE_OPERATIONS + }, # Share the bit-level fixup and guest MODE policy with the scalar executor. # FIXUP rounds OMOD explicitly; operand glue still applies CLAMP. 'v_div_fixup_f32_vop3': ( @@ -2165,11 +2184,11 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]: # v_fmac / v_mac per-isa classes only initialize src0+src1+vdst; the third FMA # operand IS vdst (no src2 Operand). The accumulate-form glue reads inst.vdst # as the third operand and applies abs/neg only to src0/src1 (per scalar body). -# NaN payload divergence accepted, same as the non-accumulate ternary slice. +# F32 NaN selection matches the non-accumulate ternary slice. SIMD_VOP3_FMAC_FP32: dict[str, str] = { - 'v_fmac_f32_vop3': '[](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }', - 'v_mac_f32_vop3': '[](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }', - 'v_fmac_dx9_zero_f32_vop3': '[](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }', + 'v_fmac_f32_vop3': '[&inst, &wf](auto a, auto b, auto c) { return amdgpu::fma_f32_simd(a, b, c, wf, amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); }', + 'v_mac_f32_vop3': '[&inst, &wf](auto a, auto b, auto c) { return amdgpu::fma_f32_simd(a, b, c, wf, amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); }', + 'v_fmac_dx9_zero_f32_vop3': SIMD_VOP3_TERNARY_FP32['v_fma_dx9_zero_f32_vop3'], } SIMD_VOP3_FMAC_FP16 = {'v_fmac_f16_vop3'} SIMD_VOP3_FMAC_FP64 = {'v_fmac_f64_vop3'} @@ -2535,6 +2554,36 @@ def _indent_probe(probe: str) -> str: return '\n'.join((' ' + line) if line else line for line in probe.splitlines()) +_PACKED_NORMALIZED_VOP3 = ( + 'v_cvt_pknorm_i16_f32_vop3', + 'v_cvt_pknorm_u16_f32_vop3', + 'v_cvt_pk_norm_i16_f32_vop3', + 'v_cvt_pk_norm_u16_f32_vop3', + 'v_cvt_pk_norm_i16_f16_vop3', + 'v_cvt_pk_norm_u16_f16_vop3', +) + + +_PACKED_RTZ_VOP3 = ('v_cvt_pkrtz_f16_f32_vop3', 'v_cvt_pk_rtz_f16_f32_vop3') + + +def _modified_conversion_op(cpp_op: str, *, bits: int, arity: int = 2) -> str: + """Apply floating source signs before a raw-word SIMD conversion.""" + operands = ('a', 'b')[:arity] + parameters = ', '.join(f'auto {operand}' for operand in operands) + sign = f'0x{1 << (bits - 1):x}u' + modifiers = ''.join( + f' if (inst.inst_.abs & {1 << i}u) {operand} &= ~{sign};' + f' if (inst.inst_.neg & {1 << i}u) {operand} ^= {sign};' + for i, operand in enumerate(operands) + ) + return ( + f'[&inst]({parameters}) {{' + + modifiers + + f' return ({cpp_op})({", ".join(operands)}); }}' + ) + + def _mode_aware_f16_result_simd_probe(default_probe: str, ovfl_probe: str) -> str: default_body = _indent_probe(default_probe) ovfl_body = _indent_probe(ovfl_probe) @@ -2581,6 +2630,9 @@ def simd_probe_line( def _guard_mode_arithmetic_probe(template_name: str, probe: str | None) -> str | None: """Retain native arithmetic only when it implements the guest FP policy.""" + if template_name == 'v_fma_mix_f32_vop3p': + # The fused MIX helper establishes MODE and applies denormal controls. + return probe arithmetic_ops = ( 'add', 'sub', @@ -2606,6 +2658,42 @@ def _guard_mode_arithmetic_probe(template_name: str, probe: str | None) -> str | dtype = next((field for field in fields if field in ('f16', 'f32', 'f64')), None) if dtype is None or (fields[1] == 'ldexp' and dtype != 'f16'): return probe + if dtype == 'f32' and fields[1] in ('sub', 'subrev'): + # Reuse the checked SIMD views and share MODE setup for ordinary + # operands. Observers and delegates retain their existing path and host + # environment; they can execute user code during register access. + source1 = 'inst.vsrc1' if fields[-1] == 'vop2' else 'inst.src1' + macro = 'VOP2_BINARY' if fields[-1] == 'vop2' else 'VOP3_BINARY_FP' + native_op = ( + 'std::minus<>{}' + if fields[1] == 'sub' + else '[](auto a, auto b) { return b - a; }' + ) + native_probe = f' ROCJITSU_TRY_SIMD_{macro}(float32_t, {native_op});' + return ( + ' if (wf.exec() != 0 && !amdgpu::simd_force_scalar() && ' + '!wf.cu().observes_register_access() && ' + '!wf.cu().debug_active() && !inst.src0.delegate() && ' + f'!{source1}.delegate() && !inst.vdst.delegate()) {{\n' + ' amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32());\n' + f'{_indent_probe(probe)}\n' + ' }\n' + ' if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), ' + f'wf.fp_denorm_mode_f32())) {{\n{native_probe}\n }}' + ) + if ( + dtype == 'f32' + and 'dx9' not in fields + and ('amdgpu::fma_f32_simd(' in probe or 'amdgpu::binary_f32_simd<' in probe) + ): + # This helper applies guest denormal controls explicitly. Establish MODE + # once for the wave, including its output scaling and clamp operations. + return ( + ' {\n' + ' amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32());\n' + f'{_indent_probe(probe)}\n' + ' }' + ) if dtype in ('f16', 'f64') and fields[1] in ('fma', 'fmac', 'fmaak', 'fmamk'): return probe mode = 'f32' if dtype == 'f32' else 'f16_f64' @@ -2743,36 +2831,20 @@ def _simd_probe_line( writer_arg = f'{result_writer}, ' if result_writer is not None else '' suffix = '_RESULT' if result_writer is not None else '' return f' ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT{suffix}({writer_arg}{lane_t}, {cpp_op});' - # VOP3 form of the f32 VOPC relational compares (17 ops: 16 relations + - # 't' constant). Per-source abs/neg modifiers applied outside the functor - # via the fp32 VOPC glue; result-writer handling matches the integer path. - specvopcv3f32 = SIMD_VOPC_VOP3_F32.get(template_name) - if specvopcv3f32 is not None: - writer_arg = f'{result_writer}, ' if result_writer is not None else '' - suffix = '_RESULT' if result_writer is not None else '' - return ( - f' ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32{suffix}({writer_arg}{specvopcv3f32});' - ) - # VOP3 form of the f16 VOPC relational compares (17 ops). The glue widens - # raw lanes to f32 then applies abs/neg in f32 domain — matching the scalar - # body's f16_to_f32 -> std::fabs/-x order. Same functor as the f32 path. - specvopcv3f16 = SIMD_VOPC_VOP3_F16.get(template_name) - if specvopcv3f16 is not None: - macro = ( - 'ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16' - if true16_vop3 - else 'ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16' - ) - writer_arg = f'{result_writer}, ' if result_writer is not None else '' - suffix = '_RESULT' if result_writer is not None else '' - return f' {macro}{suffix}({writer_arg}{specvopcv3f16});' - # VOP3 form of the f64 VOPC relational compares (17 ops). 64-bit-lane, - # per-source abs/neg modifiers applied in the f64 domain outside the functor. - specvopcv3f64 = SIMD_VOPC_VOP3_F64.get(template_name) - if specvopcv3f64 is not None: + # VOP3 form of the f16/f32/f64 VOPC compares (17 ops each). Raw lanes go + # through the integer glue; the functor applies the captured abs/neg. + specvopcv3float = SIMD_VOPC_VOP3_FLOAT.get(template_name) + if specvopcv3float is not None: + lane_t, cpp_op = specvopcv3float + if lane_t == 'uint64_t': + macro = 'ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT' + elif true16_vop3 and '_f16_' in template_name: + macro = 'ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT' + else: + macro = 'ROCJITSU_TRY_SIMD_VOPC_VOP3_INT' writer_arg = f'{result_writer}, ' if result_writer is not None else '' suffix = '_RESULT' if result_writer is not None else '' - return f' ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64{suffix}({writer_arg}{specvopcv3f64});' + return f' {macro}{suffix}({writer_arg}{lane_t}, {cpp_op});' # VOP3 integer/bitwise ternary ops (add3/or3/xor3/lshl_add/add_lshl/bfi). # Plain element-wise functor of (src0, src1, src2); no modifiers. spec3tern_true16 = SIMD_VOP3_TERNARY_TRUE16.get(template_name) @@ -2815,7 +2887,13 @@ def _simd_probe_line( # packed FMA quiets a different NaN operand vs scalar std::fma). spec3tf32 = SIMD_VOP3_TERNARY_FP32.get(template_name) if spec3tf32 is not None: - return f' ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32({spec3tf32});' + # Apply OMOD with mandatory output flushing for DX9 FMA. + policy = ( + ', true /* apply_omod */, true /* force_output_flush */' + if template_name == 'v_fma_dx9_zero_f32_vop3' + else '' + ) + return f' ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32({spec3tf32}{policy});' spec3tf16 = SIMD_VOP3_TERNARY_FP16.get(template_name) if spec3tf16 is not None: macro = ( @@ -2876,10 +2954,15 @@ def _simd_probe_line( specdotcf16 = SIMD_DOTC_F16.get(template_name) if specdotcf16 is not None: return f' ROCJITSU_TRY_SIMD_DOTC_F16({specdotcf16});' - # VOP3P fma_mix / mad_mix (six ops, three destination shapes). Same body - # for all; the routing picks the matching glue specialization. + # VOP3P fma_mix / mad_mix: select fused arithmetic and destination shape. + if template_name == 'v_fma_mix_f32_vop3p': + return ' ROCJITSU_TRY_SIMD_FUSED_MIX(F32);' if template_name in SIMD_VOP3P_FMA_MIX_F32: return ' ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F32();' + if template_name == 'v_fma_mixlo_f16_vop3p': + return ' ROCJITSU_TRY_SIMD_FUSED_MIX(F16_LO);' + if template_name == 'v_fma_mixhi_f16_vop3p': + return ' ROCJITSU_TRY_SIMD_FUSED_MIX(F16_HI);' if template_name in SIMD_VOP3P_FMA_MIX_F16_LO: return ' ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F16_LO();' if template_name in SIMD_VOP3P_FMA_MIX_F16_HI: @@ -2888,7 +2971,12 @@ def _simd_probe_line( # has no src2; the accumulate glue reads vdst instead. specfmacf32 = SIMD_VOP3_FMAC_FP32.get(template_name) if specfmacf32 is not None: - return f' ROCJITSU_TRY_SIMD_FMAC_VOP3_FP32({specfmacf32});' + policy = ( + ', true /* force_output_flush */' + if template_name == 'v_fmac_dx9_zero_f32_vop3' + else '' + ) + return f' ROCJITSU_TRY_SIMD_FMAC_VOP3_FP32({specfmacf32}{policy});' if template_name in SIMD_VOP3_FMAC_FP16: macro = ( 'ROCJITSU_TRY_SIMD_FMAC_VOP3_MODE_TRUE16_FP16' @@ -2918,12 +3006,16 @@ def _simd_probe_line( spec3bin16 = SIMD_VOP3_BINARY_TRUE16_SRC.get(template_name) if spec3bin16 is not None: cpp_t, cpp_op = spec3bin16 + if template_name in _PACKED_NORMALIZED_VOP3: + cpp_op = _modified_conversion_op(cpp_op, bits=16) return f' ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC({cpp_t}, {cpp_op});' spec3binx = SIMD_VOP3_BINARY_INT_EXTRA.get(template_name) if spec3binx is not None: if true16_vop3: return None cpp_t, cpp_op = spec3binx + if template_name in _PACKED_NORMALIZED_VOP3: + cpp_op = _modified_conversion_op(cpp_op, bits=32) return f' ROCJITSU_TRY_SIMD_VOP3_BINARY_INT({cpp_t}, {cpp_op});' # VOP3-only f32 binary (no VOP2 twin): IEEE maximum/minimum. Per-source # abs/neg + result omod/clamp applied by the f32 binary glue. @@ -2961,7 +3053,12 @@ def _simd_probe_line( if true16_vop3 else 'ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16' ) - return f' {macro}({spec3unaf16});' + rounded = ( + ', true' + if template_name.rsplit('_', 1)[0].upper() in ROUNDED_F16_OPS + else '' + ) + return f' {macro}({spec3unaf16}{rounded});' # VOP3-encoded twins of the SIMD VOP2 binary ops. Same operator/lane type; # the VOP3 form reads src0/src1 and carries abs/neg/omod/clamp modifiers. # f32 ops apply the modifiers in-vector (bit-exact); integer/bitwise ops @@ -2971,7 +3068,17 @@ def _simd_probe_line( spec2v3 = SIMD_VOP2_BINARY.get(base + '_vop2') if spec2v3 is not None: cpp_t, cpp_op = spec2v3 + if template_name in _PACKED_RTZ_VOP3: + cpp_op = _modified_conversion_op(cpp_op, bits=32) + return f' ROCJITSU_TRY_SIMD_VOP3_BINARY_INT({cpp_t}, {cpp_op});' if cpp_t == 'float32_t': + if base in ('v_add_f32', 'v_mul_f32', 'v_sub_f32', 'v_subrev_f32'): + operation = ( + 'SUB' if base == 'v_subrev_f32' else base.split('_')[1].upper() + ) + cpp_op = _binary_f32_op( + operation, modifiers=True, reverse=base == 'v_subrev_f32' + ) return f' ROCJITSU_TRY_SIMD_VOP3_BINARY_FP({cpp_t}, {cpp_op});' # f16 float binaries (v_add/sub/subrev/mul/max/min/ldexp_f16) are # uint32-typed (the functor widens f16->f32 by hand), but their VOP3 @@ -3024,7 +3131,14 @@ def _simd_probe_line( if base in _VOP3_UNARY_SKIP: return None if base in _VOP3_UNARY_FP_F32: - return f' ROCJITSU_TRY_SIMD_VOP3_UNARY_FP(float32_t, float32_t, {cpp_op});' + modifier_args = ( + ', true /* force_output_flush */' + if base.upper() in FLUSH_NEAREST_F32_OPS + else '' + ) + return f' ROCJITSU_TRY_SIMD_VOP3_UNARY_FP(float32_t, float32_t, {cpp_op}{modifier_args});' + if base in ('v_cvt_i16_f16', 'v_cvt_u16_f16'): + cpp_op = _modified_conversion_op(cpp_op, bits=16, arity=1) probe = f' ROCJITSU_TRY_SIMD_VOP1_UNARY({cpp_tin}, {cpp_tout}, {cpp_op});' if base in ( 'v_cvt_i32_f32', @@ -3035,8 +3149,7 @@ def _simd_probe_line( 'v_cvt_floor_i32_f32', ): # The VOP1 shortcut does not apply floating source modifiers. - # F16 results also use uint32_t storage, but must reach the - # mode-aware half-conversion probe below. + # Modified F32 inputs use the scalar conversion body. return f' if (!inst.inst_.abs && !inst.inst_.neg) {{\n{probe}\n }}' if base in ( 'v_cvt_f32_i32', @@ -3247,22 +3360,22 @@ def call(arity: int, half_dst: bool, half_inputs: int, functor: str) -> str: ): if 'bf16' in template_name: convert = 'util::pack_bf16_simd({x}, wf.fp16_ovfl())' - elif 'pkrtz' in template_name: + elif template_name == 'v_cvt_pkrtz_f16_f32_vop3': convert = ( 'util::f32_to_f16_rtz_simd(std::bit_cast>({x}))' ) else: convert = 'util::f32_to_f16_mode_simd(std::bit_cast>({x}), wf.fp16_ovfl())' - return call( - 2, - False, - 0, + functor = ( '[&](auto a, auto b) { return ' + convert.format(x='a') + ' | (' + convert.format(x='b') - + ' << 16); }', + + ' << 16); }' ) + if template_name == 'v_cvt_pkrtz_f16_f32_vop3': + functor = _modified_conversion_op(functor, bits=32) + return call(2, False, 0, functor) if template_name == 'v_cvt_f32_bf16_vop1': return call( 1, diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_alu.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_alu.py index 57cdd25148ec..c318ad77b645 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_alu.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_alu.py @@ -12,13 +12,14 @@ from __future__ import annotations +from amdisa.codegen.execute.cube import CUBE_OPERATIONS, cube_expression, cube_omod from amdisa.codegen.execute.fp8_formats import fp8_helper_name from amdisa.codegen.execute.vop3_modifiers import ( vop3_src_mod, vop3_dst_mod, vop3_dst_mod_f64, ) -from amdisa.semantics import F32_TO_INTEGER_DTYPES +from amdisa.semantics import F16_INPUT_CONVERSION_DTYPES, F32_TO_INTEGER_DTYPES def _read_vop3_true16_src(opnd: str, opsel: str, src_idx: int) -> str: @@ -156,8 +157,8 @@ def gen_vector_unary( f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, util::f32_to_f16_mode(s, wf.fp16_ovfl()));' ), 'f32_f16': ( - f' uint32_t raw = amdgpu::RegisterAccess(wf).read_lane({src[0]}, lane);\n' - f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, std::bit_cast(util::f16_to_f32(static_cast(raw))));' + f' float s = util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane({src[0]}, lane)));\n' + f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, amdgpu::fp_mode::cvt_f32_f16(s, wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode()));' ), 'f16_u16': ( f' uint16_t s = static_cast(amdgpu::RegisterAccess(wf).read_lane({src[0]}, lane));\n' @@ -196,7 +197,9 @@ def gen_vector_unary( } if dtype in cvt_map: body = cvt_map[dtype] - if is_vop3 and dtype in F32_TO_INTEGER_DTYPES: + if is_vop3 and dtype in ( + F32_TO_INTEGER_DTYPES | F16_INPUT_CONVERSION_DTYPES + ): source, rest = body.split('\n', 1) L.append(source) L.extend(vop3_src_mod('s', 0, has_abs)) @@ -423,25 +426,40 @@ def gen_vector_unary( if is_vop3: L.extend(vop3_src_mod('s', 0, has_abs)) math_map_f16 = { - 'rcp': '1.0f / s', - 'sqrt': 'std::sqrt(s)', - 'rsq': '1.0f / std::sqrt(s)', 'floor': 'std::floor(s)', 'ceil': 'std::ceil(s)', 'trunc': 'std::trunc(s)', 'rndne': 'util::rndne_scalar(s)', 'fract': 's - std::floor(s)', - 'exp2': 'std::exp2(s)', - 'log2': 'std::log2(s)', - 'sin': 'std::sin(s * 6.2831853071795864f)', - 'cos': 'std::cos(s * 6.2831853071795864f)', + 'exp2': 'amdgpu::transcendental::log_exp_f16(s, ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', + 'log2': 'amdgpu::transcendental::log_exp_f16(s, ' + 'wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', 'abs': 'std::fabs(s)', 'neg': '-s', } + for half_op in ('rcp', 'rsq', 'sqrt', 'sin', 'cos'): + math_map_f16[half_op] = ( + f'amdgpu::transcendental::map_f16(' + 's, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))' + ) expr = math_map_f16.get(op, f's /* TODO: {op} */') if is_vop3: L.append(f' float result = {expr};') - L.extend(vop3_dst_mod('result', omod_result_type='f16')) + if op in ('log2', 'exp2', 'rcp', 'rsq', 'sqrt', 'sin', 'cos'): + L.extend( + [ + ' const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod(' + 'wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, inst_.omod);', + ' result = amdgpu::fp_mode::apply_omod_f16(result, effective_omod, wf.fp16_ovfl());', + ' if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf);', + ] + ) + else: + L.extend(vop3_dst_mod('result', omod_result_type='f16')) L.append( ' uint32_t result_bits = util::f32_to_f16_mode(result, wf.fp16_ovfl());' ) @@ -460,19 +478,26 @@ def gen_vector_unary( if is_vop3: L.extend(vop3_src_mod('s', 0, has_abs)) math_map = { - 'rcp': 'amdgpu::transcendental::rcp_f32(s)', + 'rcp': 'amdgpu::transcendental::rcp_f32(s, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', 'rcp_iflag': 'amdgpu::transcendental::rcp_f32(s)', - 'sqrt': 'amdgpu::transcendental::sqrt_f32(s)', - 'rsq': 'amdgpu::transcendental::rsq_f32(s)', + 'sqrt': 'amdgpu::transcendental::sqrt_f32(s, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', + 'rsq': 'amdgpu::transcendental::rsq_f32(s, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', 'floor': 'std::floor(s)', 'ceil': 'std::ceil(s)', 'trunc': 'std::trunc(s)', 'rndne': 'util::rndne_scalar(s)', 'fract': 's - std::floor(s)', - 'exp2': 'amdgpu::transcendental::exp_f32(s)', - 'log2': 'amdgpu::transcendental::log_f32(s)', - 'sin': 'amdgpu::transcendental::sin_f32(s)', - 'cos': 'amdgpu::transcendental::cos_f32(s)', + 'exp2': 'amdgpu::transcendental::exp_f32(s, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', + 'log2': 'amdgpu::transcendental::log_f32(s, ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', + 'sin': 'amdgpu::transcendental::sin_f32(s, wf.fp_denorm_mode_f32(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', + 'cos': 'amdgpu::transcendental::cos_f32(s, wf.fp_denorm_mode_f32(), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))', 'abs': 'std::fabs(s)', 'neg': '-s', } @@ -592,7 +617,7 @@ def gen_vector_binop( 'max': 'std::fmax(sv0, sv1)', 'fmin': 'std::fmin(sv0, sv1)', 'fmax': 'std::fmax(sv0, sv1)', - 'fmac': f'std::fma(sv0, sv1, std::bit_cast(amdgpu::RegisterAccess(wf).read_lane({d}, lane)))', + 'fmac': f'amdgpu::fp_mode::arithmetic(sv0, sv1, std::bit_cast(amdgpu::RegisterAccess(wf).read_lane({d}, lane)), wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode())', 'ldexp': 'amdgpu::ldexp(sv0, sv1_i, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())', } expr = f_op_map.get(op, f'sv0 /* TODO: {op} */') @@ -966,7 +991,7 @@ def gen_vector_ternary( L.extend(vop3_src_mod('c', 2, has_abs)) f_map = { 'mad': 'a * b + c', - 'fma': 'std::fma(a, b, c)', + 'fma': 'amdgpu::fp_mode::arithmetic(a, b, c, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode())', 'min3': 'std::fmin(std::fmin(a, b), c)', 'max3': 'std::fmax(std::fmax(a, b), c)', 'minimum3': '[&]() { if (std::isnan(a) || std::isnan(b) || std::isnan(c)) return std::numeric_limits::quiet_NaN(); auto ab = (a == b) ? (std::signbit(a) ? a : b) : (a < b ? a : b); return (ab == c) ? (std::signbit(ab) ? ab : c) : (ab < c ? ab : c); }()', @@ -979,58 +1004,18 @@ def gen_vector_ternary( 'maxmin_num': 'std::fmax(a, std::fmin(b, c))', 'med3': 'std::fmax(std::fmin(std::fmax(a, b), c), std::fmin(a, b))', } - # Cube map operations: inputs are (x, y, z) - if op == 'cubeid': - L.append( - ' float ax = std::fabs(a), ay = std::fabs(b), az = std::fabs(c);' - ) - L.append(' float face;') - L.append(' if (az >= ax && az >= ay) face = c >= 0 ? 4.0f : 5.0f;') - L.append(' else if (ay >= ax) face = b >= 0 ? 2.0f : 3.0f;') - L.append(' else face = a >= 0 ? 0.0f : 1.0f;') - if is_vop3: - L.extend(vop3_dst_mod('face')) - L.append( - f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, std::bit_cast(face));' - ) - elif op == 'cubesc': - L.append( - ' float ax = std::fabs(a), ay = std::fabs(b), az = std::fabs(c);' - ) - L.append(' float sc;') - L.append(' if (az >= ax && az >= ay) sc = c >= 0 ? a : -a;') - L.append(' else if (ay >= ax) sc = a;') - L.append(' else sc = a >= 0 ? -c : c;') + # Cube selection and MODE policy match the typed semantics path. + if op in CUBE_OPERATIONS: + result = cube_expression(op, 'a', 'b', 'c') if is_vop3: - L.extend(vop3_dst_mod('sc')) - L.append( - f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, std::bit_cast(sc));' - ) - elif op == 'cubetc': - L.append( - ' float ax = std::fabs(a), ay = std::fabs(b), az = std::fabs(c);' - ) - L.append(' float tc;') - L.append(' if (az >= ax && az >= ay) tc = -b;') - L.append(' else if (ay >= ax) tc = b >= 0 ? c : -c;') - L.append(' else tc = -b;') + result = cube_omod(result) + L.append(f' float result = {result};') if is_vop3: - L.extend(vop3_dst_mod('tc')) - L.append( - f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, std::bit_cast(tc));' - ) - elif op == 'cubema': - L.append( - ' float ax = std::fabs(a), ay = std::fabs(b), az = std::fabs(c);' - ) - L.append(' float ma;') - L.append(' if (az >= ax && az >= ay) ma = 2.0f * az;') - L.append(' else if (ay >= ax) ma = 2.0f * ay;') - L.append(' else ma = 2.0f * ax;') - if is_vop3: - L.extend(vop3_dst_mod('ma')) + L.append( + ' if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf);' + ) L.append( - f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, std::bit_cast(ma));' + f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, std::bit_cast(result));' ) elif op in f_map: expr = f_map[op] diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_cmp.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_cmp.py index 1536da76a62a..5cfe8eb1875c 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_cmp.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_cmp.py @@ -9,6 +9,7 @@ from __future__ import annotations +from amdisa.codegen.execute import float_compare from amdisa.codegen.execute.vop3_modifiers import ( vop3_src_mod, ) @@ -190,72 +191,25 @@ def _cmp_condition( ) -> str: """Emit source reads and return the C++ condition expression. - For FP types, handles ordered comparisons (eq, lt, le, gt, ge, lg), - unordered comparisons (neq, nge, ngt, nle, nlt, nlg), and - ordered/unordered predicates (o, u) per IEEE-754. + Floating relations read the raw encodings and return a + comparison::evaluate call, which applies VOP3 ABS/NEG, MODE input + flushing and NaN ordering; the caller declares its policy. """ - is_fp = dtype in ('f32', 'f64', 'f16') - if is_fp: - if dtype == 'f64': - L.append( - f' double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64({src[0]}, lane));' - ) - L.append( - f' double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64({src[1]}, lane));' - ) - elif dtype == 'f16': - if is_vop3: - L.append( - f' float s0 = util::f16_to_f32(static_cast(::rocjitsu::amdgpu::read_vop3_true16_src({src[0]}, wf, lane, opsel, 0)));' - ) - L.append( - f' float s1 = util::f16_to_f32(static_cast(::rocjitsu::amdgpu::read_vop3_true16_src({src[1]}, wf, lane, opsel, 1)));' - ) + if dtype in float_compare.FORMATS: + if not float_compare.is_float_relation(dtype, op): + raise ValueError(f'unsupported floating compare relation: {op}') + for i in range(2): + if dtype == 'f64': + read = f'amdgpu::RegisterAccess(wf).read_lane64({src[i]}, lane)' + elif dtype == 'f16' and is_vop3: + read = f'::rocjitsu::amdgpu::read_vop3_true16_src({src[i]}, wf, lane, opsel, {i})' else: - L.append( - f' float s0 = util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane({src[0]}, lane)));' - ) - L.append( - f' float s1 = util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane({src[1]}, lane)));' - ) - else: - L.append( - f' float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane({src[0]}, lane));' - ) - L.append( - f' float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane({src[1]}, lane));' - ) + read = f'amdgpu::RegisterAccess(wf).read_lane({src[i]}, lane)' + L.append(f' const {float_compare.lane_type(dtype)} s{i} = {read};') + modifiers = None if is_vop3: - L.extend(vop3_src_mod('s0', 0, has_abs)) - L.extend(vop3_src_mod('s1', 1, has_abs)) - # Ordered comparisons (false if NaN) - ordered_map = { - 'eq': 's0 == s1', - 'ne': 's0 != s1', - 'lg': 's0 < s1 || s0 > s1', - 'lt': 's0 < s1', - 'le': 's0 <= s1', - 'gt': 's0 > s1', - 'ge': 's0 >= s1', - } - # Unordered comparisons (true if NaN) - unordered_map = { - 'neq': 's0 != s1 || std::isnan(s0) || std::isnan(s1)', - 'nge': '!(s0 >= s1)', # true when NaN (IEEE) - 'ngt': '!(s0 > s1)', - 'nle': '!(s0 <= s1)', - 'nlt': '!(s0 < s1)', - 'nlg': '!(s0 < s1 || s0 > s1)', - } - if op in ordered_map: - return ordered_map[op] - if op in unordered_map: - return unordered_map[op] - if op == 'o': - return '!std::isnan(s0) && !std::isnan(s1)' - if op == 'u': - return 'std::isnan(s0) || std::isnan(s1)' - return f's0 == s1 /* TODO: {op} */' + modifiers = ('inst_.abs' if has_abs else '0u', 'inst_.neg') + return float_compare.evaluate_expr(dtype, op, 's0', 's1', modifiers=modifiers) elif dtype in ('i64',): L.append( f' int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane64({src[0]}, lane));' @@ -363,6 +317,8 @@ def gen_vector_cmp( L.append(' uint64_t vcc = 0;') if _uses_vop3_true16_opsel(op, dtype, is_vop3): L.append(' uint32_t opsel = amdgpu::vop3_opsel(inst_);') + if float_compare.is_float_relation(dtype, op): + L.append(float_compare.policy_decl(dtype)) L.append(' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {') L.append(' if (!(exec & (1ULL << lane))) continue;') @@ -405,6 +361,8 @@ def gen_vector_cmpx( L.append(' uint64_t result = 0;') if _uses_vop3_true16_opsel(op, dtype, is_vop3): L.append(' uint32_t opsel = amdgpu::vop3_opsel(inst_);') + if float_compare.is_float_relation(dtype, op): + L.append(float_compare.policy_decl(dtype)) L.append(' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {') L.append(' if (!(exec & (1ULL << lane))) continue;') diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py index e033d953b375..67669aef720d 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py @@ -1204,10 +1204,11 @@ def gen_vector_permlane_swap(dst: list[str], src: list[str], stride: int) -> str So for the 16-lane form on a wave64 this swaps lanes 0-15<->16-31 AND 32-47<->48-63 (all four groups); for the 32-lane form it swaps 0-31<->32-63. src0[base+stride..] and vdst[base..base+stride-1] within each block are - UNCHANGED. EXEC mask is IGNORED. Both vdst and src0 are outputs (LLVM: - returns {vdst_new, src0_new}). + UNCHANGED. Each output write is masked by its destination lane's EXEC bit. + Both vdst and src0 are outputs (LLVM: returns {vdst_new, src0_new}). """ L = [] + L.append(' uint64_t exec = wf.exec();') L.append(' uint32_t tmp_dst[64] = {}, tmp_src[64] = {};') L.append(' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {') L.append( @@ -1221,11 +1222,13 @@ def gen_vector_permlane_swap(dst: list[str], src: list[str], stride: int) -> str f' for (uint32_t base = 0; base + {stride} < wf.wf_size(); base += 2u * {stride}) {{' ) L.append(f' for (uint32_t i = 0; i < {stride}; ++i) {{') + L.append(' if (exec & (1ULL << (base + i)))') L.append( - f' amdgpu::RegisterAccess(wf).write_lane({dst[1]}, base + i, tmp_dst[base + {stride} + i]);' + f' amdgpu::RegisterAccess(wf).write_lane({dst[1]}, base + i, tmp_dst[base + {stride} + i]);' ) + L.append(f' if (exec & (1ULL << (base + {stride} + i)))') L.append( - f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, base + {stride} + i, tmp_src[base + i]);' + f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, base + {stride} + i, tmp_src[base + i]);' ) L.append(' }') L.append(' }') @@ -1379,6 +1382,7 @@ def gen_vector_cvt_pk( opsel: str = '0u', dtype: str | None = None, is_vop3: bool = False, + has_abs: bool = False, fp8_format_select: str | None = None, arch_name: str = '', ) -> str: @@ -1429,11 +1433,14 @@ def gen_vector_cvt_pk( L.append( f' float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane({src[1]}, lane));' ) + if is_vop3: + L.extend(vop3_src_mod('s0', 0, has_abs)) + L.extend(vop3_src_mod('s1', 1, has_abs)) if op == 'i16': L.append(' auto cvt_i16 = [](float f) -> int16_t {') L.append(' if (std::isnan(f)) return 0;') L.append( - ' return static_cast(util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f)));' + ' return static_cast(util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0)));' ) L.append(' };') L.append(' int16_t lo = cvt_i16(s0);') @@ -1442,7 +1449,7 @@ def gen_vector_cvt_pk( L.append(' auto cvt_u16 = [](float f) -> uint16_t {') L.append(' if (std::isnan(f)) return 0;') L.append( - ' return static_cast(util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f)));' + ' return static_cast(util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0)));' ) L.append(' };') L.append(' uint16_t lo = cvt_u16(s0);') @@ -1457,6 +1464,9 @@ def gen_vector_cvt_pk( L.append( f' float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane({src[1]}, lane));' ) + if is_vop3: + L.extend(vop3_src_mod('s0', 0, has_abs)) + L.extend(vop3_src_mod('s1', 1, has_abs)) L.append(f' uint32_t lo = util::f32_to_f16_rtz(s0);') L.append(f' uint32_t hi = util::f32_to_f16_rtz(s1);') L.append( diff --git a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py index cdccdd128dd7..18395feff4e0 100644 --- a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py +++ b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py @@ -443,6 +443,11 @@ def atomic_legacy_minmax(self) -> bool: """Older float-atomic rules preserve selected input bits and propagate SNaNs.""" return True + @property + def atomic_source_nan_first(self) -> bool: + """L2 ADD NaN order; indexed LDS always prefers the incoming operand.""" + return False + def scalar_atomic_denorm_modes( self, operation: str, elem_size: int, *, ds: bool ) -> tuple[str, str]: @@ -546,6 +551,20 @@ def supports_gpr_idx(self) -> bool: """True when MODE.GPR_IDX_EN applies M0 indexing to VALU operands.""" return False + @property + def supports_vskip(self) -> bool: + """True when MODE.VSKIP suppresses vector instruction issue.""" + return False + + def vskip_affected_encoding(self, enc_name: str) -> bool: + """Whether MODE.VSKIP suppresses this encoding, including VOP subformats.""" + # XML subformats such as VOP3_SDST_ENC and VOP3P_MFMA omit ENC_. + encoding = enc_name.upper().removeprefix('ENC_') + return self.supports_vskip and ( + encoding.startswith('VOP') + or encoding in ('MUBUF', 'MTBUF', 'MIMG', 'DS', 'FLAT', 'EXP', 'VINTRP') + ) + @property def uses_packed_16bit_e32_source_selectors(self) -> bool: """True when E32 16-bit source selectors can address packed high halves.""" @@ -556,6 +575,11 @@ def renders_gfx11_image_syntax(self) -> bool: """Whether GFX11 image operands and modifiers use canonical syntax.""" return False + @property + def has_gfx11_image_address_extension(self) -> bool: + """Whether MIMG NSA appends one DWORD of address-register selectors.""" + return False + @property def split_ds_2addr_offsets(self) -> bool: """Whether DS 2ADDR instructions render two independent offsets.""" @@ -1501,6 +1525,18 @@ def waitcnt_family(self) -> str: """ return 'gfx9' + @property + def vmcnt_capacity(self) -> int: + """VMCNT's all-ones no-wait value and issue capacity, or zero if absent.""" + return 0 if self.waitcnt_family == 'gfx12' else (1 << 6) - 1 + + @property + def lgkmcnt_capacity(self) -> int: + """LGKMCNT's all-ones no-wait value and issue capacity, or zero if absent.""" + if self.waitcnt_family == 'gfx12': + return 0 + return int(self.waitcnt_lgkmcnt_mask, 0) + @property def vmem_stores_complete_in_order(self) -> bool: """Whether non-FLAT VMEM stores join the ordered VMEM completion class.""" @@ -1722,6 +1758,10 @@ def gds_uses_expcnt(self) -> bool: def supports_gpr_idx(self) -> bool: return True + @property + def supports_vskip(self) -> bool: + return True + _FLAT_SEGMENTS = frozenset({'GLBL', 'SCRATCH'}) # XML bug (P1): CDNA3/4 ENC_FLAT lists field 'SVE' at bit 13 but the # ISA PDF (CDNA3 Table 100, CDNA4 Table 101) names the field 'LDS'. @@ -2197,6 +2237,19 @@ class Rdna3Profile(_AmdgpuProfileBase): _SKIP = frozenset({'VOPDXY', 'VOPDXY_INST_LITERAL'}) _SOP1_BASE_COND = 'Nothas_lit_0_Nothas_lit_1' + def normalize_operand_type( + self, enc_name: str, field_name: str, operand_type: str + ) -> str: + # VINTERP uses the same 256..511 VGPR source selectors as VOP3. + # The GFX11 XML labels its nine-bit sources as unprefixed VGPR indices. + if enc_name.upper() == 'ENC_VINTERP' and field_name in ('src0', 'src1', 'src2'): + return 'OPR_SRC_VGPR' + return super().normalize_operand_type(enc_name, field_name, operand_type) + + @property + def has_gfx11_image_address_extension(self) -> bool: + return True + @property def vmem_writes_use_expcnt(self) -> bool: return True @@ -2537,6 +2590,10 @@ def atomic_legacy_minmax(self) -> bool: # RDNA4 chapter 13 / CDNA5 chapter 12 operate on flushed inputs. return False + @property + def atomic_source_nan_first(self) -> bool: + return True + def scalar_atomic_denorm_modes( self, operation: str, elem_size: int, *, ds: bool ) -> tuple[str, str]: @@ -2803,6 +2860,11 @@ class Cdna5Profile(Rdna4Profile): C++ lives under ``amdgpu/cdna5`` in the ``cdna5`` namespace. """ + @property + def atomic_source_nan_first(self) -> bool: + # Preserve the existing L2 policy until qualified on CDNA5 hardware. + return False + @property def vmem_stores_complete_in_order(self) -> bool: return True diff --git a/emulation/rocjitsu/lib/python/amdisa/isa_properties_codegen.py b/emulation/rocjitsu/lib/python/amdisa/isa_properties_codegen.py index 48c3c22f9b99..c60ad6d35fb7 100644 --- a/emulation/rocjitsu/lib/python/amdisa/isa_properties_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/isa_properties_codegen.py @@ -52,6 +52,8 @@ def emit_isa_properties(output_dir: str, specs) -> Path: compute_tmpring_wavesize_bits = profile.compute_tmpring_wavesize_bits wave_size = profile.wave_size wave_size_max = profile.wave_size_max + vmcnt_capacity = profile.vmcnt_capacity + lgkmcnt_capacity = profile.lgkmcnt_capacity addressable_vgprs = profile.max_addressable_vgprs_per_wf vgpr_count_granule_wave32 = profile.descriptor_vgpr_count_granule_wave32 vgpr_count_granule_wave64 = profile.descriptor_vgpr_count_granule_wave64 @@ -72,6 +74,8 @@ def emit_isa_properties(output_dir: str, specs) -> Path: f' .compute_tmpring_wavesize_bits = {compute_tmpring_wavesize_bits},', f' .wave_size = {wave_size},', f' .wave_size_max = {wave_size_max},', + f' .vmcnt_capacity = {vmcnt_capacity},', + f' .lgkmcnt_capacity = {lgkmcnt_capacity},', f' .max_addressable_vgprs_per_wf = {addressable_vgprs},', f' .descriptor_vgpr_count_granule_wave32 = {vgpr_count_granule_wave32},', f' .descriptor_vgpr_count_granule_wave64 = {vgpr_count_granule_wave64},', @@ -119,6 +123,8 @@ def emit_isa_properties(output_dir: str, specs) -> Path: ' uint32_t compute_tmpring_wavesize_bits = 0;', ' uint32_t wave_size = 0;', ' uint32_t wave_size_max = 0;', + ' uint8_t vmcnt_capacity = 0; ///< Zero when VMCNT is absent.', + ' uint8_t lgkmcnt_capacity = 0; ///< Zero when LGKMCNT is absent.', ' uint32_t max_addressable_vgprs_per_wf = 0;', ' uint32_t descriptor_vgpr_count_granule_wave32 = 0;', ' uint32_t descriptor_vgpr_count_granule_wave64 = 0;', diff --git a/emulation/rocjitsu/lib/python/amdisa/parser.py b/emulation/rocjitsu/lib/python/amdisa/parser.py index aabc06994b08..db045f5cce95 100644 --- a/emulation/rocjitsu/lib/python/amdisa/parser.py +++ b/emulation/rocjitsu/lib/python/amdisa/parser.py @@ -1129,7 +1129,10 @@ def parse_encoding_identifers( dte.sub_decode_table = f'sub_decode_{inst_enc.fmt_enc_name}'.lower() if dte.sub_decode_funcs is None: dte.is_primary = False - dte.sub_decode_funcs = list(sub_decode_funcs) + # Prefixes of one encoding share the same opcode table. + # A private copy loses high opcodes when code generation + # emits the first table with this encoding's name. + dte.sub_decode_funcs = sub_decode_funcs for j in range(1, dte.num_dupe_entries): dt[i + j] = dte diff --git a/emulation/rocjitsu/lib/python/amdisa/sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/sema_derive.py index 7d76605331b5..cd3db2850324 100644 --- a/emulation/rocjitsu/lib/python/amdisa/sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/sema_derive.py @@ -33,11 +33,18 @@ ) from amdisa.sema_effects import inline_binary_op_effects from amdisa.sema_properties import InstructionProperty, derive_properties -from amdisa.semantics import F32_TO_INTEGER_DTYPES +from amdisa.semantics import ( + F16_INPUT_CONVERSION_DTYPES, + F32_TO_INTEGER_DTYPES, + is_float_relation, +) if TYPE_CHECKING: from amdisa.semantics import InstructionSemantics +# Call name prefix of a floating VOPC relation; the relation mnemonic follows. +FLOAT_COMPARE_CALL = 'float_compare_' + def _src(idx: int, ty: SemaType = SemaType.B32) -> SemaNode: return SemaNode( @@ -1093,7 +1100,12 @@ def derive(sem: InstructionSemantics) -> SemaBlock: if op == 'cvt' and dtype: call_name = f'cvt_{dtype}' src0 = _src(0) - if dtype in F32_TO_INTEGER_DTYPES: + if dtype in F16_INPUT_CONVERSION_DTYPES: + # Expose the floating F16 source to VOP3 modifier enrichment. + src0 = _cast(_src(0, SemaType.F16), SemaType.F16) + if dtype == 'f32_f16': + call_name = 'cvt_f32_f16_valu' + elif dtype in F32_TO_INTEGER_DTYPES: # Expose the floating source to VOP3 modifier enrichment, then # pass register bits to the conversion helper. src0 = SemaNode( @@ -1106,6 +1118,7 @@ def derive(sem: InstructionSemantics) -> SemaBlock: ), ) if dtype in ( + 'f32_f16', 'f32_i32', 'f32_u32', 'f32_ubyte0', @@ -1151,7 +1164,13 @@ def derive(sem: InstructionSemantics) -> SemaBlock: 'cvt_norm_i16_f16', 'cvt_norm_u16_f16', ): - src0 = _src(0) + # Normalized conversions take a decoded float for ABS/NEG enrichment; + # packed FP8/BF8 conversions still take register bits. + src0 = ( + _cast(_src(0, SemaType.F16), SemaType.F16) + if op in ('cvt_norm_i16_f16', 'cvt_norm_u16_f16') + else _src(0) + ) body = _assign( _cast(_dst(0), SemaType.B32), SemaNode( @@ -1375,7 +1394,19 @@ def derive(sem: InstructionSemantics) -> SemaBlock: ty = _dtype_to_sema(sem.data_type) src0 = _cast(_src(0), ty) src1 = _cast(_src(1), ty) - cmp = _make_cmp(sem.operation or "", src0, src1) + op = sem.operation or "" + if is_float_relation(sem.data_type, op): + # comparison.h evaluates float relations on the raw encodings. The + # typed casts stay so enrichment still attaches the VOP3 modifiers. + name = f'{FLOAT_COMPARE_CALL}{op}' + cmp = SemaNode( + SemaNodeKind.CALL, + ty=SemaType.U1, + call_name=name, + children=(_id(name), src0, src1), + ) + else: + cmp = _make_cmp(op, src0, src1) body = _assign( SemaNode( SemaNodeKind.ARRAYDEREF, @@ -2875,6 +2906,7 @@ def derive(sem: InstructionSemantics) -> SemaBlock: return SemaBlock(sem.name, ExecModel.VECTOR, body) +@_register('lds_direct_load') @_register('lds_direct') class _LdsDirect(_ScalarDeriver): @staticmethod diff --git a/emulation/rocjitsu/lib/python/amdisa/sema_enrich.py b/emulation/rocjitsu/lib/python/amdisa/sema_enrich.py index 66e8c63bb9a0..9cce1eccf54c 100644 --- a/emulation/rocjitsu/lib/python/amdisa/sema_enrich.py +++ b/emulation/rocjitsu/lib/python/amdisa/sema_enrich.py @@ -52,6 +52,8 @@ 'V_CVT_F16_FP8', 'V_CVT_NORM_I16_F16', 'V_CVT_NORM_U16_F16', + 'V_CVT_I16_F16', + 'V_CVT_U16_F16', } ) diff --git a/emulation/rocjitsu/lib/python/amdisa/semantics.py b/emulation/rocjitsu/lib/python/amdisa/semantics.py index eb694b770835..c3e5a99a9ed0 100644 --- a/emulation/rocjitsu/lib/python/amdisa/semantics.py +++ b/emulation/rocjitsu/lib/python/amdisa/semantics.py @@ -23,6 +23,35 @@ # Floating-source conversions that accept VOP3 ABS/NEG before integer conversion. F32_TO_INTEGER_DTYPES = frozenset({'i32_f32', 'u32_f32', 'rpi_i32_f32', 'flr_i32_f32'}) +# Mixed-type conversions that accept VOP3 ABS/NEG on a floating half source. +F16_INPUT_CONVERSION_DTYPES = frozenset({'i16_f16', 'u16_f16', 'f32_f16'}) + +# Floating VOPC relations that read their sources; F and T/TRU are constants. +FLOAT_COMPARE_RELATIONS = frozenset( + { + 'lt', + 'eq', + 'le', + 'gt', + 'lg', + 'ge', + 'o', + 'u', + 'nge', + 'nlg', + 'ngt', + 'nle', + 'neq', + 'nlt', + } +) +FLOAT_COMPARE_DTYPES = frozenset({'f16', 'f32', 'f64'}) + + +def is_float_relation(dtype: str | None, op: str | None) -> bool: + """Whether a compare is a floating relation evaluated by shared/comparison.h.""" + return dtype in FLOAT_COMPARE_DTYPES and op in FLOAT_COMPARE_RELATIONS + @dataclass class InstructionSemantics: @@ -684,7 +713,9 @@ def _derive_sopc(name: str) -> InstructionSemantics | None: ) if name == 'S_SET_GPR_IDX_ON': return InstructionSemantics(name, 'gpr_idx', operation='on') - # Unrecognized SOPC instructions (S_SETVSKIP, …) → nop + if name == 'S_SETVSKIP': + return InstructionSemantics(name, 'set_vskip') + # Unrecognized SOPC instructions → nop return InstructionSemantics(name, 'nop') @@ -2579,7 +2610,14 @@ def _derive_vintrp(name: str) -> InstructionSemantics | None: def _derive_ldsdir(name: str) -> InstructionSemantics | None: """Derive semantics for an LDSDIR/VDSDIR (LDS Direct) instruction.""" - return InstructionSemantics(name, 'lds_direct') + return InstructionSemantics( + name, + ( + 'lds_direct_load' + if name.upper() in ('LDS_DIRECT_LOAD', 'DS_DIRECT_LOAD') + else 'lds_direct' + ), + ) # Map encoding format name (with ENC_ prefix stripped) to derivation function. diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index 48ec764e2085..d06acd23b241 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -1631,6 +1631,27 @@ def test_readlane_family_uses_source_vgpr_operand_type(): assert codegen._constructor_operand_type(sem, vdst) == 'OPR_VGPR' +@pytest.mark.parametrize( + 'name,semantic_class,expected_type', + [ + ('V_READFIRSTLANE_B32', 'vector_readfirstlane', 'OPR_SREG'), + ('V_READLANE_B32', 'vector_readlane', 'OPR_SREG'), + ('V_DIV_SCALE_F32', 'vector_div_scale', 'OPR_SREG_NOVCC'), + ], +) +def test_readlane_family_vcc_destination_policy_is_scoped( + name, semantic_class, expected_type +): + codegen = object.__new__(CodeGenerator) + codegen.isa_spec = SimpleNamespace(operand_types=['OPR_SREG', 'OPR_SREG_NOVCC']) + sem = InstructionSemantics(name, semantic_class) + vdst = Operand('vdst', 32, 'OPR_SREG_NOVCC', False, True, False, False, 0) + src = Operand('src1', 32, 'OPR_SREG_NOVCC', True, False, False, False, 1) + + assert codegen._constructor_operand_type(sem, vdst) == expected_type + assert codegen._constructor_operand_type(sem, src) == 'OPR_SREG_NOVCC' + + def test_pk_mov_b32_keeps_declared_scalar_or_vector_source_types(): codegen = object.__new__(CodeGenerator) codegen.isa_spec = SimpleNamespace( @@ -1672,6 +1693,15 @@ def test_readlane_family_decodes_lane_selector_as_scalar_value(): assert 'read_scalar_selected_lane(src0, lane)' in body assert 'src1.encoding_value_' not in body + inst = Instruction('V_READFIRSTLANE_B32', 'ENC_VOP1', 2, operands[:2]) + sem = InstructionSemantics('V_READFIRSTLANE_B32', 'vector_readfirstlane') + body = codegen._gen_execute_body(inst, sem, 'ENC_VOP1') + + assert 'uint64_t exec = wf.exec();' in body + assert 'exec ? static_cast(std::countr_zero(exec)) : 0' in body + assert 'read_scalar_selected_lane(src0, lane)' in body + assert 'read_lane(src0, lane)' not in body + operands = [ Operand('vdst', 32, 'OPR_VGPR', False, True, False, False, 0), Operand('src0', 32, 'OPR_SSRC', True, False, False, False, 1), @@ -3562,7 +3592,7 @@ def test_generated_pseudo_scalar_vop3_paths_ignore_exec_and_f16_opsel( 'amdgpu::RegisterAccess(wf).read_scalar(src0))' in body ) assert 'amdgpu::RegisterAccess(wf).write_scalar(' in body - assert 'amdgpu::pseudo_scalar::execute_f16(' in body + assert 'amdgpu::transcendental::execute_pseudo_f16(' in body assert 'wf.fp_round_mode_f16_f64()' in body assert 'wf.fp_denorm_mode_f16_f64()' in body @@ -4487,6 +4517,23 @@ def test_generated_rdna3_5_sendmsg_return_uses_symbolic_disassembly( assert 'return std::format("sendmsg({}, 0, 0)", value);' in sendmsg_return +def test_gfx11_graphics_sources_use_vgpr_selectors(): + for profile in (Rdna3Profile(), Rdna3_5Profile()): + for source in ('src0', 'src1', 'src2'): + assert ( + profile.normalize_operand_type('ENC_VINTERP', source, 'OPR_VGPR') + == 'OPR_SRC_VGPR' + ) + assert ( + profile.normalize_operand_type('ENC_VINTERP', 'vdst', 'OPR_VGPR') + == 'OPR_VGPR' + ) + assert ( + profile.normalize_operand_type('ENC_DS', 'data0', 'OPR_VGPR') == 'OPR_VGPR' + ) + assert profile.has_gfx11_image_address_extension + + def test_rdna3_5_disassembly_overrides_do_not_change_rdna3(): rdna3 = Rdna3Profile() rdna3_5 = Rdna3_5Profile() @@ -5874,7 +5921,7 @@ def test_generated_modern_dpp_source_policy_covers_non_vop1_families( assert 'dpp_plan_.row_bank_mask' in body, class_name assert 'dpp_plan_.source_write_mask' in body, class_name assert 'dpp_bound_ctrl_, dpp_fi_,' in body, class_name - assert 'wf.exec(), true' in body, class_name + assert re.search(r'wf\.exec\(\),\s*true', body), class_name vop3_model = (rdna4 / 'vop3.cpp').read_text() vop3_exec = _execution_source_path( @@ -6373,15 +6420,16 @@ def test_generated_rdna4_vop3_cvt_f32_f16_applies_true16_source_modifiers( rdna4_generated_root: Path, ): vop3 = (rdna4_generated_root / 'vop3_exec.cpp').read_text() - body = _generated_method_body(vop3, 'VCvtF32F16Vop3', 'VCvtU16F16Vop3') - assert 'read_vop3_true16_src(src0, wf, lane, opsel, 0)' in body - assert 'float src = util::f16_to_f32(static_cast(raw));' in body assert 'if (inst_.abs & (1u << 0))' in body - assert 'src = std::fabs(src);' in body assert 'if (inst_.neg & (1u << 0))' in body - assert 'std::bit_cast(src)' in body + assert 'util::f16_to_f32' in body + assert 'std::fabs(sv)' in body + assert 'sv = -sv' in body + assert 'amdgpu::fp_mode::cvt_f32_f16' in body + assert 'wf.fp_denorm_mode_f16_f64()' in body + assert 'wf.ieee_mode()' in body def test_generated_rdna3_dot2acc_uses_dot2c_simd_probe( diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_instruction_encoding_availability.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_instruction_encoding_availability.py index df8ad91201e7..045672da04f1 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_instruction_encoding_availability.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_instruction_encoding_availability.py @@ -8,7 +8,15 @@ from amdisa.codegen import CodeGenerator from amdisa.gpuisa import Instruction -from amdisa.isa_profile import Cdna4Profile, CdnaProfile, Rdna4Profile +from amdisa.isa_profile import ( + Cdna4Profile, + CdnaProfile, + Rdna1Profile, + Rdna2Profile, + Rdna3Profile, + Rdna3_5Profile, + Rdna4Profile, +) from amdisa.parser import Parser @@ -117,3 +125,26 @@ def test_modifier_availability_requires_explicit_encoding_provenance(): with pytest.raises(ValueError, match='V_SYNTHETIC.*encoding provenance'): generator._instruction_supports_dpp(unknown, 'ENC_VOP1') assert not generator._instruction_supports_dpp(known_absent, 'ENC_VOP1') + + +@pytest.mark.parametrize( + 'arch,profile,encoding,name', + [ + ('rdna1', Rdna1Profile(), 'ENC_VOP3', 'V_LSHLREV_B64'), + ('rdna2', Rdna2Profile(), 'ENC_VOP3', 'V_LSHLREV_B64'), + ('rdna3', Rdna3Profile(), 'ENC_MIMG', 'IMAGE_GATHER4H'), + ('rdna3_5', Rdna3_5Profile(), 'ENC_MIMG', 'IMAGE_GATHER4H'), + ], +) +def test_subdecoder_keeps_opcodes_across_primary_prefixes( + arch, profile, encoding, name +): + spec = Parser(str(_mrisa_dir() / f'amdgpu_isa_{arch}.xml'), profile).parse() + inst = _find_instruction(spec, encoding, name) + enc = spec.encoding_map[encoding] + # The generator emits one named table for every primary prefix of an encoding. + # Every such table must route the documented instruction, including opcodes + # whose prefix was first encountered without another instruction beside it. + for index in set(enc.primary_dt_ptrs) - {-1}: + entry = spec.primary_decode_table[index] + assert entry.sub_decode_funcs[inst.opcode] == f'decode{inst.fmt_name}' diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py index bcddc4c5ceb9..0029e54392a3 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py @@ -69,7 +69,8 @@ def test_pk_fmac_vop2_reads_old_destination_and_fuses_both_halves(): assert 'sdwa::output_modifier' in cpp assert ( cpp.count( - ', omod, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf))' + ', omod, false, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))' ) == 2 ) @@ -102,7 +103,8 @@ def test_pk_fma_f16_uses_mode_helper_and_clamp_for_both_halves(): assert 'wf.fp_round_mode_f16_f64()' in cpp assert 'wf.fp_denorm_mode_f16_f64()' in cpp assert ( - ', 0, inst_.clamp, wf.fp16_ovfl(), ' 'amdgpu::floating_clamp_nan_to_zero(wf))' + ', 0, inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), ' + 'amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))' ) in cpp @@ -535,10 +537,11 @@ def test_gfx1250_mad_mix_f32_uses_helper_and_fma(): op_sel_hi_2_expr='inst_.pad_14', opsel_exprs=('inst_.opsel', 'inst_.opsel_hi'), use_cdna5_helpers=True, + fused_result=True, ) assert 'read_fma_mix_source_f32(src0, wf, lane' in cpp - assert 'std::fma(a, b, c)' in cpp + assert 'arithmetic' in cpp assert 'a * b + c' not in cpp @@ -570,7 +573,8 @@ def test_gfx1250_mad_mixlo_f16_uses_helper_and_fma(): assert 'read_fma_mix_source_f32(src0, wf, lane' in cpp assert 'std::fma(a, b, c)' in cpp - assert 'util::f32_to_f16_mode(result, wf.fp16_ovfl())' in cpp + assert 'amdgpu::pseudo_scalar::round_f16_result(' in cpp + assert 'result, wf.fp_round_mode_f16_f64(), 0, false, wf.fp16_ovfl(), false' in cpp def test_mad_mixhi_f16_uses_true16_high_write(): diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_profile_properties.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_profile_properties.py index a4d37442abfc..776057d7366a 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_profile_properties.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_profile_properties.py @@ -136,6 +136,26 @@ def test_descriptor_sgpr_count_encoded(profile, expected): assert profile.descriptor_sgpr_count_encoded is expected +@pytest.mark.parametrize( + ('profile', 'vmcnt_capacity', 'lgkmcnt_capacity'), + [ + (Cdna1Profile(), 63, 15), + (Cdna2Profile(), 63, 15), + (CdnaProfile(), 63, 15), + (Cdna4Profile(), 63, 15), + (Rdna1Profile(), 63, 63), + (Rdna2Profile(), 63, 63), + (Rdna3Profile(), 63, 63), + (Rdna3_5Profile(), 63, 63), + (Rdna4Profile(), 0, 0), + (Cdna5Profile(), 0, 0), + ], +) +def test_wait_counter_capacities(profile, vmcnt_capacity, lgkmcnt_capacity): + assert profile.vmcnt_capacity == vmcnt_capacity + assert profile.lgkmcnt_capacity == lgkmcnt_capacity + + @pytest.mark.parametrize( ('profile', 'expected'), [ @@ -511,6 +531,8 @@ def test_isa_properties_codegen_uses_profile_values(tmp_path): assert 'uint32_t compute_tmpring_wavesize_bits = 0;' in output assert 'uint32_t wave_size = 0;' in output assert 'uint32_t wave_size_max = 0;' in output + assert 'uint8_t vmcnt_capacity = 0;' in output + assert 'uint8_t lgkmcnt_capacity = 0;' in output assert 'uint32_t descriptor_vgpr_count_granule_wave32 = 0;' in output assert 'uint32_t descriptor_vgpr_count_granule_wave64 = 0;' in output assert 'MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024;' in output @@ -528,6 +550,8 @@ def test_isa_properties_codegen_uses_profile_values(tmp_path): ' .compute_tmpring_wavesize_bits = 13,\n' ' .wave_size = 64,\n' ' .wave_size_max = 64,\n' + ' .vmcnt_capacity = 63,\n' + ' .lgkmcnt_capacity = 15,\n' ' .max_addressable_vgprs_per_wf = 256,\n' ' .descriptor_vgpr_count_granule_wave32 = 0,\n' ' .descriptor_vgpr_count_granule_wave64 = 8,\n' @@ -547,6 +571,8 @@ def test_isa_properties_codegen_uses_profile_values(tmp_path): ' .compute_tmpring_wavesize_bits = 18,\n' ' .wave_size = 32,\n' ' .wave_size_max = 64,\n' + ' .vmcnt_capacity = 0,\n' + ' .lgkmcnt_capacity = 0,\n' ' .max_addressable_vgprs_per_wf = 256,\n' ' .descriptor_vgpr_count_granule_wave32 = 8,\n' ' .descriptor_vgpr_count_granule_wave64 = 4,\n' @@ -566,6 +592,8 @@ def test_isa_properties_codegen_uses_profile_values(tmp_path): ' .compute_tmpring_wavesize_bits = 18,\n' ' .wave_size = 32,\n' ' .wave_size_max = 32,\n' + ' .vmcnt_capacity = 0,\n' + ' .lgkmcnt_capacity = 0,\n' ' .max_addressable_vgprs_per_wf = 1024,\n' ' .descriptor_vgpr_count_granule_wave32 = 16,\n' ' .descriptor_vgpr_count_granule_wave64 = 0,\n' @@ -1412,3 +1440,46 @@ def test_all_five_values_exist(self): def test_values_are_distinct(self): vals = [m.value for m in MemoryCoherencyModel] assert len(vals) == len(set(vals)) + + +@pytest.mark.parametrize( + ('profile', 'supported'), + [ + (Cdna1Profile(), True), + (Cdna2Profile(), True), + (CdnaProfile(), True), + (Cdna4Profile(), True), + (Rdna1Profile(), False), + (Rdna2Profile(), False), + (Rdna3Profile(), False), + (Rdna3_5Profile(), False), + (Rdna4Profile(), False), + (Cdna5Profile(), False), + ], +) +@pytest.mark.parametrize( + 'encoding', + [ + 'ENC_VOP1', + 'ENC_VOP2', + 'ENC_VOP3', + 'ENC_VOPC', + 'ENC_VOP3P', + 'VOP3_SDST_ENC', + 'VOP3P_MFMA', + 'ENC_MUBUF', + 'ENC_MTBUF', + 'ENC_MIMG', + 'ENC_DS', + 'ENC_FLAT', + 'ENC_EXP', + 'ENC_VINTRP', + ], +) +def test_vskip_vector_encodings(profile, supported, encoding): + assert profile.vskip_affected_encoding(encoding) is supported + + +@pytest.mark.parametrize('encoding', ['ENC_SOP1', 'ENC_SOPC', 'ENC_SOPP', 'ENC_SMEM']) +def test_vskip_does_not_suppress_scalar_encodings(encoding): + assert not CdnaProfile().vskip_affected_encoding(encoding) diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index 512cc5912a47..51e3acf3a5fb 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -17,6 +17,7 @@ SemaType, ) from amdisa.sema_derive import derive_sema_block +from amdisa.sema_enrich import enrich_block from amdisa.codegen.execute.sema_lower import ( LoweringContext, OperandBinding, @@ -282,7 +283,10 @@ def test_fmac_f32_reads_destination(self): all_kinds = {n.kind for n in block.body.walk()} assert SemaNodeKind.FMA in all_kinds cpp = lower_sema_block(block) - assert 'std::fma' in cpp + assert 'fp_mode::Arithmetic::FMA' in cpp + assert 'wf.fp_round_mode_f32()' in cpp + assert 'wf.fp_denorm_mode_f32()' in cpp + assert 'wf.cu().arch(), wf.ieee_mode()' in cpp def test_scalar_fma_reads_third_source(self): sem = _FakeSem('S_FMAAK_F32', 'scalar_binop', 'fma', 'f32', 'none') @@ -290,7 +294,10 @@ def test_scalar_fma_reads_third_source(self): all_kinds = {n.kind for n in block.body.walk()} assert SemaNodeKind.FMA in all_kinds cpp = lower_sema_block(block) - assert 'std::fma' in cpp + assert 'fp_mode::Arithmetic::FMA' in cpp + assert 'wf.fp_round_mode_f32()' in cpp + assert 'wf.fp_denorm_mode_f32()' in cpp + assert 'wf.cu().arch(), wf.ieee_mode()' in cpp assert 'src2' in cpp def test_scc_carry(self): @@ -805,10 +812,10 @@ def test_lowers_all(self): @pytest.mark.parametrize( ('name', 'enc', 'op', 'scale'), [ - ('V_CVT_NORM_I16_F16', 'ENC_VOP1', 'cvt_norm_i16_f16', '32767.0f'), - ('V_CVT_NORM_I16_F16', 'ENC_VOP3', 'cvt_norm_i16_f16', '32767.0f'), - ('V_CVT_NORM_U16_F16', 'ENC_VOP1', 'cvt_norm_u16_f16', '65535.0f'), - ('V_CVT_NORM_U16_F16', 'ENC_VOP3', 'cvt_norm_u16_f16', '65535.0f'), + ('V_CVT_NORM_I16_F16', 'ENC_VOP1', 'cvt_norm_i16_f16', '32767.0'), + ('V_CVT_NORM_I16_F16', 'ENC_VOP3', 'cvt_norm_i16_f16', '32767.0'), + ('V_CVT_NORM_U16_F16', 'ENC_VOP1', 'cvt_norm_u16_f16', '65535.0'), + ('V_CVT_NORM_U16_F16', 'ENC_VOP3', 'cvt_norm_u16_f16', '65535.0'), ], ) def test_cvt_norm_i16_u16_f16_lowers_to_scaled_saturating_convert( @@ -827,6 +834,50 @@ def test_cvt_norm_i16_u16_f16_lowers_to_scaled_saturating_convert( assert 'std::isnan' in cpp assert 'std::clamp' in cpp assert scale in cpp + assert 'util::rndne_scalar' in cpp + assert 'static_cast(s)' in cpp + assert '-32768' not in cpp + + @pytest.mark.parametrize('suffix', ['I16', 'U16']) + @pytest.mark.parametrize('has_abs', [False, True]) + def test_cvt_norm_i16_u16_f16_modifiers_precede_rounding(self, suffix, has_abs): + sem = derive_semantics(f'V_CVT_NORM_{suffix}_F16', 'ENC_VOP3') + fields = {'neg', 'clamp', 'omod'} + if has_abs: + fields.add('abs') + block = enrich_block(derive_sema_block(sem), enc_field_names=frozenset(fields)) + cpp = lower_sema_block(block) + assert cpp.index('util::f16_to_f32') < cpp.index('sv = -sv') + assert cpp.index('sv = -sv') < cpp.index('util::rndne_scalar') + assert ('std::fabs(sv)' in cpp) == has_abs + if has_abs: + assert cpp.index('std::fabs(sv)') < cpp.index('sv = -sv') + assert 'inst_.omod' not in cpp + assert 'inst_.clamp' not in cpp + + @pytest.mark.parametrize('suffix', ['I16', 'U16', 'F32']) + @pytest.mark.parametrize('has_abs', [False, True]) + def test_cvt_i16_u16_f32_f16_modifiers_precede_conversion(self, suffix, has_abs): + sem = derive_semantics(f'V_CVT_{suffix}_F16', 'ENC_VOP3') + fields = {'neg', 'clamp', 'omod'} + if has_abs: + fields.add('abs') + block = enrich_block(derive_sema_block(sem), enc_field_names=frozenset(fields)) + cpp = lower_sema_block(block) + assert cpp.index('util::f16_to_f32') < cpp.index('sv = -sv') + assert ('std::fabs(sv)' in cpp) == has_abs + if has_abs: + assert cpp.index('std::fabs(sv)') < cpp.index('sv = -sv') + if suffix == 'F32': + assert 'inst_.omod' in cpp + assert 'inst_.clamp' in cpp + assert 'wf.fp_denorm_mode_f32()' in cpp + assert 'wf.fp_denorm_mode_f16_f64()' in cpp + assert 'wf.ieee_mode()' in cpp + else: + assert 'inst_.omod' not in cpp + assert 'inst_.clamp' not in cpp + assert cpp.index('sv = -sv') < cpp.index('std::isnan(s)') @pytest.mark.parametrize('enc', ['ENC_VOP1', 'ENC_VOP3']) def test_cos_bf16_lowers_through_shared_transcendental(self, enc): @@ -1522,7 +1573,11 @@ def test_ignores_exec(self, name, operation): assert 'if (exec != 0)' not in cpp assert 'for (uint32_t lane = 0' not in cpp assert 'write_scalar' in cpp - assert 'amdgpu::pseudo_scalar::execute_' in cpp + assert ( + 'amdgpu::transcendental::execute_pseudo_f16' + if name.endswith('_F16') + else 'amdgpu::pseudo_scalar::execute_f32' + ) in cpp assert 'wf.fp_round_mode_' in cpp assert 'wf.fp_denorm_mode_' in cpp diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_lower.py index d76a78b38a7c..ed2066602f9f 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_lower.py @@ -160,6 +160,36 @@ def test_scalar_no_exec_loop(self): class TestLowerVectorAdd: + @pytest.mark.parametrize('operation', ['cubeid', 'cubesc', 'cubetc', 'cubema']) + def test_cube_output_scaling_uses_instruction_policy(self, operation: str): + cube = SemaNode( + SemaNodeKind.CALL, + call_name=operation, + ty=SemaType.F32, + children=( + SemaNode(SemaNodeKind.ID, id_name=operation), + *( + SemaNode(SemaNodeKind.LIT, lit_value=value, ty=SemaType.F32) + for value in ('1.0', '-2.0', '0.5') + ), + ), + ) + node = SemaNode( + SemaNodeKind.CALL, + call_name='apply_omod', + ty=SemaType.F32, + children=(SemaNode(SemaNodeKind.ID, id_name='apply_omod'), cube), + ) + + result = _lower_apply_omod(node, LoweringContext(exec_model=ExecModel.VECTOR)) + + assert 'cube::apply_omod(' in result + assert 'wf.fp_round_mode_f32()' in result + assert 'inst_.omod' in result + assert 'effective_omod' not in result + assert 'fp_denorm_mode' not in result + assert '*=' not in result + def test_apply_clamp_uses_architecture_mode_policy(self): node = SemaNode( SemaNodeKind.CALL, @@ -205,8 +235,11 @@ def test_apply_omod_uses_result_type_mode_policy( assert f'amdgpu::fp_mode::{expected_helper}' in result assert expected_denorm_mode in result - assert 'const uint32_t effective_omod' in result - assert 'finalize_omod_' in result + if result_type == SemaType.F32: + assert 'amdgpu::fp_mode::apply_omod_f32(v, ' in result + else: + assert 'const uint32_t effective_omod' in result + assert 'finalize_omod_' in result assert 'if (inst_.omod ==' not in result if result_type == SemaType.F16: assert 'false, inst_.omod' in result diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py index b83561844cc3..9e49564d0292 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py @@ -11,7 +11,11 @@ import pytest from amdisa.codegen.execute.sema_lower import _INLINE_UNARY_OPS -from amdisa.codegen.execute.simd_codegen import SIMD_VOP1_UNARY, simd_probe_line +from amdisa.codegen.execute.simd_codegen import ( + SIMD_VOP1_UNARY, + local_coverage_probe, + simd_probe_line, +) from amdisa.codegen.execute.vector_alu import gen_vector_unary from amdisa.codegen.execute.vector_special import ( _TRIG_PREOP_CHUNK_BITS, @@ -550,7 +554,7 @@ def test_vop3_pack_and_pknorm_f16_use_true16_source_halves(): assert 'read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1)' in pknorm assert 'float s0 = std::bit_cast' not in pknorm assert 'auto cvt_i16 = [](float f) -> int16_t {' in pknorm - assert 'util::round_to_nearest_even(std::clamp(f * 32767.0f' in pknorm + assert 'util::rndne_scalar(std::clamp(static_cast(f) * 32767.0' in pknorm def test_true16_special_vop3_simd_routes_use_true16_glue(): @@ -563,9 +567,81 @@ def test_true16_special_vop3_simd_routes_use_true16_glue(): assert simd_probe_line('v_pack_b32_f16_vop3').startswith( ' ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC' ) - assert simd_probe_line('v_cvt_pk_norm_i16_f16_vop3').startswith( - ' ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC' + assert 'ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC' in simd_probe_line( + 'v_cvt_pk_norm_i16_f16_vop3' ) div_fixup = simd_probe_line('v_div_fixup_f16_vop3') assert 'if (!wf.fp16_ovfl())' not in div_fixup assert 'ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP16' in div_fixup + + +@pytest.mark.parametrize( + 'name', ['v_cvt_pkrtz_f16_f32_vop3', 'v_cvt_pk_rtz_f16_f32_vop3'] +) +def test_packed_rtz_simd_applies_source_modifiers(name): + probes = [simd_probe_line(name)] + local = local_coverage_probe(name) + if name == 'v_cvt_pkrtz_f16_f32_vop3': + assert local is not None + probes.append(local) + else: + assert local is None + for probe in probes: + assert not probe.startswith(' if (!(inst.inst_.abs | inst.inst_.neg))') + assert 'f32_to_f16_rtz_simd' in probe + for index, source in enumerate(('a', 'b')): + absolute = f'if (inst.inst_.abs & {1 << index}u) {source} &= ~0x80000000u;' + negative = f'if (inst.inst_.neg & {1 << index}u) {source} ^= 0x80000000u;' + assert probe.index(absolute) < probe.index(negative) + assert probe.index(negative) < probe.index('f32_to_f16_rtz_simd') + assert 'inst_.abs' not in simd_probe_line(name.replace('_vop3', '_vop2')) + + +@pytest.mark.parametrize('has_abs', [False, True]) +def test_packed_rtz_vop3_modifiers_precede_conversion(has_abs): + cpp = gen_vector_cvt_pk( + ['vdst'], + ['src0', 'src1'], + 'vector_cvt_pkrtz_f16_f32', + None, + is_vop3=True, + has_abs=has_abs, + ) + assert ('std::fabs' in cpp) == has_abs + for source in (0, 1): + assert f'if (inst_.neg & (1u << {source})) s{source} = -s{source};' in cpp + assert cpp.index(f's{source} = -s{source}') < cpp.index('util::f32_to_f16_rtz') + + +@pytest.mark.parametrize('dtype', ['f32', 'f16']) +@pytest.mark.parametrize('op', ['u16', 'i16']) +@pytest.mark.parametrize('has_abs', [False, True]) +def test_normalized_conversion_modifiers_and_single_rounding(dtype, op, has_abs): + cpp = gen_vector_cvt_pk( + ['vdst'], + ['src0', 'src1'], + 'vector_cvt_pknorm', + op, + dtype=dtype, + is_vop3=True, + has_abs=has_abs, + ) + assert ('std::fabs(s0)' in cpp) == has_abs + assert ('std::fabs(s1)' in cpp) == has_abs + assert 'if (inst_.neg & (1u << 0)) s0 = -s0;' in cpp + assert 'if (inst_.neg & (1u << 1)) s1 = -s1;' in cpp + assert 'util::rndne_scalar(std::clamp(static_cast(f)' in cpp + for source in (0, 1): + negation = cpp.index(f's{source} = -s{source}') + assert negation < cpp.index('util::rndne_scalar') + if has_abs: + assert cpp.index(f'std::fabs(s{source})') < negation + for prefix in (['pk_norm', 'pknorm'] if dtype == 'f32' else ['pk_norm']): + probe = simd_probe_line(f'v_cvt_{prefix}_{op}_{dtype}_vop3') + assert not probe.startswith(' if (!(inst.inst_.abs | inst.inst_.neg))') + sign = '0x80000000u' if dtype == 'f32' else '0x8000u' + for index, source in enumerate(('a', 'b')): + absolute = f'if (inst.inst_.abs & {1 << index}u) {source} &= ~{sign};' + negative = f'if (inst.inst_.neg & {1 << index}u) {source} ^= {sign};' + assert probe.index(absolute) < probe.index(negative) + assert probe.index(negative) < probe.index(f'cvt_pknorm_{op}_f32_simd') diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/liveness.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/liveness.cpp index 32dd5a00e7b5..cfc3c7db14e8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/liveness.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/liveness.cpp @@ -37,6 +37,8 @@ namespace { return count <= std::numeric_limits::max(); } +// Iterative DFS from start that appends blocks in post-order, never leaving +// the allowed set. void dfs_reverse_post_order(const BasicBlock &start, const std::unordered_set &allowed, std::unordered_set &visited, @@ -62,6 +64,7 @@ void dfs_reverse_post_order(const BasicBlock &start, } } +// Ordinary registers that du overwrites on every path and in every lane. [[nodiscard]] RegisterSet kill_defs(const InstDefUse &du, ExecState exec_before) { // Ordinary registers only: special singletons (EXEC/VCC/...) live in du.defs // but are not part of scratch-allocation liveness and must not become kills. @@ -80,6 +83,7 @@ void dfs_reverse_post_order(const BasicBlock &start, return kills; } +// The 32-bit word at offset in text, or nullopt if it runs past the end. [[nodiscard]] std::optional text_word_at(std::span text, uint64_t offset) { if (text.empty() || offset + sizeof(uint32_t) > text.size()) return std::nullopt; @@ -88,6 +92,8 @@ void dfs_reverse_post_order(const BasicBlock &start, return word; } +// True if inst may reach VGPRs through M0-relative or GPR-index addressing, +// which explicit operands do not show. [[nodiscard]] bool may_access_vgprs_indirectly(const Instruction &inst, std::span text, rj_code_arch_t arch) { const std::string_view mnemonic = inst.mnemonic(); @@ -141,6 +147,7 @@ LivenessAnalysis::LivenessAnalysis(UnavailableTag) : available_(false) {} LivenessAnalysis LivenessAnalysis::unavailable() { return LivenessAnalysis(UnavailableTag{}); } +// Blocks are visited in scope order, so unreachable blocks still appear. std::vector reverse_post_order(KernelBlockScope blocks) { std::vector postorder; std::unordered_set allowed; @@ -161,6 +168,38 @@ std::vector reverse_post_order(KernelBlockScope blocks) { return postorder; } +namespace { + +// Raises bound past operand if it names an allocatable ordinary SGPR. +void raise_sgpr_bound_for_operand(uint32_t &bound, const Operand *operand) { + if (operand == nullptr) + return; + const auto ref = operand->to_register_ref(); + if (!ref || ref->cls != RegClass::SGPR) + return; + if (ref->index >= REGISTER_SET_ALLOCATABLE_SGPRS) + return; + bound = std::max(bound, static_cast(ref->index) + ref->width); +} + +} // namespace + +// Syntactic scan over explicit operands; see liveness.h for what is excluded. +uint32_t explicit_ordinary_sgpr_bound(KernelBlockScope blocks) { + uint32_t bound = 0; + for (BasicBlock *block : blocks) { + if (block == nullptr) + continue; + for (const Instruction &inst : block->instructions()) { + for (int i = 0; i < inst.num_src_operands(); ++i) + raise_sgpr_bound_for_operand(bound, inst.src_operand(i)); + for (int i = 0; i < inst.num_dst_operands(); ++i) + raise_sgpr_bound_for_operand(bound, inst.dst_operand(i)); + } + } + return bound; +} + LivenessAnalysis::LivenessAnalysis(KernelBlockScope blocks, std::unique_ptr exec, LivenessAnalysisOptions options, std::span extra_edges) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/liveness.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/liveness.h index 9feb31dc5833..ba882101b054 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/liveness.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/liveness.h @@ -160,6 +160,20 @@ struct LivenessAnalysisOptions { /// walking into other decoded code. [[nodiscard]] std::vector reverse_post_order(KernelBlockScope blocks); +/// @brief One past the highest ordinary SGPR named by an explicit operand in +/// @p blocks, counting operand width, or 0 if the scope names none. +/// +/// @details An allocation floor for callers placing new scalar storage, computed +/// without dataflow or a LivenessAnalysis instance. Implicit operands are +/// excluded: an ordinary SGPR reached implicitly is also named explicitly +/// (FLAT/GLOBAL `saddr` is both), so only special state would come in, which is +/// not allocatable. Indices at or past REGISTER_SET_ALLOCATABLE_SGPRS are +/// excluded as unreachable by any caller bound. +/// +/// A partial scope understates the result and the gap it reports may hold +/// registers the kernel is using. +[[nodiscard]] uint32_t explicit_ordinary_sgpr_bound(KernelBlockScope blocks); + /// @brief Kernel-global register usage plus deferred backward CFG liveness. /// /// @details Construction eagerly resolves gfx1250 VGPR banks and scans global diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/waitcheck/state.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/waitcheck/state.cpp index 2d8ee4987d60..d09a7c958551 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/waitcheck/state.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/waitcheck/state.cpp @@ -52,8 +52,9 @@ bool WaitcheckStateOps::register_set_less(const RegisterSet &lhs, const Register std::vector rhs_regs; lhs_regs.reserve(lhs.size()); rhs_regs.reserve(rhs.size()); - lhs.for_each([&](RegisterRef ref) { lhs_regs.push_back(ref); }); - rhs.for_each([&](RegisterRef ref) { rhs_regs.push_back(ref); }); + // Inline these callbacks to avoid packing RegisterRef through the aggregate ABI. + lhs.for_each([&](RegisterRef ref) __attribute__((always_inline)) { lhs_regs.push_back(ref); }); + rhs.for_each([&](RegisterRef ref) __attribute__((always_inline)) { rhs_regs.push_back(ref); }); auto less = [](RegisterRef lhs_ref, RegisterRef rhs_ref) { return std::make_tuple(static_cast(lhs_ref.cls), lhs_ref.index, lhs_ref.width) < std::make_tuple(static_cast(rhs_ref.cls), rhs_ref.index, rhs_ref.width); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/waitcheck/stream.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/waitcheck/stream.cpp index d9ffc36211c8..dd2c9ab5d06f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/waitcheck/stream.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/analysis/waitcheck/stream.cpp @@ -23,7 +23,8 @@ bool has_committed_generations(rj_code_arch_t arch) { std::optional first_intersection(const RegisterSet &a, const RegisterSet &b) { std::optional first; - (a & b).for_each([&](RegisterRef ref) { + // Keep the optional assignment together with RegisterRef construction. + (a & b).for_each([&](RegisterRef ref) __attribute__((always_inline)) { if (!first) first = ref; }); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/builders/smem_builders.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/builders/smem_builders.h new file mode 100644 index 000000000000..9fcea342aa84 --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/builders/smem_builders.h @@ -0,0 +1,233 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file smem_builders.h +/// @brief ISA-dispatched scalar-memory (SMEM) instruction builders. + +#pragma once + +#include +#include + +#include "rocjitsu/code/builders/instruction_builder.h" +#include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna1/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna1/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna2/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna2/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna5/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna5/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna5/operand_types.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna1/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna1/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna1/operand_types.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna2/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna2/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna2/operand_types.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3/operand_types.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3_5/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3_5/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand_types.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/operand_types.h" +#include "util/except.h" + +namespace rocjitsu { + +/// @brief Highest SGPR index the 6-bit SBASE field can name, given it holds the +/// index halved. +inline constexpr uint16_t kMaxSmemSbase = 126; + +/// @brief Highest SGPR index the 7-bit SDATA field can name. +inline constexpr uint16_t kMaxSmemSdata = 127; + +/// @brief Largest forward byte offset a 21-bit SMEM immediate can name. +/// +/// @details CDNA1-4 and RDNA1-3.5 place the offset at bit 32 with width 21 +/// (`generated//builders.h`, `build_smem`) and read it as signed, so a +/// forward offset gets the low 20 bits. The address calculation sign-extends it: +/// `shared/addr_calc_scalar.h` does `(inst.offset << 11) >> 11`. +inline constexpr uint32_t kSmem21BitMaxByteOffset = 0x0FFFFFu; + +/// @brief Largest forward byte offset a 24-bit SMEM immediate can name. +/// +/// @details RDNA4 and CDNA5 widen the field to 24 bits and rename it `ioffset`. +/// Still signed: `rdna4/addr_calc.cpp` and `cdna5/addr_calc.h` both sign-extend +/// with `(ioffset << 8) >> 8`. +inline constexpr uint32_t kSmem24BitMaxByteOffset = 0x7FFFFFu; + +/// @brief Largest forward byte offset @p arch's SMEM immediate field can name. +/// +/// @throws util::UnimplementedInst for a non-AMDGPU architecture. +[[nodiscard]] inline constexpr uint32_t max_smem_byte_offset(rj_code_arch_t arch) { + switch (arch) { + case ROCJITSU_CODE_ARCH_CDNA1: + case ROCJITSU_CODE_ARCH_CDNA2: + case ROCJITSU_CODE_ARCH_CDNA3: + case ROCJITSU_CODE_ARCH_CDNA4: + case ROCJITSU_CODE_ARCH_RDNA1: + case ROCJITSU_CODE_ARCH_RDNA2: + case ROCJITSU_CODE_ARCH_RDNA3: + case ROCJITSU_CODE_ARCH_RDNA3_5: + return kSmem21BitMaxByteOffset; + case ROCJITSU_CODE_ARCH_CDNA5: + case ROCJITSU_CODE_ARCH_RDNA4: + return kSmem24BitMaxByteOffset; + default: + throw util::UnimplementedInst("SMEM offset range for target architecture"); + } +} + +/// @brief Encode `s_load_dwordx2 s[sdst:sdst+1], s[sbase:sbase+1], @p byte_offset` +/// (spelled `s_load_b64` from RDNA3 on). +/// +/// @param sdst Base of the destination SGPR pair. Must be even. +/// @param sbase Base of the 64-bit address SGPR pair. Must be even: the encoding +/// carries the index halved and has no bit for the low one. +/// @param byte_offset Forward immediate byte offset from the base address. Must +/// be a multiple of 4 and not exceed @ref max_smem_byte_offset for @p arch. +/// +/// @details Three things the caller should not have to know. SBASE is the +/// register index halved on every generation; the decoder multiplies it back. +/// "No SGPR offset" has two spellings: CDNA1-4 gate the register and the +/// immediate with independent SOFFSET_EN and IMM bits, so an immediate-only load +/// clears the former and sets the latter, while RDNA has neither bit and instead +/// carries an always-present SOFFSET field that must name NULL. The NULL code +/// itself moved (125 on RDNA1/2, 124 from RDNA3 on), so each case takes +/// OPR_SMEM_OFFSET_NULL from its own generation's table for this field rather +/// than a shared constant or the generic scalar-source table. +/// +/// @throws util::InvalidInst if either pair base is odd or out of field range, or +/// if @p byte_offset is unaligned or out of range. +/// @throws util::UnimplementedInst for a non-AMDGPU architecture. +[[nodiscard]] inline std::array +build_s_load_dwordx2(uint16_t sdst, uint16_t sbase, uint32_t byte_offset, rj_code_arch_t arch) { + // Both operands name a 64-bit pair, so both bases must be even. The encoded + // fields are narrower than uint16_t and the generated set_field masks rather + // than rejects, so an out-of-range index would silently become a different + // register: SBASE is 6 bits holding the index halved, SDATA is 7 bits. + if ((sbase % 2) != 0) + throw util::InvalidInst("SMEM address SGPR pair base must be even"); + if ((sdst % 2) != 0) + throw util::InvalidInst("SMEM destination SGPR pair base must be even"); + if (sbase > kMaxSmemSbase) + throw util::InvalidInst("SMEM address SGPR index exceeds the SBASE field"); + if (sdst > kMaxSmemSdata) + throw util::InvalidInst("SMEM destination SGPR index exceeds the SDATA field"); + if (byte_offset > max_smem_byte_offset(arch)) + throw util::InvalidInst("SMEM immediate byte offset out of range for target architecture"); + // Scalar loads ignore the low two address bits, so an unaligned offset would + // silently load from the aligned-down address. + if ((byte_offset & 0x3u) != 0) + throw util::InvalidInst("SMEM immediate byte offset must be a multiple of 4"); + + const auto base = static_cast(sbase / 2); + const auto data = static_cast(sdst); + switch (arch) { + case ROCJITSU_CODE_ARCH_CDNA1: + return cdna1::build_smem( + cdna1::kSLoadDwordx2Smem, + {.sbase = base, .sdata = data, .soffset_en = 0, .imm = 1, .offset = byte_offset}); + case ROCJITSU_CODE_ARCH_CDNA2: + return cdna2::build_smem( + cdna2::kSLoadDwordx2Smem, + {.sbase = base, .sdata = data, .soffset_en = 0, .imm = 1, .offset = byte_offset}); + case ROCJITSU_CODE_ARCH_CDNA3: + return cdna3::build_smem( + cdna3::kSLoadDwordx2Smem, + {.sbase = base, .sdata = data, .soffset_en = 0, .imm = 1, .offset = byte_offset}); + case ROCJITSU_CODE_ARCH_CDNA4: + return cdna4::build_smem( + cdna4::kSLoadDwordx2Smem, + {.sbase = base, .sdata = data, .soffset_en = 0, .imm = 1, .offset = byte_offset}); + case ROCJITSU_CODE_ARCH_RDNA1: + return rdna1::build_smem(rdna1::kSLoadDwordx2Smem, + {.sbase = base, + .sdata = data, + .offset = byte_offset, + .soffset = static_cast(rdna1::OPR_SMEM_OFFSET_NULL)}); + case ROCJITSU_CODE_ARCH_RDNA2: + return rdna2::build_smem(rdna2::kSLoadDwordx2Smem, + {.sbase = base, + .sdata = data, + .offset = byte_offset, + .soffset = static_cast(rdna2::OPR_SMEM_OFFSET_NULL)}); + case ROCJITSU_CODE_ARCH_RDNA3: + return rdna3::build_smem(rdna3::kSLoadB64Smem, + {.sbase = base, + .sdata = data, + .offset = byte_offset, + .soffset = static_cast(rdna3::OPR_SMEM_OFFSET_NULL)}); + case ROCJITSU_CODE_ARCH_RDNA3_5: + return rdna3_5::build_smem(rdna3_5::kSLoadB64Smem, + {.sbase = base, + .sdata = data, + .offset = byte_offset, + .soffset = static_cast(rdna3_5::OPR_SMEM_OFFSET_NULL)}); + case ROCJITSU_CODE_ARCH_RDNA4: + return rdna4::build_smem(rdna4::kSLoadB64Smem, + {.sbase = base, + .sdata = data, + .ioffset = byte_offset, + .soffset = static_cast(rdna4::OPR_SMEM_OFFSET_NULL)}); + case ROCJITSU_CODE_ARCH_CDNA5: + // scale_offset is CDNA5-only and left 0: this is a plain byte offset, not an + // offset scaled by the load's data width. + return cdna5::build_smem(cdna5::kSLoadB64Smem, + {.sbase = base, + .sdata = data, + .ioffset = byte_offset, + .scale_offset = 0, + .soffset = static_cast(cdna5::OPR_SMEM_OFFSET_NULL)}); + default: + throw util::UnimplementedInst("s_load_dwordx2 for target architecture"); + } +} + +/// @brief Encode a wait for outstanding scalar-memory loads to land in their SGPRs. +/// +/// @details Through RDNA3.5 the monolithic `s_waitcnt 0` covers lgkmcnt, which is +/// where scalar loads retire. GFX12 (RDNA4, CDNA5) splits the counters and gives +/// scalar loads their own, `s_wait_kmcnt 0`; CDNA5 has no `s_waitcnt` opcode at +/// all. Neither of spill_builders.h's load waits is this counter: +/// build_wait_loads_complete drains LOADCNT, which orders VMEM, not SMEM. +/// +/// @note Scalar loads are asynchronous. Emit this between the load and the first +/// read of its destination. +/// +/// @throws util::UnimplementedInst for a non-AMDGPU architecture. +[[nodiscard]] inline uint32_t build_wait_scalar_loads_complete(rj_code_arch_t arch) { + switch (arch) { + case ROCJITSU_CODE_ARCH_CDNA1: + return build_sopp_encoding(arch, cdna1::kSWaitcntSopp, 0); + case ROCJITSU_CODE_ARCH_CDNA2: + return build_sopp_encoding(arch, cdna2::kSWaitcntSopp, 0); + case ROCJITSU_CODE_ARCH_CDNA3: + return build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0); + case ROCJITSU_CODE_ARCH_CDNA4: + return build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0); + case ROCJITSU_CODE_ARCH_RDNA1: + return build_sopp_encoding(arch, rdna1::kSWaitcntSopp, 0); + case ROCJITSU_CODE_ARCH_RDNA2: + return build_sopp_encoding(arch, rdna2::kSWaitcntSopp, 0); + case ROCJITSU_CODE_ARCH_RDNA3: + return build_sopp_encoding(arch, rdna3::kSWaitcntSopp, 0); + case ROCJITSU_CODE_ARCH_RDNA3_5: + return build_sopp_encoding(arch, rdna3_5::kSWaitcntSopp, 0); + case ROCJITSU_CODE_ARCH_RDNA4: + return build_sopp_encoding(arch, rdna4::kSWaitKmcntSopp, 0); + case ROCJITSU_CODE_ARCH_CDNA5: + return build_sopp_encoding(arch, cdna5::kSWaitKmcntSopp, 0); + default: + throw util::UnimplementedInst("scalar-load wait for target architecture"); + } +} + +} // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/kernel_descriptor_translator.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/kernel_descriptor_translator.cpp index 199636d88d7f..6ffa6af4152c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/kernel_descriptor_translator.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/kernel_descriptor_translator.cpp @@ -186,11 +186,6 @@ constexpr uint16_t kTtmpRdna4GridX = 9; return kernel_descriptor_user_sgpr_count(arch, desc); } -[[nodiscard]] bool has_kernarg_segment_ptr(const KD &desc) { - return AMDHSA_BITS_GET(desc.kernel_code_properties, - kd::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR) != 0; -} - [[nodiscard]] bool has_dispatch_ptr(const KD &desc) { return AMDHSA_BITS_GET(desc.kernel_code_properties, kd::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR) != 0; @@ -207,32 +202,6 @@ constexpr uint16_t kTtmpRdna4GridX = 9; return static_cast(sgpr); } -[[nodiscard]] uint16_t kernarg_segment_ptr_slot(const KD &desc) { - const uint32_t properties = desc.kernel_code_properties; - uint32_t sgpr = 0; - if (AMDHSA_BITS_GET(properties, kd::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER)) - sgpr += 4; - if (AMDHSA_BITS_GET(properties, kd::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR)) - sgpr += 2; - if (AMDHSA_BITS_GET(properties, kd::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR)) - sgpr += 2; - return static_cast(sgpr); -} - -[[nodiscard]] std::optional kernarg_segment_ptr_sgpr(const KD &desc) { - if (!has_kernarg_segment_ptr(desc)) - return std::nullopt; - return kernarg_segment_ptr_slot(desc); -} - -[[nodiscard]] uint32_t kernarg_preload_length(const KD &desc) { - return AMDHSA_BITS_GET(desc.kernarg_preload, kd::KERNARG_PRELOAD_SPEC_LENGTH); -} - -[[nodiscard]] uint32_t kernarg_preload_offset(const KD &desc) { - return AMDHSA_BITS_GET(desc.kernarg_preload, kd::KERNARG_PRELOAD_SPEC_OFFSET); -} - [[nodiscard]] bool uses_kernarg_preload_firmware_skip(rj_code_arch_t arch) { // CDNA3/CDNA4 implement kernarg preloading through the legacy firmware // compatibility window: old firmware enters at the descriptor entry and @@ -284,27 +253,6 @@ constexpr uint16_t kTtmpRdna4GridX = 9; return -1; } -[[nodiscard]] uint32_t source_initial_sgpr_count(const KD &desc, rj_code_arch_t arch) { - // USER_SGPR_COUNT covers only the user block. Enabled workgroup IDs and - // WORKGROUP_INFO are dense system SGPRs that follow it and must move when a - // kernarg pointer is inserted into that user block. - uint32_t sgprs = user_sgpr_count(desc, arch); - const uint32_t rsrc2 = desc.compute_pgm_rsrc2; - if (AMDHSA_BITS_GET(rsrc2, kd::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X)) - ++sgprs; - if (AMDHSA_BITS_GET(rsrc2, kd::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y)) - ++sgprs; - if (AMDHSA_BITS_GET(rsrc2, kd::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z)) - ++sgprs; - if (AMDHSA_BITS_GET(rsrc2, kd::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO)) - ++sgprs; - // Virtual-LDS translation currently has a gfx950 source. On gfx950 (and its - // gfx942 target), ENABLE_PRIVATE_SEGMENT initializes architected FLAT_SCRATCH - // special registers rather than appending an ordinary system SGPR, so it is - // deliberately absent from this repair range. - return sgprs; -} - [[nodiscard]] bool uses_gfx90a_accum_offset(rj_code_arch_t arch) { return arch == ROCJITSU_CODE_ARCH_CDNA2 || arch == ROCJITSU_CODE_ARCH_CDNA3 || arch == ROCJITSU_CODE_ARCH_CDNA4; @@ -768,7 +716,7 @@ translate_one_descriptor(rj_code_arch_t guest_arch, rj_code_arch_t host_arch, result.has_kernarg_segment_ptr = true; result.kernarg_segment_ptr_sgpr = inserted_slot; result.target_user_sgpr_count = result.source_user_sgpr_count + 2u; - const uint32_t source_initial_sgprs = source_initial_sgpr_count(src, guest_arch); + const uint32_t source_initial_sgprs = kernel_descriptor_initial_sgpr_count(guest_arch, src); if (source_initial_sgprs > inserted_slot) { const uint32_t repair_count = source_initial_sgprs - inserted_slot; if (repair_count > std::numeric_limits::max()) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/semantic/cdna4_to_cdna3.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/semantic/cdna4_to_cdna3.cpp index 0c5bd61b2e58..2b41105eb204 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/semantic/cdna4_to_cdna3.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/semantic/cdna4_to_cdna3.cpp @@ -764,10 +764,8 @@ ExpandResult lower_permlane_swap_b32_cdna4_to_cdna3(const Instruction &inst, // row-swap instruction. Run the data-gather portion with EXEC forced to all // lanes so mbcnt() produces physical lane IDs and ds_bpermute can read both // source half-waves. Only after both old values are captured do we narrow EXEC - // to the exact write lane groups. Do not intersect those masks with the - // original EXEC: generated execution semantics mark V_PERMLANE*_SWAP_B32 as - // EXEC-ignoring, and later control flow may re-enable lanes that depend on - // these swapped values. + // to each write lane group intersected with the original EXEC. Inactive + // destinations must retain their original values. emit_s_mov_b64(words, *saved_exec, kExecLo); emit_cdna3_exec_mask(words, UINT64_MAX); if (scratch->spilled) { @@ -788,15 +786,32 @@ ExpandResult lower_permlane_swap_b32_cdna4_to_cdna3(const Instruction &inst, emit_cdna3_ds(words, cdna3::kDsBpermuteB32Ds, from_src_low, partner_addr, vsrc); emit_cdna3_lgkm_wait(words); - const uint64_t group_mask = (uint64_t{1} << half_wave_lanes) - 1u; - uint64_t low_mask = 0; - for (uint8_t lane = 0; lane < 64; lane = static_cast(lane + 2 * half_wave_lanes)) - low_mask |= group_mask << lane; - const uint64_t high_mask = low_mask << half_wave_lanes; - emit_cdna3_exec_mask(words, low_mask); + const auto emit_write_mask = [&](bool high) { + for (uint8_t half = 0; half < 2; ++half) { + const uint8_t saved = static_cast(*saved_exec + half); + const uint8_t exec = static_cast(kExecLo + half); + if (half_wave_lanes == 16) { + // Pack with zero to retain the selected 16 bits of each saved EXEC + // word. Unlike S_AND_B32/B64, S_PACK leaves the guest SCC unchanged. + words.push_back( + cdna3::build_sop2(high ? cdna3::kSPackHhB32B16Sop2 : cdna3::kSPackLlB32B16Sop2, + {.ssrc0 = high ? static_cast(kInlineConst0) : saved, + .ssrc1 = high ? saved : static_cast(kInlineConst0), + .sdst = exec})[0]); + } else { + // A 32-lane group occupies a whole EXEC word; copy that word and + // clear the other one, also without changing SCC. + words.push_back(cdna3::build_sop1( + cdna3::kSMovB32Sop1, + {.ssrc0 = high == (half == 1) ? saved : static_cast(kInlineConst0), + .sdst = exec})[0]); + } + } + }; + emit_write_mask(false); emit_cdna3_vop3(words, cdna3::kVMovB32Vop3, vsrc, vgpr_src(from_dst_high)); - emit_cdna3_exec_mask(words, high_mask); + emit_write_mask(true); emit_cdna3_vop3(words, cdna3::kVMovB32Vop3, vdst, vgpr_src(from_src_low)); if (scratch->spilled) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/semantic/cdna4_to_cdna3_virtual_lds.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/semantic/cdna4_to_cdna3_virtual_lds.cpp index 928544438e56..7fd91bd6993f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/semantic/cdna4_to_cdna3_virtual_lds.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/dbt/semantic/cdna4_to_cdna3_virtual_lds.cpp @@ -3,6 +3,7 @@ #include "rocjitsu/code/dbt/semantic/cdna4_to_cdna3_virtual_lds.h" +#include "rocjitsu/code/analysis/liveness.h" #include "rocjitsu/code/basic_block.h" #include "rocjitsu/code/builders/instruction_builder.h" #include "rocjitsu/code/dbt/semantic/cdna3_lds.h" @@ -569,43 +570,12 @@ reserve_cdna3_virtual_lds_base_sgpr_pair(TranslationContext &context, KernelBloc if (!arch_is_cdna_4_or_lower(arch)) return std::nullopt; - auto note_sgpr_ref = [](uint32_t &count, RegisterRef ref) { - if (ref.cls != RegClass::SGPR) - return; - // Implicit operands include architectural special registers such as EXEC - // and VCC. Those live in the descriptor tail and must not make virtual-LDS - // scratch selection think every ordinary SGPR is already occupied. - if (ref.index >= kCdnaOrdinarySgprLimit) - return; - count = std::max(count, static_cast(ref.index) + ref.width); - }; - - uint32_t ordinary_floor = 0; - for (BasicBlock *block : blocks) { - if (block == nullptr) - continue; - for (const Instruction &inst : block->instructions()) { - for (int i = 0; i < inst.num_src_operands(); ++i) { - if (const Operand *operand = inst.src_operand(i)) { - if (auto ref = operand->to_register_ref()) - note_sgpr_ref(ordinary_floor, *ref); - } - } - for (int i = 0; i < inst.num_dst_operands(); ++i) { - if (const Operand *operand = inst.dst_operand(i)) { - if (auto ref = operand->to_register_ref()) - note_sgpr_ref(ordinary_floor, *ref); - } - } - - // Do not fold implicit uses/defs into the ordinary SGPR floor. They can - // describe architectural state such as EXEC/VCC/SCC rather than guest - // ordinary scalar registers, and counting them here forces small kernels - // into descriptor-full virtual-LDS spill mode. Explicit operands plus the - // descriptor ABI SGPR fields below are the values that matter for choosing - // a non-conflicting backing-pointer pair. - } - } + // The rest of this function bounds against kCdnaOrdinarySgprLimit; the shared + // scan excludes operands using its own constant. + static_assert(kCdnaOrdinarySgprLimit == REGISTER_SET_ALLOCATABLE_SGPRS, + "virtual-LDS reservation and the shared SGPR scan must agree on " + "where ordinary SGPRs end"); + uint32_t ordinary_floor = explicit_ordinary_sgpr_bound(blocks); ordinary_floor = std::max(ordinary_floor, translation.target_user_sgpr_count); auto include_sgpr = [&](int16_t sgpr, uint32_t width) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/kernel_descriptor_scan.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/kernel_descriptor_scan.cpp index f963d210f279..75608127e22a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/kernel_descriptor_scan.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/kernel_descriptor_scan.cpp @@ -243,4 +243,52 @@ void set_kernel_descriptor_user_sgpr_count(rj_code_arch_t arch, KD &desc, user_sgpr_count); } +uint32_t kernel_descriptor_initial_sgpr_count(rj_code_arch_t arch, const KD &desc) { + uint32_t sgprs = kernel_descriptor_user_sgpr_count(arch, desc); + const uint32_t rsrc2 = desc.compute_pgm_rsrc2; + if (AMDHSA_BITS_GET(rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X)) + ++sgprs; + if (AMDHSA_BITS_GET(rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y)) + ++sgprs; + if (AMDHSA_BITS_GET(rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z)) + ++sgprs; + if (AMDHSA_BITS_GET(rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO)) + ++sgprs; + return sgprs; +} + +bool has_kernarg_segment_ptr(const KD &desc) { + return AMDHSA_BITS_GET( + desc.kernel_code_properties, + rocr::llvm::amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR) != 0; +} + +uint16_t kernarg_segment_ptr_slot(const KD &desc) { + const uint32_t properties = desc.kernel_code_properties; + uint32_t sgpr = 0; + if (AMDHSA_BITS_GET(properties, + rocr::llvm::amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER)) + sgpr += 4; + if (AMDHSA_BITS_GET(properties, + rocr::llvm::amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR)) + sgpr += 2; + if (AMDHSA_BITS_GET(properties, rocr::llvm::amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR)) + sgpr += 2; + return static_cast(sgpr); +} + +std::optional kernarg_segment_ptr_sgpr(const KD &desc) { + if (!has_kernarg_segment_ptr(desc)) + return std::nullopt; + return kernarg_segment_ptr_slot(desc); +} + +uint32_t kernarg_preload_length(const KD &desc) { + return AMDHSA_BITS_GET(desc.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH); +} + +uint32_t kernarg_preload_offset(const KD &desc) { + return AMDHSA_BITS_GET(desc.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_OFFSET); +} + } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/kernel_descriptor_scan.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/kernel_descriptor_scan.h index 85c6248a6f47..696eea819485 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/kernel_descriptor_scan.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/code/kernel_descriptor_scan.h @@ -67,4 +67,47 @@ void set_kernel_descriptor_user_sgpr_count(rj_code_arch_t arch, rocr::llvm::amdhsa::kernel_descriptor_t &desc, uint32_t user_sgpr_count); +/// @brief USER_SGPR_COUNT plus the dense system SGPRs the hardware initializes +/// immediately after the user block: enabled workgroup IDs, then +/// WORKGROUP_INFO. A disabled field consumes no SGPR. +/// +/// @details One past the last SGPR the launch ABI writes, so DBT uses it as the +/// range to repair when it inserts a kernarg pointer into the user block, and +/// DBI as a floor below which it must not place framework storage. +/// +/// @warning ENABLE_PRIVATE_SEGMENT is deliberately absent. On CDNA3/CDNA4 it +/// initializes architected FLAT_SCRATCH rather than appending an ordinary system +/// SGPR. That is not established for the older targets this is reachable from, +/// so on CDNA1/CDNA2 the result may understate the block by two. +[[nodiscard]] uint32_t +kernel_descriptor_initial_sgpr_count(rj_code_arch_t arch, + const rocr::llvm::amdhsa::kernel_descriptor_t &desc); + +/// @brief Does @p desc request a kernarg segment pointer in its user SGPRs? +[[nodiscard]] bool has_kernarg_segment_ptr(const rocr::llvm::amdhsa::kernel_descriptor_t &desc); + +/// @brief User-SGPR index the kernarg segment pointer would occupy in @p desc, +/// whether or not the descriptor enables it. +/// +/// @note Answering for a descriptor that has not enabled the pointer is the +/// point: a caller inserting one needs the slot before setting the bit. Use +/// @ref kernarg_segment_ptr_sgpr to ask about a descriptor as it stands. +[[nodiscard]] uint16_t +kernarg_segment_ptr_slot(const rocr::llvm::amdhsa::kernel_descriptor_t &desc); + +/// @brief User-SGPR index of @p desc's kernarg segment pointer, if it has one. +/// +/// @warning Live only at the kernel entry: the register allocator reclaims the +/// pair after its last use. A consumer that needs the value later must capture +/// it at entry. +[[nodiscard]] std::optional +kernarg_segment_ptr_sgpr(const rocr::llvm::amdhsa::kernel_descriptor_t &desc); + +/// @brief KERNARG_PRELOAD_SPEC_LENGTH: user SGPRs preloaded with kernarg dwords, +/// zero when the kernel does not use preloading. +[[nodiscard]] uint32_t kernarg_preload_length(const rocr::llvm::amdhsa::kernel_descriptor_t &desc); + +/// @brief KERNARG_PRELOAD_SPEC_OFFSET: first preloaded kernarg dword index. +[[nodiscard]] uint32_t kernarg_preload_offset(const rocr::llvm::amdhsa::kernel_descriptor_t &desc); + } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/checkpoint.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/checkpoint.cpp index 9cfa7adae680..43aa596306c4 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/checkpoint.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/checkpoint.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -102,6 +103,7 @@ serialize_config(flatbuffers::FlatBufferBuilder &builder, const SoC &soc, uint32_t num_iods = soc.num_iods(); uint32_t num_ses = 0; uint32_t num_cus = 0; + uint32_t cus_per_shader_array = 0; flatbuffers::Offset fb_cu; if (num_xcds > 0) { @@ -111,6 +113,7 @@ serialize_config(flatbuffers::FlatBufferBuilder &builder, const SoC &soc, auto *se = xcd->shader_engine(0); num_cus = se->num_compute_units(); if (num_cus > 0) { + cus_per_shader_array = se->compute_unit(0)->cus_per_shader_array(); const auto &cu_cfg = se->compute_unit(0)->config(); // Store an explicit zero even though it is the stable wire default. It // means unbounded for a new checkpoint, while field absence remains the @@ -125,7 +128,7 @@ serialize_config(flatbuffers::FlatBufferBuilder &builder, const SoC &soc, } } - auto fb_se = fb::CreateShaderEngineConfig(builder, num_cus, fb_cu); + auto fb_se = fb::CreateShaderEngineConfig(builder, num_cus, fb_cu, cus_per_shader_array); auto fb_xcd = fb::CreateXcdConfig(builder, num_ses, fb_se); auto fb_gpu = fb::CreateAmdgpuConfig(builder, num_xcds, num_iods, fb_xcd); auto fb_vm = fb::CreateVirtualMachineConfig(builder, arch_str, fb_gpu); @@ -170,6 +173,9 @@ VirtualMachine::Config config_from_checkpoint(const fb::SimulationConfig *fb_con vm_config.soc.xcd.num_shader_engines = xcd->num_shader_engines(); if (auto *se = xcd->shader_engine()) { vm_config.soc.xcd.shader_engine.num_compute_units = se->num_compute_units(); + // Legacy checkpoints lack this geometry. Keep it unknown so WGP-ID + // reads remain unsupported instead of reporting an SE-local index. + vm_config.soc.xcd.shader_engine.cus_per_shader_array = se->cus_per_shader_array(); if (auto *cu = se->compute_unit()) { auto &cu_cfg = vm_config.soc.xcd.shader_engine.compute_unit; cu_cfg.num_wf_slots = cu->num_wf_slots(); @@ -362,7 +368,17 @@ LoadedConfig restore_checkpoint(const std::string &path) { result.thread_allocations.push_back( {choice->num_threads(), choice->cpu_dispatch_threads(), choice->async_helper_threads()}); const auto &xcd = vm_config.soc.xcd; - const uint32_t capacity = xcd.num_shader_engines * xcd.shader_engine.num_compute_units; + auto checked_capacity = [](uint64_t n) { + if (n > std::numeric_limits::max()) + throw std::invalid_argument("Checkpoint execution capacity exceeds uint32 range"); + return static_cast(n); + }; + const uint32_t cus = + checked_capacity(uint64_t{xcd.num_shader_engines} * xcd.shader_engine.num_compute_units); + // The stored topology is uniform: each nonempty XCD contributes CUs-1 workers + // to the shared pool, whose inclusive width also counts one caller. + const uint32_t capacity = + checked_capacity(1 + uint64_t{vm_config.soc.num_xcds} * (std::max(cus, 1u) - 1)); const uint32_t host_threads = amdgpu::available_host_threads(); result.execution_threads = resolve_execution_threads( {result.cpu_thread_budget, result.requested_engine_threads, result.cpu_dispatch_threads, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_common.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_common.h index 00700be3067f..289273b5b396 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_common.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_common.h @@ -13,6 +13,7 @@ #include "flatbuffers/idl.h" #include "simulation_config_generated.h" +#include #include #include #include @@ -93,9 +94,8 @@ inline PciDeviceConfig pci_device_from_fb(const fb::PciDeviceInfo *pci) { /// @brief Convert a FlatBuffers KFD identity table into the runtime config form. /// -/// @details This copies only scalar/string topology values. The resulting -/// config owns its marketing-name string so it is safe after the FlatBuffers -/// parser storage goes out of scope. +/// @details The resulting config owns its strings and IP versions so it is safe +/// after the FlatBuffers parser storage goes out of scope. /// @param json_path JSON path of @p device, used in validation diagnostics. inline KfdDeviceConfig kfd_device_from_fb(const fb::KfdDeviceInfo *device, std::string_view json_path) { @@ -170,6 +170,20 @@ inline KfdDeviceConfig kfd_device_from_fb(const fb::KfdDeviceInfo *device, config.capability = device->capability(); config.capability2 = device->capability2(); config.debug_prop = device->debug_prop(); + if (const auto *versions = device->ip_versions()) { + for (const auto *version : *versions) { + if (version->hardware_id() == 0 || version->major() == 0) + throw std::runtime_error(std::string(json_path) + + ".ip_versions requires nonzero hardware_id and major"); + if (std::ranges::any_of(config.ip_versions, [&](const DeviceIpVersion &existing) { + return existing.hardware_id == version->hardware_id(); + })) + throw std::runtime_error(std::string(json_path) + + ".ip_versions contains a duplicate hardware_id"); + config.ip_versions.push_back( + {version->hardware_id(), version->major(), version->minor(), version->revision()}); + } + } return config; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_loader.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_loader.cpp index 078fce302a83..3bffbe9f9a51 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_loader.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_loader.cpp @@ -26,6 +26,7 @@ #include "simdojo/sim/exec_mode.h" #include "simdojo/sim/topology.h" #include "simulation_config_generated.h" +#include "util/log.h" #include #include @@ -53,12 +54,13 @@ ExecutionThreadAllocation resolve_execution_threads(const ExecutionThreadRequest if (request.helpers < -1 || request.helpers > 128) throw std::invalid_argument("async_helper_threads must be -1 or between 0 and 128"); const uint32_t budget = request.budget ? request.budget : std::max(host_threads, 1u); - const uint32_t cpu_budget = - request.budget ? request.budget : std::min(budget, kDefaultExecutionThreadCap); xcds = std::max(xcds, 1u); - if (clocked) + if (clocked) { + const uint32_t cpu_budget = + request.budget ? request.budget : std::min(budget, kDefaultExecutionThreadCap); return {std::min(request.engines ? request.engines : cpu_budget, xcds), std::vector(dispatch_capacities.size(), 1), 0}; + } auto effective = [&](ExecutionThreadChoice choice) { ExecutionThreadAllocation result; result.engines = std::min(request.engines ? request.engines : choice.engines, xcds); @@ -80,11 +82,6 @@ ExecutionThreadAllocation resolve_execution_threads(const ExecutionThreadRequest uint64_t cpu_cost = candidate.engines; for (uint32_t width : candidate.dispatch) cpu_cost += width - 1; - // Extra affinity can supply helpers without increasing the automatic - // engine/dispatch allocation beyond its existing cap. Explicit CPU knobs - // retain their override behavior. - if (!request.engines && !request.dispatch && cpu_cost > cpu_budget) - continue; const uint64_t cost = cpu_cost + candidate.helpers; if (cost <= budget && cost >= best_cost) { result = std::move(candidate); @@ -616,9 +613,33 @@ void build_children(simdojo::CompositeComponent *parent, } } -void do_wire_cps(simdojo::CompositeComponent *root) { +void do_wire_cps(simdojo::CompositeComponent *root, uint32_t cus_per_shader_array, + uint32_t shader_arrays_per_engine) { std::vector all; root->collect_components(all); + + // Device geometry can describe a different machine from a custom topology. + // Keep it unknown unless every SE (including direct-CU groups) agrees. + const uint64_t expected_cus = uint64_t{cus_per_shader_array} * shader_arrays_per_engine; + for (auto *comp : all) { + auto *group = dynamic_cast(comp); + if (!group) + continue; + const uint64_t num_cus = + std::count_if(group->children().begin(), group->children().end(), [](const auto &child) { + return dynamic_cast(child.get()); + }); + if ((num_cus || dynamic_cast(group)) && num_cus != expected_cus) { + if (cus_per_shader_array) + util::Logger::warn("Shader-array geometry mismatch: ", group->full_path(), " has ", num_cus, + " compute units; expected ", expected_cus, + " from num_shader_arrays_per_engine * num_cu_per_sh. " + "Shader-array width remains unknown."); + cus_per_shader_array = 0; + break; + } + } + for (auto *comp : all) { auto *cp = dynamic_cast(comp); if (!cp) @@ -642,7 +663,7 @@ void do_wire_cps(simdojo::CompositeComponent *root) { auto *cu = dynamic_cast(se_child.get()); if (!cu) continue; - cu->set_shader_engine_location(shader_engine_id, cu_index++); + cu->set_shader_engine_location(shader_engine_id, cu_index++, cus_per_shader_array); cp->add_compute_unit(cu); } ++shader_engine_id; @@ -656,7 +677,7 @@ void do_wire_cps(simdojo::CompositeComponent *root) { auto *cu = dynamic_cast(child.get()); if (!cu) continue; - cu->set_shader_engine_location(0, cu_index++); + cu->set_shader_engine_location(0, cu_index++, cus_per_shader_array); cp->add_compute_unit(cu); } } @@ -708,7 +729,8 @@ void set_cu_l2(simdojo::CompositeComponent *root) { TopologyBuildResult build_topology(const fb::TopologyDef *topology_def, simdojo::ExecMode mode, rj_code_arch_t arch, rj_code_target_id_t target, - const AsyncResources &resources) { + const AsyncResources &resources, uint32_t cus_per_shader_array, + uint32_t shader_arrays_per_engine) { if (!topology_def || !topology_def->root()) throw std::runtime_error("TopologyDef missing root ComponentDef"); @@ -759,7 +781,7 @@ TopologyBuildResult build_topology(const fb::TopologyDef *topology_def, simdojo: } set_cu_l2(root); - do_wire_cps(root); + do_wire_cps(root, cus_per_shader_array, shader_arrays_per_engine); if (topology_def->links()) for (auto *ld : *topology_def->links()) @@ -859,11 +881,14 @@ ExecutionTopology execution_topology(const fb::ComponentDef *root) { } self(self, child, checked(copies * n)); } - // Match do_wire_cps(): a CP drains its sibling SEs, or direct sibling CUs - // when the parent has no shader engines. - if (has_cp) - result.dispatch_capacity = - std::max(result.dispatch_capacity, checked(has_se ? se_cus : direct_cus)); + // Match do_wire_cps(): sibling CPs share the same CUs. Each XCD caller can + // use at most CUs-1 pool workers, but concurrent XCDs share the whole pool. + // This is maximum demand; fewer engine partitions can leave workers idle. + if (has_cp) { + const uint64_t cus = has_se ? se_cus : direct_cus; + if (cus) + result.dispatch_capacity = checked(result.dispatch_capacity + copies * (cus - 1)); + } }; if (root) visit(visit, root, 1); @@ -968,7 +993,8 @@ LoadedConfig build_from_fb(const rocjitsu::fb::SimulationConfig *fb_config, uint result.engine_config.num_threads = result.execution_threads.engines; result.async_resources = make_async_execution_resources(result.execution_threads.helpers); result.build_result = - build_topology(topo_def, result.exec_mode, arch, result.target, result.async_resources); + build_topology(topo_def, result.exec_mode, arch, result.target, result.async_resources, + result.device.num_cu_per_sh, result.device.num_shader_arrays_per_engine); if (result.device.present) if (SoC *soc = result.soc()) soc->for_each_cp([&](amdgpu::CommandProcessor *cp) { @@ -994,7 +1020,8 @@ LoadedConfig build_from_fb(const rocjitsu::fb::SimulationConfig *fb_config, uint } for (uint32_t i = 1; i < result.num_gpus; ++i) { result.extra_gpu_builds.push_back( - build_topology(topo_def, result.exec_mode, arch, result.target, result.async_resources)); + build_topology(topo_def, result.exec_mode, arch, result.target, result.async_resources, + result.device.num_cu_per_sh, result.device.num_shader_arrays_per_engine)); if (auto *soc = dynamic_cast(result.extra_gpu_builds.back().root.get())) soc->for_each_cp([&](amdgpu::CommandProcessor *cp) { cp->set_scratch_slots_per_cu(result.device.max_slots_scratch_cu); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_loader.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_loader.h index 23f7fc5aa3ab..c9063c4b547e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_loader.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/config_loader.h @@ -35,11 +35,11 @@ class ExecutionResources; namespace config { -/// Default execution-thread ceiling, also used by the dispatch-only override. +/// Default clocked-engine and legacy dispatch-only ceiling. /// -/// This conservative policy limit bounds persistent worker allocation on -/// large hosts while retaining substantial CU parallelism. It is not a -/// hardware limit; embedding callers can override it. +/// These paths do not use the functional allocation table. Bound their +/// automatic worker counts on large hosts; explicit requests can override this +/// policy. Functional limits instead come from affinity and the target table. inline constexpr uint32_t kDefaultExecutionThreadCap = 32; /// Compatibility name for the older dispatch-only override API. inline constexpr uint32_t kDefaultCpuDispatchThreadCap = kDefaultExecutionThreadCap; @@ -47,7 +47,7 @@ inline constexpr uint32_t kDefaultCpuDispatchThreadCap = kDefaultExecutionThread /// A single VM-wide budget includes engines, retained workers from every SoC's /// dispatch pool, and one shared async-helper pool. Explicit knobs take priority. struct ExecutionThreadRequest { - uint32_t budget = 0; ///< Zero: affinity, with engine/dispatch capped at 32. + uint32_t budget = 0; ///< Zero uses affinity and the target allocation table. uint32_t engines = 0; ///< Zero selects from the table. uint32_t dispatch = 0; ///< Inclusive width per SoC; zero selects automatic sizing. int32_t helpers = -1; ///< -1 selects automatic sizing; zero disables helpers. @@ -205,9 +205,9 @@ struct LoadedConfig { std::vector devices; ///< Per-GPU configs (populated when num_gpus > 1). rj_code_target_id_t target = ROCJITSU_CODE_TARGET_INVALID; /// Requested dispatch width. Omitted/zero selects from thread_allocations; - /// each SoC's effective width is CU-capacity-clamped. + /// each SoC's upper bound is one plus the sum of CUs-1 across nonempty XCDs. uint32_t cpu_dispatch_threads = 0; - uint32_t cpu_thread_budget = 0; ///< Zero uses affinity; engine/dispatch stay capped at 32. + uint32_t cpu_thread_budget = 0; ///< Zero uses affinity and the target allocation table. /// Preserve old automatic-dispatch checkpoint metadata when saving again. bool legacy_auto_dispatch = false; int32_t async_helper_threads = -1; ///< -1 selects from the table; zero disables. diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/effective_config.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/effective_config.cpp index 560be4849a8f..ba2f640c094d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/effective_config.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/effective_config.cpp @@ -9,12 +9,14 @@ #include "embedded_schema.h" #include "simulation_config_generated.h" +#include #include #include #include #include #include #include +#include namespace rocjitsu { namespace config { @@ -22,10 +24,12 @@ namespace { constexpr std::string_view kBudgetField = "cpu_thread_budget"; -bool is_field_name_char(char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +bool is_identifier_start(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; } +bool is_identifier_char(char c) { return is_identifier_start(c) || (c >= '0' && c <= '9'); } + bool starts_comment(std::string_view json, size_t at) { return json[at] == '/' && at + 1 < json.size() && (json[at + 1] == '/' || json[at + 1] == '*'); } @@ -37,7 +41,7 @@ void skip_filler(std::string_view json, size_t &at) { if (c == ' ' || c == '\t' || c == '\r' || c == '\n') { ++at; } else if (json.compare(at, 2, "//") == 0) { - const size_t line_end = json.find('\n', at); + const size_t line_end = json.find_first_of("\r\n", at + 2); at = line_end == std::string_view::npos ? json.size() : line_end + 1; } else if (json.compare(at, 2, "/*") == 0) { const size_t block_end = json.find("*/", at + 2); @@ -48,36 +52,170 @@ void skip_filler(std::string_view json, size_t &at) { } } -/// @brief Advance past a quoted string, honoring backslash escapes. -void skip_string(std::string_view json, size_t &at) { - for (++at; at < json.size(); ++at) { - if (json[at] == '\\') { - ++at; +FailureOr parse_hex_digits(std::string_view json, size_t &at, size_t count, + const util::DiagnosticEmitter &emit_error) { + if (json.size() - at < count) + return emit_error.emit() << "unterminated escape sequence"; + + const std::string_view digits = json.substr(at, count); + uint32_t value = 0; + const auto [ptr, ec] = std::from_chars(digits.data(), digits.data() + digits.size(), value, 16); + if (ec != std::errc{} || ptr != digits.data() + digits.size()) + return emit_error.emit() << "escape code must be followed by hexadecimal digits"; + + at += count; + return value; +} + +Result append_utf8(uint32_t code_point, std::string &decoded, + const util::DiagnosticEmitter &emit_error) { + if (code_point <= 0x7f) { + decoded += static_cast(code_point); + } else if (code_point <= 0x7ff) { + decoded += static_cast(0xc0 | (code_point >> 6)); + decoded += static_cast(0x80 | (code_point & 0x3f)); + } else if (code_point <= 0xffff) { + decoded += static_cast(0xe0 | (code_point >> 12)); + decoded += static_cast(0x80 | ((code_point >> 6) & 0x3f)); + decoded += static_cast(0x80 | (code_point & 0x3f)); + } else if (code_point <= 0x10ffff) { + decoded += static_cast(0xf0 | (code_point >> 18)); + decoded += static_cast(0x80 | ((code_point >> 12) & 0x3f)); + decoded += static_cast(0x80 | ((code_point >> 6) & 0x3f)); + decoded += static_cast(0x80 | (code_point & 0x3f)); + } else { + return emit_error.emit() << "Unicode code point is out of range"; + } + return Result::success(); +} + +/// @brief Decode a quoted string and advance past it. +FailureOr parse_quoted_string(std::string_view json, size_t &at, + const util::DiagnosticEmitter &emit_error) { + if (at >= json.size() || (json[at] != '"' && json[at] != '\'')) + return emit_error.emit() << "expected a quoted string"; + + const char quote = json[at++]; + std::string decoded; + int unicode_high_surrogate = -1; + while (at < json.size()) { + const char c = json[at++]; + if (c == quote) { + if (unicode_high_surrogate != -1) + return emit_error.emit() << "illegal Unicode sequence"; + return decoded; + } + + if (static_cast(c) < ' ') + return emit_error.emit() << "illegal character in string constant"; + if (c != '\\') { + if (unicode_high_surrogate != -1) + return emit_error.emit() << "illegal Unicode sequence"; + decoded += c; continue; } - if (json[at] == '"') { - ++at; - return; + + if (at >= json.size()) + return emit_error.emit() << "unterminated escape sequence"; + const char escape = json[at++]; + if (unicode_high_surrogate != -1 && escape != 'u') + return emit_error.emit() << "illegal Unicode sequence"; + + switch (escape) { + case 'n': + decoded += '\n'; + break; + case 't': + decoded += '\t'; + break; + case 'r': + decoded += '\r'; + break; + case 'b': + decoded += '\b'; + break; + case 'f': + decoded += '\f'; + break; + case '"': + decoded += '"'; + break; + case '\'': + decoded += '\''; + break; + case '\\': + decoded += '\\'; + break; + case '/': + decoded += '/'; + break; + case 'x': { + FailureOr digits = parse_hex_digits(json, at, 2, emit_error); + if (digits.failed()) + return Result::failure(); + decoded += static_cast(digits.value()); + break; + } + case 'u': { + FailureOr digits = parse_hex_digits(json, at, 4, emit_error); + if (digits.failed()) + return Result::failure(); + const uint32_t value = digits.value(); + if (value >= 0xd800 && value <= 0xdbff) { + if (unicode_high_surrogate != -1) + return emit_error.emit() << "illegal Unicode sequence"; + unicode_high_surrogate = static_cast(value); + } else if (value >= 0xdc00 && value <= 0xdfff) { + if (unicode_high_surrogate == -1) + return emit_error.emit() << "illegal Unicode sequence"; + const uint32_t code_point = + 0x10000 + ((static_cast(unicode_high_surrogate) & 0x3ff) << 10) + + (value & 0x3ff); + Result appended = append_utf8(code_point, decoded, emit_error); + if (appended.failed()) + return Result::failure(); + unicode_high_surrogate = -1; + } else { + if (unicode_high_surrogate != -1) + return emit_error.emit() << "illegal Unicode sequence"; + Result appended = append_utf8(value, decoded, emit_error); + if (appended.failed()) + return Result::failure(); + } + break; + } + default: + return emit_error.emit() << "unknown escape code in string constant"; } } + + return emit_error.emit() << "unterminated string constant"; +} + +/// @brief Advance past a quoted string, honoring the loader's escapes. +Result skip_string(std::string_view json, size_t &at, const util::DiagnosticEmitter &emit_error) { + FailureOr parsed = parse_quoted_string(json, at, emit_error); + if (parsed.failed()) + return Result::failure(); + return Result::success(); } /// @brief Advance past one value, descending through nested objects and arrays. -void skip_value(std::string_view json, size_t &at) { +Result skip_value(std::string_view json, size_t &at, const util::DiagnosticEmitter &emit_error) { if (at >= json.size()) - return; + return Result::success(); - if (json[at] == '"') { - skip_string(json, at); - return; - } + if (json[at] == '"' || json[at] == '\'') + return skip_string(json, at, emit_error); if (json[at] == '{' || json[at] == '[') { size_t depth = 0; while (at < json.size()) { const char c = json[at]; - if (c == '"') { - skip_string(json, at); + if (c == '"' || c == '\'') { + Result skipped = skip_string(json, at, emit_error); + if (skipped.failed()) + return skipped; continue; } if (starts_comment(json, at)) { @@ -90,17 +228,21 @@ void skip_value(std::string_view json, size_t &at) { --depth; if (depth == 0) { ++at; - return; + return Result::success(); } } ++at; } - return; + return Result::success(); } while (at < json.size() && json[at] != ',' && json[at] != '}' && json[at] != ']' && - json[at] != ' ' && json[at] != '\t' && json[at] != '\r' && json[at] != '\n') + json[at] != ' ' && json[at] != '\t' && json[at] != '\r' && json[at] != '\n') { + if (starts_comment(json, at)) + break; ++at; + } + return Result::success(); } /// @brief Return @p json with top-level @p field set to @p value, inserting it when absent. @@ -108,12 +250,13 @@ void skip_value(std::string_view json, size_t &at) { /// @details Only the one field's text changes, so every other field keeps the /// spelling, ordering and formatting the author gave it and stays readable when a /// failing run is reproduced from the launch copy. -std::string set_top_level_field(std::string_view json, std::string_view field, - std::string_view value) { +FailureOr set_top_level_field(std::string_view json, std::string_view field, + std::string_view value, + const util::DiagnosticEmitter &emit_error) { size_t at = 0; skip_filler(json, at); if (at >= json.size() || json[at] != '{') - throw std::runtime_error("simulation config must be a JSON object"); + return emit_error.emit() << "simulation config must be a JSON object"; const size_t fields_begin = ++at; skip_filler(json, at); @@ -124,29 +267,33 @@ std::string set_top_level_field(std::string_view json, std::string_view field, if (at >= json.size() || json[at] == '}') break; - std::string_view name; - if (json[at] == '"') { - const size_t name_begin = at + 1; - skip_string(json, at); - if (at > name_begin) - name = json.substr(name_begin, at - 1 - name_begin); + std::string name; + if (json[at] == '"' || json[at] == '\'') { + FailureOr parsed_name = parse_quoted_string(json, at, emit_error); + if (parsed_name.failed()) + return Result::failure(); + name = std::move(parsed_name.value()); } else { const size_t name_begin = at; - while (at < json.size() && is_field_name_char(json[at])) + if (at >= json.size() || !is_identifier_start(json[at])) + return emit_error.emit() << "simulation config has an unreadable field name"; + while (at < json.size() && is_identifier_char(json[at])) ++at; - name = json.substr(name_begin, at - name_begin); + name.assign(json.substr(name_begin, at - name_begin)); } if (name.empty()) - throw std::runtime_error("simulation config has an unreadable field name"); + return emit_error.emit() << "simulation config has an empty field name"; skip_filler(json, at); if (at >= json.size() || json[at] != ':') - throw std::runtime_error("simulation config field '" + std::string(name) + "' has no value"); + return emit_error.emit() << "simulation config field '" << name << "' has no value"; ++at; skip_filler(json, at); const size_t value_begin = at; - skip_value(json, at); + Result skipped_value = skip_value(json, at, emit_error); + if (skipped_value.failed()) + return Result::failure(); if (name == field) { std::string rewritten(json.substr(0, value_begin)); rewritten.append(value); @@ -169,38 +316,59 @@ std::string set_top_level_field(std::string_view json, std::string_view field, } // namespace -std::string json_with_cpu_thread_budget(std::string_view json, uint32_t budget) { - std::string rewritten = set_top_level_field(json, kBudgetField, std::to_string(budget)); +FailureOr json_with_cpu_thread_budget(std::string_view json, uint32_t budget, + const util::DiagnosticEmitter &emit_error) { + FailureOr rewritten = + set_top_level_field(json, kBudgetField, std::to_string(budget), emit_error); + if (rewritten.failed()) + return Result::failure(); - // Rewriting text must never be able to start a simulation under a budget nobody - // asked for, so the result is parsed and the field read back before it is used. - const uint32_t applied = with_parsed_simulation_config_json( - rewritten, rocjitsu::kEmbeddedSchema, - [](const fb::SimulationConfig *config) { return config->cpu_thread_budget(); }); + // The schema parser reports a bad document by throwing. Callers of this function + // handle failure through the returned result, so that message is emitted here. + uint32_t applied = 0; + try { + applied = with_parsed_simulation_config_json( + rewritten.value(), rocjitsu::kEmbeddedSchema, + [](const fb::SimulationConfig *config) { return config->cpu_thread_budget(); }); + } catch (const std::runtime_error &error) { + return emit_error.emit() << error.what(); + } if (applied != budget) - throw std::runtime_error("cannot apply cpu_thread_budget to the simulation config"); - return rewritten; + return emit_error.emit() << "cannot apply cpu_thread_budget to the simulation config"; + return std::move(rewritten.value()); } -std::string write_effective_config(const std::string &source_path, uint32_t budget, pid_t pid) { - const std::string json = json_with_cpu_thread_budget(read_config_file(source_path), budget); +FailureOr write_effective_config(const std::string &source_path, uint32_t budget, + pid_t pid, + const util::DiagnosticEmitter &emit_error) { + // Reading the source throws when the file cannot be opened. The launch path + // reports that through the same result as a rewrite or publish failure. + std::string source_json; + try { + source_json = read_config_file(source_path); + } catch (const std::runtime_error &error) { + return emit_error.emit() << error.what(); + } + FailureOr json = json_with_cpu_thread_budget(source_json, budget, emit_error); + if (json.failed()) + return Result::failure(); const std::filesystem::path directory(rocjitsu::rpc_invocation_runtime_dir(pid)); std::error_code directory_error; std::filesystem::create_directories(directory, directory_error); if (directory_error) - throw std::runtime_error("cannot create runtime directory " + directory.string() + ": " + - directory_error.message()); + return emit_error.emit() << "cannot create runtime directory " << directory.string() << ": " + << directory_error.message(); const std::filesystem::path target = directory / kEffectiveConfigName; const std::filesystem::path temporary = target.string() + ".tmp"; std::ofstream output(temporary); - output << json; + output << json.value(); output.close(); if (!output.good()) { std::error_code remove_error; std::filesystem::remove(temporary, remove_error); - throw std::runtime_error("cannot write effective config " + target.string()); + return emit_error.emit() << "cannot write effective config " << target.string(); } std::error_code rename_error; @@ -208,8 +376,8 @@ std::string write_effective_config(const std::string &source_path, uint32_t budg if (rename_error) { std::error_code remove_error; std::filesystem::remove(temporary, remove_error); - throw std::runtime_error("cannot publish effective config " + target.string() + ": " + - rename_error.message()); + return emit_error.emit() << "cannot publish effective config " << target.string() << ": " + << rename_error.message(); } return target.string(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/effective_config.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/effective_config.h index aa4cac470898..95ad099d3bcf 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/effective_config.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/effective_config.h @@ -7,6 +7,9 @@ #ifndef ROCJITSU_CONFIG_EFFECTIVE_CONFIG_H_ #define ROCJITSU_CONFIG_EFFECTIVE_CONFIG_H_ +#include "rocjitsu/result.h" +#include "util/diagnostic.h" + #include #include #include @@ -25,17 +28,22 @@ inline constexpr char kEffectiveConfigName[] = "effective_config.json"; /// setting process-wide state, so simulations built from other configs in the same /// process keep the budgets their own configs ask for. The field is replaced where /// it already appears and inserted otherwise; all other text is preserved verbatim. -/// @throws std::runtime_error when @p json is not a simulation-config object, or -/// when the rewritten document does not parse back with @p budget. -std::string json_with_cpu_thread_budget(std::string_view json, uint32_t budget); +/// Failure means @p json is not a simulation-config object, or the rewritten document +/// does not parse back with @p budget. The reason is emitted through @p emit_error. +[[nodiscard]] FailureOr +json_with_cpu_thread_budget(std::string_view json, uint32_t budget, + const util::DiagnosticEmitter &emit_error = {}); /// @brief Write the effective config for one invocation and return its path. /// /// @details The copy is placed in the invocation runtime directory next to the /// config-path handoff an exec'd workload already reads, so it is reclaimed by the /// same cleanup and never lands beside a config file the launcher does not own. -/// @throws std::runtime_error when the source cannot be read or the copy written. -std::string write_effective_config(const std::string &source_path, uint32_t budget, pid_t pid); +/// Failure means the source cannot be read, the budget cannot be applied, or the +/// copy cannot be written. The reason is emitted through @p emit_error. +[[nodiscard]] FailureOr +write_effective_config(const std::string &source_path, uint32_t budget, pid_t pid, + const util::DiagnosticEmitter &emit_error = {}); } // namespace config } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/kfd_device_config.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/kfd_device_config.h index 8231429cea36..209b6e3ad9cc 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/kfd_device_config.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/config/kfd_device_config.h @@ -10,9 +10,18 @@ #include #include #include +#include namespace rocjitsu::config { +/// @brief Hardware block version for device IP discovery. +struct DeviceIpVersion { + uint16_t hardware_id = 0; + uint8_t major = 0; + uint8_t minor = 0; + uint8_t revision = 0; +}; + /// @brief Per-field topology values explicitly supplied by configuration. struct KfdDiscoveryOverrides { std::optional max_waves_per_simd; @@ -76,6 +85,7 @@ struct KfdDeviceConfig { bool present = false; ///< True if device section existed in config. uint32_t num_sdma_queues_per_engine = 0; ///< Regular SDMA queues per engine. KfdDiscoveryOverrides discovery_overrides; ///< Explicit PCI discovery topology overrides. + std::vector ip_versions; ///< Die-zero, instance-zero discovery versions. }; } // namespace rocjitsu::config diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/CMakeLists.txt b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/CMakeLists.txt index cc7996bbf0d7..793df9b48b47 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/CMakeLists.txt +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/CMakeLists.txt @@ -5,7 +5,6 @@ rj_add_object_library(rocjitsu_isa decoder.cpp execution_backend.cpp operand.cpp - register_set.cpp rj_decode.cpp target_registry.cpp ) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/encodings.cpp index 2179f1f4a510..ac7159ae07ae 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/encodings.cpp @@ -383,6 +383,10 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); + flags_ |= VSKIP_AFFECTED; } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -632,6 +636,7 @@ Vopc::Vopc(std::string_view mnemonic, const VopcMachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + flags_ |= VSKIP_AFFECTED; } bool Vopc::default_encoding() { @@ -746,6 +751,10 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); + flags_ |= VSKIP_AFFECTED; } void Vop2::implicit_uses(RegisterSet &uses) const { @@ -786,6 +795,7 @@ Vintrp::Vintrp(std::string_view mnemonic, const VintrpMachineInst *inst, Execute size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + flags_ |= VSKIP_AFFECTED; } bool Vintrp::default_encoding() { return 1; } @@ -812,6 +822,7 @@ Vop3p::Vop3p(std::string_view mnemonic, const Vop3pMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } bool Vop3::displays_vop3_op_sel() const { @@ -865,6 +876,7 @@ Vop3::Vop3(std::string_view mnemonic, const Vop3MachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } bool Ds::uses_split_ds_offsets() const { @@ -878,6 +890,7 @@ Ds::Ds(std::string_view mnemonic, const DsMachineInst *inst, ExecuteFn exec_fn) raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mubuf::Mubuf(std::string_view mnemonic, const MubufMachineInst *inst, ExecuteFn exec_fn) @@ -886,6 +899,7 @@ Mubuf::Mubuf(std::string_view mnemonic, const MubufMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mtbuf::Mtbuf(std::string_view mnemonic, const MtbufMachineInst *inst, ExecuteFn exec_fn) @@ -894,6 +908,7 @@ Mtbuf::Mtbuf(std::string_view mnemonic, const MtbufMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mimg::Mimg(std::string_view mnemonic, const MimgMachineInst *inst, ExecuteFn exec_fn) @@ -902,6 +917,7 @@ Mimg::Mimg(std::string_view mnemonic, const MimgMachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Exp::Exp(std::string_view mnemonic, const ExpMachineInst *inst, ExecuteFn exec_fn) @@ -909,6 +925,7 @@ Exp::Exp(std::string_view mnemonic, const ExpMachineInst *inst, ExecuteFn exec_f size_ = sizeof(OpEncoding); raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; + flags_ |= VSKIP_AFFECTED; } Flat::Flat(std::string_view mnemonic, const FlatMachineInst *inst, ExecuteFn exec_fn) @@ -919,6 +936,7 @@ Flat::Flat(std::string_view mnemonic, const FlatMachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } void Flat::implicit_uses(RegisterSet &uses) const { @@ -972,6 +990,7 @@ Vop3SdstEnc::Vop3SdstEnc(std::string_view mnemonic, const Vop3SdstEncMachineInst raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Vop3pMfma::Vop3pMfma(std::string_view mnemonic, const Vop3pMfmaMachineInst *inst, ExecuteFn exec_fn) @@ -980,6 +999,7 @@ Vop3pMfma::Vop3pMfma(std::string_view mnemonic, const Vop3pMfmaMachineInst *inst raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } } // namespace cdna1 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/encodings.h index 1bb5f3680cb0..ae9c53fde1bf 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/encodings.h @@ -536,6 +536,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -589,6 +590,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -651,6 +653,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/sopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/sopc_exec.cpp index 6fe9574d58d8..97e41e464167 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/sopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/sopc_exec.cpp @@ -83,8 +83,7 @@ void SBitcmp1B64Sopc::execute_impl(amdgpu::Wavefront &wf) { } void SSetvskipSopc::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + amdgpu::execute_s_setvskip_sopc(*this, wf); } void SSetGprIdxOnSopc::execute_impl(amdgpu::Wavefront &wf) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop1.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop1.cpp index ea501b14e341..542f5e2805e1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop1.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop1.cpp @@ -100,7 +100,7 @@ DecodeResult decodeVMovB32Vop1(const MachineInst *opcode, const DecodeErrorEmitt VReadfirstlaneB32Vop1::VReadfirstlaneB32Vop1(const MachineInst *inst) : Vop1("v_readfirstlane_b32_e32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadfirstlaneB32Vop1)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop1_exec.cpp index 50e231331a16..499a2d0e9e9f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop1_exec.cpp @@ -57,13 +57,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -156,6 +151,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -186,6 +182,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -216,6 +213,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -246,6 +244,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -276,6 +275,7 @@ RJ_NOINLINE void VCvtRpiI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -306,6 +306,7 @@ RJ_NOINLINE void VCvtFlrI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -518,6 +519,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -548,6 +550,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -578,6 +581,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -608,6 +612,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -638,6 +643,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -668,6 +674,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -698,6 +705,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -728,6 +736,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -758,6 +767,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -788,6 +798,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -822,6 +833,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -856,6 +868,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -886,6 +899,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1078,6 +1092,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1108,6 +1123,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1207,6 +1223,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1237,6 +1254,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1267,6 +1285,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1297,6 +1316,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1327,6 +1347,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1357,6 +1378,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1387,6 +1409,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1417,6 +1440,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1447,6 +1471,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1477,6 +1502,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1507,6 +1533,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1537,6 +1564,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1567,6 +1595,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1597,6 +1626,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1627,6 +1657,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1657,6 +1688,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1687,6 +1719,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1717,6 +1750,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop2_exec.cpp index 27d9c9bbcce3..c69bf5a8350e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop2_exec.cpp @@ -84,6 +84,8 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -120,6 +122,8 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -156,6 +160,8 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -192,6 +198,8 @@ RJ_NOINLINE void VMulLegacyF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -228,6 +236,8 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -408,6 +418,8 @@ RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -444,6 +456,8 @@ RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -824,13 +838,17 @@ void VMacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1167,6 +1185,8 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1203,6 +1223,8 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1239,6 +1261,8 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1275,6 +1299,8 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1295,13 +1321,17 @@ void VMacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1596,6 +1626,8 @@ RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1632,6 +1664,8 @@ RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1812,6 +1846,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::NONE, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -2040,13 +2075,17 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3.cpp index 20886e4bcc7a..ad181bc4e8bf 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3.cpp @@ -144,7 +144,7 @@ DecodeResult decodeVMovB32Vop3(const MachineInst *opcode, const DecodeErrorEmitt VReadfirstlaneB32Vop3::VReadfirstlaneB32Vop3(const MachineInst *inst) : Vop3("v_readfirstlane_b32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadfirstlaneB32Vop3)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; @@ -6023,7 +6023,7 @@ DecodeResult decodeVLdexpF32Vop3(const MachineInst *opcode, const DecodeErrorEmi VReadlaneB32Vop3::VReadlaneB32Vop3(const MachineInst *inst) : Vop3("v_readlane_b32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadlaneB32Vop3)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), src1(32, OperandType::OPR_SSRC_LANESEL, reinterpret_cast(inst)->src1) { dst_operands_[0] = &vdst; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp index 864769a419c6..ef89ca780b44 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp @@ -40,13 +40,8 @@ void VMovB32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::execute_v_mov_b3 void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -95,18 +90,32 @@ void VCvtF32F16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -309,8 +318,15 @@ void VCvtU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -330,8 +346,15 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -347,7 +370,13 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -356,31 +385,27 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -395,52 +420,47 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -448,7 +468,13 @@ void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -457,31 +483,27 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -496,7 +518,13 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -505,31 +533,27 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -544,7 +568,13 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -553,31 +583,27 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -951,31 +977,27 @@ void VSinF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -997,31 +1019,27 @@ void VCosF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1042,11 +1060,19 @@ void VCvtNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1062,11 +1088,19 @@ void VCvtNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1480,7 +1514,8 @@ void VMacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2261,7 +2296,8 @@ void VFmaLegacyF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2891,7 +2927,8 @@ void VFmaF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -3366,8 +3403,8 @@ void VCmpFF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -3386,32 +3423,26 @@ void VCmpLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3423,32 +3454,26 @@ void VCmpEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3460,32 +3485,26 @@ void VCmpLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3497,32 +3516,26 @@ void VCmpGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3534,35 +3547,26 @@ void VCmpLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3574,32 +3578,26 @@ void VCmpGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3611,32 +3609,26 @@ void VCmpOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3648,32 +3640,26 @@ void VCmpUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3685,32 +3671,26 @@ void VCmpNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3722,35 +3702,26 @@ void VCmpNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3762,32 +3733,26 @@ void VCmpNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3799,32 +3764,26 @@ void VCmpNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3836,32 +3795,26 @@ void VCmpNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3873,32 +3826,26 @@ void VCmpNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3910,8 +3857,8 @@ void VCmpTruF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -3948,22 +3895,14 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -3978,22 +3917,14 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4008,22 +3939,14 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4038,22 +3961,14 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4068,22 +3983,14 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4098,22 +4005,14 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4128,22 +4027,14 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4158,22 +4049,14 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4188,22 +4071,14 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4218,22 +4093,14 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4248,22 +4115,14 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4278,22 +4137,14 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4308,22 +4159,14 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4338,22 +4181,14 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4527,20 +4362,14 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4554,20 +4383,14 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4581,20 +4404,14 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4608,20 +4425,14 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4635,20 +4446,14 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4662,20 +4467,14 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4689,20 +4488,14 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4716,20 +4509,14 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4743,20 +4530,14 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4770,20 +4551,14 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4797,20 +4572,14 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4824,20 +4593,14 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4851,20 +4614,14 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4878,20 +4635,14 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5065,20 +4816,14 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5092,20 +4837,14 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5119,20 +4858,14 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5146,20 +4879,14 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5173,20 +4900,14 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5200,20 +4921,14 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5227,20 +4942,14 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5254,20 +4963,14 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5281,20 +4984,14 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5308,20 +5005,14 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5335,20 +5026,14 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5362,20 +5047,14 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5389,20 +5068,14 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5416,20 +5089,14 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vopc_exec.cpp index 10c9fab288c2..cf526d3d3728 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vopc_exec.cpp @@ -994,14 +994,14 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1029,14 +1029,14 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1056,14 +1056,14 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1091,14 +1091,14 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1118,14 +1118,14 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1153,14 +1153,14 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1180,14 +1180,14 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1215,14 +1215,14 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1242,14 +1242,14 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1277,14 +1277,14 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1304,14 +1304,14 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1339,14 +1339,14 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1366,14 +1366,14 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1401,14 +1401,14 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1428,14 +1428,14 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1463,14 +1463,14 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1490,14 +1490,14 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1525,14 +1525,14 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1552,14 +1552,14 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1587,14 +1587,14 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1614,14 +1614,14 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1649,14 +1649,14 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1676,14 +1676,14 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1711,14 +1711,14 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1738,14 +1738,14 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1773,14 +1773,14 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1800,14 +1800,14 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1835,14 +1835,14 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2542,12 +2542,14 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2575,12 +2577,14 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2600,12 +2604,14 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2633,12 +2639,14 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2658,12 +2666,14 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2691,12 +2701,14 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2716,12 +2728,14 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2749,12 +2763,14 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2774,12 +2790,14 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2807,12 +2825,14 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2832,12 +2852,14 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2865,12 +2887,14 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2890,12 +2914,14 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2923,12 +2949,14 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2948,12 +2976,14 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2981,12 +3011,14 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3006,12 +3038,14 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3039,12 +3073,14 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3064,12 +3100,14 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3097,12 +3135,14 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3122,12 +3162,14 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3155,12 +3197,14 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3180,12 +3224,14 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3213,12 +3259,14 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3238,12 +3286,14 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3271,12 +3321,14 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3296,12 +3348,14 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3329,12 +3383,14 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3478,12 +3534,14 @@ void VCmpxFF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3493,12 +3551,14 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3508,12 +3568,14 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3523,12 +3585,14 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3538,12 +3602,14 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3553,12 +3619,14 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3568,12 +3636,14 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3583,12 +3653,14 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3598,12 +3670,14 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3613,12 +3687,14 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3628,12 +3704,14 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3643,12 +3721,14 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3658,12 +3738,14 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3673,12 +3755,14 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/encodings.cpp index 20cfc4f13f3e..11cb1b3cf3ce 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/encodings.cpp @@ -383,6 +383,10 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); + flags_ |= VSKIP_AFFECTED; } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -632,6 +636,7 @@ Vopc::Vopc(std::string_view mnemonic, const VopcMachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + flags_ |= VSKIP_AFFECTED; } bool Vopc::default_encoding() { @@ -746,6 +751,10 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); + flags_ |= VSKIP_AFFECTED; } void Vop2::implicit_uses(RegisterSet &uses) const { @@ -798,6 +807,7 @@ Vop3p::Vop3p(std::string_view mnemonic, const Vop3pMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } bool Vop3::displays_vop3_op_sel() const { @@ -851,6 +861,7 @@ Vop3::Vop3(std::string_view mnemonic, const Vop3MachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } bool Ds::uses_split_ds_offsets() const { @@ -864,6 +875,7 @@ Ds::Ds(std::string_view mnemonic, const DsMachineInst *inst, ExecuteFn exec_fn) raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mubuf::Mubuf(std::string_view mnemonic, const MubufMachineInst *inst, ExecuteFn exec_fn) @@ -872,6 +884,7 @@ Mubuf::Mubuf(std::string_view mnemonic, const MubufMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mtbuf::Mtbuf(std::string_view mnemonic, const MtbufMachineInst *inst, ExecuteFn exec_fn) @@ -880,6 +893,7 @@ Mtbuf::Mtbuf(std::string_view mnemonic, const MtbufMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mimg::Mimg(std::string_view mnemonic, const MimgMachineInst *inst, ExecuteFn exec_fn) @@ -888,6 +902,7 @@ Mimg::Mimg(std::string_view mnemonic, const MimgMachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Flat::Flat(std::string_view mnemonic, const FlatMachineInst *inst, ExecuteFn exec_fn) @@ -898,6 +913,7 @@ Flat::Flat(std::string_view mnemonic, const FlatMachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } void Flat::implicit_uses(RegisterSet &uses) const { @@ -951,6 +967,7 @@ Vop3SdstEnc::Vop3SdstEnc(std::string_view mnemonic, const Vop3SdstEncMachineInst raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Vop3pMfma::Vop3pMfma(std::string_view mnemonic, const Vop3pMfmaMachineInst *inst, ExecuteFn exec_fn) @@ -959,6 +976,7 @@ Vop3pMfma::Vop3pMfma(std::string_view mnemonic, const Vop3pMfmaMachineInst *inst raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } } // namespace cdna2 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/encodings.h index 1c5f224e6153..3c4e74fa42a7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/encodings.h @@ -520,6 +520,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -573,6 +574,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -635,6 +637,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/sopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/sopc_exec.cpp index 8257657f1373..ff6a0160d035 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/sopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/sopc_exec.cpp @@ -83,8 +83,7 @@ void SBitcmp1B64Sopc::execute_impl(amdgpu::Wavefront &wf) { } void SSetvskipSopc::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + amdgpu::execute_s_setvskip_sopc(*this, wf); } void SSetGprIdxOnSopc::execute_impl(amdgpu::Wavefront &wf) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop1.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop1.cpp index c982fa0b465d..0ac3609d818b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop1.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop1.cpp @@ -100,7 +100,7 @@ DecodeResult decodeVMovB32Vop1(const MachineInst *opcode, const DecodeErrorEmitt VReadfirstlaneB32Vop1::VReadfirstlaneB32Vop1(const MachineInst *inst) : Vop1("v_readfirstlane_b32_e32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadfirstlaneB32Vop1)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop1_exec.cpp index de33227a63f5..7f7d35d81481 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop1_exec.cpp @@ -57,13 +57,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -156,6 +151,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -186,6 +182,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -216,6 +213,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -246,6 +244,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -276,6 +275,7 @@ RJ_NOINLINE void VCvtRpiI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -306,6 +306,7 @@ RJ_NOINLINE void VCvtFlrI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -518,6 +519,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -548,6 +550,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -578,6 +581,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -608,6 +612,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -638,6 +643,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -668,6 +674,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -698,6 +705,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -728,6 +736,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -758,6 +767,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -788,6 +798,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -822,6 +833,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -856,6 +868,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -886,6 +899,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1078,6 +1092,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1108,6 +1123,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1207,6 +1223,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1237,6 +1254,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1267,6 +1285,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1297,6 +1316,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1327,6 +1347,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1357,6 +1378,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1387,6 +1409,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1417,6 +1440,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1447,6 +1471,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1477,6 +1502,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1507,6 +1533,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1537,6 +1564,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1567,6 +1595,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1597,6 +1626,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1627,6 +1657,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1657,6 +1688,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1687,6 +1719,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1717,6 +1750,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop2_exec.cpp index 612592dd6e8a..acdbd2ef1dd7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop2_exec.cpp @@ -84,6 +84,8 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -120,6 +122,8 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -156,6 +160,8 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -176,13 +182,19 @@ void VFmacF64Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 9223372036854775808ull, dpp_modifiers_, + 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -217,6 +229,8 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -397,6 +411,8 @@ RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -433,6 +449,8 @@ RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -813,13 +831,17 @@ void VMacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1156,6 +1178,8 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1192,6 +1216,8 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1228,6 +1254,8 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1264,6 +1292,8 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1284,13 +1314,17 @@ void VMacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1585,6 +1619,8 @@ RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1621,6 +1657,8 @@ RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1801,6 +1839,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::NONE, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -2029,13 +2068,17 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3.cpp index 2ae43ec719bd..54e8465310e8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3.cpp @@ -55,7 +55,7 @@ DecodeResult decodeVMovB32Vop3(const MachineInst *opcode, const DecodeErrorEmitt VReadfirstlaneB32Vop3::VReadfirstlaneB32Vop3(const MachineInst *inst) : Vop3("v_readfirstlane_b32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadfirstlaneB32Vop3)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; @@ -5835,7 +5835,7 @@ DecodeResult decodeVLdexpF32Vop3(const MachineInst *opcode, const DecodeErrorEmi VReadlaneB32Vop3::VReadlaneB32Vop3(const MachineInst *inst) : Vop3("v_readlane_b32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadlaneB32Vop3)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), src1(32, OperandType::OPR_SSRC_LANESEL, reinterpret_cast(inst)->src1) { dst_operands_[0] = &vdst; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp index 5dcabae0d186..127e7398a4de 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp @@ -28,13 +28,8 @@ void VMovB32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::execute_v_mov_b3 void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -83,18 +78,32 @@ void VCvtF32F16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -297,8 +306,15 @@ void VCvtU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -318,8 +334,15 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -335,7 +358,13 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -344,31 +373,27 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -383,52 +408,47 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -436,7 +456,13 @@ void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -445,31 +471,27 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -484,7 +506,13 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -493,31 +521,27 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -532,7 +556,13 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -541,31 +571,27 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -939,31 +965,27 @@ void VSinF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -985,31 +1007,27 @@ void VCosF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1030,11 +1048,19 @@ void VCvtNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1050,11 +1076,19 @@ void VCvtNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1472,7 +1506,8 @@ void VMacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2253,7 +2288,8 @@ void VFmaLegacyF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2883,7 +2919,8 @@ void VFmaF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -3342,8 +3379,8 @@ void VCmpFF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -3362,32 +3399,26 @@ void VCmpLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3399,32 +3430,26 @@ void VCmpEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3436,32 +3461,26 @@ void VCmpLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3473,32 +3492,26 @@ void VCmpGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3510,35 +3523,26 @@ void VCmpLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3550,32 +3554,26 @@ void VCmpGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3587,32 +3585,26 @@ void VCmpOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3624,32 +3616,26 @@ void VCmpUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3661,32 +3647,26 @@ void VCmpNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3698,35 +3678,26 @@ void VCmpNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3738,32 +3709,26 @@ void VCmpNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3775,32 +3740,26 @@ void VCmpNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3812,32 +3771,26 @@ void VCmpNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3849,32 +3802,26 @@ void VCmpNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3886,8 +3833,8 @@ void VCmpTruF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -3924,22 +3871,14 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -3954,22 +3893,14 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -3984,22 +3915,14 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4014,22 +3937,14 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4044,22 +3959,14 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4074,22 +3981,14 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4104,22 +4003,14 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4134,22 +4025,14 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4164,22 +4047,14 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4194,22 +4069,14 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4224,22 +4091,14 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4254,22 +4113,14 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4284,22 +4135,14 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4314,22 +4157,14 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4503,20 +4338,14 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4530,20 +4359,14 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4557,20 +4380,14 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4584,20 +4401,14 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4611,20 +4422,14 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4638,20 +4443,14 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4665,20 +4464,14 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4692,20 +4485,14 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4719,20 +4506,14 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4746,20 +4527,14 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4773,20 +4548,14 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4800,20 +4569,14 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4827,20 +4590,14 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4854,20 +4611,14 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5041,20 +4792,14 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5068,20 +4813,14 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5095,20 +4834,14 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5122,20 +4855,14 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5149,20 +4876,14 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5176,20 +4897,14 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5203,20 +4918,14 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5230,20 +4939,14 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5257,20 +4960,14 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5284,20 +4981,14 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5311,20 +5002,14 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5338,20 +5023,14 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5365,20 +5044,14 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5392,20 +5065,14 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vopc_exec.cpp index 64e42abe8475..c8347431d1c7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vopc_exec.cpp @@ -994,14 +994,14 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1029,14 +1029,14 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1056,14 +1056,14 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1091,14 +1091,14 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1118,14 +1118,14 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1153,14 +1153,14 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1180,14 +1180,14 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1215,14 +1215,14 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1242,14 +1242,14 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1277,14 +1277,14 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1304,14 +1304,14 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1339,14 +1339,14 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1366,14 +1366,14 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1401,14 +1401,14 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1428,14 +1428,14 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1463,14 +1463,14 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1490,14 +1490,14 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1525,14 +1525,14 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1552,14 +1552,14 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1587,14 +1587,14 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1614,14 +1614,14 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1649,14 +1649,14 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1676,14 +1676,14 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1711,14 +1711,14 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1738,14 +1738,14 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1773,14 +1773,14 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1800,14 +1800,14 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1835,14 +1835,14 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2542,12 +2542,14 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2575,12 +2577,14 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2600,12 +2604,14 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2633,12 +2639,14 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2658,12 +2666,14 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2691,12 +2701,14 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2716,12 +2728,14 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2749,12 +2763,14 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2774,12 +2790,14 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2807,12 +2825,14 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2832,12 +2852,14 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2865,12 +2887,14 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2890,12 +2914,14 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2923,12 +2949,14 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2948,12 +2976,14 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2981,12 +3011,14 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3006,12 +3038,14 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3039,12 +3073,14 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3064,12 +3100,14 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3097,12 +3135,14 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3122,12 +3162,14 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3155,12 +3197,14 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3180,12 +3224,14 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3213,12 +3259,14 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3238,12 +3286,14 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3271,12 +3321,14 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3296,12 +3348,14 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3329,12 +3383,14 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3478,12 +3534,14 @@ void VCmpxFF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3493,12 +3551,14 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3508,12 +3568,14 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3523,12 +3585,14 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3538,12 +3602,14 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3553,12 +3619,14 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3568,12 +3636,14 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3583,12 +3653,14 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3598,12 +3670,14 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3613,12 +3687,14 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3628,12 +3704,14 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3643,12 +3721,14 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3658,12 +3738,14 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3673,12 +3755,14 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/encodings.cpp index 99b7a34a018d..dbb9f7e436a4 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/encodings.cpp @@ -388,6 +388,10 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); + flags_ |= VSKIP_AFFECTED; } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -637,6 +641,7 @@ Vopc::Vopc(std::string_view mnemonic, const VopcMachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + flags_ |= VSKIP_AFFECTED; } bool Vopc::default_encoding() { @@ -750,6 +755,10 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); + flags_ |= VSKIP_AFFECTED; } void Vop2::implicit_uses(RegisterSet &uses) const { @@ -802,6 +811,7 @@ Vop3p::Vop3p(std::string_view mnemonic, const Vop3pMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } bool Vop3::displays_vop3_op_sel() const { @@ -856,6 +866,7 @@ Vop3::Vop3(std::string_view mnemonic, const Vop3MachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } bool Ds::uses_split_ds_offsets() const { @@ -869,6 +880,7 @@ Ds::Ds(std::string_view mnemonic, const DsMachineInst *inst, ExecuteFn exec_fn) raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mubuf::Mubuf(std::string_view mnemonic, const MubufMachineInst *inst, ExecuteFn exec_fn) @@ -877,6 +889,7 @@ Mubuf::Mubuf(std::string_view mnemonic, const MubufMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mtbuf::Mtbuf(std::string_view mnemonic, const MtbufMachineInst *inst, ExecuteFn exec_fn) @@ -885,6 +898,7 @@ Mtbuf::Mtbuf(std::string_view mnemonic, const MtbufMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Flat::Flat(std::string_view mnemonic, const FlatMachineInst *inst, ExecuteFn exec_fn) @@ -895,6 +909,7 @@ Flat::Flat(std::string_view mnemonic, const FlatMachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } void Flat::implicit_uses(RegisterSet &uses) const { @@ -948,6 +963,7 @@ Vop3SdstEnc::Vop3SdstEnc(std::string_view mnemonic, const Vop3SdstEncMachineInst raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Vop3pMfma::Vop3pMfma(std::string_view mnemonic, const Vop3pMfmaMachineInst *inst, ExecuteFn exec_fn) @@ -956,6 +972,7 @@ Vop3pMfma::Vop3pMfma(std::string_view mnemonic, const Vop3pMfmaMachineInst *inst raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } } // namespace cdna3 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/encodings.h index e2dc0fb194dc..e8961703bf91 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/encodings.h @@ -508,6 +508,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -561,6 +562,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -623,6 +625,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/sopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/sopc_exec.cpp index 2b99014eb036..271e8c9ec66b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/sopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/sopc_exec.cpp @@ -83,8 +83,7 @@ void SBitcmp1B64Sopc::execute_impl(amdgpu::Wavefront &wf) { } void SSetvskipSopc::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + amdgpu::execute_s_setvskip_sopc(*this, wf); } void SSetGprIdxOnSopc::execute_impl(amdgpu::Wavefront &wf) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop1.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop1.cpp index 17950604c166..a1cae3eec038 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop1.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop1.cpp @@ -100,7 +100,7 @@ DecodeResult decodeVMovB32Vop1(const MachineInst *opcode, const DecodeErrorEmitt VReadfirstlaneB32Vop1::VReadfirstlaneB32Vop1(const MachineInst *inst) : Vop1("v_readfirstlane_b32_e32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadfirstlaneB32Vop1)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop1_exec.cpp index f51340143227..dfaa55a8de18 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop1_exec.cpp @@ -57,13 +57,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -156,6 +151,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -186,6 +182,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -216,6 +213,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -246,6 +244,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -276,6 +275,7 @@ RJ_NOINLINE void VCvtRpiI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -306,6 +306,7 @@ RJ_NOINLINE void VCvtFlrI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -518,6 +519,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -548,6 +550,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -578,6 +581,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -608,6 +612,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -638,6 +643,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -668,6 +674,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -698,6 +705,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -728,6 +736,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -758,6 +767,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -788,6 +798,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -822,6 +833,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -856,6 +868,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -886,6 +899,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1078,6 +1092,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1108,6 +1123,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1237,6 +1253,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1267,6 +1284,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1297,6 +1315,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1327,6 +1346,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1357,6 +1377,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1387,6 +1408,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1417,6 +1439,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1447,6 +1470,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1477,6 +1501,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1507,6 +1532,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1537,6 +1563,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1567,6 +1594,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1597,6 +1625,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1627,6 +1656,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1657,6 +1687,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1687,6 +1718,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1717,6 +1749,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1747,6 +1780,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop2_exec.cpp index 0a6e4eabb0cf..d465a8c35ede 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop2_exec.cpp @@ -84,6 +84,8 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -120,6 +122,8 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -156,6 +160,8 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -176,13 +182,19 @@ void VFmacF64Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 9223372036854775808ull, dpp_modifiers_, + 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -217,6 +229,8 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -397,6 +411,8 @@ RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -433,6 +449,8 @@ RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1131,6 +1149,8 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1167,6 +1187,8 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1203,6 +1225,8 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1239,6 +1263,8 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1259,13 +1285,17 @@ void VMacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1560,6 +1590,8 @@ RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1596,6 +1628,8 @@ RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1776,6 +1810,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::NONE, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -2004,13 +2039,17 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3.cpp index eddf59d07bae..e002b5cf8a8d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3.cpp @@ -55,7 +55,7 @@ DecodeResult decodeVMovB32Vop3(const MachineInst *opcode, const DecodeErrorEmitt VReadfirstlaneB32Vop3::VReadfirstlaneB32Vop3(const MachineInst *inst) : Vop3("v_readfirstlane_b32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadfirstlaneB32Vop3)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; @@ -5896,7 +5896,7 @@ DecodeResult decodeVLdexpF32Vop3(const MachineInst *opcode, const DecodeErrorEmi VReadlaneB32Vop3::VReadlaneB32Vop3(const MachineInst *inst) : Vop3("v_readlane_b32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadlaneB32Vop3)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), src1(32, OperandType::OPR_SSRC_LANESEL, reinterpret_cast(inst)->src1) { dst_operands_[0] = &vdst; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp index fecc6a1808dc..33b99a1a44db 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp @@ -28,13 +28,8 @@ void VMovB32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::execute_v_mov_b3 void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -83,18 +78,32 @@ void VCvtF32F16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -299,8 +308,15 @@ void VCvtU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -320,8 +336,15 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -337,7 +360,13 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -346,31 +375,27 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -385,52 +410,47 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -438,7 +458,13 @@ void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -447,31 +473,27 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -486,7 +508,13 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -495,31 +523,27 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -534,7 +558,13 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -543,31 +573,27 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -941,31 +967,27 @@ void VSinF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -987,31 +1009,27 @@ void VCosF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1032,11 +1050,19 @@ void VCvtNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1052,11 +1078,19 @@ void VCvtNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1547,7 +1581,8 @@ void VMacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2322,7 +2357,8 @@ void VFmaLegacyF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2952,7 +2988,8 @@ void VFmaF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -3477,8 +3514,8 @@ void VCmpFF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -3497,32 +3534,26 @@ void VCmpLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3534,32 +3565,26 @@ void VCmpEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3571,32 +3596,26 @@ void VCmpLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3608,32 +3627,26 @@ void VCmpGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3645,35 +3658,26 @@ void VCmpLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3685,32 +3689,26 @@ void VCmpGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3722,32 +3720,26 @@ void VCmpOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3759,32 +3751,26 @@ void VCmpUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3796,32 +3782,26 @@ void VCmpNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3833,35 +3813,26 @@ void VCmpNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3873,32 +3844,26 @@ void VCmpNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3910,32 +3875,26 @@ void VCmpNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3947,32 +3906,26 @@ void VCmpNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3984,32 +3937,26 @@ void VCmpNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -4021,8 +3968,8 @@ void VCmpTruF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -4059,22 +4006,14 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4089,22 +4028,14 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4119,22 +4050,14 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4149,22 +4072,14 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4179,22 +4094,14 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4209,22 +4116,14 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4239,22 +4138,14 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4269,22 +4160,14 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4299,22 +4182,14 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4329,22 +4204,14 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4359,22 +4226,14 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4389,22 +4248,14 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4419,22 +4270,14 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4449,22 +4292,14 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4638,20 +4473,14 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4665,20 +4494,14 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4692,20 +4515,14 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4719,20 +4536,14 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4746,20 +4557,14 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4773,20 +4578,14 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4800,20 +4599,14 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4827,20 +4620,14 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4854,20 +4641,14 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4881,20 +4662,14 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4908,20 +4683,14 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4935,20 +4704,14 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4962,20 +4725,14 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -4989,20 +4746,14 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5176,20 +4927,14 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5203,20 +4948,14 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5230,20 +4969,14 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5257,20 +4990,14 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5284,20 +5011,14 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5311,20 +5032,14 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5338,20 +5053,14 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5365,20 +5074,14 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5392,20 +5095,14 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5419,20 +5116,14 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5446,20 +5137,14 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5473,20 +5158,14 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5500,20 +5179,14 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -5527,20 +5200,14 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vopc_exec.cpp index dbff8d833f37..69a193ba5990 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vopc_exec.cpp @@ -994,14 +994,14 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1029,14 +1029,14 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1056,14 +1056,14 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1091,14 +1091,14 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1118,14 +1118,14 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1153,14 +1153,14 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1180,14 +1180,14 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1215,14 +1215,14 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1242,14 +1242,14 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1277,14 +1277,14 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1304,14 +1304,14 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1339,14 +1339,14 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1366,14 +1366,14 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1401,14 +1401,14 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1428,14 +1428,14 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1463,14 +1463,14 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1490,14 +1490,14 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1525,14 +1525,14 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1552,14 +1552,14 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1587,14 +1587,14 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1614,14 +1614,14 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1649,14 +1649,14 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1676,14 +1676,14 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1711,14 +1711,14 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1738,14 +1738,14 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1773,14 +1773,14 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1800,14 +1800,14 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1835,14 +1835,14 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2542,12 +2542,14 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2575,12 +2577,14 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2600,12 +2604,14 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2633,12 +2639,14 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2658,12 +2666,14 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2691,12 +2701,14 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2716,12 +2728,14 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2749,12 +2763,14 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2774,12 +2790,14 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2807,12 +2825,14 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2832,12 +2852,14 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2865,12 +2887,14 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2890,12 +2914,14 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2923,12 +2949,14 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2948,12 +2976,14 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2981,12 +3011,14 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3006,12 +3038,14 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3039,12 +3073,14 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3064,12 +3100,14 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3097,12 +3135,14 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3122,12 +3162,14 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3155,12 +3197,14 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3180,12 +3224,14 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3213,12 +3259,14 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3238,12 +3286,14 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3271,12 +3321,14 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3296,12 +3348,14 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3329,12 +3383,14 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3478,12 +3534,14 @@ void VCmpxFF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3493,12 +3551,14 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3508,12 +3568,14 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3523,12 +3585,14 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3538,12 +3602,14 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3553,12 +3619,14 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3568,12 +3636,14 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3583,12 +3653,14 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3598,12 +3670,14 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3613,12 +3687,14 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3628,12 +3704,14 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3643,12 +3721,14 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3658,12 +3738,14 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3673,12 +3755,14 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.cpp index ab1c5c3f232b..5662e287500c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.cpp @@ -392,6 +392,10 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); + flags_ |= VSKIP_AFFECTED; } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -641,6 +645,7 @@ Vopc::Vopc(std::string_view mnemonic, const VopcMachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + flags_ |= VSKIP_AFFECTED; } bool Vopc::default_encoding() { @@ -755,6 +760,10 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); + flags_ |= VSKIP_AFFECTED; } void Vop2::implicit_uses(RegisterSet &uses) const { @@ -807,6 +816,7 @@ Vop3p::Vop3p(std::string_view mnemonic, const Vop3pMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } bool Vop3::displays_vop3_op_sel() const { @@ -865,6 +875,7 @@ Vop3::Vop3(std::string_view mnemonic, const Vop3MachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } bool Ds::uses_split_ds_offsets() const { @@ -878,6 +889,7 @@ Ds::Ds(std::string_view mnemonic, const DsMachineInst *inst, ExecuteFn exec_fn) raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mubuf::Mubuf(std::string_view mnemonic, const MubufMachineInst *inst, ExecuteFn exec_fn) @@ -886,6 +898,7 @@ Mubuf::Mubuf(std::string_view mnemonic, const MubufMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Mtbuf::Mtbuf(std::string_view mnemonic, const MtbufMachineInst *inst, ExecuteFn exec_fn) @@ -894,6 +907,7 @@ Mtbuf::Mtbuf(std::string_view mnemonic, const MtbufMachineInst *inst, ExecuteFn raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Flat::Flat(std::string_view mnemonic, const FlatMachineInst *inst, ExecuteFn exec_fn) @@ -904,6 +918,7 @@ Flat::Flat(std::string_view mnemonic, const FlatMachineInst *inst, ExecuteFn exe raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } void Flat::implicit_uses(RegisterSet &uses) const { @@ -957,6 +972,7 @@ Vop3SdstEnc::Vop3SdstEnc(std::string_view mnemonic, const Vop3SdstEncMachineInst raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } Vop3pMfma::Vop3pMfma(std::string_view mnemonic, const Vop3pMfmaMachineInst *inst, ExecuteFn exec_fn) @@ -965,6 +981,7 @@ Vop3pMfma::Vop3pMfma(std::string_view mnemonic, const Vop3pMfmaMachineInst *inst raw_encoding_ = reinterpret_cast(&inst_); encoding_id_ = raw_encoding_[0] >> 23; opcode_ = inst_.op; + flags_ |= VSKIP_AFFECTED; } } // namespace cdna4 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.h index eb62bd9cf2c9..75975c4c09e1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.h @@ -512,6 +512,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -565,6 +566,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -627,6 +629,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/sopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/sopc_exec.cpp index 5346981a0c66..999f9f14407f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/sopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/sopc_exec.cpp @@ -83,8 +83,7 @@ void SBitcmp1B64Sopc::execute_impl(amdgpu::Wavefront &wf) { } void SSetvskipSopc::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + amdgpu::execute_s_setvskip_sopc(*this, wf); } void SSetGprIdxOnSopc::execute_impl(amdgpu::Wavefront &wf) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop1.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop1.cpp index 8546f7a18770..5c61ad7c51b3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop1.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop1.cpp @@ -100,7 +100,7 @@ DecodeResult decodeVMovB32Vop1(const MachineInst *opcode, const DecodeErrorEmitt VReadfirstlaneB32Vop1::VReadfirstlaneB32Vop1(const MachineInst *inst) : Vop1("v_readfirstlane_b32_e32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadfirstlaneB32Vop1)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop1_exec.cpp index eddcc0bb9951..e313a134e4cb 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop1_exec.cpp @@ -57,13 +57,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -156,6 +151,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -186,6 +182,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -216,6 +213,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -246,6 +244,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -276,6 +275,7 @@ RJ_NOINLINE void VCvtRpiI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -306,6 +306,7 @@ RJ_NOINLINE void VCvtFlrI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -518,6 +519,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -548,6 +550,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -578,6 +581,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -608,6 +612,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -638,6 +643,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -668,6 +674,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -698,6 +705,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -728,6 +736,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -758,6 +767,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -788,6 +798,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -822,6 +833,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -856,6 +868,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -886,6 +899,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1078,6 +1092,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1108,6 +1123,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1237,6 +1253,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1267,6 +1284,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1297,6 +1315,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1327,6 +1346,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1357,6 +1377,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1387,6 +1408,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1417,6 +1439,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1447,6 +1470,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1477,6 +1501,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1507,6 +1532,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1537,6 +1563,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1567,6 +1594,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1597,6 +1625,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1627,6 +1656,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1657,6 +1687,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1687,6 +1718,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1717,6 +1749,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1747,6 +1780,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2067,6 +2101,7 @@ RJ_NOINLINE void VPrngB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { } void VPermlane16SwapB32Vop1::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); uint32_t tmp_dst[64] = {}, tmp_src[64] = {}; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { tmp_dst[lane] = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); @@ -2074,13 +2109,16 @@ void VPermlane16SwapB32Vop1::execute_impl(amdgpu::Wavefront &wf) { } for (uint32_t base = 0; base + 16 < wf.wf_size(); base += 2u * 16) { for (uint32_t i = 0; i < 16; ++i) { - amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 16 + i]); - amdgpu::RegisterAccess(wf).write_lane(vdst, base + 16 + i, tmp_src[base + i]); + if (exec & (1ULL << (base + i))) + amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 16 + i]); + if (exec & (1ULL << (base + 16 + i))) + amdgpu::RegisterAccess(wf).write_lane(vdst, base + 16 + i, tmp_src[base + i]); } } } void VPermlane32SwapB32Vop1::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); uint32_t tmp_dst[64] = {}, tmp_src[64] = {}; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { tmp_dst[lane] = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); @@ -2088,8 +2126,10 @@ void VPermlane32SwapB32Vop1::execute_impl(amdgpu::Wavefront &wf) { } for (uint32_t base = 0; base + 32 < wf.wf_size(); base += 2u * 32) { for (uint32_t i = 0; i < 32; ++i) { - amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 32 + i]); - amdgpu::RegisterAccess(wf).write_lane(vdst, base + 32 + i, tmp_src[base + i]); + if (exec & (1ULL << (base + i))) + amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 32 + i]); + if (exec & (1ULL << (base + 32 + i))) + amdgpu::RegisterAccess(wf).write_lane(vdst, base + 32 + i, tmp_src[base + i]); } } } @@ -2128,6 +2168,7 @@ RJ_NOINLINE void VCvtF32Bf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::BF16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop2_exec.cpp index b6f64403f4bc..99393edd7d71 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop2_exec.cpp @@ -86,6 +86,8 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -122,6 +124,8 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -158,6 +162,8 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -178,13 +184,19 @@ void VFmacF64Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 9223372036854775808ull, dpp_modifiers_, + 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -219,6 +231,8 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -399,6 +413,8 @@ RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -435,6 +451,8 @@ RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1186,6 +1204,8 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1222,6 +1242,8 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1258,6 +1280,8 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1294,6 +1318,8 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1314,13 +1340,17 @@ void VMacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1615,6 +1645,8 @@ RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1651,6 +1683,8 @@ RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1831,6 +1865,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::NONE, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -2059,13 +2094,17 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, dpp_bound_ctrl_, dpp_fi_, wf.exec(), false); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -2095,16 +2134,18 @@ void VPkFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint32_t rawd = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); const uint32_t omod = amdgpu::sdwa::output_modifier(*this, wf); - uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), - static_cast(rawd), false, false, false, false, - false, false, wf.fp_round_mode_f16_f64(), - wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + uint32_t r0 = amdgpu::fp_mode::fma_f16( + static_cast(raw0), static_cast(raw1), static_cast(rawd), + false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), + wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint32_t r1 = amdgpu::fp_mode::fma_f16( static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, r0 | (r1 << 16)); } @@ -2150,16 +2191,18 @@ RJ_NOINLINE void VPkFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t rawd = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); const uint32_t omod = amdgpu::sdwa::output_modifier(*this, wf); - uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), - static_cast(rawd), false, false, false, false, - false, false, wf.fp_round_mode_f16_f64(), - wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + uint32_t r0 = amdgpu::fp_mode::fma_f16( + static_cast(raw0), static_cast(raw1), static_cast(rawd), + false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), + wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint32_t r1 = amdgpu::fp_mode::fma_f16( static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, r0 | (r1 << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3.cpp index 0898c9560e91..d299c64d48b6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3.cpp @@ -55,7 +55,7 @@ DecodeResult decodeVMovB32Vop3(const MachineInst *opcode, const DecodeErrorEmitt VReadfirstlaneB32Vop3::VReadfirstlaneB32Vop3(const MachineInst *inst) : Vop3("v_readfirstlane_b32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadfirstlaneB32Vop3)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; @@ -7737,7 +7737,7 @@ DecodeResult decodeVLdexpF32Vop3(const MachineInst *opcode, const DecodeErrorEmi VReadlaneB32Vop3::VReadlaneB32Vop3(const MachineInst *inst) : Vop3("v_readlane_b32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VReadlaneB32Vop3)), - vdst(32, OperandType::OPR_SREG_NOVCC, reinterpret_cast(inst)->vdst), + vdst(32, OperandType::OPR_SREG, reinterpret_cast(inst)->vdst), src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), src1(32, OperandType::OPR_SSRC_LANESEL, reinterpret_cast(inst)->src1) { dst_operands_[0] = &vdst; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp index 073edd76733c..8f271aa46866 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp @@ -29,13 +29,8 @@ void VMovB32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::execute_v_mov_b3 void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -84,18 +79,32 @@ void VCvtF32F16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -300,8 +309,15 @@ void VCvtU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -321,8 +337,15 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -338,7 +361,13 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -347,31 +376,27 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -386,52 +411,47 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -439,7 +459,13 @@ void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -448,31 +474,27 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -487,7 +509,13 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -496,31 +524,27 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -535,7 +559,13 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -544,31 +574,27 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -942,31 +968,27 @@ void VSinF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -988,31 +1010,27 @@ void VCosF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1033,11 +1051,19 @@ void VCvtNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1053,11 +1079,19 @@ void VCvtNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1189,6 +1223,7 @@ void VCvtPkF32Bf8Vop3::execute_impl(amdgpu::Wavefront &wf) { void VPrngB32Vop3::execute_impl(amdgpu::Wavefront &wf) { (void)wf; } void VPermlane16SwapB32Vop3::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); uint32_t tmp_dst[64] = {}, tmp_src[64] = {}; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { tmp_dst[lane] = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); @@ -1196,13 +1231,16 @@ void VPermlane16SwapB32Vop3::execute_impl(amdgpu::Wavefront &wf) { } for (uint32_t base = 0; base + 16 < wf.wf_size(); base += 2u * 16) { for (uint32_t i = 0; i < 16; ++i) { - amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 16 + i]); - amdgpu::RegisterAccess(wf).write_lane(vdst, base + 16 + i, tmp_src[base + i]); + if (exec & (1ULL << (base + i))) + amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 16 + i]); + if (exec & (1ULL << (base + 16 + i))) + amdgpu::RegisterAccess(wf).write_lane(vdst, base + 16 + i, tmp_src[base + i]); } } } void VPermlane32SwapB32Vop3::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); uint32_t tmp_dst[64] = {}, tmp_src[64] = {}; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { tmp_dst[lane] = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); @@ -1210,8 +1248,10 @@ void VPermlane32SwapB32Vop3::execute_impl(amdgpu::Wavefront &wf) { } for (uint32_t base = 0; base + 32 < wf.wf_size(); base += 2u * 32) { for (uint32_t i = 0; i < 32; ++i) { - amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 32 + i]); - amdgpu::RegisterAccess(wf).write_lane(vdst, base + 32 + i, tmp_src[base + i]); + if (exec & (1ULL << (base + i))) + amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 32 + i]); + if (exec & (1ULL << (base + 32 + i))) + amdgpu::RegisterAccess(wf).write_lane(vdst, base + 32 + i, tmp_src[base + i]); } } } @@ -1625,7 +1665,8 @@ void VMacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2103,12 +2144,14 @@ void VPkFmacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { static_cast(raw0), static_cast(raw1), static_cast(rawd), inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint32_t r1 = amdgpu::fp_mode::fma_f16( static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, r0 | (r1 << 16)); } @@ -2450,7 +2493,8 @@ void VFmaLegacyF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -3080,7 +3124,8 @@ void VFmaF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7565,9 +7610,19 @@ void VCvtPknormU16F32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VCvtPkrtzF16F32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_words_simd<2, false, false, 0>(inst, wf, [&](auto a, auto b) { - return util::f32_to_f16_rtz_simd(std::bit_cast>(a)) | - (util::f32_to_f16_rtz_simd(std::bit_cast>(b)) << 16); + if (amdgpu::try_execute_words_simd<2, false, false, 0>(inst, wf, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x80000000u; + if (inst.inst_.neg & 1u) + a ^= 0x80000000u; + if (inst.inst_.abs & 2u) + b &= ~0x80000000u; + if (inst.inst_.neg & 2u) + b ^= 0x80000000u; + return ([&](auto a, auto b) { + return util::f32_to_f16_rtz_simd(std::bit_cast>(a)) | + (util::f32_to_f16_rtz_simd(std::bit_cast>(b)) << 16); + })(a, b); })) return; uint64_t exec = wf.exec(); @@ -7576,6 +7631,14 @@ void VCvtPkrtzF16F32Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; uint32_t lo = util::f32_to_f16_rtz(s0); uint32_t hi = util::f32_to_f16_rtz(s1); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, @@ -8107,8 +8170,8 @@ void VCmpFF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -8127,32 +8190,26 @@ void VCmpLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8164,32 +8221,26 @@ void VCmpEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8201,32 +8252,26 @@ void VCmpLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8238,32 +8283,26 @@ void VCmpGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8275,35 +8314,26 @@ void VCmpLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8315,32 +8345,26 @@ void VCmpGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8352,32 +8376,26 @@ void VCmpOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8389,32 +8407,26 @@ void VCmpUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8426,32 +8438,26 @@ void VCmpNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8463,35 +8469,26 @@ void VCmpNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8503,32 +8500,26 @@ void VCmpNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8540,32 +8531,26 @@ void VCmpNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8577,32 +8562,26 @@ void VCmpNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8614,32 +8593,26 @@ void VCmpNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8651,8 +8624,8 @@ void VCmpTruF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -8671,12 +8644,12 @@ void VCmpxFF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return decltype(a == b)(false); }); + uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -8694,31 +8667,28 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a < b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -8731,31 +8701,28 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a == b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -8768,31 +8735,28 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -8805,31 +8769,28 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a > b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -8842,31 +8803,28 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -8879,31 +8837,28 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -8916,31 +8871,28 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -8953,31 +8905,28 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -8990,31 +8939,28 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9027,31 +8973,28 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9064,31 +9007,28 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9101,31 +9041,28 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9138,31 +9075,28 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a != b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9175,31 +9109,28 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9212,12 +9143,12 @@ void VCmpxTruF16Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return decltype(a == b)(true); }); + uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -9363,12 +9294,12 @@ void VCmpxFF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return decltype(a == b)(false); }); + uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -9386,28 +9317,27 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a < b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9420,28 +9350,27 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a == b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9454,28 +9383,27 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9488,28 +9416,27 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a > b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9522,28 +9449,27 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9556,28 +9482,27 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9590,28 +9515,27 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9624,28 +9548,27 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9658,28 +9581,27 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9692,28 +9614,27 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9726,28 +9647,27 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9760,28 +9680,27 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9794,28 +9713,27 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a != b; }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9828,28 +9746,27 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -9862,12 +9779,12 @@ void VCmpxTruF32Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return decltype(a == b)(true); }); + uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10013,12 +9930,12 @@ void VCmpxFF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return decltype(a == b)(false); }); + uint64_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10036,28 +9953,27 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a < b; }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10070,28 +9986,27 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a == b; }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10104,28 +10019,27 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10138,28 +10052,27 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a > b; }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10172,28 +10085,27 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10206,28 +10118,27 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10240,28 +10151,27 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10274,28 +10184,27 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10308,28 +10217,27 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10342,28 +10250,27 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10376,28 +10283,27 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10410,28 +10316,27 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10444,28 +10349,27 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return a != b; }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10478,28 +10382,27 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } amdgpu::write_wave_mask_scalar(vdst, wf, result); @@ -10512,12 +10415,12 @@ void VCmpxTruF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( [&](uint64_t value) { amdgpu::write_wave_mask_scalar(inst.vdst, wf, value); commit_result(value); }, - [](auto a, auto b) { return decltype(a == b)(true); }); + uint64_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3p_exec.cpp index 3c623926fa37..644b68549493 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3p_exec.cpp @@ -133,7 +133,8 @@ void VPkMinimum3F16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_lo ^= 0x8000u; const uint16_t rlo = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MINIMUM, first_lo, second_lo, third_lo, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint16_t first_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) first_hi ^= 0x8000u; @@ -145,7 +146,8 @@ void VPkMinimum3F16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_hi ^= 0x8000u; const uint16_t rhi = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MINIMUM, first_hi, second_hi, third_hi, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -182,7 +184,8 @@ void VPkMaximum3F16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_lo ^= 0x8000u; const uint16_t rlo = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MAXIMUM, first_lo, second_lo, third_lo, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint16_t first_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) first_hi ^= 0x8000u; @@ -194,7 +197,8 @@ void VPkMaximum3F16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_hi ^= 0x8000u; const uint16_t rhi = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MAXIMUM, first_hi, second_hi, third_hi, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vopc_exec.cpp index 331360cd11ba..3910b92e5c65 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vopc_exec.cpp @@ -1187,7 +1187,11 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1195,14 +1199,14 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1232,7 +1236,11 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1240,14 +1248,14 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1269,7 +1277,11 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) == util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1277,14 +1289,14 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1314,7 +1326,11 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) == util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1322,14 +1338,14 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1351,7 +1367,11 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1359,14 +1379,14 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1396,7 +1416,11 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1404,14 +1428,14 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1433,7 +1457,11 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1441,14 +1469,14 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1478,7 +1506,11 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1486,14 +1518,14 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1515,9 +1547,10 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return (util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_vcc_mask(value); @@ -1526,14 +1559,14 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1563,9 +1596,10 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return (util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_vcc_mask(value); @@ -1574,14 +1608,14 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1603,7 +1637,11 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1611,14 +1649,14 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1648,7 +1686,11 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1656,14 +1698,14 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1685,9 +1727,10 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !util::stdx::isnan(util::f16_to_f32_simd(a)) && - !util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_vcc_mask(value); @@ -1696,14 +1739,14 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1733,9 +1776,10 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !util::stdx::isnan(util::f16_to_f32_simd(a)) && - !util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_vcc_mask(value); @@ -1744,14 +1788,14 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1773,9 +1817,10 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return util::stdx::isnan(util::f16_to_f32_simd(a)) || - util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_vcc_mask(value); @@ -1784,14 +1829,14 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1821,9 +1866,10 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return util::stdx::isnan(util::f16_to_f32_simd(a)) || - util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_vcc_mask(value); @@ -1832,14 +1878,14 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1861,7 +1907,11 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1869,14 +1919,14 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1906,7 +1956,11 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -1914,14 +1968,14 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1943,9 +1997,10 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !((util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b))); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_vcc_mask(value); @@ -1954,14 +2009,14 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -1991,9 +2046,10 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !((util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b))); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_vcc_mask(value); @@ -2002,14 +2058,14 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2031,7 +2087,11 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -2039,14 +2099,14 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2076,7 +2136,11 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -2084,14 +2148,14 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2113,7 +2177,11 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -2121,14 +2189,14 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2158,7 +2226,11 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -2166,14 +2238,14 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2195,7 +2267,11 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) != util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -2203,14 +2279,14 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2240,7 +2316,11 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) != util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -2248,14 +2328,14 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2277,7 +2357,11 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -2285,14 +2369,14 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -2322,7 +2406,11 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -2330,14 +2418,14 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3079,8 +3167,13 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3088,12 +3181,14 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3121,8 +3216,13 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3130,12 +3230,14 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3155,8 +3257,13 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3164,12 +3271,14 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3197,8 +3306,13 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3206,12 +3320,14 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3231,8 +3347,13 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3240,12 +3361,14 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3273,8 +3396,13 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3282,12 +3410,14 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3307,8 +3437,13 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3316,12 +3451,14 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3349,8 +3486,13 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3358,12 +3500,14 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3383,8 +3527,13 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3392,12 +3541,14 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3425,8 +3576,13 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3434,12 +3590,14 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3459,8 +3617,13 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3468,12 +3631,14 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3501,8 +3666,13 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3510,12 +3680,14 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3535,8 +3707,13 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3544,12 +3721,14 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3577,8 +3756,13 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3586,12 +3770,14 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3611,8 +3797,13 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3620,12 +3811,14 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3653,8 +3846,13 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3662,12 +3860,14 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3687,8 +3887,13 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3696,12 +3901,14 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3729,8 +3936,13 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3738,12 +3950,14 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3763,8 +3977,13 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3772,12 +3991,14 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3805,8 +4026,13 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3814,12 +4040,14 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3839,8 +4067,13 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3848,12 +4081,14 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3881,8 +4116,13 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3890,12 +4130,14 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3915,8 +4157,13 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3924,12 +4171,14 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3957,8 +4206,13 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -3966,12 +4220,14 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -3991,8 +4247,13 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4000,12 +4261,14 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4033,8 +4296,13 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4042,12 +4310,14 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4067,8 +4337,13 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4076,12 +4351,14 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4109,8 +4386,13 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4118,12 +4400,14 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4293,8 +4577,13 @@ void VCmpxFF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4302,12 +4591,14 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4316,8 +4607,13 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4325,12 +4621,14 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4339,8 +4637,13 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4348,12 +4651,14 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4362,8 +4667,13 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4371,12 +4681,14 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4385,8 +4697,13 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4394,12 +4711,14 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4408,8 +4727,13 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4417,12 +4741,14 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4431,8 +4757,13 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4440,12 +4771,14 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4454,8 +4787,13 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4463,12 +4801,14 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4477,8 +4817,13 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4486,12 +4831,14 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4500,8 +4847,13 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4509,12 +4861,14 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4523,8 +4877,13 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4532,12 +4891,14 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4546,8 +4907,13 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4555,12 +4921,14 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4569,8 +4937,13 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4578,12 +4951,14 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); @@ -4592,8 +4967,13 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_vcc_mask(value); wf.set_exec(value); @@ -4601,12 +4981,14 @@ void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_vcc_mask(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/encodings.cpp index 846332464cbd..f63a49cd930d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/encodings.cpp @@ -859,6 +859,9 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -1283,6 +1286,9 @@ Vopc::Vopc(std::string_view mnemonic, const VopcMachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } bool Vopc::default_encoding() { @@ -1466,6 +1472,9 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop2::implicit_uses(RegisterSet &uses) const { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/encodings.h index d5210388ff75..2154bbff9804 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/encodings.h @@ -553,6 +553,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -611,6 +612,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -673,6 +675,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h index d95b92efa1ce..3f203f1c59b8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h @@ -139,6 +139,7 @@ class Operand final : public IsaOperand { std::optional literal32_widening_; bool packed_16bit_source_ = false; bool packed_16bit_dst_ = false; + std::optional resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const; }; } // namespace cdna5 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand_exec.cpp index 52a1dae909b1..2f242d43b133 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand_exec.cpp @@ -139,7 +139,7 @@ void Operand::read_lane_chunk_exec(const amdgpu::Wavefront &wf, uint32_t lane_ba out[i] = read_lane_exec(wf, lane_base + i); return; } - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint64_t lane_mask = count == 0 ? 0 : util::mask(static_cast(count)) << lane_base; @@ -163,7 +163,7 @@ void Operand::write_lane_chunk_exec(amdgpu::Wavefront &wf, uint32_t lane_base, u write_lane_exec(wf, lane_base + i, vals[i]); return; } - auto off = detail::resolved_vgpr_offset_for_operand(wf, *this); + auto off = resolved_vgpr_offset_exec(wf); if (!off) { for (uint32_t i = 0; i < count; ++i) if (mask & (1ULL << i)) @@ -342,13 +342,13 @@ void Operand::write_scalar64_exec(amdgpu::Wavefront &wf, uint64_t val) const { } std::optional Operand::simd_vgpr_base_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) + if (auto off = resolved_vgpr_offset_exec(wf)) return wf.vgpr_alloc().base + amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); return std::nullopt; } std::optional Operand::simd_vgpr_base_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); return wf.vgpr_alloc().base + amdgpu::apply_gpr_idx(wf, *off, role); @@ -357,7 +357,7 @@ std::optional Operand::simd_vgpr_base_mut_exec(amdgpu::Wavefront &wf) } amdgpu::ConstVgprStorage Operand::simd_vgpr_storage_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); const auto &cu = amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()); return {reinterpret_cast(cu.raw_vgpr_data(wf.vgpr_alloc().base + voff)), @@ -367,7 +367,7 @@ amdgpu::ConstVgprStorage Operand::simd_vgpr_storage_exec(const amdgpu::Wavefront } amdgpu::VgprStorage Operand::simd_vgpr_storage_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -380,7 +380,7 @@ amdgpu::VgprStorage Operand::simd_vgpr_storage_mut_exec(amdgpu::Wavefront &wf) c amdgpu::ConstVgprStoragePair64 Operand::simd_vgpr_storage64_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; const auto &cu = amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()); @@ -393,7 +393,7 @@ Operand::simd_vgpr_storage64_exec(const amdgpu::Wavefront &wf) const { } amdgpu::VgprStoragePair64 Operand::simd_vgpr_storage64_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -408,7 +408,7 @@ amdgpu::VgprStoragePair64 Operand::simd_vgpr_storage64_mut_exec(amdgpu::Wavefron void Operand::simd_notify_read_exec(const amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read( &wf, wf.vgpr_alloc().base + voff, lane_mask, byte_mask); @@ -417,7 +417,7 @@ void Operand::simd_notify_read_exec(const amdgpu::Wavefront &wf, uint64_t lane_m void Operand::simd_notify_read_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read( &wf, wf.vgpr_alloc().base + voff, lane_mask, byte_mask); @@ -426,7 +426,7 @@ void Operand::simd_notify_read_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mas void Operand::simd_notify_read64_exec(const amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read(&wf, reg, lane_mask, @@ -438,7 +438,7 @@ void Operand::simd_notify_read64_exec(const amdgpu::Wavefront &wf, uint64_t lane void Operand::simd_notify_read64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read(&wf, reg, lane_mask, @@ -450,7 +450,7 @@ void Operand::simd_notify_read64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_m void Operand::simd_notify_write_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -461,7 +461,7 @@ void Operand::simd_notify_write_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_ma void Operand::simd_notify_write64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -520,5 +520,17 @@ bool Operand::full_execution_backend_complete() { return is_complete(*static_cast(full_execution_backend())); } +std::optional Operand::resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const { + if (!reads_value()) + return std::nullopt; + auto packed = + packed_16bit_vgpr_source(packed_16bit_source_, size_bits_, opr_type_, encoding_value_); + if (!packed) + packed = packed_16bit_vgpr_dst(packed_16bit_dst_, size_bits_, opr_type_, encoding_value_); + if (packed) + return packed->reg + (wf.vgpr_msb_for_role(vgpr_msb_role()) << 8); + return detail::resolved_vgpr_offset_for_operand(wf, *this); +} + } // namespace cdna5 } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/sop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/sop2_exec.cpp index 326f8b903ef0..097e39d82607 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/sop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/sop2_exec.cpp @@ -6,6 +6,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/cdna5/sop2.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" +#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/vm/amdgpu/register_access.h" #include "rocjitsu/vm/amdgpu/wavefront.h" @@ -204,16 +205,20 @@ void SMaxNumF32Sop2::execute_impl(amdgpu::Wavefront &wf) { void SMulF32Sop2::execute_impl(amdgpu::Wavefront &wf) { amdgpu::execute_s_mul_f32_sop2(*this, wf); } void SFmaakF32Sop2::execute_impl(amdgpu::Wavefront &wf) { - float result = std::fma(std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(ssrc0)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(ssrc1)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(literal))); + float result = amdgpu::fp_mode::arithmetic( + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(ssrc0)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(ssrc1)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(literal)), wf.fp_round_mode_f32(), + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); amdgpu::RegisterAccess(wf).write_scalar(sdst, std::bit_cast(result)); } void SFmamkF32Sop2::execute_impl(amdgpu::Wavefront &wf) { - float result = std::fma(std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(ssrc0)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(literal)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(ssrc1))); + float result = amdgpu::fp_mode::arithmetic( + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(ssrc0)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(literal)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(ssrc1)), wf.fp_round_mode_f32(), + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); amdgpu::RegisterAccess(wf).write_scalar(sdst, std::bit_cast(result)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop1_exec.cpp index 5a409f719481..c000e6a613b8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop1_exec.cpp @@ -8,6 +8,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/cdna5/vop1.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" #include "rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h" +#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/isa/arch/amdgpu/shared/transcendental.h" #include "rocjitsu/vm/amdgpu/register_access.h" @@ -55,13 +56,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -82,6 +78,8 @@ RJ_NOINLINE void VCvtI32F64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -192,6 +190,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -220,6 +219,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -267,6 +267,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -302,8 +303,8 @@ void VCvtF32F16Vop1::execute_impl(amdgpu::Wavefront &wf) { } auto &inst = *this; if (amdgpu::try_execute_words_simd<1, true, false, 1>(inst, wf, [&](auto a) { - return std::bit_cast>(([](auto a) { - return util::f16_to_f32_simd(a); + return std::bit_cast>(([&wf](auto a) { + return ::rocjitsu::amdgpu::cvt_f32_f16_mode_simd(a, wf); })(std::bit_cast>(a))); })) return; @@ -313,8 +314,10 @@ void VCvtF32F16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - std::bit_cast(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))))); + std::bit_cast(std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())))); } } @@ -329,6 +332,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -336,8 +340,8 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; if (amdgpu::try_execute_words_simd<1, true, false, 1>(inst, wf, [&](auto a) { - return std::bit_cast>(([](auto a) { - return util::f16_to_f32_simd(a); + return std::bit_cast>(([&wf](auto a) { + return ::rocjitsu::amdgpu::cvt_f32_f16_mode_simd(a, wf); })(std::bit_cast>(a))); })) return; @@ -347,8 +351,10 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - std::bit_cast(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))))); + std::bit_cast(std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())))); } dpp_write_mask_scope_.restore(); } @@ -372,6 +378,7 @@ RJ_NOINLINE void VCvtNearestI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -400,6 +407,7 @@ RJ_NOINLINE void VCvtFloorI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -454,6 +462,8 @@ RJ_NOINLINE void VCvtF32F64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -480,6 +490,7 @@ RJ_NOINLINE void VCvtF64F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -618,6 +629,8 @@ RJ_NOINLINE void VCvtU32F64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -670,6 +683,8 @@ RJ_NOINLINE void VTruncF64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -696,6 +711,8 @@ RJ_NOINLINE void VCeilF64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -722,6 +739,8 @@ RJ_NOINLINE void VRndneF64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -748,6 +767,8 @@ RJ_NOINLINE void VFloorF64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -870,6 +891,7 @@ RJ_NOINLINE void VTanhF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -917,6 +939,7 @@ RJ_NOINLINE void VTanhF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -956,6 +979,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -984,6 +1008,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1012,6 +1037,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1040,6 +1066,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1068,6 +1095,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1096,6 +1124,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1124,6 +1153,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1152,6 +1182,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1180,6 +1211,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1208,6 +1240,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1240,6 +1273,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1272,6 +1306,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1300,6 +1335,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1466,6 +1502,8 @@ RJ_NOINLINE void VFrexpExpI32F64Vop1::execute_modifier_impl(amdgpu::Wavefront &w [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1492,6 +1530,8 @@ RJ_NOINLINE void VFrexpMantF64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1518,6 +1558,8 @@ RJ_NOINLINE void VFractF64Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1546,6 +1588,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1574,6 +1617,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1911,6 +1955,7 @@ RJ_NOINLINE void VMovrelsd2B32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) } void VPermlane16SwapB32Vop1::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); uint32_t tmp_dst[64] = {}, tmp_src[64] = {}; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { tmp_dst[lane] = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); @@ -1918,8 +1963,10 @@ void VPermlane16SwapB32Vop1::execute_impl(amdgpu::Wavefront &wf) { } for (uint32_t base = 0; base + 16 < wf.wf_size(); base += 2u * 16) { for (uint32_t i = 0; i < 16; ++i) { - amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 16 + i]); - amdgpu::RegisterAccess(wf).write_lane(vdst, base + 16 + i, tmp_src[base + i]); + if (exec & (1ULL << (base + i))) + amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 16 + i]); + if (exec & (1ULL << (base + 16 + i))) + amdgpu::RegisterAccess(wf).write_lane(vdst, base + 16 + i, tmp_src[base + i]); } } } @@ -1951,6 +1998,7 @@ RJ_NOINLINE void VTanhBf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2199,6 +2247,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2273,6 +2322,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2315,11 +2365,17 @@ void VRcpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2330,10 +2386,13 @@ void VRcpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2349,6 +2408,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2358,11 +2418,17 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2373,10 +2439,13 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2391,11 +2460,17 @@ void VSqrtF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2406,10 +2481,13 @@ void VSqrtF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2425,6 +2503,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2434,11 +2513,17 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2449,10 +2534,13 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2467,11 +2555,17 @@ void VRsqF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2482,10 +2576,13 @@ void VRsqF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2501,6 +2598,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2510,11 +2608,17 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2525,10 +2629,13 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2543,11 +2650,15 @@ void VLogF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2558,10 +2669,13 @@ void VLogF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2577,6 +2691,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2586,11 +2701,15 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2601,10 +2720,13 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2619,11 +2741,15 @@ void VExpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2634,10 +2760,13 @@ void VExpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2653,6 +2782,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2662,11 +2792,15 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2677,10 +2811,13 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2733,6 +2870,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2823,6 +2961,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2908,6 +3047,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2982,6 +3122,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3056,6 +3197,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3130,6 +3272,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3210,6 +3353,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3259,10 +3403,13 @@ void VSinF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -3278,6 +3425,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3289,10 +3437,13 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -3309,10 +3460,13 @@ void VCosF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -3328,6 +3482,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3339,10 +3494,13 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -3411,7 +3569,8 @@ void VCvtNormI16F16Vop1::execute_impl(amdgpu::Wavefront &wf) { static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -3428,6 +3587,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3443,7 +3603,8 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -3465,7 +3626,8 @@ void VCvtNormU16F16Vop1::execute_impl(amdgpu::Wavefront &wf) { static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } @@ -3482,6 +3644,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3497,7 +3660,8 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } @@ -3886,6 +4050,7 @@ RJ_NOINLINE void VCvtF32Bf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -4239,6 +4404,7 @@ RJ_NOINLINE void VRcpBf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -4283,6 +4449,7 @@ RJ_NOINLINE void VSqrtBf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -4327,6 +4494,7 @@ RJ_NOINLINE void VRsqBf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -4371,6 +4539,7 @@ RJ_NOINLINE void VLogBf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -4415,6 +4584,7 @@ RJ_NOINLINE void VExpBf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -4459,6 +4629,7 @@ RJ_NOINLINE void VSinBf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -4503,6 +4674,7 @@ RJ_NOINLINE void VCosBf16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop2_exec.cpp index 191b574797ad..44f5b2a0891c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop2_exec.cpp @@ -62,6 +62,7 @@ void VAddF64Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -69,7 +70,12 @@ RJ_NOINLINE void VAddF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 9223372036854775808ull, dpp_modifiers_, + 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -88,6 +94,7 @@ void VAddF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -97,7 +104,10 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -116,6 +126,7 @@ void VSubF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -125,7 +136,10 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -144,6 +158,7 @@ void VSubrevF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -153,7 +168,10 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -172,6 +190,7 @@ void VMulF64Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -179,7 +198,12 @@ RJ_NOINLINE void VMulF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 9223372036854775808ull, dpp_modifiers_, + 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -198,6 +222,7 @@ void VMulDx9ZeroF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulDx9ZeroF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -207,7 +232,10 @@ RJ_NOINLINE void VMulDx9ZeroF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -226,6 +254,7 @@ void VMulF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -235,7 +264,10 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -366,6 +398,7 @@ void VMinNumF64Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinNumF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -373,7 +406,12 @@ RJ_NOINLINE void VMinNumF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 9223372036854775808ull, dpp_modifiers_, + 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -392,6 +430,7 @@ void VMaxNumF64Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxNumF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -399,7 +438,12 @@ RJ_NOINLINE void VMaxNumF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 9223372036854775808ull, dpp_modifiers_, + 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -530,6 +574,7 @@ void VMinNumF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinNumF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -539,7 +584,10 @@ RJ_NOINLINE void VMinNumF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -558,6 +606,7 @@ void VMaxNumF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxNumF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -567,7 +616,10 @@ RJ_NOINLINE void VMaxNumF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -586,6 +638,7 @@ void VFmacF64Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -593,7 +646,12 @@ RJ_NOINLINE void VFmacF64Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { [[maybe_unused]] uint64_t dpp_old_exec_ = wf.exec(); if (inst_.src0 == amdgpu::SRC_DPP) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 9223372036854775808ull, dpp_modifiers_, + 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 9223372036854775808ull, dpp_modifiers_, + 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1164,6 +1222,7 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1173,7 +1232,10 @@ RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1193,7 +1255,8 @@ void VFmamkF32Vop2::execute_impl(amdgpu::Wavefront &wf) { amdgpu::sdwa::write_lane( *this, wf, vdst, lane, std::bit_cast(amdgpu::fp_mode::arithmetic( - s0, k, s2, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + s0, k, s2, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), + wf.ieee_mode()))); } } @@ -1208,7 +1271,8 @@ void VFmaakF32Vop2::execute_impl(amdgpu::Wavefront &wf) { amdgpu::sdwa::write_lane( *this, wf, vdst, lane, std::bit_cast(amdgpu::fp_mode::arithmetic( - s0, s1, k, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + s0, s1, k, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), + wf.ieee_mode()))); } } @@ -1222,6 +1286,7 @@ void VCvtPkRtzF16F32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCvtPkRtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1231,7 +1296,10 @@ RJ_NOINLINE void VCvtPkRtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1278,6 +1346,7 @@ void VMinNumF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinNumF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1287,7 +1356,10 @@ RJ_NOINLINE void VMinNumF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1362,6 +1434,7 @@ void VMaxNumF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxNumF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1371,7 +1444,10 @@ RJ_NOINLINE void VMaxNumF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1450,6 +1526,7 @@ void VAddF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1459,7 +1536,10 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1542,6 +1622,7 @@ void VSubF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1551,7 +1632,10 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1634,6 +1718,7 @@ void VSubrevF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1643,7 +1728,10 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1726,6 +1814,7 @@ void VMulF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1735,7 +1824,10 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1792,13 +1884,15 @@ void VFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1808,7 +1902,10 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1824,7 +1921,8 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } @@ -1842,7 +1940,8 @@ void VFmamkF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16(s0, k, s2, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } @@ -1858,7 +1957,8 @@ void VFmaakF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16(s0, s1, k, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } @@ -1920,6 +2020,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1984,16 +2085,18 @@ void VPkFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint32_t rawd = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); const uint32_t omod = amdgpu::sdwa::output_modifier(*this, wf); - uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), - static_cast(rawd), false, false, false, false, - false, false, wf.fp_round_mode_f16_f64(), - wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + uint32_t r0 = amdgpu::fp_mode::fma_f16( + static_cast(raw0), static_cast(raw1), static_cast(rawd), + false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), + wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint32_t r1 = amdgpu::fp_mode::fma_f16( static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, r0 | (r1 << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp index 0584c1be0dea..09804fb906dd 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp @@ -151,16 +151,9 @@ void VTanhF32Vop3::execute_impl(amdgpu::Wavefront &wf) { sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); }(); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -200,16 +193,9 @@ RJ_NOINLINE void VTanhF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); }(); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -983,7 +969,13 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -992,31 +984,27 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1048,7 +1036,13 @@ RJ_NOINLINE void VRcpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1057,31 +1051,27 @@ RJ_NOINLINE void VRcpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1101,52 +1091,47 @@ void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1171,52 +1156,47 @@ RJ_NOINLINE void VSqrtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1229,7 +1209,13 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1238,31 +1224,27 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1294,7 +1276,13 @@ RJ_NOINLINE void VRsqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1303,31 +1291,27 @@ RJ_NOINLINE void VRsqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1347,7 +1331,13 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1356,31 +1346,27 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1412,7 +1398,13 @@ RJ_NOINLINE void VLogF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1421,31 +1413,27 @@ RJ_NOINLINE void VLogF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1465,7 +1453,13 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1474,31 +1468,27 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1530,7 +1520,13 @@ RJ_NOINLINE void VExpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1539,31 +1535,27 @@ RJ_NOINLINE void VExpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2186,31 +2178,27 @@ void VSinF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2249,31 +2237,27 @@ RJ_NOINLINE void VSinF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2300,31 +2284,27 @@ void VCosF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2363,31 +2343,27 @@ RJ_NOINLINE void VCosF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -5217,7 +5193,8 @@ void VFmacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -5259,7 +5236,8 @@ RJ_NOINLINE void VFmacF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -5753,7 +5731,7 @@ void VSExpF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::EXP2, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -5767,7 +5745,7 @@ void VSLogF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::LOG2, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -5781,7 +5759,7 @@ void VSRcpF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::RCP, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -5795,7 +5773,7 @@ void VSRsqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::RSQ, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -5809,7 +5787,7 @@ void VSSqrtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::SQRT, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_cmp.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_cmp.cpp index b56e03a9b2f9..e74543a88e65 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_cmp.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_cmp.cpp @@ -33,32 +33,26 @@ void VCmpLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -92,32 +86,26 @@ RJ_NOINLINE void VCmpLtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -132,32 +120,26 @@ void VCmpEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -191,32 +173,26 @@ RJ_NOINLINE void VCmpEqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -231,32 +207,26 @@ void VCmpLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -290,32 +260,26 @@ RJ_NOINLINE void VCmpLeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -330,32 +294,26 @@ void VCmpGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -389,32 +347,26 @@ RJ_NOINLINE void VCmpGtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -429,35 +381,26 @@ void VCmpLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -491,35 +434,26 @@ RJ_NOINLINE void VCmpLgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -534,32 +468,26 @@ void VCmpGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -593,32 +521,26 @@ RJ_NOINLINE void VCmpGeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -633,32 +555,26 @@ void VCmpOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -692,32 +608,26 @@ RJ_NOINLINE void VCmpOF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -732,32 +642,26 @@ void VCmpUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -791,32 +695,26 @@ RJ_NOINLINE void VCmpUF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -831,32 +729,26 @@ void VCmpNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -890,32 +782,26 @@ RJ_NOINLINE void VCmpNgeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -930,35 +816,26 @@ void VCmpNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -992,35 +869,26 @@ RJ_NOINLINE void VCmpNlgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -1035,32 +903,26 @@ void VCmpNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -1094,32 +956,26 @@ RJ_NOINLINE void VCmpNgtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -1134,32 +990,26 @@ void VCmpNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -1193,32 +1043,26 @@ RJ_NOINLINE void VCmpNleF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -1233,32 +1077,26 @@ void VCmpNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -1292,32 +1130,26 @@ RJ_NOINLINE void VCmpNeqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -1332,32 +1164,26 @@ void VCmpNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -1391,32 +1217,26 @@ RJ_NOINLINE void VCmpNltF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -3777,27 +3597,24 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3831,27 +3648,24 @@ RJ_NOINLINE void VCmpxLtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3864,27 +3678,24 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3918,27 +3729,24 @@ RJ_NOINLINE void VCmpxEqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3951,27 +3759,24 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4005,27 +3810,24 @@ RJ_NOINLINE void VCmpxLeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4038,27 +3840,24 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4092,27 +3891,24 @@ RJ_NOINLINE void VCmpxGtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4125,27 +3921,24 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4179,27 +3972,24 @@ RJ_NOINLINE void VCmpxLgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4212,27 +4002,24 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4266,27 +4053,24 @@ RJ_NOINLINE void VCmpxGeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4299,28 +4083,24 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4354,28 +4134,24 @@ RJ_NOINLINE void VCmpxOF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4388,28 +4164,24 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4443,28 +4215,24 @@ RJ_NOINLINE void VCmpxUF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4477,27 +4245,24 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4531,27 +4296,24 @@ RJ_NOINLINE void VCmpxNgeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4564,28 +4326,24 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4619,28 +4377,24 @@ RJ_NOINLINE void VCmpxNlgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4653,27 +4407,24 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4707,27 +4458,24 @@ RJ_NOINLINE void VCmpxNgtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4740,27 +4488,24 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4794,27 +4539,24 @@ RJ_NOINLINE void VCmpxNleF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4827,27 +4569,24 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4881,27 +4620,24 @@ RJ_NOINLINE void VCmpxNeqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4914,27 +4650,24 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -4968,27 +4701,24 @@ RJ_NOINLINE void VCmpxNltF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5001,24 +4731,23 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5050,24 +4779,23 @@ RJ_NOINLINE void VCmpxLtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5080,24 +4808,23 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5129,24 +4856,23 @@ RJ_NOINLINE void VCmpxEqF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5159,24 +4885,23 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5208,24 +4933,23 @@ RJ_NOINLINE void VCmpxLeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5238,24 +4962,23 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5287,24 +5010,23 @@ RJ_NOINLINE void VCmpxGtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5317,24 +5039,23 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5366,24 +5087,23 @@ RJ_NOINLINE void VCmpxLgF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5396,24 +5116,23 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5445,24 +5164,23 @@ RJ_NOINLINE void VCmpxGeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5475,25 +5193,23 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5525,25 +5241,23 @@ RJ_NOINLINE void VCmpxOF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5556,25 +5270,23 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5606,25 +5318,23 @@ RJ_NOINLINE void VCmpxUF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5637,24 +5347,23 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5686,24 +5395,23 @@ RJ_NOINLINE void VCmpxNgeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5716,24 +5424,23 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5765,24 +5472,23 @@ RJ_NOINLINE void VCmpxNlgF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5795,24 +5501,23 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5844,24 +5549,23 @@ RJ_NOINLINE void VCmpxNgtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5874,24 +5578,23 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5923,24 +5626,23 @@ RJ_NOINLINE void VCmpxNleF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -5953,24 +5655,23 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6002,24 +5703,23 @@ RJ_NOINLINE void VCmpxNeqF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6032,24 +5732,23 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6081,24 +5780,23 @@ RJ_NOINLINE void VCmpxNltF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6110,24 +5808,23 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6139,24 +5836,23 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6168,24 +5864,23 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6197,24 +5892,23 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6226,24 +5920,23 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6255,24 +5948,23 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6284,25 +5976,23 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6314,25 +6004,23 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6344,24 +6032,23 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6373,24 +6060,23 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6402,24 +6088,23 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6431,24 +6116,23 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6460,24 +6144,23 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -6489,24 +6172,23 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_cvt.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_cvt.cpp index 7a763907d77f..7e507682fc43 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_cvt.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_cvt.cpp @@ -251,18 +251,32 @@ void VCvtF32F16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -287,20 +301,33 @@ RJ_NOINLINE void VCvtF32F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } - dpp_write_mask_scope_.restore(); } @@ -840,8 +867,15 @@ void VCvtU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -878,8 +912,15 @@ RJ_NOINLINE void VCvtU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -904,8 +945,15 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -944,8 +992,15 @@ RJ_NOINLINE void VCvtI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -1234,11 +1289,19 @@ void VCvtNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1271,11 +1334,19 @@ RJ_NOINLINE void VCvtNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1296,11 +1367,19 @@ void VCvtNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -1333,11 +1412,19 @@ RJ_NOINLINE void VCvtNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3909,12 +3996,22 @@ void VCvtPkNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -3924,11 +4021,19 @@ void VCvtPkNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.opsel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -3957,12 +4062,22 @@ RJ_NOINLINE void VCvtPkNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -3972,11 +4087,19 @@ RJ_NOINLINE void VCvtPkNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.opsel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -3993,10 +4116,20 @@ void VCvtPkNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -4006,11 +4139,19 @@ void VCvtPkNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.opsel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -4039,10 +4180,20 @@ RJ_NOINLINE void VCvtPkNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -4052,11 +4203,19 @@ RJ_NOINLINE void VCvtPkNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.opsel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_data.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_data.cpp index 08dbd349706c..62bb47879359 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_data.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_data.cpp @@ -54,13 +54,8 @@ RJ_NOINLINE void VMovB32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -490,6 +485,7 @@ RJ_NOINLINE void VMovrelsd2B32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) } void VPermlane16SwapB32Vop3::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); uint32_t tmp_dst[64] = {}, tmp_src[64] = {}; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { tmp_dst[lane] = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); @@ -497,8 +493,10 @@ void VPermlane16SwapB32Vop3::execute_impl(amdgpu::Wavefront &wf) { } for (uint32_t base = 0; base + 16 < wf.wf_size(); base += 2u * 16) { for (uint32_t i = 0; i < 16; ++i) { - amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 16 + i]); - amdgpu::RegisterAccess(wf).write_lane(vdst, base + 16 + i, tmp_src[base + i]); + if (exec & (1ULL << (base + i))) + amdgpu::RegisterAccess(wf).write_lane(src0, base + i, tmp_dst[base + 16 + i]); + if (exec & (1ULL << (base + 16 + i))) + amdgpu::RegisterAccess(wf).write_lane(vdst, base + 16 + i, tmp_src[base + i]); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp index 0a645e40a05b..c84e78557dcf 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp @@ -1918,7 +1918,8 @@ void VFmaF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -1960,7 +1961,8 @@ RJ_NOINLINE void VFmaF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp index 9d7b13c9f0b7..1134676ef994 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp @@ -601,11 +601,13 @@ void VPkFmaF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t rlo = amdgpu::fp_mode::fma_f16( a_lo, b_lo, c_lo, false, false, false, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint16_t rhi = amdgpu::fp_mode::fma_f16( a_hi, b_hi, c_hi, false, false, false, inst_.neg_hi & 1u, inst_.neg_hi & 2u, inst_.neg_hi & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, inst_.clamp, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -637,7 +639,7 @@ void VPkAddF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::ADD, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -647,7 +649,7 @@ void VPkAddF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::ADD, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -679,7 +681,7 @@ void VPkMulF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MUL, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -689,7 +691,7 @@ void VPkMulF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MUL, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -940,7 +942,7 @@ void VPkMinNumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MIN, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -950,7 +952,7 @@ void VPkMinNumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MIN, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -982,7 +984,7 @@ void VPkMaxNumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAX, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -992,7 +994,7 @@ void VPkMaxNumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAX, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -1024,7 +1026,7 @@ void VPkMinimumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MINIMUM, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -1034,7 +1036,7 @@ void VPkMinimumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MINIMUM, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -1066,7 +1068,7 @@ void VPkMaximumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAXIMUM, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -1076,7 +1078,7 @@ void VPkMaximumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAXIMUM, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -1086,7 +1088,7 @@ void VPkFmaF32Vop3p::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_VOP3P_PK_TERNARY_F32_SELECTORS( inst_.opsel, inst_.opsel_hi, inst_.opsel_hi_2, - [](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }); + [&wf](auto a, auto b, auto c) { return amdgpu::fma_f32_simd(a, b, c, wf); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -1120,10 +1122,12 @@ void VPkFmaF32Vop3p::execute_impl(amdgpu::Wavefront &wf) { c_hi = -c_hi; uint32_t rlo = amdgpu::fp_mode::packed_f32(a_lo, b_lo, c_lo, amdgpu::fp_mode::PackedF32Op::FMA, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), - inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint32_t rhi = amdgpu::fp_mode::packed_f32(a_hi, b_hi, c_hi, amdgpu::fp_mode::PackedF32Op::FMA, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), - inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 32)); } @@ -1158,7 +1162,8 @@ void VFmaMixF32Vop3p::execute_impl(amdgpu::Wavefront &wf) { b = -b; if (inst_.neg & 4) c = -c; - float result = std::fma(a, b, c); + float result = amdgpu::fp_mode::arithmetic( + a, b, c, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, @@ -1206,7 +1211,8 @@ RJ_NOINLINE void VFmaMixF32Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) { b = -b; if (inst_.neg & 4) c = -c; - float result = std::fma(a, b, c); + float result = amdgpu::fp_mode::arithmetic( + a, b, c, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, @@ -1244,10 +1250,10 @@ void VFmaMixloF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { b = -b; if (inst_.neg & 4) c = -c; - float result = std::fma(a, b, c); - if (inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0u, h); } } @@ -1292,10 +1298,10 @@ RJ_NOINLINE void VFmaMixloF16Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) b = -b; if (inst_.neg & 4) c = -c; - float result = std::fma(a, b, c); - if (inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0u, h); } dpp_write_mask_scope_.restore(); @@ -1330,10 +1336,10 @@ void VFmaMixhiF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { b = -b; if (inst_.neg & 4) c = -c; - float result = std::fma(a, b, c); - if (inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0x8u, h); } } @@ -1378,10 +1384,10 @@ RJ_NOINLINE void VFmaMixhiF16Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) b = -b; if (inst_.neg & 4) c = -c; - float result = std::fma(a, b, c); - if (inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0x8u, h); } dpp_write_mask_scope_.restore(); @@ -1453,10 +1459,12 @@ void VPkMulF32Vop3p::execute_impl(amdgpu::Wavefront &wf) { b_hi = -b_hi; uint32_t rlo = amdgpu::fp_mode::packed_f32(a_lo, b_lo, 0.0f, amdgpu::fp_mode::PackedF32Op::MUL, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), - inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint32_t rhi = amdgpu::fp_mode::packed_f32(a_hi, b_hi, 0.0f, amdgpu::fp_mode::PackedF32Op::MUL, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), - inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 32)); } @@ -1490,10 +1498,12 @@ void VPkAddF32Vop3p::execute_impl(amdgpu::Wavefront &wf) { b_hi = -b_hi; uint32_t rlo = amdgpu::fp_mode::packed_f32(a_lo, b_lo, 0.0f, amdgpu::fp_mode::PackedF32Op::ADD, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), - inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint32_t rhi = amdgpu::fp_mode::packed_f32(a_hi, b_hi, 0.0f, amdgpu::fp_mode::PackedF32Op::ADD, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), - inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 32)); } @@ -1848,7 +1858,8 @@ void VPkMinimum3F16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_lo ^= 0x8000u; const uint16_t rlo = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MINIMUM, first_lo, second_lo, third_lo, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint16_t first_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) first_hi ^= 0x8000u; @@ -1860,7 +1871,8 @@ void VPkMinimum3F16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_hi ^= 0x8000u; const uint16_t rhi = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MINIMUM, first_hi, second_hi, third_hi, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -1897,7 +1909,8 @@ void VPkMaximum3F16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_lo ^= 0x8000u; const uint16_t rlo = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MAXIMUM, first_lo, second_lo, third_lo, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint16_t first_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) first_hi ^= 0x8000u; @@ -1909,7 +1922,8 @@ void VPkMaximum3F16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_hi ^= 0x8000u; const uint16_t rhi = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MAXIMUM, first_hi, second_hi, third_hi, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -1946,7 +1960,8 @@ void VPkMin3NumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_lo ^= 0x8000u; const uint16_t rlo = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MIN, first_lo, second_lo, third_lo, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint16_t first_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) first_hi ^= 0x8000u; @@ -1958,7 +1973,8 @@ void VPkMin3NumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_hi ^= 0x8000u; const uint16_t rhi = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MIN, first_hi, second_hi, third_hi, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -1995,7 +2011,8 @@ void VPkMax3NumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_lo ^= 0x8000u; const uint16_t rlo = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MAX, first_lo, second_lo, third_lo, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint16_t first_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) first_hi ^= 0x8000u; @@ -2007,7 +2024,8 @@ void VPkMax3NumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { third_hi ^= 0x8000u; const uint16_t rhi = amdgpu::fp_mode::packed_select3_f16( amdgpu::fp_mode::PackedBinaryOp::MAX, first_hi, second_hi, third_hi, - wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopc_exec_cmp.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopc_exec_cmp.cpp index 1c22d2d36287..cba43003bd9d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopc_exec_cmp.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopc_exec_cmp.cpp @@ -33,6 +33,7 @@ void VCmpLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -48,7 +49,10 @@ RJ_NOINLINE void VCmpLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -70,6 +74,7 @@ void VCmpEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -85,7 +90,10 @@ RJ_NOINLINE void VCmpEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_eq_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -107,6 +115,7 @@ void VCmpLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -122,7 +131,10 @@ RJ_NOINLINE void VCmpLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_le_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -144,6 +156,7 @@ void VCmpGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -159,7 +172,10 @@ RJ_NOINLINE void VCmpGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_gt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -181,6 +197,7 @@ void VCmpLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -196,7 +213,10 @@ RJ_NOINLINE void VCmpLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lg_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -218,6 +238,7 @@ void VCmpGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -233,7 +254,10 @@ RJ_NOINLINE void VCmpGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ge_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -255,6 +279,7 @@ void VCmpOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -270,7 +295,10 @@ RJ_NOINLINE void VCmpOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_o_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -292,6 +320,7 @@ void VCmpUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -307,7 +336,10 @@ RJ_NOINLINE void VCmpUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_u_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -329,6 +361,7 @@ void VCmpNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -344,7 +377,10 @@ RJ_NOINLINE void VCmpNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nge_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -366,6 +402,7 @@ void VCmpNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -381,7 +418,10 @@ RJ_NOINLINE void VCmpNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlg_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -403,6 +443,7 @@ void VCmpNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -418,7 +459,10 @@ RJ_NOINLINE void VCmpNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ngt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -440,6 +484,7 @@ void VCmpNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -455,7 +500,10 @@ RJ_NOINLINE void VCmpNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nle_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -477,6 +525,7 @@ void VCmpNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -492,7 +541,10 @@ RJ_NOINLINE void VCmpNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_neq_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -514,6 +566,7 @@ void VCmpNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -529,7 +582,10 @@ RJ_NOINLINE void VCmpNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -551,6 +607,7 @@ void VCmpLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -566,7 +623,10 @@ RJ_NOINLINE void VCmpLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -588,6 +648,7 @@ void VCmpEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -603,7 +664,10 @@ RJ_NOINLINE void VCmpEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_eq_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -625,6 +689,7 @@ void VCmpLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -640,7 +705,10 @@ RJ_NOINLINE void VCmpLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_le_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -662,6 +730,7 @@ void VCmpGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -677,7 +746,10 @@ RJ_NOINLINE void VCmpGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_gt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -699,6 +771,7 @@ void VCmpLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -714,7 +787,10 @@ RJ_NOINLINE void VCmpLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lg_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -736,6 +812,7 @@ void VCmpGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -751,7 +828,10 @@ RJ_NOINLINE void VCmpGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ge_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -773,6 +853,7 @@ void VCmpOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -788,7 +869,10 @@ RJ_NOINLINE void VCmpOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_o_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -810,6 +894,7 @@ void VCmpUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -825,7 +910,10 @@ RJ_NOINLINE void VCmpUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_u_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -847,6 +935,7 @@ void VCmpNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -862,7 +951,10 @@ RJ_NOINLINE void VCmpNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nge_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -884,6 +976,7 @@ void VCmpNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -899,7 +992,10 @@ RJ_NOINLINE void VCmpNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlg_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -921,6 +1017,7 @@ void VCmpNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -936,7 +1033,10 @@ RJ_NOINLINE void VCmpNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ngt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -958,6 +1058,7 @@ void VCmpNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -973,7 +1074,10 @@ RJ_NOINLINE void VCmpNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nle_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -995,6 +1099,7 @@ void VCmpNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1010,7 +1115,10 @@ RJ_NOINLINE void VCmpNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_neq_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1032,6 +1140,7 @@ void VCmpNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1047,7 +1156,10 @@ RJ_NOINLINE void VCmpNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -2076,6 +2188,7 @@ RJ_NOINLINE void VCmpClassF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::execute_v_cmp_class_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { @@ -2113,6 +2226,7 @@ RJ_NOINLINE void VCmpClassF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::execute_v_cmp_class_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopc_exec_cmpx.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopc_exec_cmpx.cpp index 04a8559c4b29..2f4cd22eb9e3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopc_exec_cmpx.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopc_exec_cmpx.cpp @@ -31,19 +31,23 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -51,6 +55,7 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -66,24 +71,31 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -106,19 +118,23 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) == util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -126,6 +142,7 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -141,24 +158,31 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) == util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -181,19 +205,23 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -201,6 +229,7 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -216,24 +245,31 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -256,19 +292,23 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -276,6 +316,7 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -291,24 +332,31 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -331,22 +379,23 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return (util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -354,6 +403,7 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -369,27 +419,31 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return (util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -412,19 +466,23 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -432,6 +490,7 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -447,24 +506,31 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -487,22 +553,23 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !util::stdx::isnan(util::f16_to_f32_simd(a)) && - !util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -510,6 +577,7 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -525,27 +593,31 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !util::stdx::isnan(util::f16_to_f32_simd(a)) && - !util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -568,22 +640,23 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return util::stdx::isnan(util::f16_to_f32_simd(a)) || - util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -591,6 +664,7 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -606,27 +680,31 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return util::stdx::isnan(util::f16_to_f32_simd(a)) || - util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -649,19 +727,23 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -669,6 +751,7 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -684,24 +767,31 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -724,22 +814,23 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !((util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b))); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -747,6 +838,7 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -762,27 +854,31 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !((util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b))); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -805,19 +901,23 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -825,6 +925,7 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -840,24 +941,31 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -880,19 +988,23 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -900,6 +1012,7 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -915,24 +1028,31 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -955,19 +1075,23 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) != util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -975,6 +1099,7 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -990,24 +1115,31 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) != util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1030,19 +1162,23 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1050,6 +1186,7 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1065,24 +1202,31 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1103,18 +1247,25 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1122,6 +1273,7 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1137,21 +1289,31 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1172,18 +1334,25 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1191,6 +1360,7 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1206,21 +1376,31 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1241,18 +1421,25 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1260,6 +1447,7 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1275,21 +1463,31 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1310,18 +1508,25 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1329,6 +1534,7 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1344,21 +1550,31 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1379,18 +1595,25 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1398,6 +1621,7 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1413,21 +1637,31 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1448,18 +1682,25 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1467,6 +1708,7 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1482,21 +1724,31 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1517,18 +1769,25 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1536,6 +1795,7 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1551,21 +1811,31 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1586,18 +1856,25 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1605,6 +1882,7 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1620,21 +1898,31 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1655,18 +1943,25 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1674,6 +1969,7 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1689,21 +1985,31 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1724,18 +2030,25 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1743,6 +2056,7 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1758,21 +2072,31 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1793,18 +2117,25 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1812,6 +2143,7 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1827,21 +2159,31 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1862,18 +2204,25 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1881,6 +2230,7 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1896,21 +2246,31 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1931,18 +2291,25 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1950,6 +2317,7 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1965,21 +2333,31 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2000,18 +2378,25 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2019,6 +2404,7 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2034,21 +2420,31 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2065,18 +2461,25 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2084,18 +2487,25 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2103,18 +2513,25 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2122,18 +2539,25 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2141,18 +2565,25 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2160,18 +2591,25 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2179,18 +2617,25 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2198,18 +2643,25 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2217,18 +2669,25 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2236,18 +2695,25 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2255,18 +2721,25 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2274,18 +2747,25 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2293,18 +2773,25 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2312,18 +2799,25 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4355,6 +4849,7 @@ RJ_NOINLINE void VCmpxClassF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); [&]() -> void { auto &inst = *this; @@ -4525,6 +5020,7 @@ RJ_NOINLINE void VCmpxClassF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); [&]() -> void { auto &inst = *this; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopd_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopd_exec.cpp index 844051d77ffe..97303d6dda8c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopd_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vopd_exec.cpp @@ -139,25 +139,25 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(*slot.dst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdFmaakF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(src2), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdFmamkF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src2), std::bit_cast(src1), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdMulF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32()); + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdMulDx9ZeroF32: { @@ -170,7 +170,7 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la case kVopdAddF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32()); + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdSubF32: { @@ -209,7 +209,7 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la case kVopdFmaF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(src2), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdSubNcU32: diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/decoder.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/decoder.cpp index 58271c602ccb..ee90db9f75a2 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/decoder.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/decoder.cpp @@ -4096,7 +4096,7 @@ const std::array DecoderImpl::sub_decode_vop3 = { &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, - &DecoderImpl::decodeInvalid, + &detail::decodeVLshlrevB64Vop3, &detail::decodeVLshrrevB64Vop3, &detail::decodeVAshrrevI64Vop3, &DecoderImpl::decodeInvalid, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/encodings.cpp index ca497f179603..224f763545d3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/encodings.cpp @@ -398,6 +398,9 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -757,6 +760,9 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop2::implicit_uses(RegisterSet &uses) const { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/encodings.h index 64dc2564cd5a..943b45f76a31 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/encodings.h @@ -481,6 +481,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -534,6 +535,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -602,6 +604,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop1_exec.cpp index 8555383f870d..7adc83dfd214 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop1_exec.cpp @@ -60,13 +60,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -168,6 +163,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -201,6 +197,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -234,6 +231,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -267,6 +265,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -300,6 +299,7 @@ RJ_NOINLINE void VCvtRpiI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -333,6 +333,7 @@ RJ_NOINLINE void VCvtFlrI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -567,6 +568,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -600,6 +602,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -633,6 +636,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -666,6 +670,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -699,6 +704,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -732,6 +738,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -765,6 +772,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -798,6 +806,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -831,6 +840,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -864,6 +874,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -901,6 +912,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -938,6 +950,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -971,6 +984,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1181,6 +1195,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1214,6 +1229,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1673,6 +1689,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1706,6 +1723,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1739,6 +1757,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1772,6 +1791,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1805,6 +1825,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1838,6 +1859,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1871,6 +1893,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1904,6 +1927,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1937,6 +1961,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1970,6 +1995,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2003,6 +2029,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2036,6 +2063,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2069,6 +2097,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2102,6 +2131,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2135,6 +2165,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2168,6 +2199,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2234,6 +2266,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2267,6 +2300,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop2_exec.cpp index d313ce98cefb..b98b6f689689 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop2_exec.cpp @@ -92,6 +92,8 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -131,6 +133,8 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -170,6 +174,8 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -191,7 +197,7 @@ void VMacLegacyF32Vop2::execute_impl(amdgpu::Wavefront &wf) { std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()))); } } @@ -224,6 +230,8 @@ RJ_NOINLINE void VMulLegacyF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -263,6 +271,8 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -458,6 +468,8 @@ RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -497,6 +509,8 @@ RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -946,6 +960,7 @@ void VMacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -954,7 +969,10 @@ RJ_NOINLINE void VMacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1254,6 +1272,7 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1262,7 +1281,10 @@ RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1308,6 +1330,8 @@ RJ_NOINLINE void VCvtPkrtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1347,6 +1371,8 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1386,6 +1412,8 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1425,6 +1453,8 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1464,6 +1494,8 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1484,6 +1516,7 @@ void VFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1492,7 +1525,10 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1538,6 +1574,8 @@ RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1577,6 +1615,8 @@ RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1616,6 +1656,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::NONE, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp index 20769192dcbd..6810d420de83 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp @@ -40,13 +40,8 @@ void VMovB32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::execute_v_mov_b3 void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -346,9 +341,7 @@ void VMacLegacyF32Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, std::bit_cast([&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); @@ -367,20 +360,20 @@ void VMacLegacyF32Vop3::execute_impl(amdgpu::Wavefront &wf) { return sv; }(), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod) != 0)); const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); - if (inst_.clamp) + if (inst_.clamp) { + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); v = amdgpu::clamp_floating_result(v, wf); + } return v; }())); } @@ -1157,20 +1150,14 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1183,20 +1170,14 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1209,20 +1190,14 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1235,20 +1210,14 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1261,20 +1230,14 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1287,20 +1250,14 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1313,20 +1270,14 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1339,20 +1290,14 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1365,20 +1310,14 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1391,20 +1330,14 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1417,20 +1350,14 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1443,20 +1370,14 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1469,20 +1390,14 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1495,20 +1410,14 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1679,20 +1588,14 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1705,20 +1608,14 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1731,20 +1628,14 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1757,20 +1648,14 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1783,20 +1668,14 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1809,20 +1688,14 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1835,20 +1708,14 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1861,20 +1728,14 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1887,20 +1748,14 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1913,20 +1768,14 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1939,20 +1788,14 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1965,20 +1808,14 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1991,20 +1828,14 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -2017,20 +1848,14 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3309,22 +3134,14 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3338,22 +3155,14 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3367,22 +3176,14 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3396,22 +3197,14 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3425,22 +3218,14 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3454,22 +3239,14 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3483,22 +3260,14 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3778,22 +3547,14 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3807,22 +3568,14 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3836,22 +3589,14 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3865,22 +3610,14 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3894,22 +3631,14 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3923,22 +3652,14 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3952,22 +3673,14 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vopc_exec.cpp index f38857ea047c..e14b02cb4b63 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vopc_exec.cpp @@ -656,12 +656,14 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -688,12 +690,14 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -712,12 +716,14 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -744,12 +750,14 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -768,12 +776,14 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -800,12 +810,14 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -824,12 +836,14 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -856,12 +870,14 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -880,12 +896,14 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -912,12 +930,14 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -936,12 +956,14 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -968,12 +990,14 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -992,12 +1016,14 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1024,12 +1050,14 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1048,12 +1076,14 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1080,12 +1110,14 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1104,12 +1136,14 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1136,12 +1170,14 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1160,12 +1196,14 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1192,12 +1230,14 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1216,12 +1256,14 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1248,12 +1290,14 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1272,12 +1316,14 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1304,12 +1350,14 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1328,12 +1376,14 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1360,12 +1410,14 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1384,12 +1436,14 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1416,12 +1470,14 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1561,12 +1617,14 @@ void VCmpxFF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1575,12 +1633,14 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1589,12 +1649,14 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1603,12 +1665,14 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1617,12 +1681,14 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1631,12 +1697,14 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1645,12 +1713,14 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1659,12 +1729,14 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1673,12 +1745,14 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1687,12 +1761,14 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1701,12 +1777,14 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1715,12 +1793,14 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1729,12 +1809,14 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1743,12 +1825,14 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5131,14 +5215,14 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5165,14 +5249,14 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5191,14 +5275,14 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5225,14 +5309,14 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5251,14 +5335,14 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5285,14 +5369,14 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5311,14 +5395,14 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5345,14 +5429,14 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5371,14 +5455,14 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5405,14 +5489,14 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5431,14 +5515,14 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5465,14 +5549,14 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5491,14 +5575,14 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5525,14 +5609,14 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5977,14 +6061,14 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6011,14 +6095,14 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6037,14 +6121,14 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6071,14 +6155,14 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6097,14 +6181,14 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6131,14 +6215,14 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6157,14 +6241,14 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6191,14 +6275,14 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6217,14 +6301,14 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6251,14 +6335,14 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6277,14 +6361,14 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6311,14 +6395,14 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6337,14 +6421,14 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6371,14 +6455,14 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/decoder.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/decoder.cpp index 659a2d4a77e9..680cb256d669 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/decoder.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/decoder.cpp @@ -4104,7 +4104,7 @@ const std::array DecoderImpl::sub_decode_vop3 = { &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, - &DecoderImpl::decodeInvalid, + &detail::decodeVLshlrevB64Vop3, &detail::decodeVLshrrevB64Vop3, &detail::decodeVAshrrevI64Vop3, &DecoderImpl::decodeInvalid, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/encodings.cpp index 55db4fcdd824..9ade3222d250 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/encodings.cpp @@ -398,6 +398,9 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -756,6 +759,9 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop2::implicit_uses(RegisterSet &uses) const { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/encodings.h index 905b4973c573..78e06b9f074e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/encodings.h @@ -485,6 +485,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -538,6 +539,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bank_mask_ = 0xF; uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -606,6 +608,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop1_exec.cpp index 8c2914efd256..12ecf4db7e60 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop1_exec.cpp @@ -60,13 +60,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -168,6 +163,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -201,6 +197,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -234,6 +231,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -267,6 +265,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -300,6 +299,7 @@ RJ_NOINLINE void VCvtRpiI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -333,6 +333,7 @@ RJ_NOINLINE void VCvtFlrI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -567,6 +568,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -600,6 +602,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -633,6 +636,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -666,6 +670,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -699,6 +704,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -732,6 +738,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -765,6 +772,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -798,6 +806,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -831,6 +840,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -864,6 +874,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -901,6 +912,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -938,6 +950,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -971,6 +984,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1181,6 +1195,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1214,6 +1229,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1673,6 +1689,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1706,6 +1723,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1739,6 +1757,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1772,6 +1791,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1805,6 +1825,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1838,6 +1859,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1871,6 +1893,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1904,6 +1927,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1937,6 +1961,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1970,6 +1995,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2003,6 +2029,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2036,6 +2063,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2069,6 +2097,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2102,6 +2131,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2135,6 +2165,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2168,6 +2199,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2234,6 +2266,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2267,6 +2300,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf sdwa_src0_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src0_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop2_exec.cpp index 48a90e3c52d9..0e3e7b6ac63a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop2_exec.cpp @@ -119,6 +119,8 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -158,6 +160,8 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -197,6 +201,8 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -218,7 +224,7 @@ void VFmacLegacyF32Vop2::execute_impl(amdgpu::Wavefront &wf) { std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()))); } } @@ -251,6 +257,8 @@ RJ_NOINLINE void VMulLegacyF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -290,6 +298,8 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -512,6 +522,8 @@ RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -551,6 +563,8 @@ RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1273,6 +1287,7 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1281,7 +1296,10 @@ RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1327,6 +1345,8 @@ RJ_NOINLINE void VCvtPkrtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &w sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F32, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1366,6 +1386,8 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1405,6 +1427,8 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1444,6 +1468,8 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1483,6 +1509,8 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1503,6 +1531,7 @@ void VFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1511,7 +1540,10 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, wf.exec(), dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1557,6 +1589,8 @@ RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1596,6 +1630,8 @@ RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::F16, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; @@ -1635,6 +1671,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { sdwa_src1_abs_, amdgpu::sdwa::SourceModifierFormat::NONE, dpp_src1_, wf); } + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp index b29fca27d4d1..366026a829b2 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp @@ -40,13 +40,8 @@ void VMovB32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::execute_v_mov_b3 void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -346,9 +341,7 @@ void VFmacLegacyF32Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, std::bit_cast([&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); @@ -367,20 +360,20 @@ void VFmacLegacyF32Vop3::execute_impl(amdgpu::Wavefront &wf) { return sv; }(), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod) != 0)); const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); - if (inst_.clamp) + if (inst_.clamp) { + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); v = amdgpu::clamp_floating_result(v, wf); + } return v; }())); } @@ -491,9 +484,7 @@ void VFmaLegacyF32Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, std::bit_cast([&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); @@ -519,20 +510,20 @@ void VFmaLegacyF32Vop3::execute_impl(amdgpu::Wavefront &wf) { sv = -sv; return sv; }(), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod) != 0)); const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); - if (inst_.clamp) + if (inst_.clamp) { + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); v = amdgpu::clamp_floating_result(v, wf); + } return v; }())); } @@ -1203,20 +1194,14 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1229,20 +1214,14 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1255,20 +1234,14 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1281,20 +1254,14 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1307,20 +1274,14 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1333,20 +1294,14 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1359,20 +1314,14 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1385,20 +1334,14 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1411,20 +1354,14 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1437,20 +1374,14 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1463,20 +1394,14 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1489,20 +1414,14 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1515,20 +1434,14 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1541,20 +1454,14 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1725,20 +1632,14 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1751,20 +1652,14 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1777,20 +1672,14 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1803,20 +1692,14 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1829,20 +1712,14 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1855,20 +1732,14 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1881,20 +1752,14 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1907,20 +1772,14 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1933,20 +1792,14 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1959,20 +1812,14 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -1985,20 +1832,14 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -2011,20 +1852,14 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -2037,20 +1872,14 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -2063,20 +1892,14 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3355,22 +3178,14 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3384,22 +3199,14 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3413,22 +3220,14 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3442,22 +3241,14 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3471,22 +3262,14 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3500,22 +3283,14 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3529,22 +3304,14 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3824,22 +3591,14 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3853,22 +3612,14 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3882,22 +3633,14 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3911,22 +3654,14 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3940,22 +3675,14 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3969,22 +3696,14 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -3998,22 +3717,14 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vopc_exec.cpp index 630e22d0410a..bcc6f89d693f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vopc_exec.cpp @@ -656,12 +656,14 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -688,12 +690,14 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -712,12 +716,14 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -744,12 +750,14 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -768,12 +776,14 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -800,12 +810,14 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -824,12 +836,14 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -856,12 +870,14 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -880,12 +896,14 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -912,12 +930,14 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -936,12 +956,14 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -968,12 +990,14 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -992,12 +1016,14 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1024,12 +1050,14 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1048,12 +1076,14 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1080,12 +1110,14 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1104,12 +1136,14 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1136,12 +1170,14 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1160,12 +1196,14 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1192,12 +1230,14 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1216,12 +1256,14 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1248,12 +1290,14 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1272,12 +1316,14 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1304,12 +1350,14 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1328,12 +1376,14 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1360,12 +1410,14 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1384,12 +1436,14 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1416,12 +1470,14 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1561,12 +1617,14 @@ void VCmpxFF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1575,12 +1633,14 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1589,12 +1649,14 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1603,12 +1665,14 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1617,12 +1681,14 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1631,12 +1697,14 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1645,12 +1713,14 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1659,12 +1729,14 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1673,12 +1745,14 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1687,12 +1761,14 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1701,12 +1777,14 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1715,12 +1793,14 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1729,12 +1809,14 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -1743,12 +1825,14 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5131,14 +5215,14 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5165,14 +5249,14 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5191,14 +5275,14 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5225,14 +5309,14 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5251,14 +5335,14 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5285,14 +5369,14 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5311,14 +5395,14 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5345,14 +5429,14 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5371,14 +5455,14 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5405,14 +5489,14 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5431,14 +5515,14 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5465,14 +5549,14 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5491,14 +5575,14 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5525,14 +5609,14 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -5977,14 +6061,14 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6011,14 +6095,14 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6037,14 +6121,14 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6071,14 +6155,14 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6097,14 +6181,14 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6131,14 +6215,14 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6157,14 +6241,14 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6191,14 +6275,14 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6217,14 +6301,14 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6251,14 +6335,14 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6277,14 +6361,14 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6311,14 +6395,14 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6337,14 +6421,14 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6371,14 +6455,14 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/decoder.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/decoder.cpp index d578bf67623e..eb862da1205a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/decoder.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/decoder.cpp @@ -5461,7 +5461,7 @@ const std::array DecoderImpl::sub_decode_mimg = { &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, - &DecoderImpl::decodeInvalid, + &detail::decodeImageGather4hMimg, &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/encodings.cpp index 1c6b91892342..8773f8eca0e4 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/encodings.cpp @@ -390,6 +390,9 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -650,6 +653,9 @@ Vopc::Vopc(std::string_view mnemonic, const VopcMachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } bool Vopc::default_encoding() { @@ -753,6 +759,9 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop2::implicit_uses(RegisterSet &uses) const { @@ -1576,6 +1585,15 @@ Mimg::Mimg(std::string_view mnemonic, const MimgMachineInst *inst, ExecuteFn exe opcode_ = inst_.op; } +void Mimg::capture_nsa_words(const MachineInst *inst, const Operand *vaddr) { + if (!inst_.nsa) + return; + nsa_vaddr_operand_ = vaddr; + size_ = sizeof(OpEncoding) + sizeof(MachineInst); + std::memcpy(raw_words_.data(), inst, size_); + raw_encoding_ = raw_words_.data(); +} + bool Mimg::has_nsa() { return (inst_.nsa == 1); } Exp::Exp(std::string_view mnemonic, const ExpMachineInst *inst, ExecuteFn exec_fn) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/encodings.h index 95778541835b..5e5cc8fedf05 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/encodings.h @@ -442,6 +442,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -484,6 +485,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -528,6 +530,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -755,9 +758,12 @@ class Mtbuf : public IsaInstruction { class Mimg : public IsaInstruction { public: Mimg(std::string_view mnemonic, const MimgMachineInst *inst, ExecuteFn exec_fn); + void capture_nsa_words(const MachineInst *inst, const Operand *vaddr); bool has_nsa(); using OpEncoding = MimgMachineInst; const OpEncoding inst_; + std::array raw_words_{}; + const Operand *nsa_vaddr_operand_ = nullptr; }; class Exp : public IsaInstruction { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir.cpp index abd9a6b4dba6..2746650a6a61 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir.cpp @@ -51,6 +51,8 @@ LdsDirectLoadLdsdir::LdsDirectLoadLdsdir(const MachineInst *inst) num_dst_ = 1; dsmem.apply_fieldless_caps(false, false, false); m0.apply_fieldless_caps(false, false, false); + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::EXPCNT, + amdgpu::MemoryCompletionClass::LDS}}); flags_ |= MEMORY_WAIT_PRODUCER; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir_exec.cpp index 396be1e443e4..2866f77f2f35 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir_exec.cpp @@ -6,6 +6,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/rdna3/ldsdir.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" +#include "rocjitsu/isa/arch/amdgpu/shared/lds_direct.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/vm/amdgpu/wavefront.h" #include "util/data_types.h" @@ -23,7 +24,7 @@ void LdsParamLoadLdsdir::execute_impl(amdgpu::Wavefront &wf) { } void LdsDirectLoadLdsdir::execute_impl(amdgpu::Wavefront &wf) { - amdgpu::execute_lds_direct_load_ldsdir(*this, wf); + amdgpu::execute_lds_direct_load(*this, wf, inst_.vdst); } } // namespace rdna3 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/mimg.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/mimg.cpp index c2720f8c2526..e6bda92e3e4c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/mimg.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/mimg.cpp @@ -22,6 +22,7 @@ ImageLoadMimg::ImageLoadMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -46,6 +47,7 @@ ImageLoadMipMimg::ImageLoadMipMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -71,6 +73,7 @@ ImageLoadPckMimg::ImageLoadPckMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -96,6 +99,7 @@ ImageLoadPckSgnMimg::ImageLoadPckSgnMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -121,6 +125,7 @@ ImageLoadMipPckMimg::ImageLoadMipPckMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -146,6 +151,7 @@ ImageLoadMipPckSgnMimg::ImageLoadMipPckSgnMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -171,6 +177,7 @@ ImageStoreMimg::ImageStoreMimg(const MachineInst *inst) src_operands_[2] = &srsrc; num_src_ = 3; num_dst_ = 0; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -195,6 +202,7 @@ ImageStoreMipMimg::ImageStoreMipMimg(const MachineInst *inst) src_operands_[2] = &srsrc; num_src_ = 3; num_dst_ = 0; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -220,6 +228,7 @@ ImageStorePckMimg::ImageStorePckMimg(const MachineInst *inst) src_operands_[2] = &srsrc; num_src_ = 3; num_dst_ = 0; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -245,6 +254,7 @@ ImageStoreMipPckMimg::ImageStoreMipPckMimg(const MachineInst *inst) src_operands_[2] = &srsrc; num_src_ = 3; num_dst_ = 0; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -273,6 +283,7 @@ ImageAtomicSwapMimg::ImageAtomicSwapMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -303,6 +314,7 @@ ImageAtomicCmpswapMimg::ImageAtomicCmpswapMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -333,6 +345,7 @@ ImageAtomicAddMimg::ImageAtomicAddMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -363,6 +376,7 @@ ImageAtomicSubMimg::ImageAtomicSubMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -393,6 +407,7 @@ ImageAtomicSminMimg::ImageAtomicSminMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -423,6 +438,7 @@ ImageAtomicUminMimg::ImageAtomicUminMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -453,6 +469,7 @@ ImageAtomicSmaxMimg::ImageAtomicSmaxMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -483,6 +500,7 @@ ImageAtomicUmaxMimg::ImageAtomicUmaxMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -513,6 +531,7 @@ ImageAtomicAndMimg::ImageAtomicAndMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -543,6 +562,7 @@ ImageAtomicOrMimg::ImageAtomicOrMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -573,6 +593,7 @@ ImageAtomicXorMimg::ImageAtomicXorMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -603,6 +624,7 @@ ImageAtomicIncMimg::ImageAtomicIncMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -633,6 +655,7 @@ ImageAtomicDecMimg::ImageAtomicDecMimg(const MachineInst *inst) src_operands_[2] = &gpumem_in; num_src_ = 3; num_dst_ = 2; + capture_nsa_words(inst, &vaddr); gpumem.apply_fieldless_caps(false, false, false); gpumem_in.apply_fieldless_caps(false, false, false); flags_ |= MEMORY_WAIT_PRODUCER; @@ -660,6 +683,7 @@ ImageGetResinfoMimg::ImageGetResinfoMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -685,6 +709,7 @@ ImageMsaaLoadMimg::ImageMsaaLoadMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -710,6 +735,7 @@ ImageBvhIntersectRayMimg::ImageBvhIntersectRayMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -735,6 +761,7 @@ ImageBvh64IntersectRayMimg::ImageBvh64IntersectRayMimg(const MachineInst *inst) src_operands_[1] = &srsrc; num_src_ = 2; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -762,6 +789,7 @@ ImageSampleMimg::ImageSampleMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -789,6 +817,7 @@ ImageSampleDMimg::ImageSampleDMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -816,6 +845,7 @@ ImageSampleLMimg::ImageSampleLMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -843,6 +873,7 @@ ImageSampleBMimg::ImageSampleBMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -870,6 +901,7 @@ ImageSampleLzMimg::ImageSampleLzMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -897,6 +929,7 @@ ImageSampleCMimg::ImageSampleCMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -924,6 +957,7 @@ ImageSampleCDMimg::ImageSampleCDMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -951,6 +985,7 @@ ImageSampleCLMimg::ImageSampleCLMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -978,6 +1013,7 @@ ImageSampleCBMimg::ImageSampleCBMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1005,6 +1041,7 @@ ImageSampleCLzMimg::ImageSampleCLzMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1032,6 +1069,7 @@ ImageSampleOMimg::ImageSampleOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1059,6 +1097,7 @@ ImageSampleDOMimg::ImageSampleDOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1086,6 +1125,7 @@ ImageSampleLOMimg::ImageSampleLOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1113,6 +1153,7 @@ ImageSampleBOMimg::ImageSampleBOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1140,6 +1181,7 @@ ImageSampleLzOMimg::ImageSampleLzOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1167,6 +1209,7 @@ ImageSampleCOMimg::ImageSampleCOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1194,6 +1237,7 @@ ImageSampleCDOMimg::ImageSampleCDOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1221,6 +1265,7 @@ ImageSampleCLOMimg::ImageSampleCLOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1248,6 +1293,7 @@ ImageSampleCBOMimg::ImageSampleCBOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1275,6 +1321,7 @@ ImageSampleCLzOMimg::ImageSampleCLzOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1302,6 +1349,7 @@ ImageGather4Mimg::ImageGather4Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1329,6 +1377,7 @@ ImageGather4LMimg::ImageGather4LMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1356,6 +1405,7 @@ ImageGather4BMimg::ImageGather4BMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1383,6 +1433,7 @@ ImageGather4LzMimg::ImageGather4LzMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1410,6 +1461,7 @@ ImageGather4CMimg::ImageGather4CMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1437,6 +1489,7 @@ ImageGather4CLzMimg::ImageGather4CLzMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1464,6 +1517,7 @@ ImageGather4OMimg::ImageGather4OMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1491,6 +1545,7 @@ ImageGather4LzOMimg::ImageGather4LzOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1518,6 +1573,7 @@ ImageGather4CLzOMimg::ImageGather4CLzOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1545,6 +1601,7 @@ ImageGetLodMimg::ImageGetLodMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1572,6 +1629,7 @@ ImageSampleDG16Mimg::ImageSampleDG16Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1599,6 +1657,7 @@ ImageSampleCDG16Mimg::ImageSampleCDG16Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1626,6 +1685,7 @@ ImageSampleDOG16Mimg::ImageSampleDOG16Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1653,6 +1713,7 @@ ImageSampleCDOG16Mimg::ImageSampleCDOG16Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1680,6 +1741,7 @@ ImageSampleClMimg::ImageSampleClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1707,6 +1769,7 @@ ImageSampleDClMimg::ImageSampleDClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1734,6 +1797,7 @@ ImageSampleBClMimg::ImageSampleBClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1761,6 +1825,7 @@ ImageSampleCClMimg::ImageSampleCClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1788,6 +1853,7 @@ ImageSampleCDClMimg::ImageSampleCDClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1815,6 +1881,7 @@ ImageSampleCBClMimg::ImageSampleCBClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1842,6 +1909,7 @@ ImageSampleClOMimg::ImageSampleClOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1869,6 +1937,7 @@ ImageSampleDClOMimg::ImageSampleDClOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1896,6 +1965,7 @@ ImageSampleBClOMimg::ImageSampleBClOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1923,6 +1993,7 @@ ImageSampleCClOMimg::ImageSampleCClOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1950,6 +2021,7 @@ ImageSampleCDClOMimg::ImageSampleCDClOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1977,6 +2049,7 @@ ImageSampleCBClOMimg::ImageSampleCBClOMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2004,6 +2077,7 @@ ImageSampleCDClG16Mimg::ImageSampleCDClG16Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2031,6 +2105,7 @@ ImageSampleDClOG16Mimg::ImageSampleDClOG16Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2058,6 +2133,7 @@ ImageSampleCDClOG16Mimg::ImageSampleCDClOG16Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2085,6 +2161,7 @@ ImageSampleDClG16Mimg::ImageSampleDClG16Mimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2112,6 +2189,7 @@ ImageGather4ClMimg::ImageGather4ClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2139,6 +2217,7 @@ ImageGather4BClMimg::ImageGather4BClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2166,6 +2245,7 @@ ImageGather4CClMimg::ImageGather4CClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2193,6 +2273,7 @@ ImageGather4CLMimg::ImageGather4CLMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2220,6 +2301,7 @@ ImageGather4CBMimg::ImageGather4CBMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2247,6 +2329,7 @@ ImageGather4CBClMimg::ImageGather4CBClMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -2274,6 +2357,7 @@ ImageGather4hMimg::ImageGather4hMimg(const MachineInst *inst) src_operands_[2] = &ssamp; num_src_ = 3; num_dst_ = 1; + capture_nsa_words(inst, &vaddr); flags_ |= MEMORY_WAIT_PRODUCER; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h index 4d3210a67417..9db2b67abbb6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h @@ -139,6 +139,7 @@ class Operand final : public IsaOperand { std::optional literal32_widening_; bool packed_16bit_source_ = false; bool packed_16bit_dst_ = false; + std::optional resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const; }; } // namespace rdna3 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand_exec.cpp index 629de0ca0317..1786ee326cff 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand_exec.cpp @@ -131,7 +131,7 @@ void Operand::read_lane_chunk_exec(const amdgpu::Wavefront &wf, uint32_t lane_ba out[i] = read_lane_exec(wf, lane_base + i); return; } - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint64_t lane_mask = count == 0 ? 0 : util::mask(static_cast(count)) << lane_base; @@ -155,7 +155,7 @@ void Operand::write_lane_chunk_exec(amdgpu::Wavefront &wf, uint32_t lane_base, u write_lane_exec(wf, lane_base + i, vals[i]); return; } - auto off = detail::resolved_vgpr_offset_for_operand(wf, *this); + auto off = resolved_vgpr_offset_exec(wf); if (!off) { for (uint32_t i = 0; i < count; ++i) if (mask & (1ULL << i)) @@ -334,13 +334,13 @@ void Operand::write_scalar64_exec(amdgpu::Wavefront &wf, uint64_t val) const { } std::optional Operand::simd_vgpr_base_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) + if (auto off = resolved_vgpr_offset_exec(wf)) return wf.vgpr_alloc().base + amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); return std::nullopt; } std::optional Operand::simd_vgpr_base_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); return wf.vgpr_alloc().base + amdgpu::apply_gpr_idx(wf, *off, role); @@ -349,7 +349,7 @@ std::optional Operand::simd_vgpr_base_mut_exec(amdgpu::Wavefront &wf) } amdgpu::ConstVgprStorage Operand::simd_vgpr_storage_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); const auto &cu = amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()); return {reinterpret_cast(cu.raw_vgpr_data(wf.vgpr_alloc().base + voff)), @@ -359,7 +359,7 @@ amdgpu::ConstVgprStorage Operand::simd_vgpr_storage_exec(const amdgpu::Wavefront } amdgpu::VgprStorage Operand::simd_vgpr_storage_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -372,7 +372,7 @@ amdgpu::VgprStorage Operand::simd_vgpr_storage_mut_exec(amdgpu::Wavefront &wf) c amdgpu::ConstVgprStoragePair64 Operand::simd_vgpr_storage64_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; const auto &cu = amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()); @@ -385,7 +385,7 @@ Operand::simd_vgpr_storage64_exec(const amdgpu::Wavefront &wf) const { } amdgpu::VgprStoragePair64 Operand::simd_vgpr_storage64_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -400,7 +400,7 @@ amdgpu::VgprStoragePair64 Operand::simd_vgpr_storage64_mut_exec(amdgpu::Wavefron void Operand::simd_notify_read_exec(const amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read( &wf, wf.vgpr_alloc().base + voff, lane_mask, byte_mask); @@ -409,7 +409,7 @@ void Operand::simd_notify_read_exec(const amdgpu::Wavefront &wf, uint64_t lane_m void Operand::simd_notify_read_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read( &wf, wf.vgpr_alloc().base + voff, lane_mask, byte_mask); @@ -418,7 +418,7 @@ void Operand::simd_notify_read_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mas void Operand::simd_notify_read64_exec(const amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read(&wf, reg, lane_mask, @@ -430,7 +430,7 @@ void Operand::simd_notify_read64_exec(const amdgpu::Wavefront &wf, uint64_t lane void Operand::simd_notify_read64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read(&wf, reg, lane_mask, @@ -442,7 +442,7 @@ void Operand::simd_notify_read64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_m void Operand::simd_notify_write_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -453,7 +453,7 @@ void Operand::simd_notify_write_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_ma void Operand::simd_notify_write64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -512,5 +512,17 @@ bool Operand::full_execution_backend_complete() { return is_complete(*static_cast(full_execution_backend())); } +std::optional Operand::resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const { + if (!reads_value()) + return std::nullopt; + auto packed = + packed_16bit_vgpr_source(packed_16bit_source_, size_bits_, opr_type_, encoding_value_); + if (!packed) + packed = packed_16bit_vgpr_dst(packed_16bit_dst_, size_bits_, opr_type_, encoding_value_); + if (packed) + return packed->reg + (wf.vgpr_msb_for_role(vgpr_msb_role()) << 8); + return detail::resolved_vgpr_offset_for_operand(wf, *this); +} + } // namespace rdna3 } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vinterp.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vinterp.cpp index 8877dea0b1cf..39180f22dd07 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vinterp.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vinterp.cpp @@ -15,9 +15,9 @@ VInterpP10F32Vinterp::VInterpP10F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p10_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP10F32Vinterp)), vdst(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -42,9 +42,9 @@ VInterpP2F32Vinterp::VInterpP2F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p2_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP2F32Vinterp)), vdst(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -69,9 +69,9 @@ VInterpP10F16F32Vinterp::VInterpP10F16F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p10_f16_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP10F16F32Vinterp)), vdst(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -96,9 +96,9 @@ VInterpP2F16F32Vinterp::VInterpP2F16F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p2_f16_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP2F16F32Vinterp)), vdst(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -129,9 +129,9 @@ VInterpP10RtzF16F32Vinterp::VInterpP10RtzF16F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p10_rtz_f16_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP10RtzF16F32Vinterp)), vdst(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -157,9 +157,9 @@ VInterpP2RtzF16F32Vinterp::VInterpP2RtzF16F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p2_rtz_f16_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP2RtzF16F32Vinterp)), vdst(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop1_exec.cpp index 24eae170406a..f7dd41a99ff3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop1_exec.cpp @@ -8,6 +8,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/rdna3/vop1.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" #include "rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h" +#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/isa/arch/amdgpu/shared/transcendental.h" #include "rocjitsu/vm/amdgpu/register_access.h" @@ -55,13 +56,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -148,6 +144,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -176,6 +173,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -214,6 +212,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -252,6 +251,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -280,6 +280,7 @@ RJ_NOINLINE void VCvtNearestI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -308,6 +309,7 @@ RJ_NOINLINE void VCvtFloorI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -556,6 +558,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -584,6 +587,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -612,6 +616,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -640,6 +645,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -668,6 +674,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -696,6 +703,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -724,6 +732,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -752,6 +761,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -780,6 +790,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -808,6 +819,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -840,6 +852,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -872,6 +885,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -900,6 +914,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1080,6 +1095,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1108,6 +1124,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1575,6 +1592,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1633,6 +1651,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1669,10 +1688,13 @@ void VRcpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1688,6 +1710,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1699,10 +1722,13 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1719,10 +1745,13 @@ void VSqrtF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1738,6 +1767,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1749,10 +1779,13 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1769,10 +1802,13 @@ void VRsqF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1788,6 +1824,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1799,10 +1836,13 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1819,10 +1859,13 @@ void VLogF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1838,6 +1881,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1849,10 +1893,13 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1869,10 +1916,13 @@ void VExpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1888,6 +1938,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1899,10 +1950,13 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1940,6 +1994,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1997,6 +2052,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2051,6 +2107,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2099,6 +2156,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2147,6 +2205,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2195,6 +2254,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2247,6 +2307,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2281,10 +2342,13 @@ void VSinF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2300,6 +2364,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2311,10 +2376,13 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2331,10 +2399,13 @@ void VCosF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2350,6 +2421,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2361,10 +2433,13 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2433,7 +2508,8 @@ void VCvtNormI16F16Vop1::execute_impl(amdgpu::Wavefront &wf) { static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -2450,6 +2526,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2465,7 +2542,8 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -2487,7 +2565,8 @@ void VCvtNormU16F16Vop1::execute_impl(amdgpu::Wavefront &wf) { static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } @@ -2504,6 +2583,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2519,7 +2599,8 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop2_exec.cpp index d86131c5eee4..124b6d5fc4c0 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop2_exec.cpp @@ -90,6 +90,7 @@ void VAddF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -99,7 +100,10 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -118,6 +122,7 @@ void VSubF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -127,7 +132,10 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -146,6 +154,7 @@ void VSubrevF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -155,7 +164,10 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -178,6 +190,7 @@ void VMulDx9ZeroF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulDx9ZeroF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -187,7 +200,10 @@ RJ_NOINLINE void VMulDx9ZeroF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -206,6 +222,7 @@ void VMulF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -215,7 +232,10 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -346,6 +366,7 @@ void VMinF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -355,7 +376,10 @@ RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -374,6 +398,7 @@ void VMaxF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -383,7 +408,10 @@ RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -914,6 +942,7 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -923,7 +952,10 @@ RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -950,6 +982,7 @@ void VCvtPkRtzF16F32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCvtPkRtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -959,7 +992,10 @@ RJ_NOINLINE void VCvtPkRtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -993,6 +1029,7 @@ void VAddF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1002,7 +1039,10 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1051,6 +1091,7 @@ void VSubF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1060,7 +1101,10 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1109,6 +1153,7 @@ void VSubrevF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1118,7 +1163,10 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1167,6 +1215,7 @@ void VMulF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1176,7 +1225,10 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1216,13 +1268,15 @@ void VFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1232,7 +1286,10 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1248,7 +1305,8 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } @@ -1266,7 +1324,8 @@ void VFmamkF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16(s0, k, s2, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } @@ -1282,7 +1341,8 @@ void VFmaakF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16(s0, s1, k, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } @@ -1310,6 +1370,7 @@ void VMaxF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1319,7 +1380,10 @@ RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1364,6 +1428,7 @@ void VMinF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1373,7 +1438,10 @@ RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1428,6 +1496,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp index 83a4fd813db1..1c03c2948bef 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp @@ -56,13 +56,8 @@ RJ_NOINLINE void VMovB32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -248,18 +243,32 @@ void VCvtF32F16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -284,20 +293,33 @@ RJ_NOINLINE void VCvtF32F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } - dpp_write_mask_scope_.restore(); } @@ -1620,8 +1642,15 @@ void VCvtU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -1658,8 +1687,15 @@ RJ_NOINLINE void VCvtU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -1684,8 +1720,15 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -1724,8 +1767,15 @@ RJ_NOINLINE void VCvtI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -1746,7 +1796,13 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1755,31 +1811,27 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1811,7 +1863,13 @@ RJ_NOINLINE void VRcpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1820,31 +1878,27 @@ RJ_NOINLINE void VRcpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1864,52 +1918,47 @@ void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1934,52 +1983,47 @@ RJ_NOINLINE void VSqrtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1992,7 +2036,13 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2001,31 +2051,27 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2057,7 +2103,13 @@ RJ_NOINLINE void VRsqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2066,31 +2118,27 @@ RJ_NOINLINE void VRsqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2110,7 +2158,13 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2119,31 +2173,27 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2175,7 +2225,13 @@ RJ_NOINLINE void VLogF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2184,31 +2240,27 @@ RJ_NOINLINE void VLogF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2228,7 +2280,13 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2237,31 +2295,27 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2293,7 +2347,13 @@ RJ_NOINLINE void VExpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2302,31 +2362,27 @@ RJ_NOINLINE void VExpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3211,31 +3267,27 @@ void VSinF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3274,31 +3326,27 @@ RJ_NOINLINE void VSinF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3325,31 +3373,27 @@ void VCosF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3388,31 +3432,27 @@ RJ_NOINLINE void VCosF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3486,11 +3526,19 @@ void VCvtNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3523,11 +3571,19 @@ RJ_NOINLINE void VCvtNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3548,11 +3604,19 @@ void VCvtNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3585,11 +3649,19 @@ RJ_NOINLINE void VCvtNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -5236,7 +5308,8 @@ void VFmacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -5278,7 +5351,8 @@ RJ_NOINLINE void VFmacF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -7028,7 +7102,8 @@ void VFmaF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7070,7 +7145,8 @@ RJ_NOINLINE void VFmaF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -9986,12 +10062,22 @@ void VCvtPkNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10001,11 +10087,19 @@ void VCvtPkNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.op_sel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.op_sel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -10034,12 +10128,22 @@ RJ_NOINLINE void VCvtPkNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10049,11 +10153,19 @@ RJ_NOINLINE void VCvtPkNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.op_sel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.op_sel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -10070,10 +10182,20 @@ void VCvtPkNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10083,11 +10205,19 @@ void VCvtPkNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.op_sel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.op_sel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -10116,10 +10246,20 @@ RJ_NOINLINE void VCvtPkNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10129,11 +10269,19 @@ RJ_NOINLINE void VCvtPkNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.op_sel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.op_sel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -10832,8 +10980,8 @@ void VCmpFF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -10874,8 +11022,8 @@ RJ_NOINLINE void VCmpFF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -10897,32 +11045,26 @@ void VCmpLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -10956,32 +11098,26 @@ RJ_NOINLINE void VCmpLtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -10996,32 +11132,26 @@ void VCmpEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11055,32 +11185,26 @@ RJ_NOINLINE void VCmpEqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11095,32 +11219,26 @@ void VCmpLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11154,32 +11272,26 @@ RJ_NOINLINE void VCmpLeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11194,32 +11306,26 @@ void VCmpGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11253,32 +11359,26 @@ RJ_NOINLINE void VCmpGtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11293,35 +11393,26 @@ void VCmpLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11355,35 +11446,26 @@ RJ_NOINLINE void VCmpLgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11398,32 +11480,26 @@ void VCmpGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11457,32 +11533,26 @@ RJ_NOINLINE void VCmpGeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11497,32 +11567,26 @@ void VCmpOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11556,32 +11620,26 @@ RJ_NOINLINE void VCmpOF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11596,32 +11654,26 @@ void VCmpUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11655,32 +11707,26 @@ RJ_NOINLINE void VCmpUF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11695,32 +11741,26 @@ void VCmpNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11754,32 +11794,26 @@ RJ_NOINLINE void VCmpNgeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11794,35 +11828,26 @@ void VCmpNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11856,35 +11881,26 @@ RJ_NOINLINE void VCmpNlgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11899,32 +11915,26 @@ void VCmpNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11958,32 +11968,26 @@ RJ_NOINLINE void VCmpNgtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11998,32 +12002,26 @@ void VCmpNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12057,32 +12055,26 @@ RJ_NOINLINE void VCmpNleF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12097,32 +12089,26 @@ void VCmpNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12156,32 +12142,26 @@ RJ_NOINLINE void VCmpNeqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12196,32 +12176,26 @@ void VCmpNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12255,32 +12229,26 @@ RJ_NOINLINE void VCmpNltF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12295,8 +12263,8 @@ void VCmpTF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -12337,8 +12305,8 @@ RJ_NOINLINE void VCmpTF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -14887,22 +14855,14 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -14938,22 +14898,14 @@ RJ_NOINLINE void VCmpxLtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -14968,22 +14920,14 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15019,22 +14963,14 @@ RJ_NOINLINE void VCmpxEqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15049,22 +14985,14 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15100,22 +15028,14 @@ RJ_NOINLINE void VCmpxLeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15130,22 +15050,14 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15181,22 +15093,14 @@ RJ_NOINLINE void VCmpxGtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15211,22 +15115,14 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15262,22 +15158,14 @@ RJ_NOINLINE void VCmpxLgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15292,22 +15180,14 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15343,22 +15223,14 @@ RJ_NOINLINE void VCmpxGeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15373,22 +15245,14 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15424,22 +15288,14 @@ RJ_NOINLINE void VCmpxOF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15454,22 +15310,14 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15505,22 +15353,14 @@ RJ_NOINLINE void VCmpxUF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15535,22 +15375,14 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15586,22 +15418,14 @@ RJ_NOINLINE void VCmpxNgeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15616,22 +15440,14 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15667,22 +15483,14 @@ RJ_NOINLINE void VCmpxNlgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15697,22 +15505,14 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15748,22 +15548,14 @@ RJ_NOINLINE void VCmpxNgtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15778,22 +15570,14 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15829,22 +15613,14 @@ RJ_NOINLINE void VCmpxNleF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15859,22 +15635,14 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15910,22 +15678,14 @@ RJ_NOINLINE void VCmpxNeqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15940,22 +15700,14 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15991,22 +15743,14 @@ RJ_NOINLINE void VCmpxNltF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16124,20 +15868,14 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16170,20 +15908,14 @@ RJ_NOINLINE void VCmpxLtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16197,20 +15929,14 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16243,20 +15969,14 @@ RJ_NOINLINE void VCmpxEqF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16270,20 +15990,14 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16316,20 +16030,14 @@ RJ_NOINLINE void VCmpxLeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16343,20 +16051,14 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16389,20 +16091,14 @@ RJ_NOINLINE void VCmpxGtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16416,20 +16112,14 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16462,20 +16152,14 @@ RJ_NOINLINE void VCmpxLgF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16489,20 +16173,14 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16535,20 +16213,14 @@ RJ_NOINLINE void VCmpxGeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16562,20 +16234,14 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16608,20 +16274,14 @@ RJ_NOINLINE void VCmpxOF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16635,20 +16295,14 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16681,20 +16335,14 @@ RJ_NOINLINE void VCmpxUF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16708,20 +16356,14 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16754,20 +16396,14 @@ RJ_NOINLINE void VCmpxNgeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16781,20 +16417,14 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16827,20 +16457,14 @@ RJ_NOINLINE void VCmpxNlgF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16854,20 +16478,14 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16900,20 +16518,14 @@ RJ_NOINLINE void VCmpxNgtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16927,20 +16539,14 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16973,20 +16579,14 @@ RJ_NOINLINE void VCmpxNleF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17000,20 +16600,14 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17046,20 +16640,14 @@ RJ_NOINLINE void VCmpxNeqF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17073,20 +16661,14 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17119,20 +16701,14 @@ RJ_NOINLINE void VCmpxNltF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17211,20 +16787,14 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17237,20 +16807,14 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17263,20 +16827,14 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17289,20 +16847,14 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17315,20 +16867,14 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17341,20 +16887,14 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17367,20 +16907,14 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17393,20 +16927,14 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17419,20 +16947,14 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17445,20 +16967,14 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17471,20 +16987,14 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17497,20 +17007,14 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17523,20 +17027,14 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17549,20 +17047,14 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vopc_exec.cpp index 4a989d46e0be..29b6e3989786 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vopc_exec.cpp @@ -33,6 +33,7 @@ void VCmpFF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpFF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -48,7 +49,10 @@ RJ_NOINLINE void VCmpFF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_f_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -70,6 +74,7 @@ void VCmpLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -85,7 +90,10 @@ RJ_NOINLINE void VCmpLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -107,6 +115,7 @@ void VCmpEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -122,7 +131,10 @@ RJ_NOINLINE void VCmpEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_eq_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -144,6 +156,7 @@ void VCmpLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -159,7 +172,10 @@ RJ_NOINLINE void VCmpLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_le_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -181,6 +197,7 @@ void VCmpGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -196,7 +213,10 @@ RJ_NOINLINE void VCmpGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_gt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -218,6 +238,7 @@ void VCmpLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -233,7 +254,10 @@ RJ_NOINLINE void VCmpLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lg_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -255,6 +279,7 @@ void VCmpGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -270,7 +295,10 @@ RJ_NOINLINE void VCmpGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ge_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -292,6 +320,7 @@ void VCmpOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -307,7 +336,10 @@ RJ_NOINLINE void VCmpOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_o_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -329,6 +361,7 @@ void VCmpUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -344,7 +377,10 @@ RJ_NOINLINE void VCmpUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_u_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -366,6 +402,7 @@ void VCmpNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -381,7 +418,10 @@ RJ_NOINLINE void VCmpNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nge_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -403,6 +443,7 @@ void VCmpNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -418,7 +459,10 @@ RJ_NOINLINE void VCmpNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlg_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -440,6 +484,7 @@ void VCmpNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -455,7 +500,10 @@ RJ_NOINLINE void VCmpNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ngt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -477,6 +525,7 @@ void VCmpNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -492,7 +541,10 @@ RJ_NOINLINE void VCmpNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nle_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -514,6 +566,7 @@ void VCmpNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -529,7 +582,10 @@ RJ_NOINLINE void VCmpNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_neq_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -551,6 +607,7 @@ void VCmpNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -566,7 +623,10 @@ RJ_NOINLINE void VCmpNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -588,6 +648,7 @@ void VCmpTF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpTF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -603,7 +664,10 @@ RJ_NOINLINE void VCmpTF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_t_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -625,6 +689,7 @@ void VCmpFF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpFF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -640,7 +705,10 @@ RJ_NOINLINE void VCmpFF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_f_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -662,6 +730,7 @@ void VCmpLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -677,7 +746,10 @@ RJ_NOINLINE void VCmpLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -699,6 +771,7 @@ void VCmpEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -714,7 +787,10 @@ RJ_NOINLINE void VCmpEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_eq_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -736,6 +812,7 @@ void VCmpLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -751,7 +828,10 @@ RJ_NOINLINE void VCmpLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_le_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -773,6 +853,7 @@ void VCmpGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -788,7 +869,10 @@ RJ_NOINLINE void VCmpGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_gt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -810,6 +894,7 @@ void VCmpLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -825,7 +910,10 @@ RJ_NOINLINE void VCmpLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lg_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -847,6 +935,7 @@ void VCmpGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -862,7 +951,10 @@ RJ_NOINLINE void VCmpGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ge_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -884,6 +976,7 @@ void VCmpOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -899,7 +992,10 @@ RJ_NOINLINE void VCmpOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_o_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -921,6 +1017,7 @@ void VCmpUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -936,7 +1033,10 @@ RJ_NOINLINE void VCmpUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_u_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -958,6 +1058,7 @@ void VCmpNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -973,7 +1074,10 @@ RJ_NOINLINE void VCmpNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nge_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -995,6 +1099,7 @@ void VCmpNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1010,7 +1115,10 @@ RJ_NOINLINE void VCmpNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlg_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1032,6 +1140,7 @@ void VCmpNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1047,7 +1156,10 @@ RJ_NOINLINE void VCmpNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ngt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1069,6 +1181,7 @@ void VCmpNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1084,7 +1197,10 @@ RJ_NOINLINE void VCmpNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nle_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1106,6 +1222,7 @@ void VCmpNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1121,7 +1238,10 @@ RJ_NOINLINE void VCmpNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_neq_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1143,6 +1263,7 @@ void VCmpNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1158,7 +1279,10 @@ RJ_NOINLINE void VCmpNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1180,6 +1304,7 @@ void VCmpTF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpTF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1195,7 +1320,10 @@ RJ_NOINLINE void VCmpTF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_t_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -2396,6 +2524,7 @@ RJ_NOINLINE void VCmpClassF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::execute_v_cmp_class_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { @@ -2433,6 +2562,7 @@ RJ_NOINLINE void VCmpClassF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::execute_v_cmp_class_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { @@ -2466,6 +2596,7 @@ void VCmpxFF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxFF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2481,7 +2612,10 @@ RJ_NOINLINE void VCmpxFF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2507,14 +2641,14 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2522,6 +2656,7 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2537,17 +2672,20 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2568,14 +2706,14 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2583,6 +2721,7 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2598,17 +2737,20 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2629,14 +2771,14 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2644,6 +2786,7 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2659,17 +2802,20 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2690,14 +2836,14 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2705,6 +2851,7 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2720,17 +2867,20 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2751,14 +2901,14 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2766,6 +2916,7 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2781,17 +2932,20 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2812,14 +2966,14 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2827,6 +2981,7 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2842,17 +2997,20 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2873,14 +3031,14 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2888,6 +3046,7 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2903,17 +3062,20 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2934,14 +3096,14 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2949,6 +3111,7 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2964,17 +3127,20 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2995,14 +3161,14 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3010,6 +3176,7 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3025,17 +3192,20 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3056,14 +3226,14 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3071,6 +3241,7 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3086,17 +3257,20 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3117,14 +3291,14 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3132,6 +3306,7 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3147,17 +3322,20 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3178,14 +3356,14 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3193,6 +3371,7 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3208,17 +3387,20 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3239,14 +3421,14 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3254,6 +3436,7 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3269,17 +3452,20 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3300,14 +3486,14 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3315,6 +3501,7 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3330,17 +3517,20 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3371,6 +3561,7 @@ void VCmpxTF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxTF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3386,7 +3577,10 @@ RJ_NOINLINE void VCmpxTF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -3422,6 +3616,7 @@ void VCmpxFF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxFF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3437,7 +3632,10 @@ RJ_NOINLINE void VCmpxFF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -3463,12 +3661,14 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3476,6 +3676,7 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3491,15 +3692,20 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3520,12 +3726,14 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3533,6 +3741,7 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3548,15 +3757,20 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3577,12 +3791,14 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3590,6 +3806,7 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3605,15 +3822,20 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3634,12 +3856,14 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3647,6 +3871,7 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3662,15 +3887,20 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3691,12 +3921,14 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3704,6 +3936,7 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3719,15 +3952,20 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3748,12 +3986,14 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3761,6 +4001,7 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3776,15 +4017,20 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3805,12 +4051,14 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3818,6 +4066,7 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3833,15 +4082,20 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3862,12 +4116,14 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3875,6 +4131,7 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3890,15 +4147,20 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3919,12 +4181,14 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3932,6 +4196,7 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3947,15 +4212,20 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3976,12 +4246,14 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3989,6 +4261,7 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4004,15 +4277,20 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4033,12 +4311,14 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4046,6 +4326,7 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4061,15 +4342,20 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4090,12 +4376,14 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4103,6 +4391,7 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4118,15 +4407,20 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4147,12 +4441,14 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4160,6 +4456,7 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4175,15 +4472,20 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4204,12 +4506,14 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4217,6 +4521,7 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4232,15 +4537,20 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4271,6 +4581,7 @@ void VCmpxTF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxTF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4286,7 +4597,10 @@ RJ_NOINLINE void VCmpxTF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -4319,12 +4633,14 @@ void VCmpxFF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4333,12 +4649,14 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4347,12 +4665,14 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4361,12 +4681,14 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4375,12 +4697,14 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4389,12 +4713,14 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4403,12 +4729,14 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4417,12 +4745,14 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4431,12 +4761,14 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4445,12 +4777,14 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4459,12 +4793,14 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4473,12 +4809,14 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4487,12 +4825,14 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4501,12 +4841,14 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6371,6 +6713,7 @@ RJ_NOINLINE void VCmpxClassF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; @@ -6480,6 +6823,7 @@ RJ_NOINLINE void VCmpxClassF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vopd_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vopd_exec.cpp index c944a7970ff7..78a7709a1c56 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vopd_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vopd_exec.cpp @@ -87,25 +87,25 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(*slot.dst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdFmaakF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(src2), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdFmamkF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src2), std::bit_cast(src1), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdMulF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32()); + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdMulDx9ZeroF32: { @@ -118,7 +118,7 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la case kVopdAddF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32()); + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdSubF32: { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/decoder.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/decoder.cpp index b5a460e77aac..87e973e7d990 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/decoder.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/decoder.cpp @@ -5524,7 +5524,7 @@ const std::array DecoderImpl::sub_decode_mimg = { &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, - &DecoderImpl::decodeInvalid, + &detail::decodeImageGather4hMimg, &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, &DecoderImpl::decodeInvalid, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/encodings.cpp index dde50b632f32..aa928a8c7689 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/encodings.cpp @@ -452,6 +452,9 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -712,6 +715,9 @@ Vopc::Vopc(std::string_view mnemonic, const VopcMachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } bool Vopc::default_encoding() { @@ -815,6 +821,9 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop2::implicit_uses(RegisterSet &uses) const { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/encodings.h index 762a2ba5df81..b12c66ab881e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/encodings.h @@ -457,6 +457,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -499,6 +500,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -543,6 +545,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir.cpp index 14bcb6dad251..24b49822c562 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir.cpp @@ -51,6 +51,8 @@ LdsDirectLoadLdsdir::LdsDirectLoadLdsdir(const MachineInst *inst) num_dst_ = 1; dsmem.apply_fieldless_caps(false, false, false); m0.apply_fieldless_caps(false, false, false); + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::EXPCNT, + amdgpu::MemoryCompletionClass::LDS}}); flags_ |= MEMORY_WAIT_PRODUCER; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir_exec.cpp index d8f8794d2996..4765c91710dc 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir_exec.cpp @@ -6,6 +6,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/rdna3_5/ldsdir.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" +#include "rocjitsu/isa/arch/amdgpu/shared/lds_direct.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/vm/amdgpu/wavefront.h" #include "util/data_types.h" @@ -23,7 +24,7 @@ void LdsParamLoadLdsdir::execute_impl(amdgpu::Wavefront &wf) { } void LdsDirectLoadLdsdir::execute_impl(amdgpu::Wavefront &wf) { - amdgpu::execute_lds_direct_load_ldsdir(*this, wf); + amdgpu::execute_lds_direct_load(*this, wf, inst_.vdst); } } // namespace rdna3_5 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand.h index eb9354dee971..a3d50e1449ec 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand.h @@ -139,6 +139,7 @@ class Operand final : public IsaOperand { std::optional literal32_widening_; bool packed_16bit_source_ = false; bool packed_16bit_dst_ = false; + std::optional resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const; }; } // namespace rdna3_5 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand_exec.cpp index eb33b8e8b2b6..a02a4c89bc8f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand_exec.cpp @@ -131,7 +131,7 @@ void Operand::read_lane_chunk_exec(const amdgpu::Wavefront &wf, uint32_t lane_ba out[i] = read_lane_exec(wf, lane_base + i); return; } - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint64_t lane_mask = count == 0 ? 0 : util::mask(static_cast(count)) << lane_base; @@ -155,7 +155,7 @@ void Operand::write_lane_chunk_exec(amdgpu::Wavefront &wf, uint32_t lane_base, u write_lane_exec(wf, lane_base + i, vals[i]); return; } - auto off = detail::resolved_vgpr_offset_for_operand(wf, *this); + auto off = resolved_vgpr_offset_exec(wf); if (!off) { for (uint32_t i = 0; i < count; ++i) if (mask & (1ULL << i)) @@ -334,13 +334,13 @@ void Operand::write_scalar64_exec(amdgpu::Wavefront &wf, uint64_t val) const { } std::optional Operand::simd_vgpr_base_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) + if (auto off = resolved_vgpr_offset_exec(wf)) return wf.vgpr_alloc().base + amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); return std::nullopt; } std::optional Operand::simd_vgpr_base_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); return wf.vgpr_alloc().base + amdgpu::apply_gpr_idx(wf, *off, role); @@ -349,7 +349,7 @@ std::optional Operand::simd_vgpr_base_mut_exec(amdgpu::Wavefront &wf) } amdgpu::ConstVgprStorage Operand::simd_vgpr_storage_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); const auto &cu = amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()); return {reinterpret_cast(cu.raw_vgpr_data(wf.vgpr_alloc().base + voff)), @@ -359,7 +359,7 @@ amdgpu::ConstVgprStorage Operand::simd_vgpr_storage_exec(const amdgpu::Wavefront } amdgpu::VgprStorage Operand::simd_vgpr_storage_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -372,7 +372,7 @@ amdgpu::VgprStorage Operand::simd_vgpr_storage_mut_exec(amdgpu::Wavefront &wf) c amdgpu::ConstVgprStoragePair64 Operand::simd_vgpr_storage64_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; const auto &cu = amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()); @@ -385,7 +385,7 @@ Operand::simd_vgpr_storage64_exec(const amdgpu::Wavefront &wf) const { } amdgpu::VgprStoragePair64 Operand::simd_vgpr_storage64_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -400,7 +400,7 @@ amdgpu::VgprStoragePair64 Operand::simd_vgpr_storage64_mut_exec(amdgpu::Wavefron void Operand::simd_notify_read_exec(const amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read( &wf, wf.vgpr_alloc().base + voff, lane_mask, byte_mask); @@ -409,7 +409,7 @@ void Operand::simd_notify_read_exec(const amdgpu::Wavefront &wf, uint64_t lane_m void Operand::simd_notify_read_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read( &wf, wf.vgpr_alloc().base + voff, lane_mask, byte_mask); @@ -418,7 +418,7 @@ void Operand::simd_notify_read_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mas void Operand::simd_notify_read64_exec(const amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read(&wf, reg, lane_mask, @@ -430,7 +430,7 @@ void Operand::simd_notify_read64_exec(const amdgpu::Wavefront &wf, uint64_t lane void Operand::simd_notify_read64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read(&wf, reg, lane_mask, @@ -442,7 +442,7 @@ void Operand::simd_notify_read64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_m void Operand::simd_notify_write_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -453,7 +453,7 @@ void Operand::simd_notify_write_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_ma void Operand::simd_notify_write64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -512,5 +512,17 @@ bool Operand::full_execution_backend_complete() { return is_complete(*static_cast(full_execution_backend())); } +std::optional Operand::resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const { + if (!reads_value()) + return std::nullopt; + auto packed = + packed_16bit_vgpr_source(packed_16bit_source_, size_bits_, opr_type_, encoding_value_); + if (!packed) + packed = packed_16bit_vgpr_dst(packed_16bit_dst_, size_bits_, opr_type_, encoding_value_); + if (packed) + return packed->reg + (wf.vgpr_msb_for_role(vgpr_msb_role()) << 8); + return detail::resolved_vgpr_offset_for_operand(wf, *this); +} + } // namespace rdna3_5 } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vinterp.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vinterp.cpp index 3a4460c40c88..013e2a3e4e8e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vinterp.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vinterp.cpp @@ -15,9 +15,9 @@ VInterpP10F32Vinterp::VInterpP10F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p10_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP10F32Vinterp)), vdst(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -42,9 +42,9 @@ VInterpP2F32Vinterp::VInterpP2F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p2_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP2F32Vinterp)), vdst(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -69,9 +69,9 @@ VInterpP10F16F32Vinterp::VInterpP10F16F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p10_f16_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP10F16F32Vinterp)), vdst(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -96,9 +96,9 @@ VInterpP2F16F32Vinterp::VInterpP2F16F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p2_f16_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP2F16F32Vinterp)), vdst(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -129,9 +129,9 @@ VInterpP10RtzF16F32Vinterp::VInterpP10RtzF16F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p10_rtz_f16_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP10RtzF16F32Vinterp)), vdst(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; @@ -157,9 +157,9 @@ VInterpP2RtzF16F32Vinterp::VInterpP2RtzF16F32Vinterp(const MachineInst *inst) : Vinterp("v_interp_p2_rtz_f16_f32", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::VInterpP2RtzF16F32Vinterp)), vdst(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), - src0(16, OperandType::OPR_VGPR, reinterpret_cast(inst)->src0), - src1(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src1), - src2(32, OperandType::OPR_VGPR, reinterpret_cast(inst)->src2) { + src0(16, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src0), + src1(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src1), + src2(32, OperandType::OPR_SRC_VGPR, reinterpret_cast(inst)->src2) { dst_operands_[0] = &vdst; src_operands_[0] = &src0; src_operands_[1] = &src1; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop1_exec.cpp index 0c5ba56aebcb..ee2e13703c81 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop1_exec.cpp @@ -8,6 +8,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop1.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" #include "rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h" +#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/isa/arch/amdgpu/shared/transcendental.h" #include "rocjitsu/vm/amdgpu/register_access.h" @@ -55,13 +56,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -148,6 +144,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -176,6 +173,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -214,6 +212,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -252,6 +251,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -280,6 +280,7 @@ RJ_NOINLINE void VCvtNearestI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -308,6 +309,7 @@ RJ_NOINLINE void VCvtFloorI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -556,6 +558,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -584,6 +587,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -612,6 +616,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -640,6 +645,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -668,6 +674,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -696,6 +703,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -724,6 +732,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -752,6 +761,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -780,6 +790,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -808,6 +819,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -840,6 +852,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -872,6 +885,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -900,6 +914,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1080,6 +1095,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1108,6 +1124,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1575,6 +1592,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1633,6 +1651,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1669,10 +1688,13 @@ void VRcpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1688,6 +1710,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1699,10 +1722,13 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1719,10 +1745,13 @@ void VSqrtF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1738,6 +1767,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1749,10 +1779,13 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1769,10 +1802,13 @@ void VRsqF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1788,6 +1824,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1799,10 +1836,13 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1819,10 +1859,13 @@ void VLogF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1838,6 +1881,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1849,10 +1893,13 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1869,10 +1916,13 @@ void VExpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1888,6 +1938,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1899,10 +1950,13 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1940,6 +1994,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1997,6 +2052,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2051,6 +2107,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2099,6 +2156,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2147,6 +2205,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2195,6 +2254,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2247,6 +2307,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2281,10 +2342,13 @@ void VSinF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2300,6 +2364,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2311,10 +2376,13 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2331,10 +2399,13 @@ void VCosF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2350,6 +2421,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2361,10 +2433,13 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2433,7 +2508,8 @@ void VCvtNormI16F16Vop1::execute_impl(amdgpu::Wavefront &wf) { static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -2450,6 +2526,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2465,7 +2542,8 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -2487,7 +2565,8 @@ void VCvtNormU16F16Vop1::execute_impl(amdgpu::Wavefront &wf) { static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } @@ -2504,6 +2583,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2519,7 +2599,8 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop2_exec.cpp index c03cb59cef62..c0a9da94c95a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop2_exec.cpp @@ -90,6 +90,7 @@ void VAddF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -99,7 +100,10 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -118,6 +122,7 @@ void VSubF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -127,7 +132,10 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -146,6 +154,7 @@ void VSubrevF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -155,7 +164,10 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -178,6 +190,7 @@ void VMulDx9ZeroF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulDx9ZeroF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -187,7 +200,10 @@ RJ_NOINLINE void VMulDx9ZeroF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -206,6 +222,7 @@ void VMulF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -215,7 +232,10 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -346,6 +366,7 @@ void VMinF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -355,7 +376,10 @@ RJ_NOINLINE void VMinF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -374,6 +398,7 @@ void VMaxF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -383,7 +408,10 @@ RJ_NOINLINE void VMaxF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -914,6 +942,7 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -923,7 +952,10 @@ RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -950,6 +982,7 @@ void VCvtPkRtzF16F32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCvtPkRtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -959,7 +992,10 @@ RJ_NOINLINE void VCvtPkRtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -993,6 +1029,7 @@ void VAddF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1002,7 +1039,10 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1051,6 +1091,7 @@ void VSubF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1060,7 +1101,10 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1109,6 +1153,7 @@ void VSubrevF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1118,7 +1163,10 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1167,6 +1215,7 @@ void VMulF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1176,7 +1225,10 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1216,13 +1268,15 @@ void VFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1232,7 +1286,10 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1248,7 +1305,8 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } @@ -1266,7 +1324,8 @@ void VFmamkF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16(s0, k, s2, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } @@ -1282,7 +1341,8 @@ void VFmaakF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16(s0, s1, k, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } @@ -1310,6 +1370,7 @@ void VMaxF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1319,7 +1380,10 @@ RJ_NOINLINE void VMaxF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1364,6 +1428,7 @@ void VMinF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1373,7 +1438,10 @@ RJ_NOINLINE void VMinF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1428,6 +1496,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp index 79d8566ab47d..601e62e8b643 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp @@ -56,13 +56,8 @@ RJ_NOINLINE void VMovB32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -248,18 +243,32 @@ void VCvtF32F16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -284,20 +293,33 @@ RJ_NOINLINE void VCvtF32F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } - dpp_write_mask_scope_.restore(); } @@ -1620,8 +1642,15 @@ void VCvtU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -1658,8 +1687,15 @@ RJ_NOINLINE void VCvtU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -1684,8 +1720,15 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -1724,8 +1767,15 @@ RJ_NOINLINE void VCvtI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -1746,7 +1796,13 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1755,31 +1811,27 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1811,7 +1863,13 @@ RJ_NOINLINE void VRcpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1820,31 +1878,27 @@ RJ_NOINLINE void VRcpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1864,52 +1918,47 @@ void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1934,52 +1983,47 @@ RJ_NOINLINE void VSqrtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1992,7 +2036,13 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2001,31 +2051,27 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2057,7 +2103,13 @@ RJ_NOINLINE void VRsqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2066,31 +2118,27 @@ RJ_NOINLINE void VRsqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2110,7 +2158,13 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2119,31 +2173,27 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2175,7 +2225,13 @@ RJ_NOINLINE void VLogF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2184,31 +2240,27 @@ RJ_NOINLINE void VLogF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2228,7 +2280,13 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2237,31 +2295,27 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2293,7 +2347,13 @@ RJ_NOINLINE void VExpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2302,31 +2362,27 @@ RJ_NOINLINE void VExpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3211,31 +3267,27 @@ void VSinF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3274,31 +3326,27 @@ RJ_NOINLINE void VSinF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3325,31 +3373,27 @@ void VCosF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3388,31 +3432,27 @@ RJ_NOINLINE void VCosF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3486,11 +3526,19 @@ void VCvtNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3523,11 +3571,19 @@ RJ_NOINLINE void VCvtNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3548,11 +3604,19 @@ void VCvtNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3585,11 +3649,19 @@ RJ_NOINLINE void VCvtNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -5236,7 +5308,8 @@ void VFmacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -5278,7 +5351,8 @@ RJ_NOINLINE void VFmacF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -7028,7 +7102,8 @@ void VFmaF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7070,7 +7145,8 @@ RJ_NOINLINE void VFmaF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -9986,12 +10062,22 @@ void VCvtPkNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10001,11 +10087,19 @@ void VCvtPkNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.op_sel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.op_sel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -10034,12 +10128,22 @@ RJ_NOINLINE void VCvtPkNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10049,11 +10153,19 @@ RJ_NOINLINE void VCvtPkNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.op_sel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.op_sel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -10070,10 +10182,20 @@ void VCvtPkNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10083,11 +10205,19 @@ void VCvtPkNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.op_sel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.op_sel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -10116,10 +10246,20 @@ RJ_NOINLINE void VCvtPkNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10129,11 +10269,19 @@ RJ_NOINLINE void VCvtPkNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.op_sel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.op_sel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -10832,8 +10980,8 @@ void VCmpFF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -10874,8 +11022,8 @@ RJ_NOINLINE void VCmpFF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -10897,32 +11045,26 @@ void VCmpLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -10956,32 +11098,26 @@ RJ_NOINLINE void VCmpLtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -10996,32 +11132,26 @@ void VCmpEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11055,32 +11185,26 @@ RJ_NOINLINE void VCmpEqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11095,32 +11219,26 @@ void VCmpLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11154,32 +11272,26 @@ RJ_NOINLINE void VCmpLeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11194,32 +11306,26 @@ void VCmpGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11253,32 +11359,26 @@ RJ_NOINLINE void VCmpGtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11293,35 +11393,26 @@ void VCmpLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11355,35 +11446,26 @@ RJ_NOINLINE void VCmpLgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11398,32 +11480,26 @@ void VCmpGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11457,32 +11533,26 @@ RJ_NOINLINE void VCmpGeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11497,32 +11567,26 @@ void VCmpOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11556,32 +11620,26 @@ RJ_NOINLINE void VCmpOF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11596,32 +11654,26 @@ void VCmpUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11655,32 +11707,26 @@ RJ_NOINLINE void VCmpUF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11695,32 +11741,26 @@ void VCmpNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11754,32 +11794,26 @@ RJ_NOINLINE void VCmpNgeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11794,35 +11828,26 @@ void VCmpNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11856,35 +11881,26 @@ RJ_NOINLINE void VCmpNlgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11899,32 +11915,26 @@ void VCmpNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11958,32 +11968,26 @@ RJ_NOINLINE void VCmpNgtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -11998,32 +12002,26 @@ void VCmpNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12057,32 +12055,26 @@ RJ_NOINLINE void VCmpNleF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12097,32 +12089,26 @@ void VCmpNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12156,32 +12142,26 @@ RJ_NOINLINE void VCmpNeqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12196,32 +12176,26 @@ void VCmpNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12255,32 +12229,26 @@ RJ_NOINLINE void VCmpNltF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12295,8 +12263,8 @@ void VCmpTF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -12337,8 +12305,8 @@ RJ_NOINLINE void VCmpTF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); uint64_t vcc = 0; @@ -14887,22 +14855,14 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -14938,22 +14898,14 @@ RJ_NOINLINE void VCmpxLtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -14968,22 +14920,14 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15019,22 +14963,14 @@ RJ_NOINLINE void VCmpxEqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15049,22 +14985,14 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15100,22 +15028,14 @@ RJ_NOINLINE void VCmpxLeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15130,22 +15050,14 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15181,22 +15093,14 @@ RJ_NOINLINE void VCmpxGtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15211,22 +15115,14 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15262,22 +15158,14 @@ RJ_NOINLINE void VCmpxLgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15292,22 +15180,14 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15343,22 +15223,14 @@ RJ_NOINLINE void VCmpxGeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15373,22 +15245,14 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15424,22 +15288,14 @@ RJ_NOINLINE void VCmpxOF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15454,22 +15310,14 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15505,22 +15353,14 @@ RJ_NOINLINE void VCmpxUF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15535,22 +15375,14 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15586,22 +15418,14 @@ RJ_NOINLINE void VCmpxNgeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15616,22 +15440,14 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15667,22 +15483,14 @@ RJ_NOINLINE void VCmpxNlgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15697,22 +15505,14 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15748,22 +15548,14 @@ RJ_NOINLINE void VCmpxNgtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15778,22 +15570,14 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15829,22 +15613,14 @@ RJ_NOINLINE void VCmpxNleF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15859,22 +15635,14 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15910,22 +15678,14 @@ RJ_NOINLINE void VCmpxNeqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15940,22 +15700,14 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -15991,22 +15743,14 @@ RJ_NOINLINE void VCmpxNltF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16124,20 +15868,14 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16170,20 +15908,14 @@ RJ_NOINLINE void VCmpxLtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16197,20 +15929,14 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16243,20 +15969,14 @@ RJ_NOINLINE void VCmpxEqF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16270,20 +15990,14 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16316,20 +16030,14 @@ RJ_NOINLINE void VCmpxLeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16343,20 +16051,14 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16389,20 +16091,14 @@ RJ_NOINLINE void VCmpxGtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16416,20 +16112,14 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16462,20 +16152,14 @@ RJ_NOINLINE void VCmpxLgF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16489,20 +16173,14 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16535,20 +16213,14 @@ RJ_NOINLINE void VCmpxGeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16562,20 +16234,14 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16608,20 +16274,14 @@ RJ_NOINLINE void VCmpxOF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16635,20 +16295,14 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16681,20 +16335,14 @@ RJ_NOINLINE void VCmpxUF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16708,20 +16356,14 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16754,20 +16396,14 @@ RJ_NOINLINE void VCmpxNgeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16781,20 +16417,14 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16827,20 +16457,14 @@ RJ_NOINLINE void VCmpxNlgF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16854,20 +16478,14 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16900,20 +16518,14 @@ RJ_NOINLINE void VCmpxNgtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16927,20 +16539,14 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16973,20 +16579,14 @@ RJ_NOINLINE void VCmpxNleF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17000,20 +16600,14 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17046,20 +16640,14 @@ RJ_NOINLINE void VCmpxNeqF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17073,20 +16661,14 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17119,20 +16701,14 @@ RJ_NOINLINE void VCmpxNltF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17211,20 +16787,14 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17237,20 +16807,14 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17263,20 +16827,14 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17289,20 +16847,14 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17315,20 +16867,14 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17341,20 +16887,14 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17367,20 +16907,14 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17393,20 +16927,14 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17419,20 +16947,14 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17445,20 +16967,14 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17471,20 +16987,14 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17497,20 +17007,14 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17523,20 +17027,14 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17549,20 +17047,14 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { }; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vopc_exec.cpp index fbfd78c577c8..fb8cf869563a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vopc_exec.cpp @@ -33,6 +33,7 @@ void VCmpFF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpFF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -48,7 +49,10 @@ RJ_NOINLINE void VCmpFF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_f_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -70,6 +74,7 @@ void VCmpLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -85,7 +90,10 @@ RJ_NOINLINE void VCmpLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -107,6 +115,7 @@ void VCmpEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -122,7 +131,10 @@ RJ_NOINLINE void VCmpEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_eq_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -144,6 +156,7 @@ void VCmpLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -159,7 +172,10 @@ RJ_NOINLINE void VCmpLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_le_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -181,6 +197,7 @@ void VCmpGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -196,7 +213,10 @@ RJ_NOINLINE void VCmpGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_gt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -218,6 +238,7 @@ void VCmpLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -233,7 +254,10 @@ RJ_NOINLINE void VCmpLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lg_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -255,6 +279,7 @@ void VCmpGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -270,7 +295,10 @@ RJ_NOINLINE void VCmpGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ge_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -292,6 +320,7 @@ void VCmpOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -307,7 +336,10 @@ RJ_NOINLINE void VCmpOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_o_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -329,6 +361,7 @@ void VCmpUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -344,7 +377,10 @@ RJ_NOINLINE void VCmpUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_u_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -366,6 +402,7 @@ void VCmpNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -381,7 +418,10 @@ RJ_NOINLINE void VCmpNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nge_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -403,6 +443,7 @@ void VCmpNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -418,7 +459,10 @@ RJ_NOINLINE void VCmpNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlg_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -440,6 +484,7 @@ void VCmpNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -455,7 +500,10 @@ RJ_NOINLINE void VCmpNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ngt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -477,6 +525,7 @@ void VCmpNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -492,7 +541,10 @@ RJ_NOINLINE void VCmpNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nle_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -514,6 +566,7 @@ void VCmpNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -529,7 +582,10 @@ RJ_NOINLINE void VCmpNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_neq_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -551,6 +607,7 @@ void VCmpNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -566,7 +623,10 @@ RJ_NOINLINE void VCmpNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -588,6 +648,7 @@ void VCmpTF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpTF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -603,7 +664,10 @@ RJ_NOINLINE void VCmpTF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_t_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -625,6 +689,7 @@ void VCmpFF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpFF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -640,7 +705,10 @@ RJ_NOINLINE void VCmpFF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_f_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -662,6 +730,7 @@ void VCmpLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -677,7 +746,10 @@ RJ_NOINLINE void VCmpLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -699,6 +771,7 @@ void VCmpEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -714,7 +787,10 @@ RJ_NOINLINE void VCmpEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_eq_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -736,6 +812,7 @@ void VCmpLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -751,7 +828,10 @@ RJ_NOINLINE void VCmpLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_le_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -773,6 +853,7 @@ void VCmpGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -788,7 +869,10 @@ RJ_NOINLINE void VCmpGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_gt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -810,6 +894,7 @@ void VCmpLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -825,7 +910,10 @@ RJ_NOINLINE void VCmpLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lg_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -847,6 +935,7 @@ void VCmpGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -862,7 +951,10 @@ RJ_NOINLINE void VCmpGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ge_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -884,6 +976,7 @@ void VCmpOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -899,7 +992,10 @@ RJ_NOINLINE void VCmpOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_o_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -921,6 +1017,7 @@ void VCmpUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -936,7 +1033,10 @@ RJ_NOINLINE void VCmpUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_u_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -958,6 +1058,7 @@ void VCmpNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -973,7 +1074,10 @@ RJ_NOINLINE void VCmpNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nge_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -995,6 +1099,7 @@ void VCmpNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1010,7 +1115,10 @@ RJ_NOINLINE void VCmpNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlg_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1032,6 +1140,7 @@ void VCmpNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1047,7 +1156,10 @@ RJ_NOINLINE void VCmpNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ngt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1069,6 +1181,7 @@ void VCmpNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1084,7 +1197,10 @@ RJ_NOINLINE void VCmpNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nle_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1106,6 +1222,7 @@ void VCmpNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1121,7 +1238,10 @@ RJ_NOINLINE void VCmpNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_neq_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1143,6 +1263,7 @@ void VCmpNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1158,7 +1279,10 @@ RJ_NOINLINE void VCmpNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1180,6 +1304,7 @@ void VCmpTF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpTF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1195,7 +1320,10 @@ RJ_NOINLINE void VCmpTF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_t_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -2396,6 +2524,7 @@ RJ_NOINLINE void VCmpClassF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::execute_v_cmp_class_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { @@ -2433,6 +2562,7 @@ RJ_NOINLINE void VCmpClassF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::execute_v_cmp_class_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { @@ -2466,6 +2596,7 @@ void VCmpxFF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxFF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2481,7 +2612,10 @@ RJ_NOINLINE void VCmpxFF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2507,14 +2641,14 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2522,6 +2656,7 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2537,17 +2672,20 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2568,14 +2706,14 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2583,6 +2721,7 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2598,17 +2737,20 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2629,14 +2771,14 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2644,6 +2786,7 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2659,17 +2802,20 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2690,14 +2836,14 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2705,6 +2851,7 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2720,17 +2867,20 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2751,14 +2901,14 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2766,6 +2916,7 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2781,17 +2932,20 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2812,14 +2966,14 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2827,6 +2981,7 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2842,17 +2997,20 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2873,14 +3031,14 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2888,6 +3046,7 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2903,17 +3062,20 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2934,14 +3096,14 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2949,6 +3111,7 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2964,17 +3127,20 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2995,14 +3161,14 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3010,6 +3176,7 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3025,17 +3192,20 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3056,14 +3226,14 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3071,6 +3241,7 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3086,17 +3257,20 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3117,14 +3291,14 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3132,6 +3306,7 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3147,17 +3322,20 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3178,14 +3356,14 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3193,6 +3371,7 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3208,17 +3387,20 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3239,14 +3421,14 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3254,6 +3436,7 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3269,17 +3452,20 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3300,14 +3486,14 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3315,6 +3501,7 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3330,17 +3517,20 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3371,6 +3561,7 @@ void VCmpxTF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxTF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3386,7 +3577,10 @@ RJ_NOINLINE void VCmpxTF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -3422,6 +3616,7 @@ void VCmpxFF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxFF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3437,7 +3632,10 @@ RJ_NOINLINE void VCmpxFF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -3463,12 +3661,14 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3476,6 +3676,7 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3491,15 +3692,20 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3520,12 +3726,14 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3533,6 +3741,7 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3548,15 +3757,20 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3577,12 +3791,14 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3590,6 +3806,7 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3605,15 +3822,20 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3634,12 +3856,14 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3647,6 +3871,7 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3662,15 +3887,20 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3691,12 +3921,14 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3704,6 +3936,7 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3719,15 +3952,20 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3748,12 +3986,14 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3761,6 +4001,7 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3776,15 +4017,20 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3805,12 +4051,14 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3818,6 +4066,7 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3833,15 +4082,20 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3862,12 +4116,14 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3875,6 +4131,7 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3890,15 +4147,20 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3919,12 +4181,14 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3932,6 +4196,7 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3947,15 +4212,20 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3976,12 +4246,14 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3989,6 +4261,7 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4004,15 +4277,20 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4033,12 +4311,14 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4046,6 +4326,7 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4061,15 +4342,20 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4090,12 +4376,14 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4103,6 +4391,7 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4118,15 +4407,20 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4147,12 +4441,14 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4160,6 +4456,7 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4175,15 +4472,20 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4204,12 +4506,14 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { } uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4217,6 +4521,7 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4232,15 +4537,20 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4271,6 +4581,7 @@ void VCmpxTF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxTF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4286,7 +4597,10 @@ RJ_NOINLINE void VCmpxTF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -4319,12 +4633,14 @@ void VCmpxFF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4333,12 +4649,14 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4347,12 +4665,14 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4361,12 +4681,14 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4375,12 +4697,14 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4389,12 +4713,14 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4403,12 +4729,14 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4417,12 +4745,14 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4431,12 +4761,14 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4445,12 +4777,14 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4459,12 +4793,14 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4473,12 +4809,14 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4487,12 +4825,14 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4501,12 +4841,14 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6371,6 +6713,7 @@ RJ_NOINLINE void VCmpxClassF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; @@ -6480,6 +6823,7 @@ RJ_NOINLINE void VCmpxClassF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); uint64_t exec = wf.exec(); uint64_t result = 0; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vopd_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vopd_exec.cpp index 4fd1071bd7da..773b07651522 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vopd_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vopd_exec.cpp @@ -87,25 +87,25 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(*slot.dst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdFmaakF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(src2), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdFmamkF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src2), std::bit_cast(src1), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdMulF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32()); + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdMulDx9ZeroF32: { @@ -118,7 +118,7 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la case kVopdAddF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32()); + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdSubF32: { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/encodings.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/encodings.cpp index c3e1840fc3d2..504f4f470c28 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/encodings.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/encodings.cpp @@ -455,6 +455,9 @@ Vop1::Vop1(std::string_view mnemonic, const Vop1MachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop1::implicit_uses(RegisterSet &uses) const { @@ -689,6 +692,9 @@ Vopc::Vopc(std::string_view mnemonic, const VopcMachineInst *inst, ExecuteFn exe size_ += sizeof(MachineInst); std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } bool Vopc::default_encoding() { @@ -794,6 +800,9 @@ Vop2::Vop2(std::string_view mnemonic, const Vop2MachineInst *inst, ExecuteFn exe literal_ = reinterpret_cast(inst)[1]; std::memcpy(raw_words_.data(), inst, size_); raw_encoding_ = raw_words_.data(); + if (has_encoded_dpp()) + dpp_modifiers_ = amdgpu::dpp::SourceModifiers::decode( + *reinterpret_cast(inst)); } void Vop2::implicit_uses(RegisterSet &uses) const { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/encodings.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/encodings.h index 1b72b434644a..09aa1127de8a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/encodings.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/encodings.h @@ -475,6 +475,7 @@ class Vop1 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -517,6 +518,7 @@ class Vopc : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; @@ -561,6 +563,7 @@ class Vop2 : public IsaInstruction { uint32_t dpp_bound_ctrl_ = 0; uint32_t dpp_fi_ = 1; uint32_t dpp8_lane_sel_ = 0; + amdgpu::dpp::SourceModifiers dpp_modifiers_; uint32_t sdwa_src0_sel_ = amdgpu::sdwa::DWORD; bool sdwa_src0_sext_ = false; bool sdwa_src0_neg_ = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h index 4c5f6e332763..892c549642b6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h @@ -139,6 +139,7 @@ class Operand final : public IsaOperand { std::optional literal32_widening_; bool packed_16bit_source_ = false; bool packed_16bit_dst_ = false; + std::optional resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const; }; } // namespace rdna4 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand_exec.cpp index 337dc08e8e95..312c4232b30d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand_exec.cpp @@ -131,7 +131,7 @@ void Operand::read_lane_chunk_exec(const amdgpu::Wavefront &wf, uint32_t lane_ba out[i] = read_lane_exec(wf, lane_base + i); return; } - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint64_t lane_mask = count == 0 ? 0 : util::mask(static_cast(count)) << lane_base; @@ -155,7 +155,7 @@ void Operand::write_lane_chunk_exec(amdgpu::Wavefront &wf, uint32_t lane_base, u write_lane_exec(wf, lane_base + i, vals[i]); return; } - auto off = detail::resolved_vgpr_offset_for_operand(wf, *this); + auto off = resolved_vgpr_offset_exec(wf); if (!off) { for (uint32_t i = 0; i < count; ++i) if (mask & (1ULL << i)) @@ -334,13 +334,13 @@ void Operand::write_scalar64_exec(amdgpu::Wavefront &wf, uint64_t val) const { } std::optional Operand::simd_vgpr_base_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) + if (auto off = resolved_vgpr_offset_exec(wf)) return wf.vgpr_alloc().base + amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); return std::nullopt; } std::optional Operand::simd_vgpr_base_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); return wf.vgpr_alloc().base + amdgpu::apply_gpr_idx(wf, *off, role); @@ -349,7 +349,7 @@ std::optional Operand::simd_vgpr_base_mut_exec(amdgpu::Wavefront &wf) } amdgpu::ConstVgprStorage Operand::simd_vgpr_storage_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); const auto &cu = amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()); return {reinterpret_cast(cu.raw_vgpr_data(wf.vgpr_alloc().base + voff)), @@ -359,7 +359,7 @@ amdgpu::ConstVgprStorage Operand::simd_vgpr_storage_exec(const amdgpu::Wavefront } amdgpu::VgprStorage Operand::simd_vgpr_storage_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -372,7 +372,7 @@ amdgpu::VgprStorage Operand::simd_vgpr_storage_mut_exec(amdgpu::Wavefront &wf) c amdgpu::ConstVgprStoragePair64 Operand::simd_vgpr_storage64_exec(const amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; const auto &cu = amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()); @@ -385,7 +385,7 @@ Operand::simd_vgpr_storage64_exec(const amdgpu::Wavefront &wf) const { } amdgpu::VgprStoragePair64 Operand::simd_vgpr_storage64_mut_exec(amdgpu::Wavefront &wf) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -400,7 +400,7 @@ amdgpu::VgprStoragePair64 Operand::simd_vgpr_storage64_mut_exec(amdgpu::Wavefron void Operand::simd_notify_read_exec(const amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read( &wf, wf.vgpr_alloc().base + voff, lane_mask, byte_mask); @@ -409,7 +409,7 @@ void Operand::simd_notify_read_exec(const amdgpu::Wavefront &wf, uint64_t lane_m void Operand::simd_notify_read_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read( &wf, wf.vgpr_alloc().base + voff, lane_mask, byte_mask); @@ -418,7 +418,7 @@ void Operand::simd_notify_read_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mas void Operand::simd_notify_read64_exec(const amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read(&wf, reg, lane_mask, @@ -430,7 +430,7 @@ void Operand::simd_notify_read64_exec(const amdgpu::Wavefront &wf, uint64_t lane void Operand::simd_notify_read64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, vgpr_msb_role()); uint32_t reg = wf.vgpr_alloc().base + voff; amdgpu::OperandExecutionAccess::raw_compute_unit(wf.cu()).notify_vgpr_read(&wf, reg, lane_mask, @@ -442,7 +442,7 @@ void Operand::simd_notify_read64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_m void Operand::simd_notify_write_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -453,7 +453,7 @@ void Operand::simd_notify_write_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_ma void Operand::simd_notify_write64_mut_exec(amdgpu::Wavefront &wf, uint64_t lane_mask, uint8_t byte_mask) const { - if (auto off = detail::resolved_vgpr_offset_for_operand(wf, *this)) { + if (auto off = resolved_vgpr_offset_exec(wf)) { amdgpu::VgprMsbRole role = vgpr_msb_role() == amdgpu::VgprMsbRole::None ? amdgpu::VgprMsbRole::Dst : vgpr_msb_role(); uint32_t voff = amdgpu::apply_gpr_idx(wf, *off, role); @@ -512,5 +512,17 @@ bool Operand::full_execution_backend_complete() { return is_complete(*static_cast(full_execution_backend())); } +std::optional Operand::resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const { + if (!reads_value()) + return std::nullopt; + auto packed = + packed_16bit_vgpr_source(packed_16bit_source_, size_bits_, opr_type_, encoding_value_); + if (!packed) + packed = packed_16bit_vgpr_dst(packed_16bit_dst_, size_bits_, opr_type_, encoding_value_); + if (packed) + return packed->reg + (wf.vgpr_msb_for_role(vgpr_msb_role()) << 8); + return detail::resolved_vgpr_offset_for_operand(wf, *this); +} + } // namespace rdna4 } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vbuffer_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vbuffer_exec.cpp index 8208209e6f1c..6f170c5ff9f2 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vbuffer_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vbuffer_exec.cpp @@ -1577,6 +1577,7 @@ void BufferAtomicAddF32Vbuffer::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::FADD; + d->atomic_source_nan_first = true; d->atomic_denorm_mode = 3; d->atomic_lds_denorm_mode = 3; d->atomic_legacy_minmax = false; @@ -1606,6 +1607,7 @@ void BufferAtomicPkAddF16Vbuffer::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::PK_ADD_F16; + d->atomic_source_nan_first = true; d->wait_counter_type = (amdgpu::gfx12_atomic_returns(inst_.th) ? amdgpu::WaitCounterType::LOADCNT : amdgpu::WaitCounterType::STORECNT); @@ -1632,6 +1634,7 @@ void BufferAtomicPkAddBf16Vbuffer::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::PK_ADD_BF16; + d->atomic_source_nan_first = true; d->wait_counter_type = (amdgpu::gfx12_atomic_returns(inst_.th) ? amdgpu::WaitCounterType::LOADCNT : amdgpu::WaitCounterType::STORECNT); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir.cpp index 5dac3ef9a33d..29a0b197cf34 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir.cpp @@ -51,6 +51,8 @@ DsDirectLoadVdsdir::DsDirectLoadVdsdir(const MachineInst *inst) num_dst_ = 1; dsmem.apply_fieldless_caps(false, false, false); m0.apply_fieldless_caps(false, false, false); + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::EXPCNT, + amdgpu::MemoryCompletionClass::LDS}}); flags_ |= MEMORY_WAIT_PRODUCER; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir_exec.cpp index 9b84c54e49b4..0a324396c5b5 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir_exec.cpp @@ -5,6 +5,7 @@ // See lib/python/amdisa/README.md for regeneration instructions. #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/vdsdir.h" +#include "rocjitsu/isa/arch/amdgpu/shared/lds_direct.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/vm/amdgpu/wavefront.h" #include "util/data_types.h" @@ -22,7 +23,7 @@ void DsParamLoadVdsdir::execute_impl(amdgpu::Wavefront &wf) { } void DsDirectLoadVdsdir::execute_impl(amdgpu::Wavefront &wf) { - (void)wf; // Interpolation/LDS-direct: no-op in compute simulation. + amdgpu::execute_lds_direct_load(*this, wf, inst_.vdst); } } // namespace rdna4 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vflat_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vflat_exec.cpp index 66c6f191b391..cd4ad3924553 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vflat_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vflat_exec.cpp @@ -1200,6 +1200,7 @@ void FlatAtomicAddF32Vflat::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::FADD; + d->atomic_source_nan_first = true; d->atomic_denorm_mode = 3; d->atomic_lds_denorm_mode = 3; d->atomic_legacy_minmax = false; @@ -1229,6 +1230,7 @@ void FlatAtomicPkAddF16Vflat::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::PK_ADD_F16; + d->atomic_source_nan_first = true; d->wait_counter_type = (amdgpu::gfx12_atomic_returns(inst_.th) ? amdgpu::WaitCounterType::LOADCNT : amdgpu::WaitCounterType::STORECNT); @@ -1255,6 +1257,7 @@ void FlatAtomicPkAddBf16Vflat::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::PK_ADD_BF16; + d->atomic_source_nan_first = true; d->wait_counter_type = (amdgpu::gfx12_atomic_returns(inst_.th) ? amdgpu::WaitCounterType::LOADCNT : amdgpu::WaitCounterType::STORECNT); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vglobal_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vglobal_exec.cpp index 0c6494b984f4..a33945e3c7e2 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vglobal_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vglobal_exec.cpp @@ -1305,6 +1305,7 @@ void GlobalAtomicAddF32Vglobal::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::FADD; + d->atomic_source_nan_first = true; d->atomic_denorm_mode = 3; d->atomic_lds_denorm_mode = 3; d->atomic_legacy_minmax = false; @@ -1362,6 +1363,7 @@ void GlobalAtomicPkAddF16Vglobal::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::PK_ADD_F16; + d->atomic_source_nan_first = true; d->wait_counter_type = (amdgpu::gfx12_atomic_returns(inst_.th) ? amdgpu::WaitCounterType::LOADCNT : amdgpu::WaitCounterType::STORECNT); @@ -1388,6 +1390,7 @@ void GlobalAtomicPkAddBf16Vglobal::execute_impl(amdgpu::Wavefront &wf) { d->num_elems = 1; d->is_load = amdgpu::gfx12_atomic_returns(inst_.th); d->atomic_op = amdgpu::AtomicOp::PK_ADD_BF16; + d->atomic_source_nan_first = true; d->wait_counter_type = (amdgpu::gfx12_atomic_returns(inst_.th) ? amdgpu::WaitCounterType::LOADCNT : amdgpu::WaitCounterType::STORECNT); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop1_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop1_exec.cpp index ee796573d941..008267f310d6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop1_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop1_exec.cpp @@ -8,6 +8,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/vop1.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" #include "rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h" +#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/isa/arch/amdgpu/shared/transcendental.h" #include "rocjitsu/vm/amdgpu/register_access.h" @@ -55,13 +56,8 @@ RJ_NOINLINE void VMovB32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop1::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -148,6 +144,7 @@ RJ_NOINLINE void VCvtU32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -176,6 +173,7 @@ RJ_NOINLINE void VCvtI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -223,6 +221,7 @@ RJ_NOINLINE void VCvtF16F32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -258,8 +257,8 @@ void VCvtF32F16Vop1::execute_impl(amdgpu::Wavefront &wf) { } auto &inst = *this; if (amdgpu::try_execute_words_simd<1, true, false, 1>(inst, wf, [&](auto a) { - return std::bit_cast>(([](auto a) { - return util::f16_to_f32_simd(a); + return std::bit_cast>(([&wf](auto a) { + return ::rocjitsu::amdgpu::cvt_f32_f16_mode_simd(a, wf); })(std::bit_cast>(a))); })) return; @@ -269,8 +268,10 @@ void VCvtF32F16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - std::bit_cast(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))))); + std::bit_cast(std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())))); } } @@ -285,6 +286,7 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -292,8 +294,8 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; if (amdgpu::try_execute_words_simd<1, true, false, 1>(inst, wf, [&](auto a) { - return std::bit_cast>(([](auto a) { - return util::f16_to_f32_simd(a); + return std::bit_cast>(([&wf](auto a) { + return ::rocjitsu::amdgpu::cvt_f32_f16_mode_simd(a, wf); })(std::bit_cast>(a))); })) return; @@ -303,8 +305,10 @@ RJ_NOINLINE void VCvtF32F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - std::bit_cast(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))))); + std::bit_cast(std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())))); } dpp_write_mask_scope_.restore(); } @@ -328,6 +332,7 @@ RJ_NOINLINE void VCvtNearestI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -356,6 +361,7 @@ RJ_NOINLINE void VCvtFloorI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -616,6 +622,7 @@ RJ_NOINLINE void VFractF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -644,6 +651,7 @@ RJ_NOINLINE void VTruncF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -672,6 +680,7 @@ RJ_NOINLINE void VCeilF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -700,6 +709,7 @@ RJ_NOINLINE void VRndneF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -728,6 +738,7 @@ RJ_NOINLINE void VFloorF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -756,6 +767,7 @@ RJ_NOINLINE void VExpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -784,6 +796,7 @@ RJ_NOINLINE void VLogF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -812,6 +825,7 @@ RJ_NOINLINE void VRcpF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -840,6 +854,7 @@ RJ_NOINLINE void VRcpIflagF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -868,6 +883,7 @@ RJ_NOINLINE void VRsqF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -900,6 +916,7 @@ RJ_NOINLINE void VSqrtF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -932,6 +949,7 @@ RJ_NOINLINE void VSinF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -960,6 +978,7 @@ RJ_NOINLINE void VCosF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1140,6 +1159,7 @@ RJ_NOINLINE void VFrexpExpI32F32Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1168,6 +1188,7 @@ RJ_NOINLINE void VFrexpMantF32Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1707,6 +1728,7 @@ RJ_NOINLINE void VCvtU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1781,6 +1803,7 @@ RJ_NOINLINE void VCvtI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1823,11 +1846,17 @@ void VRcpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -1838,10 +1867,13 @@ void VRcpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1857,6 +1889,7 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1866,11 +1899,17 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -1881,10 +1920,13 @@ RJ_NOINLINE void VRcpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1899,11 +1941,17 @@ void VSqrtF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -1914,10 +1962,13 @@ void VSqrtF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -1933,6 +1984,7 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1942,11 +1994,17 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -1957,10 +2015,13 @@ RJ_NOINLINE void VSqrtF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -1975,11 +2036,17 @@ void VRsqF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -1990,10 +2057,13 @@ void VRsqF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2009,6 +2079,7 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2018,11 +2089,17 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2033,10 +2110,13 @@ RJ_NOINLINE void VRsqF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2051,11 +2131,15 @@ void VLogF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2066,10 +2150,13 @@ void VLogF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2085,6 +2172,7 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2094,11 +2182,15 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2109,10 +2201,13 @@ RJ_NOINLINE void VLogF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2127,11 +2222,15 @@ void VExpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2142,10 +2241,13 @@ void VExpF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2161,6 +2263,7 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2170,11 +2273,15 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { if (amdgpu::try_execute_words_simd<1, true, true, 1>(inst, wf, [&](auto a) { return std::bit_cast>(([&](auto a) { if (wf.fp16_ovfl()) - return ([](auto a) { - return util::f32_to_f16_ovfl_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); - return ([](auto a) { - return util::f32_to_f16_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + return ([&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); })(a); })(std::bit_cast>(a))); })) @@ -2185,10 +2292,13 @@ RJ_NOINLINE void VExpF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2241,6 +2351,7 @@ RJ_NOINLINE void VFrexpMantF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2331,6 +2442,7 @@ RJ_NOINLINE void VFrexpExpI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2416,6 +2528,7 @@ RJ_NOINLINE void VFloorF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2490,6 +2603,7 @@ RJ_NOINLINE void VCeilF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2564,6 +2678,7 @@ RJ_NOINLINE void VTruncF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2638,6 +2753,7 @@ RJ_NOINLINE void VRndneF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2718,6 +2834,7 @@ RJ_NOINLINE void VFractF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2767,10 +2884,13 @@ void VSinF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2786,6 +2906,7 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2797,10 +2918,13 @@ RJ_NOINLINE void VSinF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2817,10 +2941,13 @@ void VCosF16Vop1::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -2836,6 +2963,7 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2847,10 +2975,13 @@ RJ_NOINLINE void VCosF16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane( *this, wf, vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } dpp_write_mask_scope_.restore(); @@ -2919,7 +3050,8 @@ void VCvtNormI16F16Vop1::execute_impl(amdgpu::Wavefront &wf) { static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -2936,6 +3068,7 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -2951,7 +3084,8 @@ RJ_NOINLINE void VCvtNormI16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -2973,7 +3107,8 @@ void VCvtNormU16F16Vop1::execute_impl(amdgpu::Wavefront &wf) { static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } @@ -2990,6 +3125,7 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -3005,7 +3141,8 @@ RJ_NOINLINE void VCvtNormU16F16Vop1::execute_modifier_impl(amdgpu::Wavefront &wf static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop2_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop2_exec.cpp index a818f3584f4f..6d391d204346 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop2_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop2_exec.cpp @@ -64,6 +64,7 @@ void VAddF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -73,7 +74,10 @@ RJ_NOINLINE void VAddF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -92,6 +96,7 @@ void VSubF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -101,7 +106,10 @@ RJ_NOINLINE void VSubF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -120,6 +128,7 @@ void VSubrevF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -129,7 +138,10 @@ RJ_NOINLINE void VSubrevF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -150,6 +162,7 @@ void VMulDx9ZeroF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulDx9ZeroF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -159,7 +172,10 @@ RJ_NOINLINE void VMulDx9ZeroF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -178,6 +194,7 @@ void VMulF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -187,7 +204,10 @@ RJ_NOINLINE void VMulF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -438,6 +458,7 @@ void VMinNumF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinNumF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -447,7 +468,10 @@ RJ_NOINLINE void VMinNumF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -466,6 +490,7 @@ void VMaxNumF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxNumF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -475,7 +500,10 @@ RJ_NOINLINE void VMaxNumF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -898,6 +926,7 @@ void VFmacF32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -907,7 +936,10 @@ RJ_NOINLINE void VFmacF32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -934,6 +966,7 @@ void VCvtPkRtzF16F32Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCvtPkRtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -943,7 +976,10 @@ RJ_NOINLINE void VCvtPkRtzF16F32Vop2::execute_modifier_impl(amdgpu::Wavefront &w amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -990,6 +1026,7 @@ void VMinNumF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMinNumF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -999,7 +1036,10 @@ RJ_NOINLINE void VMinNumF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1074,6 +1114,7 @@ void VMaxNumF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMaxNumF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1083,7 +1124,10 @@ RJ_NOINLINE void VMaxNumF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1162,6 +1206,7 @@ void VAddF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1171,7 +1216,10 @@ RJ_NOINLINE void VAddF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1254,6 +1302,7 @@ void VSubF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1263,7 +1312,10 @@ RJ_NOINLINE void VSubF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1346,6 +1398,7 @@ void VSubrevF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1355,7 +1408,10 @@ RJ_NOINLINE void VSubrevF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1438,6 +1494,7 @@ void VMulF16Vop2::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1447,7 +1504,10 @@ RJ_NOINLINE void VMulF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1504,13 +1564,15 @@ void VFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1520,7 +1582,10 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_write_mask_scope_.bind(wf, @@ -1536,7 +1601,8 @@ RJ_NOINLINE void VFmacF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } @@ -1554,7 +1620,8 @@ void VFmamkF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16(s0, k, s2, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } @@ -1570,7 +1637,8 @@ void VFmaakF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16(s0, s1, k, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, result); } } @@ -1632,6 +1700,7 @@ RJ_NOINLINE void VLdexpF16Vop2::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::dpp::ScopedVgprWriteMask dpp_write_mask_scope_; if (inst_.src0 == amdgpu::SRC_DPP) @@ -1696,16 +1765,18 @@ void VPkFmacF16Vop2::execute_impl(amdgpu::Wavefront &wf) { uint32_t rawd = amdgpu::RegisterAccess(wf).read_lane(vdst, lane); const uint32_t omod = amdgpu::sdwa::output_modifier(*this, wf); - uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), - static_cast(rawd), false, false, false, false, - false, false, wf.fp_round_mode_f16_f64(), - wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + uint32_t r0 = amdgpu::fp_mode::fma_f16( + static_cast(raw0), static_cast(raw1), static_cast(rawd), + false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), + wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint32_t r1 = amdgpu::fp_mode::fma_f16( static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, r0 | (r1 << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp index 74c0d45ce73b..078d82eed2fe 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp @@ -58,13 +58,8 @@ RJ_NOINLINE void VMovB32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { void VReadfirstlaneB32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); - uint32_t val = 0; - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (exec & (1ULL << lane)) { - val = amdgpu::RegisterAccess(wf).read_lane(src0, lane); - break; - } - } + uint32_t lane = exec ? static_cast(std::countr_zero(exec)) : 0; + uint32_t val = amdgpu::RegisterAccess(wf).read_scalar_selected_lane(src0, lane); amdgpu::RegisterAccess(wf).write_scalar(vdst, val); } @@ -250,18 +245,32 @@ void VCvtF32F16Vop3::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -286,20 +295,33 @@ RJ_NOINLINE void VCvtF32F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { auto &inst = *this; ROCJITSU_TRY_SIMD_CVT_F32_F16_VOP3_TRUE16(); uint64_t exec = wf.exec(); - const uint32_t opsel = ::rocjitsu::amdgpu::vop3_opsel(inst_); + [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - uint32_t raw = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); - float src = util::f16_to_f32(static_cast(raw)); - if (inst_.abs & (1u << 0)) - src = std::fabs(src); - if (inst_.neg & (1u << 0)) - src = -src; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - std::bit_cast(src)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst_.omod)); + }(); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } - dpp_write_mask_scope_.restore(); } @@ -1628,8 +1650,15 @@ void VCvtU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -1666,8 +1695,15 @@ RJ_NOINLINE void VCvtU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -1692,8 +1728,15 @@ void VCvtI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -1732,8 +1775,15 @@ RJ_NOINLINE void VCvtI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -1754,7 +1804,13 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1763,31 +1819,27 @@ void VRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1819,7 +1871,13 @@ RJ_NOINLINE void VRcpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -1828,31 +1886,27 @@ RJ_NOINLINE void VRcpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -1872,52 +1926,47 @@ void VSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1942,52 +1991,47 @@ RJ_NOINLINE void VSqrtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = - static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( - *this, wf, - [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); - if (inst_.clamp) - v = amdgpu::clamp_floating_result(v, wf); - return v; - }(), - wf.fp16_ovfl()), - amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), - wf.ieee_mode(), false, inst_.omod)))); + uint32_t src_half = static_cast< + uint32_t>(static_cast(amdgpu::fp_mode::finalize_omod_f16( + amdgpu::sdwa::finish_rounded_f16( + *this, wf, + [&]() { + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); + if (inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }(), + wf.fp16_ovfl()), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod)))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -2000,7 +2044,13 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2009,31 +2059,27 @@ void VRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2065,7 +2111,13 @@ RJ_NOINLINE void VRsqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2074,31 +2126,27 @@ RJ_NOINLINE void VRsqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2118,7 +2166,13 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2127,31 +2181,27 @@ void VLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2183,7 +2233,13 @@ RJ_NOINLINE void VLogF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2192,31 +2248,27 @@ RJ_NOINLINE void VLogF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2236,7 +2288,13 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2245,31 +2303,27 @@ void VExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -2301,7 +2355,13 @@ RJ_NOINLINE void VExpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_TRUE16_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2310,31 +2370,27 @@ RJ_NOINLINE void VExpF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3219,31 +3275,27 @@ void VSinF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3282,31 +3334,27 @@ RJ_NOINLINE void VSinF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3333,31 +3381,27 @@ void VCosF16Vop3::execute_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3396,31 +3440,27 @@ RJ_NOINLINE void VCosF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { { uint32_t src_half = static_cast(static_cast(amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( *this, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( + src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst_.omod), + wf.fp16_ovfl()); if (inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -3494,11 +3534,19 @@ void VCvtNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3531,11 +3579,19 @@ RJ_NOINLINE void VCvtNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3556,11 +3612,19 @@ void VCvtNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -3593,11 +3657,19 @@ RJ_NOINLINE void VCvtNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf continue; { uint32_t src_half = static_cast(static_cast([&]() -> uint32_t { - float s = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + float s = [&]() { + float sv = util::f16_to_f32(static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); + if (inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); @@ -5772,7 +5844,8 @@ void VFmacF16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -5814,7 +5887,8 @@ RJ_NOINLINE void VFmacF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, inst_.abs & 1u, inst_.abs & 2u, false, inst_.neg & 1u, inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8120,7 +8194,8 @@ void VFmaF16Vop3::execute_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -8162,7 +8237,8 @@ RJ_NOINLINE void VFmaF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { src0_bits, src1_bits, src2_bits, inst_.abs & 1u, inst_.abs & 2u, inst_.abs & 4u, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -10329,7 +10405,7 @@ void VSExpF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSExpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::EXP2, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -10343,7 +10419,7 @@ void VSLogF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSLogF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::LOG2, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -10357,7 +10433,7 @@ void VSRcpF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSRcpF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::RCP, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -10371,7 +10447,7 @@ void VSRsqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSRsqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::RSQ, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -10385,7 +10461,7 @@ void VSSqrtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { void VSSqrtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::RegisterAccess(wf).write_scalar( vdst, - amdgpu::pseudo_scalar::execute_f16( + amdgpu::transcendental::execute_pseudo_f16( amdgpu::pseudo_scalar::Operation::SQRT, util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_scalar(src0))), (inst_.abs & 1u) != 0, (inst_.neg & 1u) != 0, wf.fp_round_mode_f16_f64(), @@ -11184,12 +11260,22 @@ void VCvtPkNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11199,11 +11285,19 @@ void VCvtPkNormI16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.opsel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -11232,12 +11326,22 @@ RJ_NOINLINE void VCvtPkNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(util::f16_to_f32_simd(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11247,11 +11351,19 @@ RJ_NOINLINE void VCvtPkNormI16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.opsel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -11268,10 +11380,20 @@ void VCvtPkNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11281,11 +11403,19 @@ void VCvtPkNormU16F16Vop3::execute_impl(amdgpu::Wavefront &wf) { ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.opsel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -11314,10 +11444,20 @@ RJ_NOINLINE void VCvtPkNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_TRUE16_SRC(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + if (inst.inst_.abs & 2u) + b &= ~0x8000u; + if (inst.inst_.neg & 2u) + b ^= 0x8000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(util::f16_to_f32_simd(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11327,11 +11467,19 @@ RJ_NOINLINE void VCvtPkNormU16F16Vop3::execute_modifier_impl(amdgpu::Wavefront & ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, inst_.opsel, 0))); float s1 = util::f16_to_f32(static_cast( ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, inst_.opsel, 1))); + if (inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst_.neg & (1u << 0)) + s0 = -s0; + if (inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -12614,32 +12762,26 @@ void VCmpLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12673,32 +12815,26 @@ RJ_NOINLINE void VCmpLtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12713,32 +12849,26 @@ void VCmpEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12772,32 +12902,26 @@ RJ_NOINLINE void VCmpEqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12812,32 +12936,26 @@ void VCmpLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12871,32 +12989,26 @@ RJ_NOINLINE void VCmpLeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12911,32 +13023,26 @@ void VCmpGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -12970,32 +13076,26 @@ RJ_NOINLINE void VCmpGtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13010,35 +13110,26 @@ void VCmpLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13072,35 +13163,26 @@ RJ_NOINLINE void VCmpLgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13115,32 +13197,26 @@ void VCmpGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13174,32 +13250,26 @@ RJ_NOINLINE void VCmpGeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13214,32 +13284,26 @@ void VCmpOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13273,32 +13337,26 @@ RJ_NOINLINE void VCmpOF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13313,32 +13371,26 @@ void VCmpUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13372,32 +13424,26 @@ RJ_NOINLINE void VCmpUF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13412,32 +13458,26 @@ void VCmpNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13471,32 +13511,26 @@ RJ_NOINLINE void VCmpNgeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13511,35 +13545,26 @@ void VCmpNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13573,35 +13598,26 @@ RJ_NOINLINE void VCmpNlgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - commit_result, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13616,32 +13632,26 @@ void VCmpNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13675,32 +13685,26 @@ RJ_NOINLINE void VCmpNgtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13715,32 +13719,26 @@ void VCmpNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13774,32 +13772,26 @@ RJ_NOINLINE void VCmpNleF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13814,32 +13806,26 @@ void VCmpNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13873,32 +13859,26 @@ RJ_NOINLINE void VCmpNeqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13913,32 +13893,26 @@ void VCmpNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -13972,32 +13946,26 @@ RJ_NOINLINE void VCmpNltF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::write_wave_mask_scalar(vdst, wf, final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(commit_result, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); [[maybe_unused]] uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - if (inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32(static_cast( - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.abs, inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -16358,27 +16326,24 @@ void VCmpxLtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16412,27 +16377,24 @@ RJ_NOINLINE void VCmpxLtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16445,27 +16407,24 @@ void VCmpxEqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16499,27 +16458,24 @@ RJ_NOINLINE void VCmpxEqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16532,27 +16488,24 @@ void VCmpxLeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16586,27 +16539,24 @@ RJ_NOINLINE void VCmpxLeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16619,27 +16569,24 @@ void VCmpxGtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16673,27 +16620,24 @@ RJ_NOINLINE void VCmpxGtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16706,27 +16650,24 @@ void VCmpxLgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16760,27 +16701,24 @@ RJ_NOINLINE void VCmpxLgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16793,27 +16731,24 @@ void VCmpxGeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16847,27 +16782,24 @@ RJ_NOINLINE void VCmpxGeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16880,28 +16812,24 @@ void VCmpxOF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16935,28 +16863,24 @@ RJ_NOINLINE void VCmpxOF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -16969,28 +16893,24 @@ void VCmpxUF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17024,28 +16944,24 @@ RJ_NOINLINE void VCmpxUF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17058,27 +16974,24 @@ void VCmpxNgeF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17112,27 +17025,24 @@ RJ_NOINLINE void VCmpxNgeF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17145,28 +17055,24 @@ void VCmpxNlgF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17200,28 +17106,24 @@ RJ_NOINLINE void VCmpxNlgF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17234,27 +17136,24 @@ void VCmpxNgtF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17288,27 +17187,24 @@ RJ_NOINLINE void VCmpxNgtF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17321,27 +17217,24 @@ void VCmpxNleF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17375,27 +17268,24 @@ RJ_NOINLINE void VCmpxNleF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17408,27 +17298,24 @@ void VCmpxNeqF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17462,27 +17349,24 @@ RJ_NOINLINE void VCmpxNeqF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17495,27 +17379,24 @@ void VCmpxNltF16Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17549,27 +17430,24 @@ RJ_NOINLINE void VCmpxNltF16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; uint32_t opsel = amdgpu::vop3_opsel(inst_); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0))); - float s1 = util::f16_to_f32( - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); + const uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17582,24 +17460,23 @@ void VCmpxLtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17631,24 +17508,23 @@ RJ_NOINLINE void VCmpxLtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17661,24 +17537,23 @@ void VCmpxEqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17710,24 +17585,23 @@ RJ_NOINLINE void VCmpxEqF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17740,24 +17614,23 @@ void VCmpxLeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17789,24 +17662,23 @@ RJ_NOINLINE void VCmpxLeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17819,24 +17691,23 @@ void VCmpxGtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17868,24 +17739,23 @@ RJ_NOINLINE void VCmpxGtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17898,24 +17768,23 @@ void VCmpxLgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17947,24 +17816,23 @@ RJ_NOINLINE void VCmpxLgF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -17977,24 +17845,23 @@ void VCmpxGeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18026,24 +17893,23 @@ RJ_NOINLINE void VCmpxGeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18056,25 +17922,23 @@ void VCmpxOF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18106,25 +17970,23 @@ RJ_NOINLINE void VCmpxOF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18137,25 +17999,23 @@ void VCmpxUF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18187,25 +18047,23 @@ RJ_NOINLINE void VCmpxUF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18218,24 +18076,23 @@ void VCmpxNgeF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18267,24 +18124,23 @@ RJ_NOINLINE void VCmpxNgeF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18297,24 +18153,23 @@ void VCmpxNlgF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18346,24 +18201,23 @@ RJ_NOINLINE void VCmpxNlgF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18376,24 +18230,23 @@ void VCmpxNgtF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18425,24 +18278,23 @@ RJ_NOINLINE void VCmpxNgtF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18455,24 +18307,23 @@ void VCmpxNleF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18504,24 +18355,23 @@ RJ_NOINLINE void VCmpxNleF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18534,24 +18384,23 @@ void VCmpxNeqF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18583,24 +18432,23 @@ RJ_NOINLINE void VCmpxNeqF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18613,24 +18461,23 @@ void VCmpxNltF32Vop3::execute_impl(amdgpu::Wavefront &wf) { } auto commit_result = [&](uint64_t raw_result) { wf.set_exec(raw_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18662,24 +18509,23 @@ RJ_NOINLINE void VCmpxNltF32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18691,24 +18537,23 @@ void VCmpxLtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18720,24 +18565,23 @@ void VCmpxEqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18749,24 +18593,23 @@ void VCmpxLeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18778,24 +18621,23 @@ void VCmpxGtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18807,24 +18649,23 @@ void VCmpxLgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18836,24 +18677,23 @@ void VCmpxGeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18865,25 +18705,23 @@ void VCmpxOF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18895,25 +18733,23 @@ void VCmpxUF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( - [&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18925,24 +18761,23 @@ void VCmpxNgeF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18954,24 +18789,23 @@ void VCmpxNlgF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -18983,24 +18817,23 @@ void VCmpxNgtF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -19012,24 +18845,23 @@ void VCmpxNleF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -19041,24 +18873,23 @@ void VCmpxNeqF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); @@ -19070,24 +18901,23 @@ void VCmpxNltF64Vop3::execute_impl(amdgpu::Wavefront &wf) { wf.set_exec(final_result); }; auto &inst = *this; - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT([&](uint64_t value) { commit_result(value); }, - [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + [&](uint64_t value) { commit_result(value); }, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)); - if (inst_.abs & (1u << 0)) - s0 = std::fabs(s0); - if (inst_.neg & (1u << 0)) - s0 = -s0; - if (inst_.abs & (1u << 1)) - s1 = std::fabs(s1); - if (inst_.neg & (1u << 1)) - s1 = -s1; - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(src1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, inst_.abs, inst_.neg, compare_policy)) result |= (1ULL << lane); } commit_result(result); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp index aa134af4570b..ed372f211d00 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp @@ -502,11 +502,13 @@ void VPkFmaF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t rlo = amdgpu::fp_mode::fma_f16( a_lo, b_lo, c_lo, false, false, false, inst_.neg & 1u, inst_.neg & 2u, inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint16_t rhi = amdgpu::fp_mode::fma_f16( a_hi, b_hi, c_hi, false, false, false, inst_.neg_hi & 1u, inst_.neg_hi & 2u, inst_.neg_hi & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, inst_.clamp, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -538,7 +540,7 @@ void VPkAddF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::ADD, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -548,7 +550,7 @@ void VPkAddF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::ADD, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -580,7 +582,7 @@ void VPkMulF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MUL, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -590,7 +592,7 @@ void VPkMulF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MUL, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -812,7 +814,7 @@ void VPkMinNumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MIN, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -822,7 +824,7 @@ void VPkMinNumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MIN, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -854,7 +856,7 @@ void VPkMaxNumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAX, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -864,7 +866,7 @@ void VPkMaxNumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAX, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -896,7 +898,7 @@ void VPkMinimumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MINIMUM, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -906,7 +908,7 @@ void VPkMinimumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MINIMUM, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -938,7 +940,7 @@ void VPkMaximumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAXIMUM, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -948,7 +950,7 @@ void VPkMaximumF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAXIMUM, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -960,9 +962,7 @@ void VFmaMixF32Vop3p::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F32(); - } + ROCJITSU_TRY_SIMD_FUSED_MIX(F32); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -994,7 +994,8 @@ void VFmaMixF32Vop3p::execute_impl(amdgpu::Wavefront &wf) { b = -b; if (inst_.neg & 4) c = -c; - float result = a * b + c; + float result = amdgpu::fp_mode::arithmetic( + a, b, c, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, @@ -1019,9 +1020,7 @@ RJ_NOINLINE void VFmaMixF32Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) { dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F32(); - } + ROCJITSU_TRY_SIMD_FUSED_MIX(F32); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -1053,7 +1052,8 @@ RJ_NOINLINE void VFmaMixF32Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) { b = -b; if (inst_.neg & 4) c = -c; - float result = a * b + c; + float result = amdgpu::fp_mode::arithmetic( + a, b, c, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); if (inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); amdgpu::sdwa::write_lane(*this, wf, vdst, lane, @@ -1068,7 +1068,7 @@ void VFmaMixloF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F16_LO(); + ROCJITSU_TRY_SIMD_FUSED_MIX(F16_LO); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -1100,10 +1100,10 @@ void VFmaMixloF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { b = -b; if (inst_.neg & 4) c = -c; - float result = a * b + c; - if (inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0u, h); } } @@ -1125,7 +1125,7 @@ RJ_NOINLINE void VFmaMixloF16Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F16_LO(); + ROCJITSU_TRY_SIMD_FUSED_MIX(F16_LO); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -1157,10 +1157,10 @@ RJ_NOINLINE void VFmaMixloF16Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) b = -b; if (inst_.neg & 4) c = -c; - float result = a * b + c; - if (inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0u, h); } dpp_write_mask_scope_.restore(); @@ -1172,7 +1172,7 @@ void VFmaMixhiF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F16_HI(); + ROCJITSU_TRY_SIMD_FUSED_MIX(F16_HI); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -1204,10 +1204,10 @@ void VFmaMixhiF16Vop3p::execute_impl(amdgpu::Wavefront &wf) { b = -b; if (inst_.neg & 4) c = -c; - float result = a * b + c; - if (inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0x8u, h); } } @@ -1229,7 +1229,7 @@ RJ_NOINLINE void VFmaMixhiF16Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) dpp_write_mask_scope_.bind(wf, wf.exec() & dpp_plan_.row_bank_mask & dpp_plan_.source_write_mask); auto &inst = *this; - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F16_HI(); + ROCJITSU_TRY_SIMD_FUSED_MIX(F16_HI); uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -1261,10 +1261,10 @@ RJ_NOINLINE void VFmaMixhiF16Vop3p::execute_modifier_impl(amdgpu::Wavefront &wf) b = -b; if (inst_.neg & 4) c = -c; - float result = a * b + c; - if (inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, 0x8u, h); } dpp_write_mask_scope_.restore(); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vopc_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vopc_exec.cpp index 0f0fe5815961..88b938017e2d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vopc_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vopc_exec.cpp @@ -33,6 +33,7 @@ void VCmpLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -48,7 +49,10 @@ RJ_NOINLINE void VCmpLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -70,6 +74,7 @@ void VCmpEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -85,7 +90,10 @@ RJ_NOINLINE void VCmpEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_eq_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -107,6 +115,7 @@ void VCmpLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -122,7 +131,10 @@ RJ_NOINLINE void VCmpLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_le_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -144,6 +156,7 @@ void VCmpGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -159,7 +172,10 @@ RJ_NOINLINE void VCmpGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_gt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -181,6 +197,7 @@ void VCmpLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -196,7 +213,10 @@ RJ_NOINLINE void VCmpLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lg_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -218,6 +238,7 @@ void VCmpGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -233,7 +254,10 @@ RJ_NOINLINE void VCmpGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ge_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -255,6 +279,7 @@ void VCmpOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -270,7 +295,10 @@ RJ_NOINLINE void VCmpOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_o_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -292,6 +320,7 @@ void VCmpUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -307,7 +336,10 @@ RJ_NOINLINE void VCmpUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_u_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -329,6 +361,7 @@ void VCmpNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -344,7 +377,10 @@ RJ_NOINLINE void VCmpNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nge_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -366,6 +402,7 @@ void VCmpNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -381,7 +418,10 @@ RJ_NOINLINE void VCmpNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlg_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -403,6 +443,7 @@ void VCmpNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -418,7 +459,10 @@ RJ_NOINLINE void VCmpNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ngt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -440,6 +484,7 @@ void VCmpNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -455,7 +500,10 @@ RJ_NOINLINE void VCmpNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nle_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -477,6 +525,7 @@ void VCmpNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -492,7 +541,10 @@ RJ_NOINLINE void VCmpNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_neq_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -514,6 +566,7 @@ void VCmpNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -529,7 +582,10 @@ RJ_NOINLINE void VCmpNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlt_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -551,6 +607,7 @@ void VCmpLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -566,7 +623,10 @@ RJ_NOINLINE void VCmpLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -588,6 +648,7 @@ void VCmpEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -603,7 +664,10 @@ RJ_NOINLINE void VCmpEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_eq_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -625,6 +689,7 @@ void VCmpLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -640,7 +705,10 @@ RJ_NOINLINE void VCmpLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_le_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -662,6 +730,7 @@ void VCmpGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -677,7 +746,10 @@ RJ_NOINLINE void VCmpGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_gt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -699,6 +771,7 @@ void VCmpLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -714,7 +787,10 @@ RJ_NOINLINE void VCmpLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_lg_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -736,6 +812,7 @@ void VCmpGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -751,7 +828,10 @@ RJ_NOINLINE void VCmpGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ge_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -773,6 +853,7 @@ void VCmpOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -788,7 +869,10 @@ RJ_NOINLINE void VCmpOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_o_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -810,6 +894,7 @@ void VCmpUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -825,7 +910,10 @@ RJ_NOINLINE void VCmpUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_u_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -847,6 +935,7 @@ void VCmpNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -862,7 +951,10 @@ RJ_NOINLINE void VCmpNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nge_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -884,6 +976,7 @@ void VCmpNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -899,7 +992,10 @@ RJ_NOINLINE void VCmpNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlg_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -921,6 +1017,7 @@ void VCmpNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -936,7 +1033,10 @@ RJ_NOINLINE void VCmpNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_ngt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -958,6 +1058,7 @@ void VCmpNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -973,7 +1074,10 @@ RJ_NOINLINE void VCmpNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nle_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -995,6 +1099,7 @@ void VCmpNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1010,7 +1115,10 @@ RJ_NOINLINE void VCmpNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_neq_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -1032,6 +1140,7 @@ void VCmpNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -1047,7 +1156,10 @@ RJ_NOINLINE void VCmpNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); amdgpu::execute_v_cmp_nlt_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { uint64_t dpp_row_bank_mask = dpp_row_bank_mask_; @@ -2076,6 +2188,7 @@ RJ_NOINLINE void VCmpClassF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::execute_v_cmp_class_f16_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { @@ -2113,6 +2226,7 @@ RJ_NOINLINE void VCmpClassF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); amdgpu::execute_v_cmp_class_f32_vopc(*this, wf); if (inst_.src0 == amdgpu::SRC_DPP) { @@ -2137,19 +2251,23 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2157,6 +2275,7 @@ void VCmpxLtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2172,24 +2291,31 @@ RJ_NOINLINE void VCmpxLtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2212,19 +2338,23 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) == util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2232,6 +2362,7 @@ void VCmpxEqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2247,24 +2378,31 @@ RJ_NOINLINE void VCmpxEqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) == util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2287,19 +2425,23 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2307,6 +2449,7 @@ void VCmpxLeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2322,24 +2465,31 @@ RJ_NOINLINE void VCmpxLeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2362,19 +2512,23 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2382,6 +2536,7 @@ void VCmpxGtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2397,24 +2552,31 @@ RJ_NOINLINE void VCmpxGtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2437,22 +2599,23 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return (util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2460,6 +2623,7 @@ void VCmpxLgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2475,27 +2639,31 @@ RJ_NOINLINE void VCmpxLgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return (util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2518,19 +2686,23 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2538,6 +2710,7 @@ void VCmpxGeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2553,24 +2726,31 @@ RJ_NOINLINE void VCmpxGeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2593,22 +2773,23 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !util::stdx::isnan(util::f16_to_f32_simd(a)) && - !util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2616,6 +2797,7 @@ void VCmpxOF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2631,27 +2813,31 @@ RJ_NOINLINE void VCmpxOF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !util::stdx::isnan(util::f16_to_f32_simd(a)) && - !util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2674,22 +2860,23 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return util::stdx::isnan(util::f16_to_f32_simd(a)) || - util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2697,6 +2884,7 @@ void VCmpxUF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2712,27 +2900,31 @@ RJ_NOINLINE void VCmpxUF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return util::stdx::isnan(util::f16_to_f32_simd(a)) || - util::stdx::isnan(util::f16_to_f32_simd(b)); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2755,19 +2947,23 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2775,6 +2971,7 @@ void VCmpxNgeF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2790,24 +2987,31 @@ RJ_NOINLINE void VCmpxNgeF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2830,22 +3034,23 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !((util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b))); + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2853,6 +3058,7 @@ void VCmpxNlgF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2868,27 +3074,31 @@ RJ_NOINLINE void VCmpxNlgF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { - return !((util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b))); + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2911,19 +3121,23 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2931,6 +3145,7 @@ void VCmpxNgtF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -2946,24 +3161,31 @@ RJ_NOINLINE void VCmpxNgtF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -2986,19 +3208,23 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3006,6 +3232,7 @@ void VCmpxNleF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3021,24 +3248,31 @@ RJ_NOINLINE void VCmpxNleF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3061,19 +3295,23 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) != util::f16_to_f32_simd(b); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3081,6 +3319,7 @@ void VCmpxNeqF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3096,24 +3335,31 @@ RJ_NOINLINE void VCmpxNeqF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return util::f16_to_f32_simd(a) != util::f16_to_f32_simd(b); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3136,19 +3382,23 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)); }, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3156,6 +3406,7 @@ void VCmpxNltF16Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3171,24 +3422,31 @@ RJ_NOINLINE void VCmpxNltF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 32768ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; if (amdgpu::try_execute_vopc_simd( inst, wf, - [](auto a, auto b) { return !(util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)); }, + [compare_policy = + amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = - util::f16_to_f32(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))); - float s1 = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane))); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3209,18 +3467,25 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3228,6 +3493,7 @@ void VCmpxLtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3243,21 +3509,31 @@ RJ_NOINLINE void VCmpxLtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3278,18 +3554,25 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3297,6 +3580,7 @@ void VCmpxEqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3312,21 +3596,31 @@ RJ_NOINLINE void VCmpxEqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 == s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3347,18 +3641,25 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3366,6 +3667,7 @@ void VCmpxLeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3381,21 +3683,31 @@ RJ_NOINLINE void VCmpxLeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 <= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3416,18 +3728,25 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3435,6 +3754,7 @@ void VCmpxGtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3450,21 +3770,31 @@ RJ_NOINLINE void VCmpxGtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3485,18 +3815,25 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3504,6 +3841,7 @@ void VCmpxLgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3519,21 +3857,31 @@ RJ_NOINLINE void VCmpxLgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3554,18 +3902,25 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3573,6 +3928,7 @@ void VCmpxGeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3588,21 +3944,31 @@ RJ_NOINLINE void VCmpxGeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 >= s1) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3623,18 +3989,25 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3642,6 +4015,7 @@ void VCmpxOF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3657,21 +4031,31 @@ RJ_NOINLINE void VCmpxOF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3692,18 +4076,25 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3711,6 +4102,7 @@ void VCmpxUF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3726,21 +4118,31 @@ RJ_NOINLINE void VCmpxUF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3761,18 +4163,25 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3780,6 +4189,7 @@ void VCmpxNgeF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3795,21 +4205,31 @@ RJ_NOINLINE void VCmpxNgeF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 >= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3830,18 +4250,25 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3849,6 +4276,7 @@ void VCmpxNlgF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3864,21 +4292,31 @@ RJ_NOINLINE void VCmpxNlgF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3899,18 +4337,25 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3918,6 +4363,7 @@ void VCmpxNgtF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -3933,21 +4379,31 @@ RJ_NOINLINE void VCmpxNgtF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 > s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3968,18 +4424,25 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -3987,6 +4450,7 @@ void VCmpxNleF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4002,21 +4466,31 @@ RJ_NOINLINE void VCmpxNleF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 <= s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4037,18 +4511,25 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4056,6 +4537,7 @@ void VCmpxNeqF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4071,21 +4553,31 @@ RJ_NOINLINE void VCmpxNeqF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4106,18 +4598,25 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { return; } auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4125,6 +4624,7 @@ void VCmpxNltF32Vopc::execute_impl(amdgpu::Wavefront &wf) { RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { std::optional dpp_src0_; + std::optional dpp_src1_; amdgpu::dpp::DppPlan dpp_plan_; if (inst_.src0 == amdgpu::SRC_DPP) dpp_plan_ = amdgpu::dpp::make_dpp_plan(wf.wf_size(), dpp_ctrl_, dpp_row_mask_, dpp_bank_mask_, @@ -4140,21 +4640,31 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); + amdgpu::dpp::apply_source_modifiers(vsrc1, dpp_src1_, wf, 2147483648ull, dpp_modifiers_, 1); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); + ScopedOperandDelegate dpp_src1_binding_(vsrc1, dpp_src1_ ? &*dpp_src1_ : nullptr); [&]() -> void { auto &inst = *this; - if (amdgpu::try_execute_vopc_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); - float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane)); - if (!(s0 < s1)) + const uint32_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); + const uint32_t s1 = amdgpu::RegisterAccess(wf).read_lane(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4171,18 +4681,25 @@ RJ_NOINLINE void VCmpxNltF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) { void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a < b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4190,18 +4707,25 @@ void VCmpxLtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a == b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 == s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4209,18 +4733,25 @@ void VCmpxEqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a <= b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 <= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4228,18 +4759,25 @@ void VCmpxLeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a > b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4247,18 +4785,25 @@ void VCmpxGtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return (a < b) || (a > b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 < s1 || s0 > s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4266,18 +4811,25 @@ void VCmpxLgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a >= b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 >= s1) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4285,18 +4837,25 @@ void VCmpxGeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!std::isnan(s0) && !std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4304,18 +4863,25 @@ void VCmpxOF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4323,18 +4889,25 @@ void VCmpxUF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a >= b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 >= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4342,18 +4915,25 @@ void VCmpxNgeF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !((a < b) || (a > b)); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1 || s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4361,18 +4941,25 @@ void VCmpxNlgF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a > b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 > s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4380,18 +4967,25 @@ void VCmpxNgtF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a <= b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 <= s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4399,18 +4993,25 @@ void VCmpxNleF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return a != b; }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (s0 != s1 || std::isnan(s0) || std::isnan(s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -4418,18 +5019,25 @@ void VCmpxNeqF64Vopc::execute_impl(amdgpu::Wavefront &wf) { void VCmpxNltF64Vopc::execute_impl(amdgpu::Wavefront &wf) { auto &inst = *this; - if (amdgpu::try_execute_vopc64_simd( - inst, wf, [](auto a, auto b) { return !(a < b); }, + if (amdgpu::try_execute_vopc64_simd( + inst, wf, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64())](auto a, + auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }, [&](uint64_t value) { wf.set_exec(value); })) return; uint64_t exec = wf.exec(); uint64_t result = 0; + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - double s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)); - double s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane)); - if (!(s0 < s1)) + const uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane64(src0, lane); + const uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane64(vsrc1, lane); + if (amdgpu::comparison::evaluate( + s0, s1, compare_policy)) result |= (1ULL << lane); } wf.set_exec(result); @@ -6461,6 +7069,7 @@ RJ_NOINLINE void VCmpxClassF16Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 32768ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); [&]() -> void { auto &inst = *this; @@ -6631,6 +7240,7 @@ RJ_NOINLINE void VCmpxClassF32Vopc::execute_modifier_impl(amdgpu::Wavefront &wf) amdgpu::dpp::apply_dpp(src0, dpp_plan_, dpp_old_exec_, dpp_src0_, wf); if (amdgpu::dpp::is_src_dpp8(inst_.src0)) amdgpu::dpp::apply_dpp8(src0, dpp8_lane_sel_, dpp_fi_, dpp_src0_, wf); + amdgpu::dpp::apply_source_modifiers(src0, dpp_src0_, wf, 2147483648ull, dpp_modifiers_, 0); ScopedOperandDelegate dpp_src0_binding_(src0, dpp_src0_ ? &*dpp_src0_ : nullptr); [&]() -> void { auto &inst = *this; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vopd_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vopd_exec.cpp index 35dc8b45dda1..891daa7c4956 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vopd_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vopd_exec.cpp @@ -87,25 +87,25 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(*slot.dst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdFmaakF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), std::bit_cast(src2), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdFmamkF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src2), std::bit_cast(src1), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdMulF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32()); + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdMulDx9ZeroF32: { @@ -118,7 +118,7 @@ uint32_t Vopd::execute_slot(const Slot &slot, amdgpu::Wavefront &wf, uint32_t la case kVopdAddF32: { float result = amdgpu::fp_mode::arithmetic( std::bit_cast(src0), std::bit_cast(src1), 0.0f, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32()); + wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); return std::bit_cast(result); } case kVopdSubF32: { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index a1c426e48c7c..f74f0e66a840 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -9,10 +9,12 @@ #include "rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h" #include "rocjitsu/isa/arch/amdgpu/shared/alu_exceptions.h" +#include "rocjitsu/isa/arch/amdgpu/shared/cube.h" #include "rocjitsu/isa/arch/amdgpu/shared/division.h" #include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" #include "rocjitsu/isa/arch/amdgpu/shared/gfx11_dot2.h" #include "rocjitsu/isa/arch/amdgpu/shared/gfx12_dot.h" +#include "rocjitsu/isa/arch/amdgpu/shared/lds_direct.h" #include "rocjitsu/isa/arch/amdgpu/shared/pseudo_scalar.h" #include "rocjitsu/isa/arch/amdgpu/shared/simd_glue.h" #include "rocjitsu/isa/arch/amdgpu/shared/transcendental.h" @@ -276,10 +278,6 @@ template inline void execute_image_bvh_intersect_ray_mimg([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) {} -template -inline void execute_lds_direct_load_ldsdir([[maybe_unused]] Inst &inst, - [[maybe_unused]] Wavefront &wf) {} - template inline void execute_lds_param_load_ldsdir([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) {} @@ -1689,10 +1687,11 @@ inline void execute_s_floor_f32_sop1([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_s_fmaak_f32_sop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - float result = - std::fma(std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc0)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc1)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.simm32))); + float result = amdgpu::fp_mode::arithmetic( + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc0)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc1)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.simm32)), + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); amdgpu::RegisterAccess(wf).write_scalar(inst.sdst, std::bit_cast(result)); } @@ -1707,17 +1706,21 @@ inline void execute_s_fmac_f16_sop2([[maybe_unused]] Inst &inst, [[maybe_unused] template inline void execute_s_fmac_f32_sop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - float result = std::fma(std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc0)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc1)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.sdst))); + float result = amdgpu::fp_mode::arithmetic( + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc0)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc1)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.sdst)), + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); amdgpu::RegisterAccess(wf).write_scalar(inst.sdst, std::bit_cast(result)); } template inline void execute_s_fmamk_f32_sop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - float result = std::fma(std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc0)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.simm32)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc1))); + float result = amdgpu::fp_mode::arithmetic( + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc0)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.simm32)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc1)), + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); amdgpu::RegisterAccess(wf).write_scalar(inst.sdst, std::bit_cast(result)); } @@ -2556,6 +2559,14 @@ inline void execute_s_setkill_sopp([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_s_setprio_sopp([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) {} +template +inline void execute_s_setvskip_sopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + const uint32_t source = amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc0); + const uint32_t bit = amdgpu::RegisterAccess(wf).read_scalar(inst.ssrc1) & 31u; + const uint32_t vskip = ((source >> bit) & 1u) * Wavefront::VSKIP_BIT; + wf.set_mode_raw((wf.mode_raw() & ~Wavefront::VSKIP_BIT) | vskip); +} + template inline void execute_s_sext_i32_i16_sop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { @@ -3080,8 +3091,11 @@ inline void execute_v_add_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_add_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP2_BINARY(float32_t, std::plus<>{}); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP2_BINARY(float32_t, [&wf](auto a, auto b) { + return amdgpu::binary_f32_simd(a, b, wf); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -3092,14 +3106,18 @@ inline void execute_v_add_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] std::bit_cast(amdgpu::fp_mode::arithmetic( std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), 0.0f, - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()))); } } template inline void execute_v_add_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP3_BINARY_FP(float32_t, std::plus<>{}); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP3_BINARY_FP(float32_t, [&inst, &wf](auto a, auto b) { + return amdgpu::binary_f32_simd( + a, b, wf, amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -3109,7 +3127,6 @@ inline void execute_v_add_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] inst, wf, inst.vdst, lane, std::bit_cast([&]() { amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = @@ -3129,17 +3146,16 @@ inline void execute_v_add_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sv = -sv; return sv; }(), - 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), + wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod) != 0)); const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -3868,16 +3884,9 @@ inline void execute_v_ceil_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -4443,30 +4452,23 @@ inline void execute_v_cmp_class_f64_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_eq_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -4474,18 +4476,21 @@ inline void execute_v_cmp_eq_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_eq_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return util::f16_to_f32_simd(a) == util::f16_to_f32_simd(b); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) == - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -4494,28 +4499,23 @@ inline void execute_v_cmp_eq_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_eq_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -4523,14 +4523,21 @@ inline void execute_v_cmp_eq_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_eq_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) == - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -4539,30 +4546,23 @@ inline void execute_v_cmp_eq_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_eq_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() == - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -4570,14 +4570,21 @@ inline void execute_v_cmp_eq_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_eq_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return a == b; }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) == - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -4777,8 +4784,8 @@ inline void execute_v_cmp_eq_u64_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_f_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, - [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT(commit_result, uint32_t, + [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -4809,8 +4816,8 @@ inline void execute_v_cmp_f_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_f_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, - [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT(commit_result, uint32_t, + [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -4839,8 +4846,8 @@ inline void execute_v_cmp_f_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_f_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, - [](auto a, auto b) { return decltype(a == b)(false); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT(commit_result, uint64_t, + [](auto a, auto b) { return decltype(a == b)(false); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -5021,30 +5028,23 @@ inline void execute_v_cmp_f_u64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_ge_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5052,18 +5052,21 @@ inline void execute_v_cmp_ge_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_ge_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) >= - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -5072,28 +5075,23 @@ inline void execute_v_cmp_ge_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_ge_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5101,14 +5099,21 @@ inline void execute_v_cmp_ge_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_ge_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) >= - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -5117,30 +5122,23 @@ inline void execute_v_cmp_ge_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_ge_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5148,14 +5146,21 @@ inline void execute_v_cmp_ge_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_ge_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return a >= b; }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) >= - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -5355,30 +5360,23 @@ inline void execute_v_cmp_ge_u64_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_gt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5386,17 +5384,21 @@ inline void execute_v_cmp_gt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_gt_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC( - uint32_t, [](auto a, auto b) { return util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) > - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -5405,28 +5407,23 @@ inline void execute_v_cmp_gt_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_gt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5434,14 +5431,21 @@ inline void execute_v_cmp_gt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_gt_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) > - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -5450,30 +5454,23 @@ inline void execute_v_cmp_gt_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_gt_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5481,14 +5478,21 @@ inline void execute_v_cmp_gt_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_gt_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return a > b; }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) > - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -5688,30 +5692,23 @@ inline void execute_v_cmp_gt_u64_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_le_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5719,18 +5716,21 @@ inline void execute_v_cmp_le_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_le_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) <= - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -5739,28 +5739,23 @@ inline void execute_v_cmp_le_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_le_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5768,14 +5763,21 @@ inline void execute_v_cmp_le_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_le_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) <= - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -5784,30 +5786,23 @@ inline void execute_v_cmp_le_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_le_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -5815,14 +5810,21 @@ inline void execute_v_cmp_le_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_le_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return a <= b; }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) <= - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6022,34 +6024,23 @@ inline void execute_v_cmp_le_u64_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lg_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6057,22 +6048,21 @@ inline void execute_v_cmp_lg_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lg_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return (util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - auto b = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6081,32 +6071,23 @@ inline void execute_v_cmp_lg_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lg_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6114,17 +6095,21 @@ inline void execute_v_cmp_lg_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lg_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - auto b = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6133,34 +6118,23 @@ inline void execute_v_cmp_lg_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lg_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, - [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6168,17 +6142,21 @@ inline void execute_v_cmp_lg_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lg_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return (a < b) || (a > b); }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - auto a = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - auto b = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)); - return (a < b) || (a > b); - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6187,30 +6165,23 @@ inline void execute_v_cmp_lg_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6218,17 +6189,21 @@ inline void execute_v_cmp_lt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lt_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC( - uint32_t, [](auto a, auto b) { return util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) < - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6237,28 +6212,23 @@ inline void execute_v_cmp_lt_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6266,14 +6236,21 @@ inline void execute_v_cmp_lt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lt_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) < - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6282,30 +6259,23 @@ inline void execute_v_cmp_lt_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lt_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6313,14 +6283,21 @@ inline void execute_v_cmp_lt_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_lt_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return a < b; }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) < - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6711,30 +6688,23 @@ inline void execute_v_cmp_ne_u64_vopc([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cmp_neq_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6743,18 +6713,21 @@ inline void execute_v_cmp_neq_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_neq_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return util::f16_to_f32_simd(a) != util::f16_to_f32_simd(b); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) != - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6763,28 +6736,23 @@ inline void execute_v_cmp_neq_f16_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_neq_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6793,14 +6761,21 @@ inline void execute_v_cmp_neq_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_neq_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) != - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6809,30 +6784,23 @@ inline void execute_v_cmp_neq_f32_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_neq_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if (([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() != - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }())) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6841,14 +6809,21 @@ inline void execute_v_cmp_neq_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_neq_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return a != b; }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) != - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6857,30 +6832,23 @@ inline void execute_v_cmp_neq_f64_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nge_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6889,18 +6857,21 @@ inline void execute_v_cmp_nge_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nge_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return !(util::f16_to_f32_simd(a) >= util::f16_to_f32_simd(b)); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) >= - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6909,28 +6880,23 @@ inline void execute_v_cmp_nge_f16_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nge_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6939,14 +6905,21 @@ inline void execute_v_cmp_nge_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nge_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) >= - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -6955,31 +6928,23 @@ inline void execute_v_cmp_nge_f32_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nge_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, - [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() >= - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -6988,14 +6953,21 @@ inline void execute_v_cmp_nge_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nge_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return !(a >= b); }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) >= - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7004,30 +6976,23 @@ inline void execute_v_cmp_nge_f64_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_ngt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7036,18 +7001,21 @@ inline void execute_v_cmp_ngt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_ngt_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return !(util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b)); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) > - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7056,28 +7024,23 @@ inline void execute_v_cmp_ngt_f16_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_ngt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7086,14 +7049,21 @@ inline void execute_v_cmp_ngt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_ngt_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) > - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7102,30 +7072,23 @@ inline void execute_v_cmp_ngt_f32_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_ngt_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() > - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7134,14 +7097,21 @@ inline void execute_v_cmp_ngt_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_ngt_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return !(a > b); }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) > - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7150,30 +7120,23 @@ inline void execute_v_cmp_ngt_f64_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nle_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7182,18 +7145,21 @@ inline void execute_v_cmp_nle_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nle_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return !(util::f16_to_f32_simd(a) <= util::f16_to_f32_simd(b)); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) <= - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7202,28 +7168,23 @@ inline void execute_v_cmp_nle_f16_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nle_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7232,14 +7193,21 @@ inline void execute_v_cmp_nle_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nle_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) <= - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7248,31 +7216,23 @@ inline void execute_v_cmp_nle_f32_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nle_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, - [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() <= - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7281,14 +7241,21 @@ inline void execute_v_cmp_nle_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nle_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return !(a <= b); }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) <= - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7297,34 +7264,23 @@ inline void execute_v_cmp_nle_f64_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nlg_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7333,22 +7289,21 @@ inline void execute_v_cmp_nlg_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nlg_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return !((util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)) || - (util::f16_to_f32_simd(a) > util::f16_to_f32_simd(b))); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - auto b = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7357,32 +7312,23 @@ inline void execute_v_cmp_nlg_f16_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nlg_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7391,17 +7337,21 @@ inline void execute_v_cmp_nlg_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nlg_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - auto b = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7410,34 +7360,23 @@ inline void execute_v_cmp_nlg_f32_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nlg_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, - [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto b = [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7446,17 +7385,21 @@ inline void execute_v_cmp_nlg_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nlg_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return !((a < b) || (a > b)); }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - auto a = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - auto b = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)); - return (a < b) || (a > b); - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7465,30 +7408,23 @@ inline void execute_v_cmp_nlg_f64_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nlt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7497,18 +7433,21 @@ inline void execute_v_cmp_nlt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nlt_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return !(util::f16_to_f32_simd(a) < util::f16_to_f32_simd(b)); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) < - util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7517,28 +7456,23 @@ inline void execute_v_cmp_nlt_f16_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nlt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7547,14 +7481,21 @@ inline void execute_v_cmp_nlt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nlt_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(float32_t, [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) < - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7563,30 +7504,23 @@ inline void execute_v_cmp_nlt_f32_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_nlt_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }() < - [&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7595,14 +7529,21 @@ inline void execute_v_cmp_nlt_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unus template inline void execute_v_cmp_nlt_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64(double, [](auto a, auto b) { return !(a < b); }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)) < - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7611,31 +7552,23 @@ inline void execute_v_cmp_nlt_f64_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_o_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7643,19 +7576,21 @@ inline void execute_v_cmp_o_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_o_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return !util::stdx::isnan(util::f16_to_f32_simd(a)) && - !util::stdx::isnan(util::f16_to_f32_simd(b)); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))) && - !std::isnan(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7664,29 +7599,23 @@ inline void execute_v_cmp_o_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_o_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7694,15 +7623,21 @@ inline void execute_v_cmp_o_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_o_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC( - float32_t, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) && - !std::isnan(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7711,31 +7646,23 @@ inline void execute_v_cmp_o_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_o_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( - commit_result, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) && - !std::isnan([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -7743,17 +7670,21 @@ inline void execute_v_cmp_o_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_o_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64( - double, [](auto a, auto b) { return !util::stdx::isnan(a) && !util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((!std::isnan( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane))) && - !std::isnan( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -7775,8 +7706,8 @@ inline void execute_v_cmp_t_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_t_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, - [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT(commit_result, uint32_t, + [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -7804,8 +7735,8 @@ inline void execute_v_cmp_t_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_t_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, - [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT(commit_result, uint64_t, + [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -7985,8 +7916,8 @@ inline void execute_v_cmp_t_u64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_tru_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(commit_result, - [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT(commit_result, uint32_t, + [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -8018,8 +7949,8 @@ inline void execute_v_cmp_tru_f16_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_tru_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(commit_result, - [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT(commit_result, uint32_t, + [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -8049,8 +7980,8 @@ inline void execute_v_cmp_tru_f32_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_tru_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(commit_result, - [](auto a, auto b) { return decltype(a == b)(true); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT(commit_result, uint64_t, + [](auto a, auto b) { return decltype(a == b)(true); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -8080,31 +8011,23 @@ inline void execute_v_cmp_tru_f64_vopc([[maybe_unused]] Inst &inst, template inline void execute_v_cmp_u_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8112,19 +8035,21 @@ inline void execute_v_cmp_u_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_u_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC(uint32_t, [](auto a, auto b) { - return util::stdx::isnan(util::f16_to_f32_simd(a)) || - util::stdx::isnan(util::f16_to_f32_simd(b)); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))) || - std::isnan(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -8133,29 +8058,23 @@ inline void execute_v_cmp_u_f16_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_u_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC_VOP3_INT_RESULT( + commit_result, uint32_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8163,15 +8082,21 @@ inline void execute_v_cmp_u_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_u_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC( - float32_t, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC(uint32_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f32())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f32()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))) || - std::isnan(std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -8180,31 +8105,23 @@ inline void execute_v_cmp_u_f32_vopc([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_u_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf, [[maybe_unused]] CommitResult commit_result) { - ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT( - commit_result, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT( + commit_result, uint64_t, + [compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()), + abs_mods = inst.inst_.abs, neg_mods = inst.inst_.neg](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, abs_mods, neg_mods, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()) || - std::isnan([&]() { - double sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }()))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src1, lane)), + inst.inst_.abs, inst.inst_.neg, compare_policy)) vcc |= (1ULL << lane); } commit_result(vcc); @@ -8212,17 +8129,21 @@ inline void execute_v_cmp_u_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_cmp_u_f64_vopc([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOPC64( - double, [](auto a, auto b) { return util::stdx::isnan(a) || util::stdx::isnan(b); }); + ROCJITSU_TRY_SIMD_VOPC64(uint64_t, [compare_policy = amdgpu::comparison::Policy::make( + wf.fp_denorm_mode_f16_f64())](auto a, auto b) { + return amdgpu::comparison::evaluate( + a, b, compare_policy); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); + const auto compare_policy = amdgpu::comparison::Policy::make(wf.fp_denorm_mode_f16_f64()); uint64_t vcc = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - if ((std::isnan( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane))) || - std::isnan( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane))))) + if (amdgpu::comparison::evaluate( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.vsrc1, lane)), + compare_policy)) vcc |= (1ULL << lane); } wf.set_vcc_mask(vcc); @@ -8270,10 +8191,13 @@ inline void execute_v_cos_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, - amdgpu::transcendental::cos_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -8287,31 +8211,26 @@ inline void execute_v_cos_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sdwa::write_lane( inst, wf, inst.vdst, lane, amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = util::f16_to_f32(static_cast( - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst.inst_.omod), + wf.fp16_ovfl()); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -8324,45 +8243,44 @@ inline void execute_v_cos_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_cos_f32_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + fp_mode::ScopedEnvironment environment(0); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::transcendental::cos_f32( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + wf.fp_denorm_mode_f32(), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())))); } } template inline void execute_v_cos_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + fp_mode::ScopedEnvironment environment(0); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { float v = [&]() { - float v = amdgpu::transcendental::cos_f32([&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + float v = amdgpu::transcendental::cos_f32( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f32(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), + inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -8412,60 +8330,50 @@ inline void execute_v_ctz_i32_b32_vop3([[maybe_unused]] Inst &inst, template inline void execute_v_cubeid_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32( - [](auto x, auto y, auto z) { return util::cube_id_f32_simd(x, y, z); }); + [&inst, &wf](auto x, auto y, auto z) { + return ::rocjitsu::amdgpu::cube::apply_omod( + ::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::ID>( + x, y, z, wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode()), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode(), inst.inst_.omod); + }, + false); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - float v = [&]() { - float v = [&]() { - auto x = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto y = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - auto z = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); - if (inst.inst_.abs & (1u << 2)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 2)) - sv = -sv; - return sv; - }(); - float ax = std::fabs(x), ay = std::fabs(y), az = std::fabs(z); - if (ax >= ay && ax >= az) - return x >= 0 ? 0.0f : 1.0f; - if (ay >= ax && ay >= az) - return y >= 0 ? 2.0f : 3.0f; - return z >= 0 ? 4.0f : 5.0f; - }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = ::rocjitsu::amdgpu::cube::apply_omod( + ::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::ID>( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 1)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 1)) + sv = -sv; + return sv; + }(), + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); + if (inst.inst_.abs & (1u << 2)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 2)) + sv = -sv; + return sv; + }(), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode()), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode(), inst.inst_.omod); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -8475,59 +8383,51 @@ inline void execute_v_cubeid_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cubema_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32([](auto x, auto y, auto z) { - return 2.0f * util::stdx::fmax(util::stdx::abs(x), - util::stdx::fmax(util::stdx::abs(y), util::stdx::abs(z))); - }); + ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32( + [&inst, &wf](auto x, auto y, auto z) { + return ::rocjitsu::amdgpu::cube::apply_omod( + ::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::MA>( + x, y, z, wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode()), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode(), inst.inst_.omod); + }, + false); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - float v = [&]() { - float v = [&]() { - auto x = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto y = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - auto z = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); - if (inst.inst_.abs & (1u << 2)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 2)) - sv = -sv; - return sv; - }(); - float ax = std::fabs(x), ay = std::fabs(y), az = std::fabs(z); - return 2.0f * std::fmax(ax, std::fmax(ay, az)); - }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = ::rocjitsu::amdgpu::cube::apply_omod( + ::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::MA>( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 1)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 1)) + sv = -sv; + return sv; + }(), + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); + if (inst.inst_.abs & (1u << 2)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 2)) + sv = -sv; + return sv; + }(), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode()), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode(), inst.inst_.omod); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -8538,60 +8438,50 @@ inline void execute_v_cubema_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cubesc_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32( - [](auto x, auto y, auto z) { return util::cube_sc_f32_simd(x, y, z); }); + [&inst, &wf](auto x, auto y, auto z) { + return ::rocjitsu::amdgpu::cube::apply_omod( + ::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::SC>( + x, y, z, wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode()), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode(), inst.inst_.omod); + }, + false); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - float v = [&]() { - float v = [&]() { - auto x = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto y = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - auto z = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); - if (inst.inst_.abs & (1u << 2)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 2)) - sv = -sv; - return sv; - }(); - float ax = std::fabs(x), ay = std::fabs(y), az = std::fabs(z); - if (ax >= ay && ax >= az) - return x >= 0 ? z : -z; - if (ay >= ax && ay >= az) - return x; - return z >= 0 ? -x : x; - }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = ::rocjitsu::amdgpu::cube::apply_omod( + ::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::SC>( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 1)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 1)) + sv = -sv; + return sv; + }(), + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); + if (inst.inst_.abs & (1u << 2)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 2)) + sv = -sv; + return sv; + }(), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode()), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode(), inst.inst_.omod); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -8602,60 +8492,50 @@ inline void execute_v_cubesc_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_cubetc_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32( - [](auto x, auto y, auto z) { return util::cube_tc_f32_simd(x, y, z); }); + [&inst, &wf](auto x, auto y, auto z) { + return ::rocjitsu::amdgpu::cube::apply_omod( + ::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::TC>( + x, y, z, wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode()), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode(), inst.inst_.omod); + }, + false); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - float v = [&]() { - float v = [&]() { - auto x = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }(); - auto y = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - if (inst.inst_.abs & (1u << 1)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 1)) - sv = -sv; - return sv; - }(); - auto z = [&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); - if (inst.inst_.abs & (1u << 2)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 2)) - sv = -sv; - return sv; - }(); - float ax = std::fabs(x), ay = std::fabs(y), az = std::fabs(z); - if (ax >= ay && ax >= az) - return -y; - if (ay >= ax && ay >= az) - return y >= 0 ? -z : z; - return -y; - }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = ::rocjitsu::amdgpu::cube::apply_omod( + ::rocjitsu::amdgpu::cube::execute<::rocjitsu::amdgpu::cube::Operation::TC>( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 1)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 1)) + sv = -sv; + return sv; + }(), + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); + if (inst.inst_.abs & (1u << 2)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 2)) + sv = -sv; + return sv; + }(), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode()), + wf.fp_round_mode_f32(), wf.cu().arch(), wf.ieee_mode(), inst.inst_.omod); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -8822,16 +8702,19 @@ inline void execute_v_cvt_f16_u16_vop3([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_f32_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, float32_t, - [](auto a) { return util::f16_to_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, float32_t, [&wf](auto a) { + return ::rocjitsu::amdgpu::cvt_f32_f16_mode_simd(a, wf); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - std::bit_cast(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + std::bit_cast(std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())))); } } @@ -8844,9 +8727,27 @@ inline void execute_v_cvt_f32_f16_vop3([[maybe_unused]] Inst &inst, if (!(exec & (1ULL << lane))) continue; sdwa::write_lane( - inst, wf, inst.vdst, lane, - std::bit_cast(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + inst, wf, inst.vdst, lane, std::bit_cast([&]() { + float v = [&]() { + float v = std::bit_cast(amdgpu::fp_mode::cvt_f32_f16( + [&]() { + float sv = util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); + }(); + if (inst.inst_.clamp) + v = amdgpu::clamp_floating_result(v, wf); + return v; + }())); } } @@ -8916,16 +8817,9 @@ inline void execute_v_cvt_f32_i32_vop3([[maybe_unused]] Inst &inst, float v = [&]() { float v = std::bit_cast(std::bit_cast(static_cast( static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -8968,16 +8862,9 @@ inline void execute_v_cvt_f32_u32_vop3([[maybe_unused]] Inst &inst, float v = [&]() { float v = std::bit_cast(std::bit_cast( static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -9020,16 +8907,9 @@ inline void execute_v_cvt_f32_ubyte0_vop3([[maybe_unused]] Inst &inst, float v = [&]() { float v = std::bit_cast(std::bit_cast( static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) & 0xFFu))); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -9072,16 +8952,9 @@ inline void execute_v_cvt_f32_ubyte1_vop3([[maybe_unused]] Inst &inst, float v = [&]() { float v = std::bit_cast(std::bit_cast(static_cast( (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) >> 8) & 0xFFu))); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -9124,16 +8997,9 @@ inline void execute_v_cvt_f32_ubyte2_vop3([[maybe_unused]] Inst &inst, float v = [&]() { float v = std::bit_cast(std::bit_cast(static_cast( (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) >> 16) & 0xFFu))); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -9176,16 +9042,9 @@ inline void execute_v_cvt_f32_ubyte3_vop3([[maybe_unused]] Inst &inst, float v = [&]() { float v = std::bit_cast(std::bit_cast(static_cast( (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) >> 24) & 0xFFu))); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -9443,9 +9302,15 @@ inline void execute_v_cvt_i16_f16_vop1([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_i16_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - auto s = util::f16_to_f32_simd(a); - return ::rocjitsu::amdgpu::simd_cvt_i16_f32_to_u32(s); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&inst](auto a) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + return ([](auto a) { + auto s = util::f16_to_f32_simd(a); + return ::rocjitsu::amdgpu::simd_cvt_i16_f32_to_u32(s); + })(a); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -9453,8 +9318,15 @@ inline void execute_v_cvt_i16_f16_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, [&]() -> uint32_t { - float s = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + float s = [&]() { + float sv = util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; if (s >= 32768.0f) @@ -9673,7 +9545,8 @@ inline void execute_v_cvt_norm_i16_f16_vop1([[maybe_unused]] Inst &inst, static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -9688,11 +9561,19 @@ inline void execute_v_cvt_norm_i16_f16_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, [&]() -> uint32_t { - float s = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + float s = [&]() { + float sv = util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 32767.0f, -32768.0f, 32767.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 32767.0, -32767.0, 32767.0)); return static_cast(static_cast(static_cast(scaled))); }()); } @@ -9711,7 +9592,8 @@ inline void execute_v_cvt_norm_u16_f16_vop1([[maybe_unused]] Inst &inst, static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } @@ -9726,11 +9608,19 @@ inline void execute_v_cvt_norm_u16_f16_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, [&]() -> uint32_t { - float s = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + float s = [&]() { + float sv = util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s)) return 0u; - float scaled = std::clamp(s * 65535.0f, 0.0f, 65535.0f); + double scaled = + util::rndne_scalar(std::clamp(static_cast(s) * 65535.0, 0.0, 65535.0)); return static_cast(static_cast(scaled)); }()); } @@ -9771,16 +9661,9 @@ inline void execute_v_cvt_off_f32_i4_vop3([[maybe_unused]] Inst &inst, nibble -= 16; return static_cast(nibble) * 0.0625f; }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -9822,12 +9705,22 @@ inline void execute_v_cvt_pk_i16_i32_vop3([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_pk_norm_i16_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(std::bit_cast>(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(std::bit_cast>(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x80000000u; + if (inst.inst_.neg & 1u) + a ^= 0x80000000u; + if (inst.inst_.abs & 2u) + b &= ~0x80000000u; + if (inst.inst_.neg & 2u) + b ^= 0x80000000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(std::bit_cast>(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(std::bit_cast>(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -9835,11 +9728,19 @@ inline void execute_v_cvt_pk_norm_i16_f32_vop3([[maybe_unused]] Inst &inst, continue; float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst.inst_.neg & (1u << 0)) + s0 = -s0; + if (inst.inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst.inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -9852,10 +9753,20 @@ inline void execute_v_cvt_pk_norm_i16_f32_vop3([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_pk_norm_u16_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(std::bit_cast>(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(std::bit_cast>(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x80000000u; + if (inst.inst_.neg & 1u) + a ^= 0x80000000u; + if (inst.inst_.abs & 2u) + b &= ~0x80000000u; + if (inst.inst_.neg & 2u) + b ^= 0x80000000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(std::bit_cast>(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(std::bit_cast>(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -9863,11 +9774,19 @@ inline void execute_v_cvt_pk_norm_u16_f32_vop3([[maybe_unused]] Inst &inst, continue; float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst.inst_.neg & (1u << 0)) + s0 = -s0; + if (inst.inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst.inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -9901,10 +9820,20 @@ inline void execute_v_cvt_pk_rtz_f16_f32_vop2([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_pk_rtz_f16_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [](auto a, auto b) { - auto lo = util::f32_to_f16_rtz_simd(std::bit_cast>(a)); - auto hi = util::f32_to_f16_rtz_simd(std::bit_cast>(b)); - return lo | (hi << 16); + ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x80000000u; + if (inst.inst_.neg & 1u) + a ^= 0x80000000u; + if (inst.inst_.abs & 2u) + b &= ~0x80000000u; + if (inst.inst_.neg & 2u) + b ^= 0x80000000u; + return ([](auto a, auto b) { + auto lo = util::f32_to_f16_rtz_simd(std::bit_cast>(a)); + auto hi = util::f32_to_f16_rtz_simd(std::bit_cast>(b)); + return lo | (hi << 16); + })(a, b); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -9912,6 +9841,14 @@ inline void execute_v_cvt_pk_rtz_f16_f32_vop3([[maybe_unused]] Inst &inst, continue; float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst.inst_.neg & (1u << 0)) + s0 = -s0; + if (inst.inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst.inst_.neg & (1u << 1)) + s1 = -s1; uint32_t lo = util::f32_to_f16_rtz(s0); uint32_t hi = util::f32_to_f16_rtz(s1); sdwa::write_lane(inst, wf, inst.vdst, lane, @@ -9981,12 +9918,22 @@ inline void execute_v_cvt_pkaccum_u8_f32_vop3([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_pknorm_i16_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [](auto a, auto b) { - using U = util::native; - auto lo = util::cvt_pknorm_i16_f32_simd(std::bit_cast>(a)); - auto hi = util::cvt_pknorm_i16_f32_simd(std::bit_cast>(b)); - return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | - (util::stdx::static_simd_cast(lo) & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x80000000u; + if (inst.inst_.neg & 1u) + a ^= 0x80000000u; + if (inst.inst_.abs & 2u) + b &= ~0x80000000u; + if (inst.inst_.neg & 2u) + b ^= 0x80000000u; + return ([](auto a, auto b) { + using U = util::native; + auto lo = util::cvt_pknorm_i16_f32_simd(std::bit_cast>(a)); + auto hi = util::cvt_pknorm_i16_f32_simd(std::bit_cast>(b)); + return ((util::stdx::static_simd_cast(hi) & 0xFFFFu) << 16) | + (util::stdx::static_simd_cast(lo) & 0xFFFFu); + })(a, b); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -9994,11 +9941,19 @@ inline void execute_v_cvt_pknorm_i16_f32_vop3([[maybe_unused]] Inst &inst, continue; float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst.inst_.neg & (1u << 0)) + s0 = -s0; + if (inst.inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst.inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_i16 = [](float f) -> int16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 32767.0f, -32768.0f, 32767.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0))); }; int16_t lo = cvt_i16(s0); int16_t hi = cvt_i16(s1); @@ -10011,10 +9966,20 @@ inline void execute_v_cvt_pknorm_i16_f32_vop3([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_pknorm_u16_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [](auto a, auto b) { - auto lo = util::cvt_pknorm_u16_f32_simd(std::bit_cast>(a)); - auto hi = util::cvt_pknorm_u16_f32_simd(std::bit_cast>(b)); - return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x80000000u; + if (inst.inst_.neg & 1u) + a ^= 0x80000000u; + if (inst.inst_.abs & 2u) + b &= ~0x80000000u; + if (inst.inst_.neg & 2u) + b ^= 0x80000000u; + return ([](auto a, auto b) { + auto lo = util::cvt_pknorm_u16_f32_simd(std::bit_cast>(a)); + auto hi = util::cvt_pknorm_u16_f32_simd(std::bit_cast>(b)); + return ((hi & 0xFFFFu) << 16) | (lo & 0xFFFFu); + })(a, b); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10022,11 +9987,19 @@ inline void execute_v_cvt_pknorm_u16_f32_vop3([[maybe_unused]] Inst &inst, continue; float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst.inst_.neg & (1u << 0)) + s0 = -s0; + if (inst.inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst.inst_.neg & (1u << 1)) + s1 = -s1; auto cvt_u16 = [](float f) -> uint16_t { if (std::isnan(f)) return 0; return static_cast( - util::round_to_nearest_even(std::clamp(f * 65535.0f, 0.0f, 65535.0f))); + util::rndne_scalar(std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0))); }; uint16_t lo = cvt_u16(s0); uint16_t hi = cvt_u16(s1); @@ -10060,10 +10033,20 @@ inline void execute_v_cvt_pkrtz_f16_f32_vop2([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_pkrtz_f16_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [](auto a, auto b) { - auto lo = util::f32_to_f16_rtz_simd(std::bit_cast>(a)); - auto hi = util::f32_to_f16_rtz_simd(std::bit_cast>(b)); - return lo | (hi << 16); + ROCJITSU_TRY_SIMD_VOP3_BINARY_INT(uint32_t, [&inst](auto a, auto b) { + if (inst.inst_.abs & 1u) + a &= ~0x80000000u; + if (inst.inst_.neg & 1u) + a ^= 0x80000000u; + if (inst.inst_.abs & 2u) + b &= ~0x80000000u; + if (inst.inst_.neg & 2u) + b ^= 0x80000000u; + return ([](auto a, auto b) { + auto lo = util::f32_to_f16_rtz_simd(std::bit_cast>(a)); + auto hi = util::f32_to_f16_rtz_simd(std::bit_cast>(b)); + return lo | (hi << 16); + })(a, b); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10071,6 +10054,14 @@ inline void execute_v_cvt_pkrtz_f16_f32_vop3([[maybe_unused]] Inst &inst, continue; float s0 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); float s1 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); + if (inst.inst_.abs & (1u << 0)) + s0 = std::fabs(s0); + if (inst.inst_.neg & (1u << 0)) + s0 = -s0; + if (inst.inst_.abs & (1u << 1)) + s1 = std::fabs(s1); + if (inst.inst_.neg & (1u << 1)) + s1 = -s1; uint32_t lo = util::f32_to_f16_rtz(s0); uint32_t hi = util::f32_to_f16_rtz(s1); sdwa::write_lane(inst, wf, inst.vdst, lane, @@ -10173,9 +10164,15 @@ inline void execute_v_cvt_u16_f16_vop1([[maybe_unused]] Inst &inst, template inline void execute_v_cvt_u16_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - auto s = util::f16_to_f32_simd(a); - return ::rocjitsu::amdgpu::simd_cvt_u16_f32_to_u32(s); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&inst](auto a) { + if (inst.inst_.abs & 1u) + a &= ~0x8000u; + if (inst.inst_.neg & 1u) + a ^= 0x8000u; + return ([](auto a) { + auto s = util::f16_to_f32_simd(a); + return ::rocjitsu::amdgpu::simd_cvt_u16_f32_to_u32(s); + })(a); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -10183,8 +10180,15 @@ inline void execute_v_cvt_u16_f16_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, [&]() -> uint32_t { - float s = util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + float s = [&]() { + float sv = util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(); if (std::isnan(s) || s < 0.0f) return 0u; if (s >= 65536.0f) @@ -11146,12 +11150,16 @@ inline void execute_v_dot8c_i32_i4_vop3([[maybe_unused]] Inst &inst, template inline void execute_v_exp_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { if (wf.fp16_ovfl()) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_ovfl_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } else { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_simd(util::exp_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -11160,17 +11168,26 @@ inline void execute_v_exp_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, - amdgpu::transcendental::exp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } template inline void execute_v_exp_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16([](auto a) { return util::exp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -11178,31 +11195,26 @@ inline void execute_v_exp_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sdwa::write_lane( inst, wf, inst.vdst, lane, amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst.inst_.omod), + wf.fp16_ovfl()); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -11215,48 +11227,52 @@ inline void execute_v_exp_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_exp_f32_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [](auto a) { return util::exp_f32_simd(a); }); + fp_mode::ScopedEnvironment environment(0); + ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [&wf](auto a) { + return util::exp_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::transcendental::exp_f32( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())))); } } template inline void execute_v_exp_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP(float32_t, float32_t, - [](auto a) { return util::exp_f32_simd(a); }); + fp_mode::ScopedEnvironment environment(0); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP( + float32_t, float32_t, + [&wf](auto a) { + return util::exp_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true /* force_output_flush */); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { float v = [&]() { - float v = amdgpu::transcendental::exp_f32([&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + float v = amdgpu::transcendental::exp_f32( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), + inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -11481,16 +11497,9 @@ inline void execute_v_floor_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -11554,13 +11563,9 @@ template inline void execute_v_fma_dx9_zero_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32([&wf](auto a, auto b, auto c) { - return decltype(a)([&](auto i) { - return amdgpu::fp_mode::arithmetic( - static_cast(a[i]), static_cast(b[i]), static_cast(c[i]), - wf.fp_round_mode_f32(), 0); - }); - }); + ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32( + [&wf](auto a, auto b, auto c) { return amdgpu::fma_dx9_zero_f32_simd(a, b, c, wf); }, + true /* apply_omod */, true /* force_output_flush */); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11568,9 +11573,7 @@ inline void execute_v_fma_dx9_zero_f32_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = @@ -11599,20 +11602,20 @@ inline void execute_v_fma_dx9_zero_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }(), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), + inst.inst_.omod) != 0)); + const uint32_t effective_omod = + amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), inst.inst_.omod); + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); - if (inst.inst_.clamp) + if (inst.inst_.clamp) { + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); v = amdgpu::clamp_floating_result(v, wf); + } return v; }())); } @@ -11637,16 +11640,20 @@ inline void execute_v_fma_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] src0_bits, src1_bits, src2_bits, inst.inst_.abs & 1u, inst.inst_.abs & 2u, inst.inst_.abs & 4u, inst.inst_.neg & 1u, inst.inst_.neg & 2u, inst.inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst.inst_.clamp, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane(inst, wf, inst.vdst, lane, result); } } template inline void execute_v_fma_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32( - [](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP32([&inst, &wf](auto a, auto b, auto c) { + return amdgpu::fma_f32_simd(a, b, c, wf, + amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11654,9 +11661,7 @@ inline void execute_v_fma_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = @@ -11685,20 +11690,20 @@ inline void execute_v_fma_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sv = -sv; return sv; }(), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod) != 0)); const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); - if (inst.inst_.clamp) + if (inst.inst_.clamp) { + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); v = amdgpu::clamp_floating_result(v, wf); + } return v; }())); } @@ -11744,9 +11749,7 @@ inline void execute_v_fma_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_fma_mix_f32_vop3p([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F32(); - } + ROCJITSU_TRY_SIMD_FUSED_MIX(F32); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -11778,7 +11781,8 @@ inline void execute_v_fma_mix_f32_vop3p([[maybe_unused]] Inst &inst, b = -b; if (inst.inst_.neg & 4) c = -c; - float result = a * b + c; + float result = amdgpu::fp_mode::arithmetic( + a, b, c, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()); if (inst.inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); sdwa::write_lane(inst, wf, inst.vdst, lane, @@ -11789,7 +11793,7 @@ inline void execute_v_fma_mix_f32_vop3p([[maybe_unused]] Inst &inst, template inline void execute_v_fma_mixhi_f16_vop3p([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F16_HI(); + ROCJITSU_TRY_SIMD_FUSED_MIX(F16_HI); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -11821,10 +11825,10 @@ inline void execute_v_fma_mixhi_f16_vop3p([[maybe_unused]] Inst &inst, b = -b; if (inst.inst_.neg & 4) c = -c; - float result = a * b + c; - if (inst.inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(inst.vdst, wf, lane, 0x8u, h); } } @@ -11832,7 +11836,7 @@ inline void execute_v_fma_mixhi_f16_vop3p([[maybe_unused]] Inst &inst, template inline void execute_v_fma_mixlo_f16_vop3p([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3P_FMA_MIX_F16_LO(); + ROCJITSU_TRY_SIMD_FUSED_MIX(F16_LO); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -11864,10 +11868,10 @@ inline void execute_v_fma_mixlo_f16_vop3p([[maybe_unused]] Inst &inst, b = -b; if (inst.inst_.neg & 4) c = -c; - float result = a * b + c; - if (inst.inst_.clamp) - result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + amdgpu::fp_mode::ScopedEnvironment environment(0); + uint16_t h = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a, b, c, wf.fp_round_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf)); ::rocjitsu::amdgpu::write_vop3_true16_dst(inst.vdst, wf, lane, 0u, h); } } @@ -11885,17 +11889,19 @@ inline void execute_v_fmaak_f16_vop2([[maybe_unused]] Inst &inst, [[maybe_unused uint16_t result = amdgpu::fp_mode::fma_f16(s0, s1, k, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane(inst, wf, inst.vdst, lane, result); } } template inline void execute_v_fmaak_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); ROCJITSU_TRY_SIMD_VOP2_TERNARY( float32_t, amdgpu::RegisterAccess(wf).read_scalar(inst.simm32), - [](auto a, auto b, auto, auto k) { return util::stdx::fma(a, b, k); }); + [&wf](auto a, auto b, auto, auto k) { return amdgpu::fma_f32_simd(a, b, k, wf); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11907,7 +11913,8 @@ inline void execute_v_fmaak_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::fp_mode::arithmetic( - s0, s1, k, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + s0, s1, k, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), + wf.ieee_mode()))); } } @@ -11915,8 +11922,9 @@ template inline void execute_v_fmac_dx9_zero_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP2_TERNARY_ACC( - float32_t, 0u, [](auto a, auto b, auto d, auto) { return util::stdx::fma(a, b, d); }); + ROCJITSU_TRY_SIMD_VOP2_TERNARY_ACC(float32_t, 0u, [&wf](auto a, auto b, auto d, auto) { + return amdgpu::fma_dx9_zero_f32_simd(a, b, d, wf); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11924,11 +11932,12 @@ inline void execute_v_fmac_dx9_zero_f32_vop2([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - std::bit_cast(amdgpu::fp_mode::arithmetic( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + std::bit_cast( + amdgpu::fp_mode::arithmetic( + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vdst, lane)), + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()))); } } @@ -11937,7 +11946,8 @@ inline void execute_v_fmac_dx9_zero_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { ROCJITSU_TRY_SIMD_FMAC_VOP3_FP32( - [](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }); + [&wf](auto a, auto b, auto c) { return amdgpu::fma_dx9_zero_f32_simd(a, b, c, wf); }, + true /* force_output_flush */); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -11945,10 +11955,8 @@ inline void execute_v_fmac_dx9_zero_f32_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); - float v = amdgpu::fp_mode::arithmetic( + float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); @@ -11968,20 +11976,20 @@ inline void execute_v_fmac_dx9_zero_f32_vop3([[maybe_unused]] Inst &inst, return sv; }(), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), + inst.inst_.omod) != 0)); + const uint32_t effective_omod = + amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), inst.inst_.omod); + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); - if (inst.inst_.clamp) + if (inst.inst_.clamp) { + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); v = amdgpu::clamp_floating_result(v, wf); + } return v; }())); } @@ -12004,7 +12012,8 @@ inline void execute_v_fmac_f16_vop2([[maybe_unused]] Inst &inst, [[maybe_unused] uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane(inst, wf, inst.vdst, lane, result); } } @@ -12028,16 +12037,19 @@ inline void execute_v_fmac_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] src0_bits, src1_bits, accumulator, inst.inst_.abs & 1u, inst.inst_.abs & 2u, false, inst.inst_.neg & 1u, inst.inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane(inst, wf, inst.vdst, lane, result); } } template inline void execute_v_fmac_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP2_TERNARY_ACC( - float32_t, 0u, [](auto a, auto b, auto d, auto) { return util::stdx::fma(a, b, d); }); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP2_TERNARY_ACC(float32_t, 0u, [&wf](auto a, auto b, auto d, auto) { + return amdgpu::fma_f32_simd(a, b, d, wf); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -12049,15 +12061,18 @@ inline void execute_v_fmac_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused] std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()))); } } template inline void execute_v_fmac_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_FMAC_VOP3_FP32( - [](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_FMAC_VOP3_FP32([&inst, &wf](auto a, auto b, auto c) { + return amdgpu::fma_f32_simd(a, b, c, wf, + amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -12065,9 +12080,7 @@ inline void execute_v_fmac_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = @@ -12088,20 +12101,20 @@ inline void execute_v_fmac_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] return sv; }(), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod) != 0)); const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); - if (inst.inst_.clamp) + if (inst.inst_.clamp) { + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); v = amdgpu::clamp_floating_result(v, wf); + } return v; }())); } @@ -12171,17 +12184,19 @@ inline void execute_v_fmamk_f16_vop2([[maybe_unused]] Inst &inst, [[maybe_unused uint16_t result = amdgpu::fp_mode::fma_f16(s0, k, s2, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane(inst, wf, inst.vdst, lane, result); } } template inline void execute_v_fmamk_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); ROCJITSU_TRY_SIMD_VOP2_TERNARY( float32_t, amdgpu::RegisterAccess(wf).read_scalar(inst.simm32), - [](auto a, auto b, auto, auto k) { return util::stdx::fma(a, k, b); }); + [&wf](auto a, auto b, auto, auto k) { return amdgpu::fma_f32_simd(a, k, b, wf); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -12193,7 +12208,8 @@ inline void execute_v_fmamk_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::fp_mode::arithmetic( - s0, k, s2, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + s0, k, s2, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), + wf.ieee_mode()))); } } @@ -12314,16 +12330,9 @@ inline void execute_v_fract_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused }(); return v - std::floor(v); }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -12729,16 +12738,9 @@ inline void execute_v_frexp_mant_f32_vop3([[maybe_unused]] Inst &inst, }(), wf.fp_denorm_mode_f32()) .mantissa; - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -13018,12 +13020,16 @@ inline void execute_v_lerp_u8_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_log_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { if (wf.fp16_ovfl()) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_ovfl_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_ovfl_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } else { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_simd(util::log_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_simd(amdgpu::transcendental::log_exp_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -13032,17 +13038,26 @@ inline void execute_v_log_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, - amdgpu::transcendental::log_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))), + amdgpu::transcendental::log_exp_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } template inline void execute_v_log_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16([](auto a) { return util::log_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16( + [&wf](auto a) { + return amdgpu::transcendental::log_exp_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -13050,31 +13065,26 @@ inline void execute_v_log_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sdwa::write_lane( inst, wf, inst.vdst, lane, amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = util::f16_to_f32(static_cast( - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::log_exp_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst.inst_.omod), + wf.fp16_ovfl()); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -13087,48 +13097,52 @@ inline void execute_v_log_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_log_f32_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [](auto a) { return util::log_f32_simd(a); }); + fp_mode::ScopedEnvironment environment(0); + ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [&wf](auto a) { + return util::log_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::transcendental::log_f32( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())))); } } template inline void execute_v_log_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP(float32_t, float32_t, - [](auto a) { return util::log_f32_simd(a); }); + fp_mode::ScopedEnvironment environment(0); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP( + float32_t, float32_t, + [&wf](auto a) { + return util::log_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true /* force_output_flush */); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { float v = [&]() { - float v = amdgpu::transcendental::log_f32([&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + float v = amdgpu::transcendental::log_f32( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), + inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -13386,16 +13400,19 @@ inline void execute_v_mac_f16_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] uint16_t result = amdgpu::fp_mode::fma_f16( src0_bits, src1_bits, accumulator, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane(inst, wf, inst.vdst, lane, result); } } template inline void execute_v_mac_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP2_TERNARY_ACC( - float32_t, 0u, [](auto a, auto b, auto d, auto) { return util::stdx::fma(a, b, d); }); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP2_TERNARY_ACC(float32_t, 0u, [&wf](auto a, auto b, auto d, auto) { + return amdgpu::fma_f32_simd(a, b, d, wf); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -13407,15 +13424,18 @@ inline void execute_v_mac_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()))); } } template inline void execute_v_mac_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_FMAC_VOP3_FP32( - [](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_FMAC_VOP3_FP32([&inst, &wf](auto a, auto b, auto c) { + return amdgpu::fma_f32_simd(a, b, c, wf, + amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -13423,9 +13443,7 @@ inline void execute_v_mac_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = @@ -13446,20 +13464,20 @@ inline void execute_v_mac_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] return sv; }(), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vdst, lane)), - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod) != 0)); const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); - if (inst.inst_.clamp) + if (inst.inst_.clamp) { + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); v = amdgpu::clamp_floating_result(v, wf); + } return v; }())); } @@ -13554,16 +13572,9 @@ inline void execute_v_mad_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -13709,16 +13720,9 @@ inline void execute_v_mad_legacy_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -13808,7 +13812,8 @@ inline void execute_v_mad_mixhi_f16_vop3p([[maybe_unused]] Inst &inst, float result = a * b + c; if (inst.inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + uint16_t h = amdgpu::pseudo_scalar::round_f16_result(result, wf.fp_round_mode_f16_f64(), 0, + false, wf.fp16_ovfl(), false); ::rocjitsu::amdgpu::write_vop3_true16_dst(inst.vdst, wf, lane, 0x8u, h); } } @@ -13851,7 +13856,8 @@ inline void execute_v_mad_mixlo_f16_vop3p([[maybe_unused]] Inst &inst, float result = a * b + c; if (inst.inst_.clamp) result = amdgpu::clamp_floating_result(result, wf); - uint16_t h = util::f32_to_f16_mode(result, wf.fp16_ovfl()); + uint16_t h = amdgpu::pseudo_scalar::round_f16_result(result, wf.fp_round_mode_f16_f64(), 0, + false, wf.fp16_ovfl(), false); ::rocjitsu::amdgpu::write_vop3_true16_dst(inst.vdst, wf, lane, 0u, h); } } @@ -13974,10 +13980,11 @@ inline void execute_v_madak_f16_vop2([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_madak_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); ROCJITSU_TRY_SIMD_VOP2_TERNARY( float32_t, amdgpu::RegisterAccess(wf).read_scalar(inst.simm32), - [](auto a, auto b, auto, auto k) { return util::stdx::fma(a, b, k); }); + [&wf](auto a, auto b, auto, auto k) { return amdgpu::fma_f32_simd(a, b, k, wf); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -13989,7 +13996,8 @@ inline void execute_v_madak_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::fp_mode::arithmetic( - s0, s1, k, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + s0, s1, k, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), + wf.ieee_mode()))); } } @@ -14030,10 +14038,11 @@ inline void execute_v_madmk_f16_vop2([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_madmk_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); ROCJITSU_TRY_SIMD_VOP2_TERNARY( float32_t, amdgpu::RegisterAccess(wf).read_scalar(inst.simm32), - [](auto a, auto b, auto, auto k) { return util::stdx::fma(a, k, b); }); + [&wf](auto a, auto b, auto, auto k) { return amdgpu::fma_f32_simd(a, k, b, wf); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -14045,7 +14054,8 @@ inline void execute_v_madmk_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::fp_mode::arithmetic( - s0, k, s2, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + s0, k, s2, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), + wf.ieee_mode()))); } } @@ -14154,16 +14164,9 @@ inline void execute_v_max3_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -14253,16 +14256,9 @@ inline void execute_v_max3_num_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -14447,16 +14443,9 @@ inline void execute_v_max_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -14625,16 +14614,9 @@ inline void execute_v_max_num_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -14813,16 +14795,9 @@ inline void execute_v_maximum3_f32_vop3([[maybe_unused]] Inst &inst, auto ab = (a == b) ? (std::signbit(a) ? b : a) : (a > b ? a : b); return (ab == c) ? (std::signbit(ab) ? c : ab) : (ab > c ? ab : c); }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -14868,16 +14843,9 @@ inline void execute_v_maximum_f32_vop3([[maybe_unused]] Inst &inst, return std::signbit(a) ? b : a; return a > b ? a : b; }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -14986,16 +14954,9 @@ inline void execute_v_maximumminimum_f32_vop3([[maybe_unused]] Inst &inst, auto ab = (a == b) ? (std::signbit(a) ? b : a) : (a > b ? a : b); return (ab == c) ? (std::signbit(ab) ? ab : c) : (ab < c ? ab : c); }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -15043,16 +15004,9 @@ inline void execute_v_maxmin_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -15116,16 +15070,9 @@ inline void execute_v_maxmin_num_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -15280,25 +15227,18 @@ inline void execute_v_med3_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] }(); auto c = [&]() { float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); - if (inst.inst_.abs & (1u << 2)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 2)) - sv = -sv; - return sv; - }(); - return std::fmax(std::fmin(std::fmax(a, b), c), std::fmin(a, b)); - }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); + if (inst.inst_.abs & (1u << 2)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 2)) + sv = -sv; + return sv; + }(); + return std::fmax(std::fmin(std::fmax(a, b), c), std::fmin(a, b)); + }(); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -15391,16 +15331,9 @@ inline void execute_v_med3_num_f32_vop3([[maybe_unused]] Inst &inst, }(); return std::fmax(std::fmin(std::fmax(a, b), c), std::fmin(a, b)); }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -15555,16 +15488,9 @@ inline void execute_v_min3_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -15654,16 +15580,9 @@ inline void execute_v_min3_num_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -15848,16 +15767,9 @@ inline void execute_v_min_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -16026,16 +15938,9 @@ inline void execute_v_min_num_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -16214,16 +16119,9 @@ inline void execute_v_minimum3_f32_vop3([[maybe_unused]] Inst &inst, auto ab = (a == b) ? (std::signbit(a) ? a : b) : (a < b ? a : b); return (ab == c) ? (std::signbit(ab) ? ab : c) : (ab < c ? ab : c); }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -16269,16 +16167,9 @@ inline void execute_v_minimum_f32_vop3([[maybe_unused]] Inst &inst, return std::signbit(a) ? a : b; return a < b ? a : b; }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -16387,16 +16278,9 @@ inline void execute_v_minimummaximum_f32_vop3([[maybe_unused]] Inst &inst, auto ab = (a == b) ? (std::signbit(a) ? a : b) : (a < b ? a : b); return (ab == c) ? (std::signbit(ab) ? c : ab) : (ab > c ? ab : c); }(); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -16444,16 +16328,9 @@ inline void execute_v_minmax_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -16517,16 +16394,9 @@ inline void execute_v_minmax_num_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -16678,16 +16548,9 @@ inline void execute_v_mul_dx9_zero_f32_vop3([[maybe_unused]] Inst &inst, return sv; }(), 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -16802,9 +16665,11 @@ inline void execute_v_mul_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] uint32_t alu_causes = classify_mul_f32_vop2(inst, wf); wf.set_trapsts(wf.trapsts() | alu_causes); if (!alu_exception_trap_enables(wf)) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP2_BINARY(float32_t, std::multiplies<>{}); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP2_BINARY(float32_t, [&wf](auto a, auto b) { + return amdgpu::binary_f32_simd(a, b, wf); + }); } } uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -16816,7 +16681,7 @@ inline void execute_v_mul_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] std::bit_cast(amdgpu::fp_mode::arithmetic( std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.vsrc1, lane)), 0.0f, - wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()))); + wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), wf.ieee_mode()))); } } @@ -16825,9 +16690,12 @@ inline void execute_v_mul_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] uint32_t alu_causes = classify_mul_f32_vop3(inst, wf); wf.set_trapsts(wf.trapsts() | alu_causes); if (!alu_exception_trap_enables(wf)) { - if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32())) { - ROCJITSU_TRY_SIMD_VOP3_BINARY_FP(float32_t, std::multiplies<>{}); + { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP3_BINARY_FP(float32_t, [&inst, &wf](auto a, auto b) { + return amdgpu::binary_f32_simd( + a, b, wf, amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); + }); } } uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -16838,7 +16706,6 @@ inline void execute_v_mul_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] inst, wf, inst.vdst, lane, std::bit_cast([&]() { amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = [&]() { - amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); float v = amdgpu::fp_mode::arithmetic( [&]() { float sv = @@ -16858,17 +16725,16 @@ inline void execute_v_mul_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sv = -sv; return sv; }(), - 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); + 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32(), wf.cu().arch(), + wf.ieee_mode(), + (amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod) != 0)); const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (effective_omod == 0) + return v; + amdgpu::fp_mode::detail::ScopedFenv environment(wf.fp_round_mode_f32()); + return amdgpu::fp_mode::apply_omod_f32(v, effective_omod); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -17165,16 +17031,9 @@ inline void execute_v_mul_legacy_f32_vop3([[maybe_unused]] Inst &inst, return sv; }(), 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -17477,7 +17336,7 @@ inline void execute_v_pk_add_f16_vop3p([[maybe_unused]] Inst &inst, const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::ADD, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst.inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -17487,7 +17346,7 @@ inline void execute_v_pk_add_f16_vop3p([[maybe_unused]] Inst &inst, const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::ADD, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); sdwa::write_lane(inst, wf, inst.vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -17525,10 +17384,12 @@ inline void execute_v_pk_add_f32_vop3p([[maybe_unused]] Inst &inst, b_hi = -b_hi; uint32_t rlo = amdgpu::fp_mode::packed_f32( a_lo, b_lo, 0.0f, amdgpu::fp_mode::PackedF32Op::ADD, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint32_t rhi = amdgpu::fp_mode::packed_f32( a_hi, b_hi, 0.0f, amdgpu::fp_mode::PackedF32Op::ADD, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); sdwa::write_lane64( inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 32)); } @@ -17662,11 +17523,13 @@ inline void execute_v_pk_fma_f16_vop3p([[maybe_unused]] Inst &inst, uint16_t rlo = amdgpu::fp_mode::fma_f16( a_lo, b_lo, c_lo, false, false, false, inst.inst_.neg & 1u, inst.inst_.neg & 2u, inst.inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, - inst.inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst.inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint16_t rhi = amdgpu::fp_mode::fma_f16( a_hi, b_hi, c_hi, false, false, false, inst.inst_.neg_hi & 1u, inst.inst_.neg_hi & 2u, inst.inst_.neg_hi & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, - inst.inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + inst.inst_.clamp, wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane( inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17676,7 +17539,7 @@ template inline void execute_v_pk_fma_f32_vop3p([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { ROCJITSU_TRY_SIMD_VOP3P_PK_TERNARY_F32( - [](auto a, auto b, auto c) { return util::stdx::fma(a, b, c); }); + [&wf](auto a, auto b, auto c) { return amdgpu::fma_f32_simd(a, b, c, wf); }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -17716,10 +17579,12 @@ inline void execute_v_pk_fma_f32_vop3p([[maybe_unused]] Inst &inst, c_hi = -c_hi; uint32_t rlo = amdgpu::fp_mode::packed_f32( a_lo, b_lo, c_lo, amdgpu::fp_mode::PackedF32Op::FMA, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint32_t rhi = amdgpu::fp_mode::packed_f32( a_hi, b_hi, c_hi, amdgpu::fp_mode::PackedF32Op::FMA, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); sdwa::write_lane64( inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 32)); } @@ -17741,16 +17606,18 @@ inline void execute_v_pk_fmac_f16_vop2([[maybe_unused]] Inst &inst, uint32_t rawd = amdgpu::RegisterAccess(wf).read_lane(inst.vdst, lane); const uint32_t omod = amdgpu::sdwa::output_modifier(inst, wf); - uint32_t r0 = amdgpu::fp_mode::fma_f16(static_cast(raw0), static_cast(raw1), - static_cast(rawd), false, false, false, false, - false, false, wf.fp_round_mode_f16_f64(), - wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + uint32_t r0 = amdgpu::fp_mode::fma_f16( + static_cast(raw0), static_cast(raw1), static_cast(rawd), + false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), + wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint32_t r1 = amdgpu::fp_mode::fma_f16( static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, false, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane(inst, wf, inst.vdst, lane, r0 | (r1 << 16)); } @@ -17780,13 +17647,15 @@ inline void execute_v_pk_fmac_f16_vop3([[maybe_unused]] Inst &inst, static_cast(raw0), static_cast(raw1), static_cast(rawd), inst.inst_.abs & 1u, inst.inst_.abs & 2u, false, inst.inst_.neg & 1u, inst.inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst.inst_.clamp, - wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); uint32_t r1 = amdgpu::fp_mode::fma_f16( static_cast(raw0 >> 16), static_cast(raw1 >> 16), static_cast(rawd >> 16), inst.inst_.abs & 1u, inst.inst_.abs & 2u, false, inst.inst_.neg & 1u, inst.inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); sdwa::write_lane(inst, wf, inst.vdst, lane, r0 | (r1 << 16)); } @@ -17962,7 +17831,7 @@ inline void execute_v_pk_max_f16_vop3p([[maybe_unused]] Inst &inst, const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAX, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst.inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -17972,7 +17841,7 @@ inline void execute_v_pk_max_f16_vop3p([[maybe_unused]] Inst &inst, const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MAX, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); sdwa::write_lane(inst, wf, inst.vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -18070,7 +17939,7 @@ inline void execute_v_pk_min_f16_vop3p([[maybe_unused]] Inst &inst, const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MIN, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst.inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -18080,7 +17949,7 @@ inline void execute_v_pk_min_f16_vop3p([[maybe_unused]] Inst &inst, const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MIN, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); sdwa::write_lane(inst, wf, inst.vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -18199,7 +18068,7 @@ inline void execute_v_pk_mul_f16_vop3p([[maybe_unused]] Inst &inst, const uint16_t rlo = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MUL, a_lo, b_lo, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); if (inst.inst_.neg_hi & 1u) a_hi ^= 0x8000u; @@ -18209,7 +18078,7 @@ inline void execute_v_pk_mul_f16_vop3p([[maybe_unused]] Inst &inst, const uint16_t rhi = amdgpu::fp_mode::packed_binary_f16( amdgpu::fp_mode::PackedBinaryOp::MUL, a_hi, b_hi, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), inst.inst_.clamp, wf.fp16_ovfl(), - amdgpu::floating_clamp_nan_to_zero(wf)); + amdgpu::floating_clamp_nan_to_zero(wf), wf.cu().arch(), wf.ieee_mode()); sdwa::write_lane(inst, wf, inst.vdst, lane, rlo | (static_cast(rhi) << 16)); } @@ -18247,10 +18116,12 @@ inline void execute_v_pk_mul_f32_vop3p([[maybe_unused]] Inst &inst, b_hi = -b_hi; uint32_t rlo = amdgpu::fp_mode::packed_f32( a_lo, b_lo, 0.0f, amdgpu::fp_mode::PackedF32Op::MUL, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); uint32_t rhi = amdgpu::fp_mode::packed_f32( a_hi, b_hi, 0.0f, amdgpu::fp_mode::PackedF32Op::MUL, wf.fp_round_mode_f32(), - wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f32(), inst.inst_.clamp, amdgpu::floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); sdwa::write_lane64( inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 32)); } @@ -18350,12 +18221,18 @@ inline void execute_v_pk_sub_u16_vop3p([[maybe_unused]] Inst &inst, template inline void execute_v_rcp_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { if (wf.fp16_ovfl()) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_ovfl_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } else { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_simd(util::rcp_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -18364,17 +18241,26 @@ inline void execute_v_rcp_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, - amdgpu::transcendental::rcp_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } template inline void execute_v_rcp_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16([](auto a) { return util::rcp_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -18382,31 +18268,26 @@ inline void execute_v_rcp_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sdwa::write_lane( inst, wf, inst.vdst, lane, amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = util::f16_to_f32(static_cast( - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst.inst_.omod), + wf.fp16_ovfl()); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -18419,48 +18300,52 @@ inline void execute_v_rcp_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_rcp_f32_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [](auto a) { return util::rcp_f32_simd(a); }); + fp_mode::ScopedEnvironment environment(0); + ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [&wf](auto a) { + return util::rcp_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::transcendental::rcp_f32( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())))); } } template inline void execute_v_rcp_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP(float32_t, float32_t, - [](auto a) { return util::rcp_f32_simd(a); }); + fp_mode::ScopedEnvironment environment(0); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP( + float32_t, float32_t, + [&wf](auto a) { + return util::rcp_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true /* force_output_flush */); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { float v = [&]() { - float v = amdgpu::transcendental::rcp_f32([&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + float v = amdgpu::transcendental::rcp_f32( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), + inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -18565,16 +18450,9 @@ inline void execute_v_rcp_iflag_f32_vop3([[maybe_unused]] Inst &inst, sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -18679,22 +18557,15 @@ inline void execute_v_rndne_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused float v = util::rndne_scalar([&]() { float sv = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }()); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -18757,12 +18628,18 @@ inline void execute_v_rndne_f64_vop3([[maybe_unused]] Inst &inst, [[maybe_unused template inline void execute_v_rsq_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { if (wf.fp16_ovfl()) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_ovfl_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } else { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_simd(util::rsq_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -18771,17 +18648,26 @@ inline void execute_v_rsq_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, - amdgpu::transcendental::rsq_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } template inline void execute_v_rsq_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16([](auto a) { return util::rsq_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -18789,31 +18675,26 @@ inline void execute_v_rsq_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sdwa::write_lane( inst, wf, inst.vdst, lane, amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = util::f16_to_f32(static_cast( - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst.inst_.omod), + wf.fp16_ovfl()); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -18826,48 +18707,52 @@ inline void execute_v_rsq_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_rsq_f32_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [](auto a) { return util::rsq_f32_simd(a); }); + fp_mode::ScopedEnvironment environment(0); + ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [&wf](auto a) { + return util::rsq_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::transcendental::rsq_f32( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())))); } } template inline void execute_v_rsq_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP(float32_t, float32_t, - [](auto a) { return util::rsq_f32_simd(a); }); + fp_mode::ScopedEnvironment environment(0); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP( + float32_t, float32_t, + [&wf](auto a) { + return util::rsq_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true /* force_output_flush */); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { float v = [&]() { - float v = amdgpu::transcendental::rsq_f32([&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + float v = amdgpu::transcendental::rsq_f32( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), + inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -19098,10 +18983,13 @@ inline void execute_v_sin_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, - amdgpu::transcendental::sin_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } @@ -19115,31 +19003,26 @@ inline void execute_v_sin_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sdwa::write_lane( inst, wf, inst.vdst, lane, amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = util::f16_to_f32(static_cast( - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst.inst_.omod), + wf.fp16_ovfl()); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -19159,7 +19042,8 @@ inline void execute_v_sin_f32_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::transcendental::sin_f32( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + wf.fp_denorm_mode_f32(), amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())))); } } @@ -19172,25 +19056,21 @@ inline void execute_v_sin_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { float v = [&]() { - float v = amdgpu::transcendental::sin_f32([&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + float v = amdgpu::transcendental::sin_f32( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f32(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -19202,12 +19082,18 @@ inline void execute_v_sin_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_sqrt_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { if (wf.fp16_ovfl()) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_ovfl_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_ovfl_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } else { - ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [](auto a) { - return util::f32_to_f16_simd(util::sqrt_f32_simd(util::f16_to_f32_simd(a))); + ROCJITSU_TRY_SIMD_VOP1_UNARY(uint32_t, uint32_t, [&wf](auto a) { + return util::f32_to_f16_simd( + amdgpu::transcendental::map_f16_simd( + util::f16_to_f32_simd(a), wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode()))); }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -19216,22 +19102,26 @@ inline void execute_v_sqrt_f16_vop1([[maybe_unused]] Inst &inst, [[maybe_unused] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, - amdgpu::transcendental::sqrt_f32(util::f16_to_f32( - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)))), + amdgpu::transcendental::map_f16( + util::f16_to_f32( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), wf.fp16_ovfl())); } } template inline void execute_v_sqrt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16([](auto a) { - auto r = util::stdx::sqrt(a); - util::stdx::where(util::stdx::isnan(a), r) = a; - util::stdx::where(a < 0.0f, r) = std::numeric_limits::quiet_NaN(); - return r; - }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP16( + [&wf](auto a) { + return amdgpu::transcendental::map_f16_simd( + a, wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true); uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) @@ -19239,31 +19129,26 @@ inline void execute_v_sqrt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] sdwa::write_lane( inst, wf, inst.vdst, lane, amdgpu::fp_mode::finalize_omod_f16( - amdgpu::sdwa::round_f16_result( + amdgpu::sdwa::finish_rounded_f16( inst, wf, [&]() { - float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = util::f16_to_f32(static_cast( - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_f16_omod( - wf.cu().arch(), wf.fp_denorm_mode_f16_f64(), wf.ieee_mode(), false, - inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; - }(); + float v = amdgpu::fp_mode::apply_omod_f16( + amdgpu::transcendental::map_f16( + [&]() { + float sv = util::f16_to_f32(static_cast( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + wf.fp_denorm_mode_f16_f64(), wf.fp16_ovfl(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())), + amdgpu::fp_mode::effective_f16_omod(wf.cu().arch(), + wf.fp_denorm_mode_f16_f64(), + wf.ieee_mode(), false, inst.inst_.omod), + wf.fp16_ovfl()); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); return v; @@ -19276,57 +19161,61 @@ inline void execute_v_sqrt_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] template inline void execute_v_sqrt_f32_vop1([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + fp_mode::ScopedEnvironment environment(0); uint32_t alu_causes = classify_sqrt_f32_vop1(inst, wf); wf.set_trapsts(wf.trapsts() | alu_causes); if (!alu_exception_trap_enables(wf)) { - ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, - [](auto a) { return util::sqrt_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP1_UNARY(float32_t, float32_t, [&wf](auto a) { + return util::sqrt_f32_simd(a, amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast(amdgpu::transcendental::sqrt_f32( - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))))); + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())))); } } template inline void execute_v_sqrt_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + fp_mode::ScopedEnvironment environment(0); uint32_t alu_causes = classify_sqrt_f32_vop3(inst, wf); wf.set_trapsts(wf.trapsts() | alu_causes); if (!alu_exception_trap_enables(wf)) { - ROCJITSU_TRY_SIMD_VOP3_UNARY_FP(float32_t, float32_t, - [](auto a) { return util::sqrt_f32_simd(a); }); + ROCJITSU_TRY_SIMD_VOP3_UNARY_FP( + float32_t, float32_t, + [&wf](auto a) { + return util::sqrt_f32_simd(a, + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + }, + true /* force_output_flush */); } uint64_t exec = dpp::execution_lane_mask(inst, wf); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( + sdwa::write_lane( inst, wf, inst.vdst, lane, std::bit_cast([&]() { float v = [&]() { - float v = amdgpu::transcendental::sqrt_f32([&]() { - float sv = - std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); - if (inst.inst_.abs & (1u << 0)) - sv = std::fabs(sv); - if (inst.inst_.neg & (1u << 0)) - sv = -sv; - return sv; - }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + float v = amdgpu::transcendental::sqrt_f32( + [&]() { + float sv = + std::bit_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); + if (inst.inst_.abs & (1u << 0)) + sv = std::fabs(sv); + if (inst.inst_.neg & (1u << 0)) + sv = -sv; + return sv; + }(), + amdgpu::fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), 0, wf.ieee_mode(), + inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -19596,6 +19485,14 @@ inline void execute_v_sub_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_sub_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + if (wf.exec() != 0 && !amdgpu::simd_force_scalar() && !wf.cu().observes_register_access() && + !wf.cu().debug_active() && !inst.src0.delegate() && !inst.vsrc1.delegate() && + !inst.vdst.delegate()) { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP2_BINARY(float32_t, [&wf](auto a, auto b) { + return amdgpu::binary_f32_simd(a, b, wf); + }); + } if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { ROCJITSU_TRY_SIMD_VOP2_BINARY(float32_t, std::minus<>{}); } @@ -19614,6 +19511,15 @@ inline void execute_v_sub_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] template inline void execute_v_sub_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + if (wf.exec() != 0 && !amdgpu::simd_force_scalar() && !wf.cu().observes_register_access() && + !wf.cu().debug_active() && !inst.src0.delegate() && !inst.src1.delegate() && + !inst.vdst.delegate()) { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP3_BINARY_FP(float32_t, [&inst, &wf](auto a, auto b) { + return amdgpu::binary_f32_simd( + a, b, wf, amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); + }); + } if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { ROCJITSU_TRY_SIMD_VOP3_BINARY_FP(float32_t, std::minus<>{}); } @@ -19646,16 +19552,9 @@ inline void execute_v_sub_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] return sv; }(), 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -20129,6 +20028,14 @@ inline void execute_v_subrev_f16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_subrev_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + if (wf.exec() != 0 && !amdgpu::simd_force_scalar() && !wf.cu().observes_register_access() && + !wf.cu().debug_active() && !inst.src0.delegate() && !inst.vsrc1.delegate() && + !inst.vdst.delegate()) { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP2_BINARY(float32_t, [&wf](auto a, auto b) { + return amdgpu::binary_f32_simd(b, a, wf); + }); + } if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { ROCJITSU_TRY_SIMD_VOP2_BINARY(float32_t, [](auto a, auto b) { return b - a; }); } @@ -20147,6 +20054,15 @@ inline void execute_v_subrev_f32_vop2([[maybe_unused]] Inst &inst, [[maybe_unuse template inline void execute_v_subrev_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] Wavefront &wf) { + if (wf.exec() != 0 && !amdgpu::simd_force_scalar() && !wf.cu().observes_register_access() && + !wf.cu().debug_active() && !inst.src0.delegate() && !inst.src1.delegate() && + !inst.vdst.delegate()) { + amdgpu::fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + ROCJITSU_TRY_SIMD_VOP3_BINARY_FP(float32_t, [&inst, &wf](auto a, auto b) { + return amdgpu::binary_f32_simd( + b, a, wf, amdgpu::effective_vop3_omod_f32(wf, inst.inst_.omod)); + }); + } if (amdgpu::fp_mode::native_arithmetic_matches(wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32())) { ROCJITSU_TRY_SIMD_VOP3_BINARY_FP(float32_t, [](auto a, auto b) { return b - a; }); } @@ -20179,16 +20095,9 @@ inline void execute_v_subrev_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse return sv; }(), 0.0f, wf.fp_round_mode_f32(), wf.fp_denorm_mode_f32()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); @@ -20427,16 +20336,9 @@ inline void execute_v_trunc_f32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused sv = -sv; return sv; }()); - const uint32_t effective_omod = amdgpu::fp_mode::effective_omod( - wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), inst.inst_.omod); - if (effective_omod == 1) - v *= 2.0f; - else if (effective_omod == 2) - v *= 4.0f; - else if (effective_omod == 3) - v *= 0.5f; - v = amdgpu::fp_mode::finalize_omod_f32(v, effective_omod); - return v; + return amdgpu::fp_mode::apply_omod_f32( + v, amdgpu::fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod)); }(); if (inst.inst_.clamp) v = amdgpu::clamp_floating_result(v, wf); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/isa_properties.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/isa_properties.h index 0b079d48b54b..d6b82b8c7980 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/isa_properties.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/isa_properties.h @@ -38,6 +38,8 @@ struct IsaProperties { uint32_t compute_tmpring_wavesize_bits = 0; uint32_t wave_size = 0; uint32_t wave_size_max = 0; + uint8_t vmcnt_capacity = 0; ///< Zero when VMCNT is absent. + uint8_t lgkmcnt_capacity = 0; ///< Zero when LGKMCNT is absent. uint32_t max_addressable_vgprs_per_wf = 0; uint32_t descriptor_vgpr_count_granule_wave32 = 0; uint32_t descriptor_vgpr_count_granule_wave64 = 0; @@ -60,6 +62,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 13, .wave_size = 64, .wave_size_max = 64, + .vmcnt_capacity = 63, + .lgkmcnt_capacity = 15, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 0, .descriptor_vgpr_count_granule_wave64 = 4, @@ -77,6 +81,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 13, .wave_size = 64, .wave_size_max = 64, + .vmcnt_capacity = 63, + .lgkmcnt_capacity = 15, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 0, .descriptor_vgpr_count_granule_wave64 = 8, @@ -94,6 +100,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 13, .wave_size = 64, .wave_size_max = 64, + .vmcnt_capacity = 63, + .lgkmcnt_capacity = 15, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 0, .descriptor_vgpr_count_granule_wave64 = 8, @@ -111,6 +119,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 13, .wave_size = 64, .wave_size_max = 64, + .vmcnt_capacity = 63, + .lgkmcnt_capacity = 15, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 0, .descriptor_vgpr_count_granule_wave64 = 8, @@ -128,6 +138,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 13, .wave_size = 32, .wave_size_max = 64, + .vmcnt_capacity = 63, + .lgkmcnt_capacity = 63, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 8, .descriptor_vgpr_count_granule_wave64 = 4, @@ -145,6 +157,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 13, .wave_size = 32, .wave_size_max = 64, + .vmcnt_capacity = 63, + .lgkmcnt_capacity = 63, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 8, .descriptor_vgpr_count_granule_wave64 = 4, @@ -162,6 +176,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 15, .wave_size = 32, .wave_size_max = 64, + .vmcnt_capacity = 63, + .lgkmcnt_capacity = 63, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 8, .descriptor_vgpr_count_granule_wave64 = 4, @@ -179,6 +195,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 15, .wave_size = 32, .wave_size_max = 64, + .vmcnt_capacity = 63, + .lgkmcnt_capacity = 63, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 8, .descriptor_vgpr_count_granule_wave64 = 4, @@ -196,6 +214,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 18, .wave_size = 32, .wave_size_max = 64, + .vmcnt_capacity = 0, + .lgkmcnt_capacity = 0, .max_addressable_vgprs_per_wf = 256, .descriptor_vgpr_count_granule_wave32 = 8, .descriptor_vgpr_count_granule_wave64 = 4, @@ -213,6 +233,8 @@ inline constexpr uint32_t MAX_SUPPORTED_ADDRESSABLE_VGPRS_PER_WF = 1024; .compute_tmpring_wavesize_bits = 18, .wave_size = 32, .wave_size_max = 32, + .vmcnt_capacity = 0, + .lgkmcnt_capacity = 0, .max_addressable_vgprs_per_wf = 1024, .descriptor_vgpr_count_granule_wave32 = 16, .descriptor_vgpr_count_granule_wave64 = 0, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_buffer.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_buffer.h index 6f92fa086e19..ff80023329ee 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_buffer.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_buffer.h @@ -13,6 +13,7 @@ #include "rocjitsu/isa/arch/amdgpu/shared/buffer_address.h" #include "rocjitsu/isa/arch/amdgpu/shared/scalar_operand_read.h" +#include "rocjitsu/isa/arch/amdgpu/shared/scalar_static_resolve.h" #include "rocjitsu/isa/isa_traits.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" #include "rocjitsu/vm/amdgpu/mem_state.h" @@ -42,6 +43,14 @@ inline bool buffer_resource_range_is_backed(const Wavefront &wf, uint32_t select return scalar_selector_range_is_backed(wf, selector, 4); } +/// MUBUF/MTBUF SOFFSET accepts scalar registers and inline integer/float constants. +/// Constants supply their raw 32-bit value as an unsigned byte offset. +inline std::optional buffer_scalar_offset(Wavefront &wf, uint32_t selector) { + if (auto constant = resolve_src_scalar_statically(selector)) + return static_cast(*constant); + return try_read_scalar_selector(wf, selector); +} + constexpr uint32_t buffer_offset_part(uint32_t voffset, int64_t inst_offset) { // Hardware forms this sub-expression in 32-bit offset space before it is // widened and added to the descriptor base address. @@ -292,16 +301,12 @@ void mubuf_calculate_addresses(const MubufInst &inst, amdgpu::Wavefront &wf, Vec uint32_t srd2 = amdgpu::read_scalar_selector(wf, sb_sel + 2); uint32_t srd3 = amdgpu::read_scalar_selector(wf, sb_sel + 3); uint64_t base_addr = (static_cast(srd1 & 0xFFFF) << 32) | srd0; - // soffset field: 0-105 = SGPR index, 128 (0x80) = inline constant 0. - uint32_t soffset_val = 0; - if (inst.soffset != 0x80) { - auto soffset = amdgpu::try_read_scalar_selector(wf, inst.soffset); - if (!soffset) { - reject_vector_memory_access(d); - return; - } - soffset_val = *soffset; + auto soffset = buffer_scalar_offset(wf, inst.soffset); + if (!soffset) { + reject_vector_memory_access(d); + return; } + const uint32_t soffset_val = *soffset; // Buffer bounds checking: OOB loads return 0, OOB stores are dropped. // num_records is the buffer size in bytes, or in records when index-checked. uint32_t num_records = srd2; @@ -441,15 +446,12 @@ void mtbuf_calculate_addresses(const MtbufInst &inst, amdgpu::Wavefront &wf, Vec uint32_t srd2 = amdgpu::read_scalar_selector(wf, sb_sel + 2); uint32_t srd3 = amdgpu::read_scalar_selector(wf, sb_sel + 3); uint64_t base_addr = (static_cast(srd1 & 0xFFFF) << 32) | srd0; - uint32_t soffset_val = 0; - if (inst.soffset != 0x80) { - auto soffset = amdgpu::try_read_scalar_selector(wf, inst.soffset); - if (!soffset) { - reject_vector_memory_access(d); - return; - } - soffset_val = *soffset; + auto soffset = buffer_scalar_offset(wf, inst.soffset); + if (!soffset) { + reject_vector_memory_access(d); + return; } + const uint32_t soffset_val = *soffset; uint32_t num_records = srd2; uint32_t stride = (srd1 >> 16) & 0x3FFF; const auto rdna_oob_select = static_cast((srd3 >> 28) & 0x3); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/alu_exceptions.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/alu_exceptions.h index fd9194378788..23ae3fafc447 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/alu_exceptions.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/alu_exceptions.h @@ -9,6 +9,7 @@ #include "rocjitsu/vm/amdgpu/register_access.h" #include "rocjitsu/vm/amdgpu/wavefront.h" #include +#include #include #include #include @@ -38,6 +39,49 @@ inline uint32_t alu_exception_trap_enables(const Wavefront &wf) { // earlier instruction latched. Trap delivery needs the former, so a classifier // that grows a new early return has to report on that path too. +namespace detail { + +// A host F32-to-F64 conversion can flush subnormals under DAZ. Form those +// values from their integer significands; nonzero finite F32 values are normal in F64. +inline double widen_alu_f32(float value) { + const uint32_t bits = std::bit_cast(value); + const uint32_t magnitude = bits & 0x7fffffffu; + if (magnitude < 0x00800000u) { + const double widened = static_cast(magnitude) * 0x1p-149; + return (bits >> 31) ? -widened : widened; + } + return static_cast(value); +} + +template +uint32_t classify_mul_lanes(const Lhs &lhs, const Rhs &rhs, Wavefront &wf, Classify classify) { + uint32_t causes = 0; + const uint64_t exec = wf.exec(); + RegisterAccess regs(wf); + auto visit = [&](auto read_lhs, auto read_rhs) { + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) + if (exec & (1ULL << lane)) + causes |= + classify(std::bit_cast(read_lhs(lane)), std::bit_cast(read_rhs(lane))); + }; + // Views resolve contiguous VGPR storage and scalar broadcasts once while + // retaining register ownership checks and plugin-visible active-lane reads. + // Delegates and sub-dword selections keep their per-lane operand semantics. + if (exec && lhs.simd_capable() && rhs.simd_capable()) { + auto lhs_view = regs.read_operand(lhs, exec); + auto rhs_view = regs.read_operand(rhs, exec); + visit([&](uint32_t lane) { return lhs_view.lane(lane); }, + [&](uint32_t lane) { return rhs_view.lane(lane); }); + } else { + visit([&](uint32_t lane) { return regs.read_lane(lhs, lane); }, + [&](uint32_t lane) { return regs.read_lane(rhs, lane); }); + } + wf.raise_alu_causes(causes); + return causes; +} + +} // namespace detail + /// @brief EXCP causes raised by `lhs * rhs`, optionally scaled by an output /// modifier. /// @details @p omod_scale must be applied to the exact and the rounded value @@ -45,44 +89,41 @@ inline uint32_t alu_exception_trap_enables(const Wavefront &wf) { /// product instead reports every nonzero product as inexact the moment an /// output modifier is present, because scaling by 2 changes the value it is /// being compared against. +/// @param omod_scale One of the architectural scales: 1, 2, 4, or 0.5. inline uint32_t classify_mul_f32(float lhs, float rhs, float omod_scale = 1.0f) { + assert(omod_scale == 1.0f || omod_scale == 2.0f || omod_scale == 4.0f || omod_scale == 0.5f); uint32_t causes = 0; if (std::fpclassify(lhs) == FP_SUBNORMAL || std::fpclassify(rhs) == FP_SUBNORMAL) causes |= 1u << 1; - const long double exact = - static_cast(lhs) * static_cast(rhs) * omod_scale; + // An F32 product needs at most 48 significand bits. The power-of-two OMOD + // scales (1, 2, 4, 0.5) preserve that precision and stay normal in F64. + const double exact = detail::widen_alu_f32(lhs) * detail::widen_alu_f32(rhs) * omod_scale; const float result = (lhs * rhs) * omod_scale; if (std::isfinite(lhs) && std::isfinite(rhs) && std::isinf(result)) causes |= 1u << 3; - if (exact != 0.0L && (result == 0.0f || std::fpclassify(result) == FP_SUBNORMAL)) + if (exact != 0.0 && (result == 0.0f || std::fpclassify(result) == FP_SUBNORMAL)) causes |= 1u << 4; - if (std::isfinite(exact) && static_cast(result) != exact) + if (std::isfinite(exact) && detail::widen_alu_f32(result) != exact) causes |= 1u << 5; return causes; } template uint32_t classify_mul_f32_vop2(const Inst &inst, Wavefront &wf) { - uint32_t causes = 0; - const uint64_t exec = wf.exec(); - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (!(exec & (1ULL << lane))) - continue; - const float lhs = std::bit_cast(RegisterAccess(wf).read_lane(inst.src0, lane)); - const float rhs = std::bit_cast(RegisterAccess(wf).read_lane(inst.vsrc1, lane)); - causes |= classify_mul_f32(lhs, rhs); - } - wf.raise_alu_causes(causes); - return causes; + return detail::classify_mul_lanes( + inst.src0, inst.vsrc1, wf, [](float lhs, float rhs) { return classify_mul_f32(lhs, rhs); }); } template uint32_t classify_mul_f32_vop3(const Inst &inst, Wavefront &wf) { - uint32_t causes = 0; - const uint64_t exec = wf.exec(); - for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { - if (!(exec & (1ULL << lane))) - continue; - float lhs = std::bit_cast(RegisterAccess(wf).read_lane(inst.src0, lane)); - float rhs = std::bit_cast(RegisterAccess(wf).read_lane(inst.src1, lane)); + // OMOD scales the exact and rounded products alike. Its policy is uniform + // across the wave, including which causes the target suppresses. + const uint32_t omod = fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), + wf.ieee_mode(), inst.inst_.omod); + const float omod_scale = omod == 1 ? 2.0f : omod == 2 ? 4.0f : omod == 3 ? 0.5f : 1.0f; + const uint32_t suppressed = omod != 0 && (wf.cu().arch() == ROCJITSU_CODE_ARCH_RDNA4 || + wf.cu().arch() == ROCJITSU_CODE_ARCH_CDNA5) + ? (1u << 4) | (1u << 5) + : 0; + return detail::classify_mul_lanes(inst.src0, inst.src1, wf, [&](float lhs, float rhs) { if (inst.inst_.abs & 1u) lhs = std::fabs(lhs); if (inst.inst_.neg & 1u) @@ -91,25 +132,8 @@ template uint32_t classify_mul_f32_vop3(const Inst &inst, Wavefr rhs = std::fabs(rhs); if (inst.inst_.neg & 2u) rhs = -rhs; - // OMOD scales the result, so it has to take part in deriving the causes - // rather than being compared against the unscaled product afterwards. - const uint32_t omod = fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), - wf.ieee_mode(), inst.inst_.omod); - float omod_scale = 1.0f; - if (omod == 1) - omod_scale = 2.0f; - else if (omod == 2) - omod_scale = 4.0f; - else if (omod == 3) - omod_scale = 0.5f; - uint32_t lane_causes = classify_mul_f32(lhs, rhs, omod_scale); - if (omod != 0 && - (wf.cu().arch() == ROCJITSU_CODE_ARCH_RDNA4 || wf.cu().arch() == ROCJITSU_CODE_ARCH_CDNA5)) - lane_causes &= ~((1u << 4) | (1u << 5)); - causes |= lane_causes; - } - wf.raise_alu_causes(causes); - return causes; + return classify_mul_f32(lhs, rhs, omod_scale) & ~suppressed; + }); } template uint32_t classify_sqrt_f32_vop1(const Inst &inst, Wavefront &wf) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/comparison.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/comparison.h new file mode 100644 index 000000000000..4f0c6decc76b --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/comparison.h @@ -0,0 +1,187 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +/// @file comparison.h +/// @brief Floating-point VOPC relations evaluated in hardware stage order. +/// +/// Each source passes through the same stages before the relation sees it: +/// +/// 1. Source modifiers: ABS clears, then NEG flips, the sign bit. +/// 2. Input flush: when MODE disables input denormals, a subnormal becomes a +/// zero of the same sign. The output is a lane mask, so nothing is flushed +/// afterwards. +/// 3. Order key: NaNs are set aside and the remaining encodings map to +/// unsigned keys that order like their values, with -0 equal to +0. +/// 4. Relation: the operation template parameter, called directly on the keys. +/// +/// The VOPC encoding lists the sixteen float relations as eight ordered +/// predicates followed by their negations in reverse order: opcode 15 - k is +/// the negation of opcode k. An ordered predicate is false when either source +/// is NaN, so its negation is true there: +/// +/// F LT EQ LE GT LG GE O (0-7, ordered) +/// T NLT NEQ NLE NGT NLG NGE U (15-8, negated) +/// +/// Every stage works on raw encodings in unsigned integer lanes, never on host +/// floats, so host DAZ cannot flush a subnormal the guest MODE preserves. The +/// same templates accept a scalar lane or a std::experimental::simd of lanes; +/// F16 occupies the low half of a 32-bit lane as in the VOPC SIMD path. Lane +/// values above the format width are ignored. CLASS tests read the raw +/// encoding and do not use these stages. + +#include +#include +#include + +namespace rocjitsu::amdgpu::comparison { + +/// @brief Binary interchange format carried in an unsigned lane type. +template struct Format { + using Lane = LaneType; + static constexpr unsigned kExponentBits = ExponentBits; + static constexpr unsigned kMantissaBits = MantissaBits; + static constexpr unsigned kWidth = 1 + ExponentBits + MantissaBits; + static constexpr Lane kSign = Lane{1} << (kWidth - 1); + static constexpr Lane kMagnitude = kSign - 1; + static constexpr Lane kBits = kSign | kMagnitude; + static constexpr Lane kExponentMax = (Lane{1} << ExponentBits) - 1; + static constexpr Lane kInfinity = kExponentMax << MantissaBits; + + static_assert(std::is_unsigned_v && kWidth <= 8 * sizeof(Lane)); +}; + +using F16 = Format; +using F32 = Format; +using F64 = Format; + +/// @brief Per-instruction compare policy, fixed before any lane is evaluated. +struct Policy { + bool flush_inputs = false; + + /// @details Every ISA manual applies the MODE denormal controls to all + /// floating-point operations, with no exception for compares; gfx1201 + /// captures across every MODE.FP_DENORM setting confirm it. + /// @param denorm_mode MODE.FP_DENORM field of the source format; bit 0 allows input denormals. + static constexpr Policy make(uint32_t denorm_mode) { return {(denorm_mode & 1u) == 0}; } +}; + +/// @brief A relation: an operation on order keys, optionally negated. +template struct Relation { + using Operation = Op; + static constexpr bool kNegated = Negated; +}; + +namespace detail { + +/// @brief Whether V is the format's lane type or a SIMD vector of it. +template +inline constexpr bool is_lane_v = std::is_same_v || requires { + requires std::is_same_v; +}; + +/// @brief All ones where the exponent field is nonzero, zero elsewhere. +/// @details The field plus its maximum carries into the next bit exactly when +/// the field is nonzero, which avoids a mask type. +template constexpr V normal_or_special(V bits) { + const V field = (bits & Fmt::kInfinity) >> Fmt::kMantissaBits; + return typename Fmt::Lane{0} - ((field + Fmt::kExponentMax) >> Fmt::kExponentBits); +} + +/// @brief All ones where the magnitude is nonzero, zero for either signed zero. +template constexpr V nonzero(V bits) { + const V magnitude = bits & Fmt::kMagnitude; + return typename Fmt::Lane{0} - ((magnitude + Fmt::kMagnitude) >> (Fmt::kWidth - 1)); +} + +/// @brief The ordered predicate F: false on every lane, in the key comparison's result type. +struct Never { + template constexpr auto operator()(const V &a, const V &) const { return a != a; } +}; + +/// @brief The ordered predicate O: true on every lane that reaches the operation. +struct Always { + template constexpr auto operator()(const V &a, const V &) const { return a == a; } +}; + +} // namespace detail + +using F = Relation; +using Lt = Relation>; +using Eq = Relation>; +using Le = Relation>; +using Gt = Relation>; +using Lg = Relation>; +using Ge = Relation>; +using O = Relation; +using U = Relation; +using Nge = Relation, true>; +using Nlg = Relation, true>; +using Ngt = Relation, true>; +using Nle = Relation, true>; +using Neq = Relation, true>; +using Nlt = Relation, true>; +using T = Relation; + +/// @brief Stage 1: apply VOP3 or DPP source modifiers to one source. +template constexpr V modify(V bits, bool absolute, bool negate) { + static_assert(detail::is_lane_v); + if (absolute) + bits = bits & Fmt::kMagnitude; + if (negate) + bits = bits ^ Fmt::kSign; + return bits; +} + +/// @brief Stage 2: flush a subnormal source to a zero of the same sign. +/// @details NaN, infinity, zero and normal encodings pass through unchanged. +template constexpr V flush_input(V bits, const Policy &policy) { + static_assert(detail::is_lane_v); + if (!policy.flush_inputs) + return bits; + return bits & (detail::normal_or_special(bits) | Fmt::kSign); +} + +/// @brief Whether a source encoding is NaN. +template constexpr auto is_nan(V bits) { + static_assert(detail::is_lane_v); + return (bits & Fmt::kMagnitude) > Fmt::kInfinity; +} + +/// @brief Stage 3: map a non-NaN encoding to a key that orders like its value. +/// @details Both zeros map to the same key. A positive encoding sets the sign +/// bit; a negative one inverts every bit of the format, reversing magnitude order. +template constexpr V order_key(V bits) { + static_assert(detail::is_lane_v); + bits = bits & Fmt::kBits; + bits = bits & detail::nonzero(bits); + const V negative = typename Fmt::Lane{0} - (bits >> (Fmt::kWidth - 1)); + return bits ^ ((negative & Fmt::kBits) | Fmt::kSign); +} + +/// @brief Evaluate a relation on two sources that already carry their modifiers. +/// @returns bool for scalar lanes, or the key comparison's mask for SIMD lanes. +template +constexpr auto evaluate(V a, V b, const Policy &policy) { + static_assert(detail::is_lane_v); + a = flush_input(a & Fmt::kBits, policy); + b = flush_input(b & Fmt::kBits, policy); + const auto ordered = !(is_nan(a) || is_nan(b)); + const auto holds = ordered && typename Rel::Operation{}(order_key(a), order_key(b)); + if constexpr (Rel::kNegated) + return !holds; + else + return holds; +} + +/// @brief Evaluate a relation after applying per-source VOP3 modifiers. +/// @param abs VOP3 ABS field; bit 0 applies to src0 and bit 1 to src1. +/// @param neg VOP3 NEG field, with the same bit assignment. +template +constexpr auto evaluate(V a, V b, uint32_t abs, uint32_t neg, const Policy &policy) { + return evaluate(modify(a, abs & 1u, neg & 1u), + modify(b, (abs & 2u) != 0, (neg & 2u) != 0), policy); +} + +} // namespace rocjitsu::amdgpu::comparison diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/cube.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/cube.h new file mode 100644 index 000000000000..b944fb40fe5c --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/cube.h @@ -0,0 +1,146 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +/// @file cube.h +/// @brief Shared scalar and SIMD cube coordinate instruction semantics. + +#include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" +#include "util/simd.h" + +#include +#include +#include + +namespace rocjitsu::amdgpu::cube { + +/// @brief Cube face, S/T coordinate, and signed major-axis operations. +enum class Operation : uint8_t { ID, SC, TC, MA }; + +namespace detail { + +inline constexpr uint32_t kSign = 0x80000000u; +inline constexpr uint32_t kMagnitude = 0x7fffffffu; +inline constexpr uint32_t kInfinity = 0x7f800000u; +inline constexpr uint32_t kExponentStep = 0x00800000u; +inline constexpr uint32_t kMaxFinite = 0x7f7fffffu; +inline constexpr uint32_t kDoubleOverflow = 0x7f000000u; +inline constexpr uint32_t kQuietNaN = 0x00400000u; +inline constexpr uint32_t kFaceNegativeZ = 0x40a00000u; +inline constexpr uint32_t kFacePositiveZ = 0x40800000u; +inline constexpr uint32_t kFaceNegativeY = 0x40400000u; +inline constexpr uint32_t kFacePositiveY = 0x40000000u; +inline constexpr uint32_t kFaceNegativeX = 0x3f800000u; +inline constexpr uint32_t kFacePositiveX = 0x00000000u; + +template Bits select(Mask condition, Bits yes, Bits no) { + if constexpr (std::is_same_v) { + return condition ? yes : no; + } else { + util::stdx::where(condition, no) = yes; + return no; + } +} + +// Comparisons flush subnormal magnitudes regardless of MODE, but SC/TC copy +// the selected raw source. Integer comparisons also preserve signaling NaNs. +template Bits comparison_magnitude(Bits bits) { + const Bits magnitude = bits & Bits(kMagnitude); + return select(magnitude < Bits(kExponentStep), Bits(0), magnitude); +} + +template Bits overflow(Bits sign, uint32_t round_mode) { + Bits magnitude = Bits(kInfinity); + if (round_mode == 1) + magnitude = select(sign != Bits(0), Bits(kMaxFinite), magnitude); + else if (round_mode == 2) + magnitude = select(sign == Bits(0), Bits(kMaxFinite), magnitude); + else if (round_mode == 3) + magnitude = Bits(kMaxFinite); + return sign | magnitude; +} + +template +Bits execute_bits(Bits x, Bits y, Bits z, uint32_t round_mode, bool quiet_nan) { + const Bits ax = comparison_magnitude(x); + const Bits ay = comparison_magnitude(y); + const Bits az = comparison_magnitude(z); + const auto z_axis = (az <= Bits(kInfinity)) && (ax <= Bits(kInfinity)) && + (ay <= Bits(kInfinity)) && (az >= ax) && (az >= ay); + const auto y_axis = (ay <= Bits(kInfinity)) && (ax <= Bits(kInfinity)) && (ay >= ax); + const Bits major = select(z_axis, z, select(y_axis, y, x)); + const Bits magnitude = comparison_magnitude(major); + const auto negative = ((major & Bits(kSign)) != Bits(0)) && (magnitude != Bits(0)) && + (magnitude <= Bits(kInfinity)); + Bits result; + if constexpr (Op == Operation::ID) { + // Z wins ties, then Y, then X. A zero or unordered major is nonnegative. + result = select( + z_axis, select(negative, Bits(kFaceNegativeZ), Bits(kFacePositiveZ)), + select(y_axis, select(negative, Bits(kFaceNegativeY), Bits(kFacePositiveY)), + select(negative, Bits(kFaceNegativeX), Bits(kFacePositiveX)))); + } else if constexpr (Op == Operation::SC) { + const Bits sign = select(negative, Bits(kSign), Bits(0)); + result = select(z_axis, x ^ sign, select(y_axis, x, z ^ sign ^ Bits(kSign))); + } else if constexpr (Op == Operation::TC) { + const Bits sign = select(negative, Bits(kSign), Bits(0)); + result = select(!z_axis && y_axis, z ^ sign, y ^ Bits(kSign)); + } else { + const Bits sign = major & Bits(kSign); + // Multiplication by two is exact until overflow. Directed rounding chooses + // the appropriate finite bound; only round-down retains negative zero. + result = select(magnitude >= Bits(kDoubleOverflow), overflow(sign, round_mode), + major + Bits(kExponentStep)); + result = select(magnitude >= Bits(kInfinity), major, result); + result = select(magnitude == Bits(0), round_mode == 2 ? sign : Bits(0), result); + } + if (quiet_nan) + result = select((result & Bits(kMagnitude)) > Bits(kInfinity), result | Bits(kQuietNaN), + result); + return result; +} + +} // namespace detail + +/// @brief Execute a cube operation after source modifiers and before output modifiers. +/// @details Scalar and SIMD paths share bit-level selection, rounding and NaN handling. +template +Float execute(Float x, Float y, Float z, uint32_t round_mode, rj_code_arch_t arch, bool ieee_mode) { + using Bits = std::conditional_t, uint32_t, util::native>; + const bool quiet_nan = + arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5 || ieee_mode; + return std::bit_cast( + detail::execute_bits(std::bit_cast(x), std::bit_cast(y), + std::bit_cast(z), round_mode, quiet_nan)); +} + +/// @brief Apply cube OMOD independently of the denormal MODE bits. +/// @details Flushes subnormals before scaling, preserves NaN payloads, and uses +/// guest rounding on overflow. +template +Float apply_omod(Float value, uint32_t round_mode, rj_code_arch_t arch, bool ieee_mode, + uint32_t omod) { + omod = fp_mode::effective_omod(arch, 0, ieee_mode, omod); + if (omod == 0) + return value; + using Bits = std::conditional_t, uint32_t, util::native>; + const Bits bits = std::bit_cast(value); + const Bits sign = bits & Bits(detail::kSign); + const Bits exponent = bits & Bits(detail::kInfinity); + Bits result; + if (omod == 3) { + result = detail::select(exponent == Bits(detail::kExponentStep), sign, + bits - Bits(detail::kExponentStep)); + } else { + result = detail::select( + exponent >= Bits(detail::kInfinity - omod * detail::kExponentStep), + detail::overflow(sign, round_mode), bits + Bits(omod * detail::kExponentStep)); + } + result = detail::select(exponent == Bits(detail::kInfinity), bits, result); + result = detail::select(exponent == Bits(0), Bits(0), result); + return std::bit_cast(result); +} + +} // namespace rocjitsu::amdgpu::cube diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h index 0cbba5fa0088..f61468988a5b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h @@ -24,6 +24,7 @@ #include "rocjitsu/vm/amdgpu/wavefront.h" #include "util/except.h" #include +#include #include #include #include @@ -66,6 +67,15 @@ class StagedOperand final : public Operand { hi_[lane] = static_cast(value >> 32); } + void apply_float_sign_modifiers(uint64_t sign, bool absolute, bool negate) { + const uint64_t clear = absolute ? ~sign : ~uint64_t{0}; + const uint64_t flip = negate ? sign : 0; + for (int lane = 0; lane < lane_count_; ++lane) { + const uint64_t value = uint64_t{lo_[lane]} | (uint64_t{hi_[lane]} << 32); + set_lane64(lane, (value & clear) ^ flip); + } + } + private: uint32_t read_lane(const amdgpu::Wavefront &, uint32_t lane) const override { return lane < static_cast(lane_count_) ? lo_[lane] : 0; @@ -448,6 +458,18 @@ inline void stage_dpp_operand(const Operand &source, const DppPlan &plan, else if ((plan.zero_source_mask & lane_bit) == 0) storage->set_lane64(lane, src_view.lane(lane)); } + } else if (!source.simd_capable()) { + // Packed half selectors return logical, right-aligned values. Gather only + // physical reads, once per lane, before applying the permutation. + assert(source_byte_mask == 0); + std::array values{}; + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) + if (source_lane_mask & (uint64_t{1} << lane)) + values[lane] = regs.read_lane(source, lane); + const uint64_t reads = read_destinations & plan.physical_read_dest_mask; + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) + if (reads & (uint64_t{1} << lane)) + storage->set_lane(lane, values[plan.source_lanes[lane]]); } else { if (source_byte_mask == 0) source_byte_mask = source.size_bits_ == 16 ? rocjitsu::ExecutionPlugin::kLowHalfByteMask @@ -482,6 +504,37 @@ inline void apply_dpp(const Operand &source, const DppPlan &plan, uint64_t read_ stage_dpp_operand(source, plan, read_destinations, storage, wf, source_byte_mask); } +/// Apply the DPP16 extension's floating source modifiers after permutation. +/// Stage an unpermuted second source only when it has a modifier. +inline void apply_source_modifiers(const Operand &source, std::optional &storage, + amdgpu::Wavefront &wf, uint64_t sign, SourceModifiers modifiers, + uint32_t source_index) { + const bool absolute = modifiers.absolute & (1u << source_index); + const bool negate = modifiers.negate & (1u << source_index); + if (!absolute && !negate) + return; + if (!storage) { + RegisterAccess regs(wf); + storage.emplace(source, static_cast(wf.wf_size())); + if (source.size_bits_ > 32) { + const auto values = regs.read_operand64(source, wf.exec()); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) + storage->set_lane64(lane, values.lane(lane)); + } else if (!source.simd_capable()) { + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) + if (wf.exec() & (uint64_t{1} << lane)) + storage->set_lane(lane, regs.read_lane(source, lane)); + } else { + const uint8_t bytes = source.size_bits_ == 16 ? ExecutionPlugin::kLowHalfByteMask + : ExecutionPlugin::kFullByteMask; + const auto values = regs.read_operand(source, wf.exec(), bytes); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) + storage->set_lane(lane, values.lane(lane)); + } + } + storage->apply_float_sign_modifiers(sign, absolute, negate); +} + inline uint32_t dpp8_src_lane(uint32_t lane, uint32_t lane_sel) { uint32_t sel = (lane_sel >> ((lane & 7u) * 3u)) & 7u; return (lane & ~7u) | sel; @@ -500,6 +553,9 @@ inline void apply_dpp8(const Operand &source, uint32_t lane_sel, uint32_t fi, namespace sdwa { +/// @brief Instruction policy for SDWA scaling, shared with its VOP3 form. +enum class OutputPolicy : uint8_t { MODE, FLUSH_NEAREST }; + /// @brief Return the architectural source bytes selected by an SDWA selector. inline uint8_t sdwa_src_byte_mask(uint32_t sel) { if (sel <= BYTE_3) @@ -734,8 +790,9 @@ template inline uint32_t clamp_result(uint32_t result, con } /// @brief Return the enabled output modifier for the SDWA result format. -template +template inline uint32_t output_modifier(const Inst &inst, const Wavefront &wf) { + static_assert(Policy == OutputPolicy::MODE || Format == ResultFormat::F32); if constexpr (Format != ResultFormat::NONE && requires { inst.sdwa_omod_; inst.inst_.src0; @@ -748,7 +805,9 @@ inline uint32_t output_modifier(const Inst &inst, const Wavefront &wf) { if constexpr (Format == ResultFormat::F16 || Format == ResultFormat::PK_F16) return fp_mode::effective_f16_omod(wf.cu().arch(), denorm_mode, wf.ieee_mode(), Format == ResultFormat::PK_F16, inst.sdwa_omod_); - return fp_mode::effective_omod(wf.cu().arch(), denorm_mode, wf.ieee_mode(), inst.sdwa_omod_); + return fp_mode::effective_omod(wf.cu().arch(), + Policy == OutputPolicy::FLUSH_NEAREST ? 0u : denorm_mode, + wf.ieee_mode(), inst.sdwa_omod_); } return 0; } @@ -766,6 +825,14 @@ inline uint16_t round_f16_result(const Inst &inst, const Wavefront &wf, float va omod); } +/// @brief Apply SDWA scaling to an already rounded half transcendental result. +template +inline uint16_t finish_rounded_f16(const Inst &inst, const Wavefront &wf, float value, + bool fp16_ovfl) { + return util::f32_to_f16( + fp_mode::apply_omod_f16(value, output_modifier(inst, wf), fp16_ovfl)); +} + /// @brief Apply SDWA scaling before guest-mode rounding of a wide F16 arithmetic result. template inline uint16_t finish_arithmetic_f16(const Inst &inst, const Wavefront &wf, double value, @@ -819,9 +886,9 @@ inline uint32_t scale_f32(uint32_t value, uint32_t omod, uint32_t round_mode) { } /// @brief Scale a floating-point SDWA result before destination placement. -template +template inline uint32_t scale_result(const Inst &inst, const Wavefront &wf, uint32_t value) { - const uint32_t omod = output_modifier(inst, wf); + const uint32_t omod = output_modifier(inst, wf); if (omod == 0) return value; if constexpr (Format == ResultFormat::F16) { @@ -832,6 +899,11 @@ inline uint32_t scale_result(const Inst &inst, const Wavefront &wf, uint32_t val const uint32_t high = fp_mode::finalize_omod_f16(static_cast(value >> 16), omod); return low | (high << 16); } else if constexpr (Format == ResultFormat::F32) { + if constexpr (Policy == OutputPolicy::FLUSH_NEAREST) { + // The generated LOG/EXP body protects the complete instruction with + // nearest rounding. Reuse its VOP3 finalizer, including NaN passthrough. + return std::bit_cast(fp_mode::apply_omod_f32(std::bit_cast(value), omod)); + } return scale_f32(value, omod, wf.fp_round_mode_f32()); } return value; @@ -856,7 +928,7 @@ template inline bool supports_direct_simd_store(const Inst &inst /// /// Scaling and clamp use the semantic result width before destination placement. /// Preserved destination bytes are neither clamped nor reported as architectural writes. -template +template inline void write_lane(Inst &inst, amdgpu::Wavefront &wf, const Op &op, uint32_t lane, uint32_t value) { if constexpr (requires { @@ -868,7 +940,7 @@ inline void write_lane(Inst &inst, amdgpu::Wavefront &wf, const Op &op, uint32_t }) { if (inst.inst_.src0 == amdgpu::SRC_SDWA && op.is_vgpr() && inst.dst_operand(0) == static_cast(&op)) { - value = scale_result(inst, wf, value); + value = scale_result(inst, wf, value); if (inst.sdwa_clamp_) value = clamp_result(value, wf); const uint8_t byte_mask = sdwa_dst_byte_mask(inst.sdwa_dst_sel_, inst.sdwa_dst_unused_); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h index c78818be5bd4..78514aa16597 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h @@ -131,6 +131,16 @@ inline ExactF64Sum add_exact(double lhs, double rhs) { return {value, error}; } +/// Retain an exact sum's sticky information before rounding to a narrower format. +/// An inexact F64 result gets an odd low bit, avoiding double rounding at F16 +/// boundaries while preserving directed rounding for tiny contributions. +inline double round_to_odd(ExactF64Sum sum) { + if (sum.error == 0.0 || !std::isfinite(sum.value) || (std::bit_cast(sum.value) & 1u)) + return sum.value; + return std::nextafter(sum.value, + std::copysign(std::numeric_limits::infinity(), sum.error)); +} + /// @brief Compare an exact two-component sum with an exactly representable value. inline int compare_exact(ExactF64Sum sum, double value) { if (sum.value < value) @@ -227,6 +237,25 @@ inline uint16_t round_exact_to_bf16(ExactF64Sum sum, uint32_t round_mode) { } // namespace detail +/// @brief Whether floating operations with exception support quiet signaling NaNs. +inline bool quiets_nan(rj_code_arch_t arch, bool ieee_mode) { + return arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5 || ieee_mode; +} + +/// @brief Apply V_CVT_F32_F16 policy to an already decoded and modified half. +/// @details Raw half decoding preserves signaling NaNs and subnormals; the +/// instruction applies MODE input flushing and target-specific NaN quieting. +inline uint32_t cvt_f32_f16(float source, rj_code_arch_t arch, uint32_t denorm_mode, + bool ieee_mode) { + uint32_t bits = std::bit_cast(source); + const uint32_t magnitude = bits & 0x7fffffffu; + if (!(denorm_mode & 1u) && magnitude < 0x38800000u) + return bits & 0x80000000u; + if (quiets_nan(arch, ieee_mode) && magnitude > 0x7f800000u) + bits |= 0x00400000u; + return bits; +} + /// @brief Return the OMOD value supported by an ordinary floating-point result. inline uint32_t effective_omod(rj_code_arch_t arch, uint32_t denorm_mode, bool ieee_mode, uint32_t omod) { @@ -274,6 +303,37 @@ inline uint16_t finalize_omod_bf16(uint16_t value, uint32_t omod) { namespace detail { +/// Round an F32-source FMA directly to F16 in a clean nearest environment. +inline uint16_t fma_f32_to_f16_nearest_environment(float a, float b, float c, uint32_t round_mode, + bool clamp, bool fp16_ovfl, + bool clamp_nan_to_zero) { + double value; + if (std::isfinite(a) && std::isfinite(b) && std::isfinite(c)) { + const auto exact = add_exact(double(a) * double(b), double(c)); + value = round_to_odd(exact); + if (exact.value == 0 && exact.error == 0) { + // Finite F32 products are exact, normal F64 values or signed zero. + // Matching signed zeros keep their sign; cancellation and opposite + // signed zeros are negative only when rounding toward negative infinity. + const uint32_t product_sign = (std::bit_cast(a) ^ std::bit_cast(b)) >> 31; + const uint32_t addend_bits = std::bit_cast(c); + const bool matching_zeros = + (addend_bits & 0x7fffffffu) == 0 && product_sign == (addend_bits >> 31); + const bool negative = matching_zeros ? product_sign != 0 : (round_mode & 3u) == 2u; + value = std::bit_cast(uint64_t{negative} << 63); + } + } else { + const uint32_t a_abs = std::bit_cast(a) & 0x7fffffffu; + const uint32_t b_abs = std::bit_cast(b) & 0x7fffffffu; + // The invalid product wins even when the addend supplies a NaN payload. + const bool invalid_product = + (a_abs == 0 && b_abs == 0x7f800000u) || (b_abs == 0 && a_abs == 0x7f800000u); + value = invalid_product ? std::bit_cast(uint64_t{0xfff8000000000000}) + : std::fma(double(a), double(b), double(c)); + } + return pseudo_scalar::round_f16_result(value, round_mode, 0, clamp, fp16_ovfl, clamp_nan_to_zero); +} + /// @brief Execute F32-source fused multiply-add in an established clean nearest environment. inline uint16_t fma_f32_to_bf16_nearest_environment(float multiplicand, float multiplier, float addend, uint32_t round_mode, bool clamp, @@ -328,10 +388,29 @@ enum class PackedBinaryOp { ADD, MUL, MIN, MAX, MINIMUM, MAXIMUM }; /// @brief Packed F16 binary arithmetic uses the FP16/64 MODE controls. inline uint16_t packed_binary_f16(PackedBinaryOp operation, uint16_t a_bits, uint16_t b_bits, uint32_t round_mode, uint32_t denorm_mode, bool clamp, - bool fp16_ovfl, bool clamp_nan_to_zero) { + bool fp16_ovfl, bool clamp_nan_to_zero, rj_code_arch_t arch, + bool ieee_mode) { detail::ScopedFenv nearest_environment(0); a_bits = detail::flush_input_f16(a_bits, denorm_mode); b_bits = detail::flush_input_f16(b_bits, denorm_mode); + const bool a_nan = (a_bits & 0x7fffu) > 0x7c00u; + const bool b_nan = (b_bits & 0x7fffu) > 0x7c00u; + const bool select = operation == PackedBinaryOp::MIN || operation == PackedBinaryOp::MAX; + const bool arithmetic = operation == PackedBinaryOp::ADD || operation == PackedBinaryOp::MUL; + if ((select || arithmetic) && (a_nan || b_nan)) { + const bool a_signaling = a_nan && !(a_bits & 0x0200u); + const bool b_signaling = b_nan && !(b_bits & 0x0200u); + const bool signaling_select = select && ieee_mode && arch != ROCJITSU_CODE_ARCH_RDNA4 && + arch != ROCJITSU_CODE_ARCH_CDNA5 && (a_signaling || b_signaling); + if (arithmetic || (a_nan && b_nan) || signaling_select) { + // Select before host arithmetic: host NaN payload priority can change + // with inlining or sanitizer instrumentation. Legacy IEEE min/max gives + // signaling inputs priority; arithmetic and newer min/max prefer src0. + const uint16_t selected = (signaling_select ? a_signaling : a_nan) ? a_bits : b_bits; + return clamp && clamp_nan_to_zero ? 0 + : selected | (quiets_nan(arch, ieee_mode) ? 0x0200u : 0u); + } + } const double first = util::f16_to_f32(a_bits); const double second = util::f16_to_f32(b_bits); double value = std::numeric_limits::quiet_NaN(); @@ -372,10 +451,11 @@ inline uint16_t packed_binary_f16(PackedBinaryOp operation, uint16_t a_bits, uin /// @brief Select three packed F16 operands, applying output controls only to the final result. inline uint16_t packed_select3_f16(PackedBinaryOp operation, uint16_t first, uint16_t second, uint16_t third, uint32_t denorm_mode, bool clamp, - bool clamp_nan_to_zero) { - const uint16_t pair = - packed_binary_f16(operation, first, second, 0, denorm_mode | 2u, false, false, false); - return packed_binary_f16(operation, pair, third, 0, denorm_mode, clamp, false, clamp_nan_to_zero); + bool clamp_nan_to_zero, rj_code_arch_t arch, bool ieee_mode) { + const uint16_t pair = packed_binary_f16(operation, first, second, 0, denorm_mode | 2u, false, + false, false, arch, ieee_mode); + return packed_binary_f16(operation, pair, third, 0, denorm_mode, clamp, false, clamp_nan_to_zero, + arch, ieee_mode); } /// @brief Packed BF16 numeric min/max preserve denormals and order signed zeros explicitly. @@ -400,10 +480,71 @@ inline uint16_t clamp_bf16(uint16_t value, bool clamp, bool clamp_nan_to_zero) { return value > 0x3f80u ? 0x3f80u : value; } +/// @brief F32 FMA with architectural NaN selection in the active FP environment. +inline float fma_f32(float a, float b, float c, rj_code_arch_t arch, bool ieee_mode, + uint32_t denorm_mode, bool force_output_flush = false) { +#if defined(__clang__) +#pragma STDC FENV_ACCESS ON +#endif + auto input_bits = [denorm_mode](float value) { + const uint32_t bits = std::bit_cast(value); + return !(denorm_mode & 1u) && (bits & 0x7f800000u) == 0 ? bits & 0x80000000u : bits; + }; + const uint32_t a_bits = input_bits(a), b_bits = input_bits(b), c_bits = input_bits(c); + const uint32_t a_abs = a_bits & 0x7fffffffu, b_abs = b_bits & 0x7fffffffu; + // An invalid product takes precedence even when the addend is a NaN. + if ((a_abs == 0 && b_abs == 0x7f800000u) || (b_abs == 0 && a_abs == 0x7f800000u)) + return std::bit_cast(0xffc00000u); + for (const uint32_t bits : {a_bits, b_bits, c_bits}) + if ((bits & 0x7fffffffu) > 0x7f800000u) + return std::bit_cast(bits | (quiets_nan(arch, ieee_mode) ? 0x00400000u : 0u)); + volatile float left = std::bit_cast(a_bits); + volatile float right = std::bit_cast(b_bits); + volatile float addend = std::bit_cast(c_bits); + const float result = std::fma(left, right, addend); + if (force_output_flush || !(denorm_mode & 2u)) { + const uint32_t bits = std::bit_cast(result); + const uint32_t magnitude = bits & 0x7fffffffu; + if (magnitude < 0x00800000u) + return std::bit_cast(bits & 0x80000000u); + if (magnitude == 0x00800000u) { + // Detect tininess after rounding the 24-bit significand, before packing + // it into F32's exponent range. Subnormal F32 rounding has a wider step + // and can otherwise promote a still-tiny intermediate to minimum normal. + const int rounding = std::fegetround(); + detail::ScopedFenv environment(0); + volatile double wide_left = left; + volatile double wide_right = right; + volatile double wide_addend = addend; + // A product of two F32 operands is exact in F64. TwoSum retains the + // residual of the addition, including products far below the addend. + volatile double product = wide_left * wide_right; + volatile double sum = product + wide_addend; + volatile double addend_part = sum - product; + const double residual = (product - (sum - addend_part)) + (wide_addend - addend_part); + const bool negative = bits >> 31; + const double magnitude_sum = negative ? -sum : sum; + const double magnitude_residual = negative ? -residual : residual; + double threshold = static_cast(std::numeric_limits::min()); + const bool away_from_zero = rounding == (negative ? FE_DOWNWARD : FE_UPWARD); + if (rounding == FE_TONEAREST) + threshold -= 0x1p-151; + else if (away_from_zero) + threshold -= 0x1p-150; + if (magnitude_sum < threshold || + (magnitude_sum == threshold && + (magnitude_residual < 0 || (away_from_zero && magnitude_residual == 0)))) + return std::bit_cast(bits & 0x80000000u); + } + } + // Opposite infinities also produce the architectural indefinite value. + return std::isnan(result) ? std::bit_cast(0xffc00000u) : result; +} + /// @brief Packed F32 operations use FP32 MODE, independently of the host environment. inline uint32_t packed_f32(float first, float second, float third, PackedF32Op operation, uint32_t round_mode, uint32_t denorm_mode, bool clamp, - bool clamp_nan_to_zero) { + bool clamp_nan_to_zero, rj_code_arch_t arch, bool ieee_mode) { detail::ScopedFenv environment(round_mode); auto flush = [](float value) { const uint32_t bits = std::bit_cast(value); @@ -421,7 +562,7 @@ inline uint32_t packed_f32(float first, float second, float third, PackedF32Op o result = first_input * second_input; break; case PackedF32Op::FMA: - result = std::fma(first_input, second_input, third_input); + result = fma_f32(first_input, second_input, third_input, arch, ieee_mode, denorm_mode); break; } if (clamp) { @@ -470,7 +611,7 @@ enum class ScalarAtomicOp { FADD, FMIN, FMAX, FCMPSWAP }; /// @brief Execute scalar floating atomics with explicit ISA policy and bit-preserving selection. template Bits atomic_scalar(ScalarAtomicOp operation, Bits old_bits, Bits source_bits, Bits compare_bits, - uint32_t denorm_mode, bool legacy_minmax) { + uint32_t denorm_mode, bool legacy_minmax, bool source_nan_first = false) { static_assert(std::is_same_v || std::is_same_v); using Float = std::conditional_t; constexpr Bits kSign = Bits{1} << (sizeof(Bits) * 8 - 1); @@ -496,6 +637,8 @@ Bits atomic_scalar(ScalarAtomicOp operation, Bits old_bits, Bits source_bits, Bi if (operation == ScalarAtomicOp::FADD) { // DS and cache ADD use fixed RNE, independent of both MODE.round and the host. // Select NaNs explicitly so host operand scheduling cannot change payload priority. + if (source_nan_first && is_nan(source_input)) + return source_input | kQuiet; if (is_nan(old_input)) return old_input | kQuiet; if (is_nan(source_input)) @@ -540,7 +683,7 @@ Bits atomic_scalar(ScalarAtomicOp operation, Bits old_bits, Bits source_bits, Bi /// NaN selection. FP16_OVFL applies to VALU results, not memory atomics. The /// caller supplies DS input/output denormal controls, or 3 for no flushing. inline uint32_t atomic_add_packed_16(uint32_t old_val, uint32_t src_val, bool bf16, - uint32_t denorm_mode) { + uint32_t denorm_mode, bool source_nan_first = false) { detail::ScopedFenv nearest_environment(0); const uint16_t exponent_mask = bf16 ? 0x7f80u : 0x7c00u; auto flush_denorm = [&](uint16_t bits) -> uint16_t { @@ -559,7 +702,9 @@ inline uint32_t atomic_add_packed_16(uint32_t old_val, uint32_t src_val, bool bf uint16_t sum; // Float memory add propagates the first NaN, quieting it without changing // the sign or payload. Invalid infinity addition produces negative QNaN. - if (is_nan(a)) + if (source_nan_first && is_nan(b)) + sum = b | quiet_bit; + else if (is_nan(a)) sum = a | quiet_bit; else if (is_nan(b)) sum = b | quiet_bit; @@ -613,6 +758,47 @@ inline float finalize_omod_f32(float value, uint32_t omod) { return std::bit_cast(bits); } +/// Apply FP32 output scaling in the caller's established rounding environment. +/// Zeros and tiny arithmetic results become +0 before scaling; a normal result +/// that becomes tiny when halved retains its sign and flushes before packing. +inline float apply_omod_f32(float value, uint32_t omod) { + if (omod == 0) + return value; + const uint32_t bits = std::bit_cast(value); + const uint32_t magnitude = bits & 0x7fffffffu; + if (magnitude > 0x7f800000u) + return value; + if (magnitude < 0x00800000u) + return 0.0f; + if (omod == 3 && magnitude < 0x01000000u) + return std::bit_cast(bits & 0x80000000u); + if (omod == 1) + value *= 2.0f; + else if (omod == 2) + value *= 4.0f; + else if (omod == 3) + value *= 0.5f; + return value; +} + +/// @brief Scale an already rounded half result represented in F32. +/// @details Preserve NaNs, flush input subnormals, and round scaling to F16. +inline float apply_omod_f16(float value, uint32_t omod, bool fp16_ovfl) { + if (omod == 0) + return value; + const uint32_t magnitude = std::bit_cast(value) & 0x7fffffffu; + if (magnitude > 0x7f800000u) + return value; + if (magnitude < 0x38800000u) + return 0.0f; + // A half value scaled by a power of two is exact in F32, including infinity. + value *= omod == 3 ? 0.5f : omod == 2 ? 4.0f : 2.0f; + uint16_t result = util::f32_to_f16_mode(value, fp16_ovfl); + if ((result & 0x7c00u) == 0) + result &= 0x8000u; + return util::f16_to_f32(result); +} + inline double finalize_omod_f64(double value, uint32_t omod) { if (omod == 0) return value; @@ -651,30 +837,66 @@ inline Float evaluate_arithmetic(Float lhs, Float rhs, Float addend) { return left * right; else if constexpr (operation == Arithmetic::MUL_LEGACY) return left == 0 || right == 0 ? Float{0} : left * right; - else if constexpr (operation == Arithmetic::FMA_DX9_ZERO) - return left == 0 || right == 0 ? static_cast(accumulator) - : std::fma(left, right, accumulator); else return std::fma(left, right, accumulator); } +template +inline Float arithmetic_with_policy(Float lhs, Float rhs, Float addend, uint32_t round_mode, + uint32_t denorm_mode, Evaluate evaluate) { + detail::ScopedFenv environment(round_mode); + if ((denorm_mode & 1u) == 0) { + lhs = detail::flush_denormal(lhs); + rhs = detail::flush_denormal(rhs); + addend = detail::flush_denormal(addend); + } + const Float result = evaluate(lhs, rhs, addend); + return (denorm_mode & 2u) == 0 ? detail::flush_denormal(result) : result; +} + } // namespace detail /// @brief Evaluate F32/F64 arithmetic independently of the caller's host environment. template inline Float arithmetic(Float lhs, Float rhs, Float addend, uint32_t round_mode, uint32_t denorm_mode) { - detail::ScopedFenv environment(round_mode); - // DX9 FMA flushes both inputs and outputs regardless of MODE. + static_assert(operation != Arithmetic::FMA_DX9_ZERO, + "DX9 FMA requires the architectural NaN and underflow policy"); + return detail::arithmetic_with_policy(lhs, rhs, addend, round_mode, denorm_mode, + detail::evaluate_arithmetic); +} + +/// @brief F32 arithmetic combines MODE with architectural NaN and tininess policies. +template +inline float arithmetic(float lhs, float rhs, float addend, uint32_t round_mode, + uint32_t denorm_mode, rj_code_arch_t arch, bool ieee_mode, + bool force_output_flush = false) { + static_assert(operation == Arithmetic::ADD || operation == Arithmetic::MUL || + operation == Arithmetic::FMA || operation == Arithmetic::FMA_DX9_ZERO); + // DX9 FMA flushes all inputs and the output, independently of MODE. if constexpr (operation == Arithmetic::FMA_DX9_ZERO) denorm_mode = 0; - if ((denorm_mode & 1u) == 0) { - lhs = detail::flush_denormal(lhs); - rhs = detail::flush_denormal(rhs); - addend = detail::flush_denormal(addend); - } - const Float result = detail::evaluate_arithmetic(lhs, rhs, addend); - return (denorm_mode & 2u) == 0 ? detail::flush_denormal(result) : result; + return detail::arithmetic_with_policy( + lhs, rhs, addend, round_mode, denorm_mode, [=](float a, float b, float c) { + // Exact FMA forms preserve binary arithmetic's rounding, NaN order, + // signed zeros and tininess detection before exponent packing. + if constexpr (operation == Arithmetic::ADD) { + c = b; + b = 1.0f; + } else if constexpr (operation == Arithmetic::MUL) { + c = std::bit_cast((std::bit_cast(a) ^ std::bit_cast(b)) & + 0x80000000u); + } + if constexpr (operation == Arithmetic::FMA_DX9_ZERO) + if ((std::bit_cast(a) & 0x7fffffffu) == 0 || + (std::bit_cast(b) & 0x7fffffffu) == 0) { + // DX9 supplies a positive zero product, then performs the addition. + // This still applies signed-zero rounding and the addend's NaN policy. + a = 0.0f; + b = 1.0f; + } + return fma_f32(a, b, c, arch, ieee_mode, denorm_mode, force_output_flush); + }); } /// @brief Evaluate F16 arithmetic before output modifiers and destination rounding. @@ -725,19 +947,82 @@ inline bool native_arithmetic_matches(uint32_t round_mode, uint32_t denorm_mode) #endif } +/// @brief Round an F16 FMA significand before testing tininess and packing the exponent. +inline uint16_t round_fma_f16(double value, uint32_t round_mode, uint32_t denorm_mode, bool clamp, + bool fp16_ovfl, bool clamp_nan_to_zero) { + uint16_t result = + pseudo_scalar::round_f16_result(value, round_mode, 0, clamp, fp16_ovfl, clamp_nan_to_zero); + if (!(denorm_mode & 2u)) { + if ((result & 0x7fffu) == 0x0400u) { + const bool away = round_mode == ((result & 0x8000u) ? 2u : 1u); + const double threshold = 0x1p-14 - (round_mode == 0 ? 0x1p-26 : away ? 0x1p-25 : 0.0); + if (std::abs(value) < threshold || (away && std::abs(value) == threshold)) + result &= 0x8000u; + } + if ((result & 0x7c00u) == 0) + result &= 0x8000u; + } + return result; +} + +/// @brief Apply F16 FMA arithmetic rounding and result modifiers. +/// @details Active OMOD scales the rounded F16 result, so it cannot recover an +/// arithmetic overflow or flushed tiny result. Newly tiny negative results +/// retain their sign, while zeros entering the modifier become positive zero. +inline uint16_t finish_fma_f16(double value, uint32_t round_mode, uint32_t denorm_mode, + uint32_t omod, bool clamp, bool fp16_ovfl, bool clamp_nan_to_zero) { + if (omod == 0) + return round_fma_f16(value, round_mode, denorm_mode, clamp, fp16_ovfl, clamp_nan_to_zero); + uint16_t result = round_fma_f16(value, round_mode, 0, false, fp16_ovfl, false); + const uint16_t magnitude = result & 0x7fffu; + if (magnitude < 0x0400u) + result = 0; + else if (omod == 3 && magnitude < 0x0800u) + result &= 0x8000u; + else + result = pseudo_scalar::round_f16_result(util::f16_to_f32(result), round_mode, omod, false, + fp16_ovfl, false); + if (clamp) + result = pseudo_scalar::round_f16_result(util::f16_to_f32(result), round_mode, 0, true, + fp16_ovfl, clamp_nan_to_zero); + return result; +} + /// @brief Execute an F16 fused multiply-add and return its raw F16 encoding. inline uint16_t fma_f16(uint16_t src0, uint16_t src1, uint16_t src2, bool abs0, bool abs1, bool abs2, bool neg0, bool neg1, bool neg2, uint32_t round_mode, uint32_t denorm_mode, uint32_t omod, bool clamp, bool fp16_ovfl, - bool clamp_nan_to_zero) { + bool clamp_nan_to_zero, bool quiet_nan) { src0 = detail::flush_input_f16(detail::modify_f16(src0, abs0, neg0), denorm_mode); src1 = detail::flush_input_f16(detail::modify_f16(src1, abs1, neg1), denorm_mode); src2 = detail::flush_input_f16(detail::modify_f16(src2, abs2, neg2), denorm_mode); + const uint16_t magnitude0 = src0 & 0x7fffu, magnitude1 = src1 & 0x7fffu; + if ((magnitude0 == 0 && magnitude1 == 0x7c00u) || (magnitude1 == 0 && magnitude0 == 0x7c00u)) + return clamp && clamp_nan_to_zero ? 0u : 0xfe00u; + for (uint16_t bits : {src0, src1, src2}) + if ((bits & 0x7fffu) > 0x7c00u) + return clamp && clamp_nan_to_zero ? 0u : bits | (quiet_nan ? 0x0200u : 0u); + + // Handle infinities before host arithmetic so invalid cancellation always + // produces the architectural indefinite, independent of the host's NaN sign. + if (magnitude0 == 0x7c00u || magnitude1 == 0x7c00u) { + const uint16_t product = ((src0 ^ src1) & 0x8000u) | 0x7c00u; + if ((src2 & 0x7fffu) == 0x7c00u && ((product ^ src2) & 0x8000u)) + return clamp && clamp_nan_to_zero ? 0u : 0xfe00u; + return clamp ? ((product & 0x8000u) ? 0u : 0x3c00u) : product; + } + if ((src2 & 0x7fffu) == 0x7c00u) + return clamp ? ((src2 & 0x8000u) ? 0u : 0x3c00u) : src2; + const double multiplicand = static_cast(util::f16_to_f32(src0)); const double multiplier = static_cast(util::f16_to_f32(src1)); const double addend = static_cast(util::f16_to_f32(src2)); - double value = std::fma(multiplicand, multiplier, addend); + detail::ScopedFenv environment(0); + // The product is exact in F64, but a tiny product added to a large F16 + // accumulator can still exceed F64's significand range. + volatile double product = multiplicand * multiplier; + double value = detail::round_to_odd(detail::add_exact(product, addend)); if (value == 0.0) { // F16 products cannot underflow in double: zero here is exact. Cancellation // uses the guest rounding mode; matching zero signs retain their sign. @@ -747,11 +1032,7 @@ inline uint16_t fma_f16(uint16_t src0, uint16_t src1, uint16_t src2, bool abs0, const bool negative_zero = matching_zeros ? negative_product : round_mode == 2; value = negative_zero ? -0.0 : 0.0; } - uint16_t result = - pseudo_scalar::round_f16_result(value, round_mode, omod, clamp, fp16_ovfl, clamp_nan_to_zero); - if ((denorm_mode & 2u) == 0 && (result & 0x7c00u) == 0 && (result & 0x03ffu) != 0) - result &= 0x8000u; - return finalize_omod_f16(result, omod); + return finish_fma_f16(value, round_mode, denorm_mode, omod, clamp, fp16_ovfl, clamp_nan_to_zero); } /// @brief Execute an F64 fused multiply-add under MODE.FP_ROUND and MODE.FP_DENORM. diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h index d5c48b690b20..5fcf59547269 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h @@ -24,6 +24,17 @@ constexpr uint32_t SRC_DPP8_HI = SRC_DPP8_FI_1; namespace dpp { +/// @brief Floating source modifiers decoded from a DPP16 extension. +struct SourceModifiers { + uint32_t absolute = 0; + uint32_t negate = 0; + + template static SourceModifiers decode(const Encoding &encoding) { + return {.absolute = uint32_t{encoding.src0_abs} | (uint32_t{encoding.src1_abs} << 1), + .negate = uint32_t{encoding.src0_neg} | (uint32_t{encoding.src1_neg} << 1)}; + } +}; + /// @brief DPP control value ranges encoded in VOP instruction modifiers. enum DppCtrl : uint32_t { QUAD_PERM_MAX = 0xFF, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/lds_direct.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/lds_direct.h new file mode 100644 index 000000000000..32344ac1dfb2 --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/lds_direct.h @@ -0,0 +1,36 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "rocjitsu/isa/instruction.h" +#include "rocjitsu/vm/amdgpu/mem_state.h" +#include "rocjitsu/vm/amdgpu/wavefront.h" + +namespace rocjitsu::amdgpu { + +/// RDNA3+ direct LDS loads broadcast one M0-selected value to whole active quads. +inline void execute_lds_direct_load(Instruction &inst, Wavefront &wf, uint32_t dst) { + if (!wf.exec()) + return; + const uint32_t type = (wf.m0() >> 16) & 7, address = wf.m0() & 0xffff; + if ((address & 3) || type == 3 || type >= 6 || dst >= wf.num_vgprs()) { + wf.report_instruction_execution_error(InstructionExecutionError::UnsupportedOperandValue); + return; + } + auto d = std::make_unique(LOCAL_MEM); + d->wf_size = wf.wf_size(); + d->dst_reg_base = wf.vgpr_alloc().base + dst; + d->elem_size = type == 2 ? 4 : (type & 1 ? 2 : 1); + d->num_elems = 1; + d->sign_extend = type & 4; + d->wait_counter_type = WaitCounterType::EXPCNT; + for (uint32_t quad = 0; quad < wf.wf_size(); quad += 4) + if (wf.exec() & (uint64_t{15} << quad)) + d->exec_mask |= uint64_t{15} << quad; + d->lane_mask = d->exec_mask; + d->per_lane_addr.fill(wf.lds_base() + address); + inst.set_data(std::move(d)); +} + +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/memory_issue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/memory_issue.h index 042a72259789..c4bea6b403e8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/memory_issue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/memory_issue.h @@ -74,8 +74,9 @@ class MemoryCounterObligation { /// FLAT and pre-GFX12 stores can contribute to multiple simultaneous domains. /// Each obligation carries its own completion-order class because operations /// sharing one counter need not form one FIFO, and one instruction's different -/// counter domains can have different ordering guarantees. The memory route -/// does not change these obligations. exec_masked distinguishes ordinary +/// counter domains can have different ordering guarantees. Decoded FLAT metadata +/// lists possible domains; address routing selects the domains actually used. +/// exec_masked distinguishes ordinary /// vector memory operations from scalar memory and the few vector operations /// that execute independently of EXEC. /// @@ -92,6 +93,33 @@ struct MemoryIssueInfo { } [[nodiscard]] bool empty() const { return num_counter_obligations_ == 0; } + /// Select FLAT's actual memory domains without changing completion ordering + /// or independent obligations such as source-register export completion. + [[nodiscard]] MemoryIssueInfo for_flat_memory_domains(bool vmem, bool lds) const { + MemoryIssueInfo result; + result.exec_masked = exec_masked; + for (const auto obligation : counter_obligations()) { + switch (obligation.wait_counter_type()) { + case WaitCounterType::VMCNT: + case WaitCounterType::VSCNT: + case WaitCounterType::LOADCNT: + case WaitCounterType::STORECNT: + if (!vmem) + continue; + break; + case WaitCounterType::LGKMCNT: + case WaitCounterType::DSCNT: + if (!lds) + continue; + break; + default: + break; + } + result.counter_obligations_[result.num_counter_obligations_++] = obligation; + } + return result; + } + std::array counter_obligations_{}; uint8_t num_counter_obligations_ = 0; bool exec_masked = true; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mixed_fma_simd.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mixed_fma_simd.h new file mode 100644 index 000000000000..1c4a5d9411ab --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mixed_fma_simd.h @@ -0,0 +1,141 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" +#include "util/simd.h" + +namespace rocjitsu::amdgpu { + +// The compatible ABI avoids broken Clang/libstdc++ AVX-512 64-bit masks +// while retaining packed arithmetic instead of a per-lane fallback. +using MixedFmaAbi = std::conditional_t, + util::stdx::simd_abi::native>; +using MixedFmaDouble = util::stdx::simd; +using MixedFmaBits = util::stdx::simd; +inline constexpr std::size_t mixed_fma_width64 = MixedFmaDouble::size(); + +/// Narrow finite F64 lanes directly to F16, without an intermediate F32 rounding. +inline MixedFmaBits round_finite_f16_simd(MixedFmaDouble value, uint32_t round_mode, + bool fp16_ovfl) { + using U = MixedFmaBits; + const U bits = std::bit_cast(value); + const U sign = (bits >> 48) & U(0x8000u); + const U exponent = (bits >> 52) & U(0x7ffu); + const U significand = (bits & U(0x000fffffffffffffull)) | U(1ull << 52); + U shift(42); + util::stdx::where(exponent < U(1009), shift) = U(1051) - exponent; + util::stdx::where(exponent < U(998), shift) = U(53); + const U remainder = significand & ((U(1) << shift) - U(1)); + U rounded = significand >> shift; + util::stdx::where(exponent >= U(1009), rounded) += (exponent - U(1009)) << 10; + const auto inexact = remainder != U(0); + switch (round_mode & 3u) { + case 0: { + const U halfway = U(1) << (shift - U(1)); + const auto increment = + (remainder > halfway) || ((remainder == halfway) && ((rounded & U(1)) != U(0))); + util::stdx::where(increment && (exponent >= U(998)), rounded) += U(1); + break; + } + case 1: + util::stdx::where(inexact && (sign == U(0)), rounded) += U(1); + break; + case 2: + util::stdx::where(inexact && (sign != U(0)), rounded) += U(1); + break; + default: + break; + } + U overflow(0x7bffu); + if (!fp16_ovfl) { + if ((round_mode & 3u) == 0) + overflow = U(0x7c00u); + else if ((round_mode & 3u) == 1) + util::stdx::where(sign == U(0), overflow) = U(0x7c00u); + else if ((round_mode & 3u) == 2) + util::stdx::where(sign != U(0), overflow) = U(0x7c00u); + } + util::stdx::where((exponent > U(1038)) || (rounded >= U(0x7c00u)), rounded) = overflow; + util::stdx::where((bits & U(0x7fffffffffffffffull)) == U(0), rounded) = U(0); + return sign | rounded; +} + +/// MIX F16 keeps the exact F32 product and addition residual in F64 SIMD lanes. +/// Only nonfinite inputs need scalar NaN selection. Legacy MAD passes its +/// already rounded F32 result as c and uses the same vector narrowing policy. +template +#if defined(__GNUC__) && !defined(__clang__) +[[gnu::optimize("rounding-math")]] +#endif +inline util::native +mixed_fma_f16_simd(util::native a, util::native b, util::native c, + uint32_t round_mode, bool clamp, bool fp16_ovfl, bool clamp_nan_to_zero) { + using D = MixedFmaDouble; + using U = MixedFmaBits; + constexpr std::size_t W = util::native_width_v; + constexpr std::size_t WD = mixed_fma_width64; + using NarrowFloat = util::stdx::fixed_size_simd; + using NarrowWord = util::stdx::fixed_size_simd; + alignas(util::native) float av[W], bv[W], cv[W]; + alignas(util::native) uint32_t out[W]; + a.copy_to(av, util::stdx::vector_aligned); + b.copy_to(bv, util::stdx::vector_aligned); + c.copy_to(cv, util::stdx::vector_aligned); + fp_mode::ScopedEnvironment environment(0); + for (std::size_t base = 0; base < W; base += WD) { + const D da = + util::stdx::static_simd_cast(NarrowFloat(av + base, util::stdx::element_aligned)); + const D db = + util::stdx::static_simd_cast(NarrowFloat(bv + base, util::stdx::element_aligned)); + const D dc = + util::stdx::static_simd_cast(NarrowFloat(cv + base, util::stdx::element_aligned)); + D value = dc; + auto special = util::stdx::isinf(dc) || util::stdx::isnan(dc); + if constexpr (Fused) { + const D product = da * db; + value = product + dc; + const D addend_virtual = value - product; + const D residual = (product - (value - addend_virtual)) + (dc - addend_virtual); + U bits = std::bit_cast(value); + const U error = std::bit_cast(residual); + const auto step = ((error & U(0x7fffffffffffffffull)) != U(0)) && ((bits & U(1)) == U(0)); + const auto same_sign = ((bits ^ error) >> 63) == U(0); + util::stdx::where(step && same_sign, bits) += U(1); + util::stdx::where(step && !same_sign, bits) -= U(1); + const U product_sign = (std::bit_cast(da) ^ std::bit_cast(db)) & U(1ull << 63); + const U addend = std::bit_cast(dc); + const auto matching_zeros = ((addend & U(0x7fffffffffffffffull)) == U(0)) && + ((addend & U(1ull << 63)) == product_sign); + U zero_sign((round_mode & 3u) == 2 ? 1ull << 63 : 0); + util::stdx::where(matching_zeros, zero_sign) = product_sign; + util::stdx::where((bits & U(0x7fffffffffffffffull)) == U(0), bits) = zero_sign; + value = std::bit_cast(bits); + special |= util::stdx::isinf(da) || util::stdx::isnan(da) || util::stdx::isinf(db) || + util::stdx::isnan(db); + } + if (clamp) { + util::stdx::where(value <= D(0), value) = D(0); + util::stdx::where(value > D(1), value) = D(1); + } + const auto half = util::stdx::static_simd_cast( + round_finite_f16_simd(value, round_mode, fp16_ovfl)); + half.copy_to(out + base, util::stdx::element_aligned); + if (util::stdx::any_of(special)) + for (std::size_t lane = 0; lane < WD; ++lane) + if (special[lane]) { + const auto i = base + lane; + if constexpr (Fused) + out[i] = fp_mode::detail::fma_f32_to_f16_nearest_environment( + av[i], bv[i], cv[i], round_mode, clamp, fp16_ovfl, clamp_nan_to_zero); + else + out[i] = pseudo_scalar::round_f16_result(cv[i], round_mode, 0, clamp, fp16_ovfl, + clamp_nan_to_zero); + } + } + return util::native(out, util::stdx::vector_aligned); +} + +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mma_exec.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mma_exec.h index 4eedaf48e8ef..979ad925367e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mma_exec.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mma_exec.h @@ -339,6 +339,11 @@ inline InputLoc gfx12_wmma_input_loc(uint32_t wave_size, uint32_t dim, uint32_t require_gfx12_wmma_wave_size(wave_size); if (wave_size == WMMA_WAVE32) return wmma_input_loc(dim, K, i, k, data_bits); + if (dim == 16 && data_bits == 4 && (K == 16 || K == 32)) { + const uint32_t block = k / 8u; + const uint32_t lane_block = K == 16 ? block : ((block & 1u) << 1u) | (block >> 1u); + return wmma_packed_input_loc(i + 16u * lane_block, k & 7u, data_bits); + } if (dim == 16 && K == 16) { const uint32_t lane = i + 16u * ((k >> 2) & 1u) + 32u * ((k >> 3) & 1u); const uint32_t slot = 2u * ((k >> 1) & 1u) + (k & 1u); @@ -561,6 +566,8 @@ inline SwmmacIndexLoc swmmac_index_loc(uint32_t M, uint32_t K, uint32_t elem_bit } return {row + 16u * ((group / 2u) & 1u), 2u * (group & 1u) + 4u * (group / 4u) + slot}; } + if (M == 16 && K == 64 && elem_bits == 4 && index_entries == 16) + return {row + 16u * ((compressed_k / 8u) & 1u), 8u * (compressed_k / 16u) + compressed_k % 8u}; // This generic routing is also intentional for gfx1250 K=128 8-bit // SWMMAC. Hardware-reference Tensile kernels require contiguous 32-entry // selector blocks even though sparse A changes lane halves every 16 packed @@ -576,6 +583,11 @@ inline SwmmacIndexLoc swmmac_index_loc(uint32_t wave_size, uint32_t M, uint32_t require_gfx12_wmma_wave_size(wave_size); if (wave_size == WMMA_WAVE32) return swmmac_index_loc(M, K, elem_bits, row, compressed_k, index_entries); + if (M == 16 && elem_bits == 4 && index_entries == 16 && (K == 32 || K == 64)) { + const uint32_t block = compressed_k / 8u; + const uint32_t lane_block = K == 32 ? block : ((block & 1u) << 1u) | (block >> 1u); + return {row + 16u * lane_block, compressed_k & 7u}; + } if (M == 16 && K == 32 && index_entries == 16) { const uint32_t group = compressed_k / 2u; const uint32_t slot = compressed_k & 1u; @@ -611,6 +623,11 @@ inline SwmmacIndexLoc swmmac_index_loc(uint32_t wave_size, uint32_t M, uint32_t require_gfx12_wmma_wave_size(wave_size); if (wave_size == WMMA_WAVE32) return swmmac_a_input_loc(M, K, row, compressed_k, elem_bits); + if (M == 16 && elem_bits == 4 && (K == 32 || K == 64)) { + const uint32_t block = compressed_k / 8u; + const uint32_t lane_block = K == 32 ? block : ((block & 1u) << 1u) | (block >> 1u); + return wmma_packed_input_loc(row + 16u * lane_block, compressed_k & 7u, elem_bits); + } if (M == 16 && K == 32) { const uint32_t group = compressed_k / 2u; const uint32_t slot = compressed_k & 1u; @@ -636,7 +653,7 @@ inline SwmmacIndexLoc swmmac_index_loc(uint32_t wave_size, uint32_t M, uint32_t return wmma_packed_input_loc(lane, slot, elem_bits); } if (N == 16 && K == 32) { - if (elem_bits == 8) + if (elem_bits == 4 || elem_bits == 8) return wmma_packed_input_loc(col + 16u * (dense_k / 16u), dense_k % 16u, elem_bits); if (elem_bits >= 16) { const uint32_t lane = col + 16u * ((dense_k / 8u) & 1u); @@ -653,6 +670,12 @@ inline SwmmacIndexLoc swmmac_index_loc(uint32_t wave_size, uint32_t M, uint32_t require_gfx12_wmma_wave_size(wave_size); if (wave_size == WMMA_WAVE32) return swmmac_b_input_loc(N, K, col, dense_k, elem_bits); + if (N == 16 && elem_bits == 4 && (K == 32 || K == 64)) { + const uint32_t values_per_lane = K == 32 ? 8u : 16u; + const uint32_t block = dense_k / values_per_lane; + const uint32_t lane_block = ((block & 1u) << 1u) | (block >> 1u); + return wmma_packed_input_loc(col + 16u * lane_block, dense_k % values_per_lane, elem_bits); + } if (N == 16 && K == 32) { if (elem_bits == 8) { const uint32_t lane = col + 32u * ((dense_k >> 3) & 1u) + 16u * (dense_k >> 4); @@ -2385,7 +2408,7 @@ void convert_f8_matrix_region(const RegisterAccess::VgprReadRegion &source, floa /// Fast path for the dense f32-out fp8/bf8-input WMMA shapes /// (v_wmma_f32_16x16x{64,128}_{fp8,bf8}_{fp8,bf8}, gfx1250, wave32). Identical /// to the f16/bf16 specializations except the bulk input convert is the f8 LUT -/// gather; A/B formats are compile-time selected. Sparse SWMMAC stays generic. +/// gather; A/B formats are compile-time selected. /// Falls back to the generic exec_wmma_f32_mixed without AVX-512 / under /// force-scalar. template @@ -2826,6 +2849,191 @@ try_exec_swmmac_16x16x64_16bit(auto &cu, uint32_t M, uint32_t N, uint32_t K, uin } } +enum class SwmmacK128Input { FP8, BF8 }; +enum class SwmmacK128Result { F32, F16 }; + +/// AVX-512 fast path for gfx1250 16x16x128 FP8/BF8 SWMMAC. The sparse +/// selectors occupy two VGPRs per lane. Snapshot both words and every matrix +/// operand before writing D so D may alias A, B, C, or the index tuple. +#if defined(__AVX512F__) && __has_include() +RJ_NOINLINE inline void exec_swmmac_16x16x128_8bit_fast_body( + auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t acc_base, uint32_t index_base, + SwmmacK128Input a_format, SwmmacK128Input b_format, SwmmacK128Result result_format, + uint32_t const_acc, bool fp16_ovfl) { + constexpr uint32_t SPEC_M = 16; + constexpr uint32_t SPEC_N = 16; + constexpr uint32_t SPEC_K = 128; + constexpr uint32_t COMPRESSED_K = SPEC_K / 2; + constexpr uint32_t WF = WMMA_WAVE32; + constexpr uint32_t IN_BITS = 8; + constexpr uint32_t INDEX_ENTRIES = 32; + + const uint32_t acc_bits = result_format == SwmmacK128Result::F32 ? uint32_t{32} : uint32_t{16}; + auto reads = read_mixed_matrix_fast_path_regions( + cu, s0, s1, acc_base, static_cast(SPEC_M) * COMPRESSED_K, IN_BITS, + static_cast(SPEC_N) * SPEC_K, IN_BITS, static_cast(SPEC_M) * SPEC_N, + acc_bits, const_acc, WF); + RegisterAccess regs(cu); + auto index_read = regs.read_vgpr_region(index_base, /*reg_count=*/2, mfma_full_lane_mask(WF)); + + alignas(64) float a_physical[SPEC_M * COMPRESSED_K]; + alignas(64) float b_physical[SPEC_N * SPEC_K]; + alignas(64) uint32_t c_words[SPEC_M * SPEC_N] = {}; + alignas(64) uint32_t index_words[2 * WF] = {}; + if (a_format == SwmmacK128Input::FP8) + convert_f8_matrix_region(reads.a, a_physical, SPEC_M * COMPRESSED_K); + else + convert_f8_matrix_region(reads.a, a_physical, SPEC_M * COMPRESSED_K); + if (b_format == SwmmacK128Input::FP8) + convert_f8_matrix_region(reads.b, b_physical, SPEC_N * SPEC_K); + else + convert_f8_matrix_region(reads.b, b_physical, SPEC_N * SPEC_K); + if (reads.acc) + copy_matrix_region_words(*reads.acc, c_words); + index_read.copy_to({index_words, 2 * WF}); + + alignas(64) float a_matrix[SPEC_M * COMPRESSED_K]; + alignas(64) float b_matrix[SPEC_K * SPEC_N]; + alignas(64) float c_matrix[SPEC_M * SPEC_N]; + alignas(64) uint32_t dense_k[SPEC_M * COMPRESSED_K]; + static_assert(sizeof(a_physical) + sizeof(b_physical) + sizeof(c_words) + sizeof(index_words) + + sizeof(a_matrix) + sizeof(b_matrix) + sizeof(c_matrix) + sizeof(dense_k) + + (SPEC_M * SPEC_N / 2) * sizeof(uint32_t) <= + 48 * 1024, + "SWMMAC AVX-512 staging buffers exceed the 48 KiB stack budget"); + + for (uint32_t row = 0; row < SPEC_M; ++row) { + for (uint32_t ck = 0; ck < COMPRESSED_K; ++ck) { + const auto a_loc = swmmac_a_input_loc(WF, SPEC_M, SPEC_K, row, ck, IN_BITS); + a_matrix[row * COMPRESSED_K + ck] = + a_physical[(a_loc.vgpr_offset * WF + a_loc.lane) * 4 + a_loc.sub_element]; + const auto index_loc = swmmac_index_loc(WF, SPEC_M, SPEC_K, IN_BITS, row, ck, INDEX_ENTRIES); + const uint64_t index_set = static_cast(index_words[index_loc.lane]) | + (static_cast(index_words[WF + index_loc.lane]) << 32); + dense_k[row * COMPRESSED_K + ck] = + swmmac_dense_k(index_set, ck, index_loc.local_compressed_k); + } + } + for (uint32_t k = 0; k < SPEC_K; ++k) + for (uint32_t col = 0; col < SPEC_N; ++col) { + const auto b_loc = swmmac_b_input_loc(WF, SPEC_N, SPEC_K, col, k, IN_BITS); + b_matrix[k * SPEC_N + col] = + b_physical[(b_loc.vgpr_offset * WF + b_loc.lane) * 4 + b_loc.sub_element]; + } + + auto initial_acc = [&](uint32_t row, uint32_t col) { + if (const_acc != ACC_FROM_VGPR) + return std::bit_cast(const_acc); + if (result_format == SwmmacK128Result::F32) { + const auto out = wmma_output_loc_32(SPEC_M, SPEC_N, row, col); + return std::bit_cast(c_words[out.reg * WF + out.lane]); + } + const auto out = wmma_output_loc_16(SPEC_M, SPEC_N, row, col); + const uint32_t raw = c_words[out.reg * WF + out.lane]; + return util::f16_to_f32(static_cast(raw >> (out.sub_element * 16))); + }; + for (uint32_t row = 0; row < SPEC_M; ++row) + for (uint32_t col = 0; col < SPEC_N; ++col) + c_matrix[row * SPEC_N + col] = initial_acc(row, col); + + // The selected B row is shared by all columns, while each output keeps the + // scalar executor's compressed-K FMA order. + for (uint32_t row = 0; row < SPEC_M; ++row) { + util::native c_row; + c_row.copy_from(&c_matrix[row * SPEC_N], util::stdx::vector_aligned); + for (uint32_t ck = 0; ck < COMPRESSED_K; ++ck) { + util::native b_row; + b_row.copy_from(&b_matrix[dense_k[row * COMPRESSED_K + ck] * SPEC_N], + util::stdx::vector_aligned); + c_row = + util::native_fma(util::native(a_matrix[row * COMPRESSED_K + ck]), b_row, c_row); + } + const uint64_t nan_lanes = util::simd_mask_to_bits(util::stdx::isnan(c_row)); + c_row.copy_to(&c_matrix[row * SPEC_N], util::stdx::vector_aligned); + uint64_t pending_nan_lanes = nan_lanes; + while (pending_nan_lanes != 0) { + const uint32_t col = static_cast(std::countr_zero(pending_nan_lanes)); + pending_nan_lanes &= pending_nan_lanes - 1; + float acc = initial_acc(row, col); + for (uint32_t ck = 0; ck < COMPRESSED_K; ++ck) { + const uint32_t k = dense_k[row * COMPRESSED_K + ck]; + acc = matrix_fma(a_matrix[row * COMPRESSED_K + ck], b_matrix[k * SPEC_N + col], acc); + } + c_matrix[row * SPEC_N + col] = acc; + } + } + + const uint32_t result_bits = result_format == SwmmacK128Result::F32 ? uint32_t{32} : uint32_t{16}; + auto writes = write_wmma_output_region(cu, dst, SPEC_M, SPEC_N, result_bits, WF); + if (result_format == SwmmacK128Result::F32) { + for (uint32_t row = 0; row < SPEC_M; ++row) + for (uint32_t col = 0; col < SPEC_N; ++col) { + const auto out = wmma_output_loc_32(SPEC_M, SPEC_N, row, col); + writes.set_linear_word(out.reg * WF + out.lane, + std::bit_cast(c_matrix[row * SPEC_N + col])); + } + } else { + constexpr uint32_t PACKED_REGS = (SPEC_M * SPEC_N / WF) / 2; + alignas(64) uint32_t packed_words[PACKED_REGS * WF] = {}; + for (uint32_t row = 0; row < SPEC_M; ++row) + for (uint32_t col = 0; col < SPEC_N; ++col) { + const auto out = wmma_output_loc_16(SPEC_M, SPEC_N, row, col); + packed_words[out.reg * WF + out.lane] |= + static_cast(wmma_round_f16(c_matrix[row * SPEC_N + col], fp16_ovfl)) + << (out.sub_element * 16); + } + for (uint32_t reg = 0; reg < PACKED_REGS; ++reg) + for (uint32_t lane = 0; lane < WF; ++lane) + writes.set_linear_word(reg * WF + lane, packed_words[reg * WF + lane]); + } +} +#endif + +/// Keep eligibility checks outside the stack-heavy implementation so +/// forced-scalar and unsupported executions avoid its scratch frame. +template +[[gnu::always_inline]] inline bool try_exec_swmmac_16x16x128_8bit( + auto &cu, uint32_t M, uint32_t N, uint32_t K, uint32_t in_bits, uint32_t dst, uint32_t s0, + uint32_t s1, uint32_t acc_base, uint32_t index_base, uint32_t index_entries, uint32_t index_key, + ExtractA, ExtractB, SwmmacK128Result result_format, uint32_t const_acc = ACC_FROM_VGPR, + uint32_t wave_size = WMMA_WAVE32, bool fp16_ovfl = false) { +#if defined(__AVX512F__) && __has_include() + using A = std::remove_cvref_t; + using B = std::remove_cvref_t; + constexpr bool a_fp8 = std::is_same_v || std::is_same_v; + constexpr bool b_fp8 = std::is_same_v || std::is_same_v; + constexpr bool a_bf8 = std::is_same_v || std::is_same_v; + constexpr bool b_bf8 = std::is_same_v || std::is_same_v; + if constexpr (util::has_stdx_simd && (a_fp8 || a_bf8) && (b_fp8 || b_bf8)) { + if (util::force_scalar() || util::native::size() != 16 || M != 16 || N != 16 || + K != 128 || in_bits != 8 || index_entries != 32 || index_key != 0 || + wave_size != WMMA_WAVE32 || cu.wf_size() != WMMA_WAVE32) + return false; + exec_swmmac_16x16x128_8bit_fast_body( + cu, dst, s0, s1, acc_base, index_base, a_fp8 ? SwmmacK128Input::FP8 : SwmmacK128Input::BF8, + b_fp8 ? SwmmacK128Input::FP8 : SwmmacK128Input::BF8, result_format, const_acc, fp16_ovfl); + return true; + } +#endif + (void)cu; + (void)M; + (void)N; + (void)K; + (void)in_bits; + (void)dst; + (void)s0; + (void)s1; + (void)acc_base; + (void)index_base; + (void)index_entries; + (void)index_key; + (void)result_format; + (void)const_acc; + (void)wave_size; + (void)fp16_ovfl; + return false; +} + template void exec_swmmac_f32_mixed(auto &cu, uint32_t M, uint32_t N, uint32_t K, uint32_t a_bits, uint32_t b_bits, uint32_t dst, uint32_t s0, uint32_t s1, @@ -2942,6 +3150,10 @@ void exec_swmmac_f32(auto &cu, uint32_t M, uint32_t N, uint32_t K, uint32_t in_b wave_size)) return; } + if (try_exec_swmmac_16x16x128_8bit(cu, M, N, K, in_bits, dst, s0, s1, acc_base, index_base, + index_entries, index_key, ea, eb, SwmmacK128Result::F32, + const_acc, wave_size)) + return; exec_swmmac_f32_mixed(cu, M, N, K, in_bits, in_bits, dst, s0, s1, acc_base, index_base, index_entries, index_key, ea, eb, const_acc, wave_size); } @@ -3548,6 +3760,10 @@ void exec_swmmac_f16(auto &cu, uint32_t M, uint32_t N, uint32_t K, uint32_t in_b wave_size, fp16_ovfl)) return; } + if (try_exec_swmmac_16x16x128_8bit(cu, M, N, K, in_bits, dst, s0, s1, acc_base, index_base, + index_entries, index_key, ea, eb, SwmmacK128Result::F16, + const_acc, wave_size, fp16_ovfl)) + return; exec_swmmac_packed16( cu, M, N, K, in_bits, dst, s0, s1, acc_base, index_base, index_entries, index_key, ea, eb, [](auto &cu, uint32_t reg, uint32_t lane, uint32_t sub) { @@ -4693,11 +4909,220 @@ struct SmfmacReadBf16 { }; inline constexpr SmfmacReadBf16 smfmac_read_bf16{}; +enum class SmfmacLayout { Cdna3F16, Cdna4F16, Cdna3Fp8, Cdna4Fp8 }; + +struct SmfmacSparseLoc { + uint32_t lane; + uint32_t nibble; + uint32_t element; +}; + +struct SmfmacDenseLoc { + uint32_t lane; + uint32_t element; +}; + +/// The sparse A element and its 2:4 selector occupy the same source lane. +/// Keep the four ISA layouts here so scalar/SIMD comparisons exercise exactly +/// the register permutations used by the original scalar helpers below. +template +inline SmfmacSparseLoc smfmac_sparse_loc(uint32_t row, uint32_t q, uint32_t s) { + if constexpr (Layout == SmfmacLayout::Cdna3F16 || Layout == SmfmacLayout::Cdna4F16) { + constexpr uint32_t groups_per_lane = Layout == SmfmacLayout::Cdna3F16 ? 2 : 4; + return {(q / groups_per_lane) * M + row, q % groups_per_lane, + (2 * q + s) % (2 * groups_per_lane)}; + } else if constexpr (Layout == SmfmacLayout::Cdna3Fp8) { + constexpr uint32_t half_groups = K / 8; + return {((q % half_groups) / 2) * M + row, 2 * (q / half_groups) + (q % 2), + 4 * (q / half_groups) + (2 * q + s) % 4}; + } else if constexpr (M == 16) { + const uint32_t lane_group = 2 * (q / 16) + ((q / 4) % 2); + const uint32_t nibble = 2 * ((q / 8) % 2) + 4 * ((q % 4) / 2) + (q % 2); + const uint32_t byte = 4 * (2 * ((q / 2) % 2) + ((q / 8) % 2)) + (2 * q + s) % 4; + return {lane_group * M + row, nibble, byte}; + } else { + static_assert(M == 32); + const uint32_t lane_group = q / 8; + const uint32_t nibble = (q % 2) + 2 * ((q / 4) % 2) + 4 * ((q / 2) % 2); + const uint32_t byte = 4 * (2 * ((q / 2) % 2) + ((q / 4) % 2)) + (2 * q + s) % 4; + return {lane_group * M + row, nibble, byte}; + } +} + +template +inline SmfmacDenseLoc smfmac_dense_loc(uint32_t k, uint32_t col) { + if constexpr (Layout == SmfmacLayout::Cdna3F16) { + if constexpr (N == 16) + return {(k / 8) * 16 + col, k % 8}; + else + return {16 * ((col / 16) + 2 * (k / 8)) + (col % 16), k % 8}; + } else if constexpr (N == 16) { + return {16 * ((k % 32) / 8) + col, 8 * (k / 32) + (k % 8)}; + } else { + static_assert(N == 32); + return {32 * ((k % 16) / 8) + col, 8 * (k / 16) + (k % 8)}; + } +} + +template +inline constexpr uint32_t smfmac_input_bits = + std::is_same_v, SmfmacReadF16> || + std::is_same_v, SmfmacReadBf16> + ? 16u + : 8u; + +template inline float smfmac_decode_word(uint32_t word, uint32_t element) { + using Reader = std::remove_cvref_t; + if constexpr (std::is_same_v) { + return util::f16_to_f32(static_cast(word >> (16 * (element % 2)))); + } else if constexpr (std::is_same_v) { + return util::bf16_to_f32(static_cast(word >> (16 * (element % 2)))); + } else { + const auto byte = static_cast(word >> (8 * (element % 4))); + if constexpr (std::is_same_v || std::is_same_v) + return util::fp8_e4m3_ocp_to_f32(byte); + else if constexpr (std::is_same_v || + std::is_same_v) + return util::bf8_e5m2_ocp_to_f32(byte); + else if constexpr (std::is_same_v) + return util::fp8_e4m3_fnuz_to_f32(byte); + else if constexpr (std::is_same_v) + return util::bf8_e5m2_fnuz_to_f32(byte); + else + static_assert(util::always_false_v, "unsupported SMFMAC input format"); + } +} + +template +inline float smfmac_decode_snapshot(const uint32_t (&words)[Words], uint32_t element, + uint32_t lane) { + constexpr uint32_t elements_per_word = 32 / smfmac_input_bits; + return smfmac_decode_word(words[(element / elements_per_word) * 64 + lane], element); +} + +/// AVX-512 sparse F32 matrix kernel. Snapshot every operand region before +/// publishing D: tied accumulators and overlapping A/B/index registers then +/// have the same read-before-write behavior as the scalar helpers. Selected K +/// positions are resolved once per row; the FMA loop retains q,s order. +#if defined(__AVX512F__) && __has_include() +template +RJ_NOINLINE inline void smfmac_avx512_fast_body(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, + uint32_t idx_base) { + static_assert((M == 16 && N == 16) || (M == 32 && N == 32)); + static_assert(K % 4 == 0); + constexpr uint32_t compressed_k = K / 2; + constexpr uint32_t a_regs = (M * compressed_k * smfmac_input_bits + 2047) / 2048; + constexpr uint32_t b_regs = (N * K * smfmac_input_bits + 2047) / 2048; + constexpr uint32_t d_regs = M * N / 64; + static_assert(((a_regs + b_regs + 1 + d_regs) * 64 + M * compressed_k + K * N + M * N) * + sizeof(uint32_t) + + M * compressed_k <= + 48 * 1024, + "SMFMAC AVX-512 staging buffers exceed the 48 KiB stack budget"); + constexpr uint64_t full_lanes = ~uint64_t{0}; + RegisterAccess regs(cu); + auto a_read = regs.read_vgpr_region(s0, a_regs, full_lanes); + auto b_read = regs.read_vgpr_region(s1, b_regs, full_lanes); + auto index_read = regs.read_vgpr_region(idx_base, 1, full_lanes); + auto d_read = regs.read_vgpr_region(dst, d_regs, full_lanes); + alignas(64) uint32_t a_words[a_regs * 64]; + alignas(64) uint32_t b_words[b_regs * 64]; + alignas(64) uint32_t index_words[64]; + alignas(64) uint32_t d_words[d_regs * 64]; + copy_matrix_region_words(a_read, a_words); + copy_matrix_region_words(b_read, b_words); + copy_matrix_region_words(index_read, index_words); + copy_matrix_region_words(d_read, d_words); + + alignas(64) float a_values[M * compressed_k]; + alignas(64) float b_values[K * N]; + alignas(64) float c_values[M * N]; + uint8_t selected_k[M * compressed_k]; + for (uint32_t row = 0; row < M; ++row) { + for (uint32_t col = 0; col < N; ++col) { + const auto out = output_loc_32(M, N, row, col, 0); + c_values[row * N + col] = std::bit_cast(d_words[out.reg * 64 + out.lane]); + } + for (uint32_t q = 0; q < K / 4; ++q) { + const auto first = smfmac_sparse_loc(row, q, 0); + const uint32_t field = (index_words[first.lane] >> (4 * first.nibble)) & 0xFu; + for (uint32_t s = 0; s < 2; ++s) { + const auto loc = smfmac_sparse_loc(row, q, s); + const uint32_t t = row * compressed_k + 2 * q + s; + a_values[t] = smfmac_decode_snapshot(a_words, loc.element, loc.lane); + selected_k[t] = static_cast(4 * q + ((field >> (2 * s)) & 3u)); + } + } + } + for (uint32_t k = 0; k < K; ++k) + for (uint32_t col = 0; col < N; ++col) { + const auto loc = smfmac_dense_loc(k, col); + b_values[k * N + col] = smfmac_decode_snapshot(b_words, loc.element, loc.lane); + } + + for (uint32_t row = 0; row < M; ++row) + for (uint32_t c0 = 0; c0 < N; c0 += 16) { + util::native c_row; + c_row.copy_from(&c_values[row * N + c0], util::stdx::vector_aligned); + for (uint32_t t = 0; t < compressed_k; ++t) { + util::native b_row; + b_row.copy_from(&b_values[selected_k[row * compressed_k + t] * N + c0], + util::stdx::vector_aligned); + c_row = + util::native_fma(util::native(a_values[row * compressed_k + t]), b_row, c_row); + } + c_row.copy_to(&c_values[row * N + c0], util::stdx::vector_aligned); + } + + // Packed host FMAs can select a different NaN payload than scalar FMA. + // Replay only exceptional outputs through the shared matrix NaN policy. + auto writes = regs.write_vgpr_region(dst, d_regs, full_lanes); + for (uint32_t row = 0; row < M; ++row) + for (uint32_t col = 0; col < N; ++col) { + float result = c_values[row * N + col]; + const auto out = output_loc_32(M, N, row, col, 0); + if (std::isnan(result)) { + result = std::bit_cast(d_words[out.reg * 64 + out.lane]); + for (uint32_t t = 0; t < compressed_k; ++t) + result = matrix_fma(a_values[row * compressed_k + t], + b_values[selected_k[row * compressed_k + t] * N + col], result); + } + writes.set_linear_word(out.reg * 64 + out.lane, std::bit_cast(result)); + } +} +#endif + +/// Keep eligibility checks outside the stack-heavy implementation so +/// forced-scalar and non-AVX-512 executions avoid its scratch frame. +template +[[gnu::always_inline]] inline bool smfmac_try_avx512(auto &cu, uint32_t dst, uint32_t s0, + uint32_t s1, uint32_t idx_base, ExtractA, + ExtractB) { +#if defined(__AVX512F__) && __has_include() + if (util::force_scalar() || cu.wf_size() != 64 || util::native::size() != 16) + return false; + + smfmac_avx512_fast_body(cu, dst, s0, s1, idx_base); + return true; +#else + (void)cu; + (void)dst; + (void)s0; + (void)s1; + (void)idx_base; + return false; +#endif +} + /// SMFMAC 16x16x32 f16/bf16 (CDNA3 mai-insts). K=32, 8 sparse groups. /// A = v2 (4 halves/lane), B = v4 (8 halves/lane), D = v4 f32. template void exec_smfmac_f32_16x16x32_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t idx_base, Extract ex) { + if (smfmac_try_avx512(cu, dst, s0, s1, idx_base, ex, ex)) + return; struct Result { uint32_t reg, lane, val; }; @@ -4718,7 +5143,7 @@ void exec_smfmac_f32_16x16x32_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t int p0 = field & 3, p1 = (field >> 2) & 3; for (int s = 0; s < 2; ++s) { float av = ex(cu, s0, (2 * q + s) % 4, laneA); - acc += av * Bcol[4 * q + (s == 0 ? p0 : p1)]; + acc = matrix_fma(av, Bcol[4 * q + (s == 0 ? p0 : p1)], acc); } } results.push_back({out.reg, out.lane, std::bit_cast(acc)}); @@ -4733,6 +5158,8 @@ void exec_smfmac_f32_16x16x32_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t template void exec_smfmac_f32_32x32x16_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t idx_base, Extract ex) { + if (smfmac_try_avx512(cu, dst, s0, s1, idx_base, ex, ex)) + return; struct Result { uint32_t reg, lane, val; }; @@ -4756,7 +5183,7 @@ void exec_smfmac_f32_32x32x16_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t int p0 = field & 3, p1 = (field >> 2) & 3; for (int s = 0; s < 2; ++s) { float av = ex(cu, s0, (2 * q + s) % 4, laneA); - acc += av * Bcol[4 * q + (s == 0 ? p0 : p1)]; + acc = matrix_fma(av, Bcol[4 * q + (s == 0 ? p0 : p1)], acc); } } results.push_back({out.reg, out.lane, std::bit_cast(acc)}); @@ -4771,6 +5198,8 @@ void exec_smfmac_f32_32x32x16_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t template void exec_smfmac_f32_16x16x64_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t idx_base, Extract ex) { + if (smfmac_try_avx512(cu, dst, s0, s1, idx_base, ex, ex)) + return; struct Result { uint32_t reg, lane, val; }; @@ -4793,7 +5222,7 @@ void exec_smfmac_f32_16x16x64_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t int p0 = field & 3, p1 = (field >> 2) & 3; for (int s = 0; s < 2; ++s) { float av = ex(cu, s0, (2 * q + s) % 8, laneA); - acc += av * Bcol[4 * q + (s == 0 ? p0 : p1)]; + acc = matrix_fma(av, Bcol[4 * q + (s == 0 ? p0 : p1)], acc); } } results.push_back({out.reg, out.lane, std::bit_cast(acc)}); @@ -4808,6 +5237,8 @@ void exec_smfmac_f32_16x16x64_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t template void exec_smfmac_f32_32x32x32_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t idx_base, Extract ex) { + if (smfmac_try_avx512(cu, dst, s0, s1, idx_base, ex, ex)) + return; struct Result { uint32_t reg, lane, val; }; @@ -4833,7 +5264,7 @@ void exec_smfmac_f32_32x32x32_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t int p0 = field & 3, p1 = (field >> 2) & 3; for (int s = 0; s < 2; ++s) { float av = ex(cu, s0, (2 * q + s) % 8, laneA); - acc += av * Bcol[4 * q + (s == 0 ? p0 : p1)]; + acc = matrix_fma(av, Bcol[4 * q + (s == 0 ? p0 : p1)], acc); } } results.push_back({out.reg, out.lane, std::bit_cast(acc)}); @@ -4848,6 +5279,8 @@ void exec_smfmac_f32_32x32x32_f16(auto &cu, uint32_t dst, uint32_t s0, uint32_t template void exec_smfmac_f32_16x16x64_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t idx_base, ExtractA ea, ExtractB eb) { + if (smfmac_try_avx512(cu, dst, s0, s1, idx_base, ea, eb)) + return; struct Result { uint32_t reg, lane, val; }; @@ -4875,7 +5308,7 @@ void exec_smfmac_f32_16x16x64_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t int cc = 2 * q + s; int byte = 4 * (cc / 16) + (cc % 16) % 4; float av = ea(cu, s0, byte, laneA); - acc += av * Bcol[4 * q + (s == 0 ? p0 : p1)]; + acc = matrix_fma(av, Bcol[4 * q + (s == 0 ? p0 : p1)], acc); } } results.push_back({out.reg, out.lane, std::bit_cast(acc)}); @@ -4890,6 +5323,8 @@ void exec_smfmac_f32_16x16x64_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t template void exec_smfmac_f32_32x32x32_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t idx_base, ExtractA ea, ExtractB eb) { + if (smfmac_try_avx512(cu, dst, s0, s1, idx_base, ea, eb)) + return; struct Result { uint32_t reg, lane, val; }; @@ -4919,7 +5354,7 @@ void exec_smfmac_f32_32x32x32_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t int cc = 2 * q + s; int byte = 4 * (cc / 8) + (cc % 8) % 4; float av = ea(cu, s0, byte, laneA); - acc += av * Bcol[4 * q + (s == 0 ? p0 : p1)]; + acc = matrix_fma(av, Bcol[4 * q + (s == 0 ? p0 : p1)], acc); } } results.push_back({out.reg, out.lane, std::bit_cast(acc)}); @@ -4934,6 +5369,8 @@ void exec_smfmac_f32_32x32x32_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t template void exec_smfmac_f32_16x16x128_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t idx_base, ExtractA ea, ExtractB eb) { + if (smfmac_try_avx512(cu, dst, s0, s1, idx_base, ea, eb)) + return; struct Result { uint32_t reg, lane, val; }; @@ -4961,7 +5398,7 @@ void exec_smfmac_f32_16x16x128_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t int hb = 2 * ((cc >> 2) & 1) + ((cc >> 4) & 1); int byte = 4 * hb + (cc & 3); float av = ea(cu, s0, byte, ga * 16 + row); - acc += av * Bcol[4 * q + (s == 0 ? p0 : p1)]; + acc = matrix_fma(av, Bcol[4 * q + (s == 0 ? p0 : p1)], acc); } } results.push_back({out.reg, out.lane, std::bit_cast(acc)}); @@ -4976,6 +5413,8 @@ void exec_smfmac_f32_16x16x128_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t template void exec_smfmac_f32_32x32x64_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t idx_base, ExtractA ea, ExtractB eb) { + if (smfmac_try_avx512(cu, dst, s0, s1, idx_base, ea, eb)) + return; struct Result { uint32_t reg, lane, val; }; @@ -5005,7 +5444,7 @@ void exec_smfmac_f32_32x32x64_fp8(auto &cu, uint32_t dst, uint32_t s0, uint32_t int hb = 2 * ((cc >> 2) & 1) + ((cc >> 3) & 1); int byte = 4 * hb + (cc & 3); float av = ea(cu, s0, byte, ga * 32 + row); - acc += av * Bcol[4 * q + (s == 0 ? p0 : p1)]; + acc = matrix_fma(av, Bcol[4 * q + (s == 0 ? p0 : p1)], acc); } } results.push_back({out.reg, out.lane, std::bit_cast(acc)}); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/pseudo_scalar.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/pseudo_scalar.h index b6cd151fc271..782ae23300a4 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/pseudo_scalar.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/pseudo_scalar.h @@ -5,13 +5,17 @@ /// @file Shared pseudo-scalar transcendental implementations. /// -/// @details Architectural numeric behavior follows the AMD RDNA4 Instruction Set Architecture -/// Reference Guide: section 7.10 requires the usual DENORMAL and ROUND mode bits, and section -/// 7.2.3.1 requires nonzero OMOD to flush output denormals and map negative zero to positive zero. -/// The functional examples for the vector V_LOG_F32, V_RSQ_F32, and V_SQRT_F32 equivalents specify -/// negative quiet NaNs for invalid domains. The host standard-library functions below provide the -/// approximate transcendental values; the surrounding logic applies these architectural rules. - +/// @details Physical gfx1201 F32 operations match their vector equivalents: fixed rounding +/// and unconditional denormal flushing. Half execution shares the vector mapping in +/// transcendental.h. Section 7.2.3.1 requires nonzero OMOD to flush +/// output denormals and map negative zero to positive zero. Vector LOG, RSQ and SQRT +/// functional examples specify negative quiet NaNs for invalid domains. + +#include "util/amdgpu_exp.h" +#include "util/amdgpu_log.h" +#include "util/amdgpu_rcp.h" +#include "util/amdgpu_rsq.h" +#include "util/amdgpu_sqrt.h" #include "util/data_types.h" #include @@ -50,60 +54,13 @@ inline float flush_input_f16(float value, uint32_t denorm_mode) { inline float quiet_nan(float value) { uint32_t bits = std::bit_cast(value); - if ((bits & 0x7f800000u) == 0x7f800000u && (bits & 0x007fffffu) != 0) + // Test signaling NaNs specifically: a general NaN check can become a host + // floating-point comparison that raises FE_INVALID before the quieting step. + if ((bits & 0x7fc00000u) == 0x7f800000u && (bits & 0x003fffffu) != 0) bits |= 0x00400000u; return std::bit_cast(bits); } -inline EvaluationResult evaluate(Operation operation, double value) { - if (std::isnan(value)) - return {value, ResultProvenance::VALUE}; - - switch (operation) { - case Operation::EXP2: { - if (std::isinf(value)) - return {value < 0.0 ? 0.0 : value, ResultProvenance::VALUE}; - const double result = std::exp2(value); - if (std::isinf(result)) - return {result, ResultProvenance::FINITE_OVERFLOW}; - if (result == 0.0) - return {result, ResultProvenance::FINITE_UNDERFLOW}; - return {result, ResultProvenance::VALUE}; - } - case Operation::LOG2: - if (value == 0.0) - return {-std::numeric_limits::infinity(), ResultProvenance::VALUE}; - if (value < 0.0) - return {std::numeric_limits::quiet_NaN(), ResultProvenance::INVALID_DOMAIN}; - if (std::isinf(value)) - return {value, ResultProvenance::VALUE}; - return {std::log2(value), ResultProvenance::VALUE}; - case Operation::RCP: - if (value == 0.0) - return {std::copysign(std::numeric_limits::infinity(), value), - ResultProvenance::VALUE}; - if (std::isinf(value)) - return {std::copysign(0.0, value), ResultProvenance::VALUE}; - return {1.0 / value, ResultProvenance::VALUE}; - case Operation::RSQ: - if (value == 0.0) - return {std::copysign(std::numeric_limits::infinity(), value), - ResultProvenance::VALUE}; - if (value < 0.0) - return {std::numeric_limits::quiet_NaN(), ResultProvenance::INVALID_DOMAIN}; - if (std::isinf(value)) - return {0.0, ResultProvenance::VALUE}; - return {1.0 / std::sqrt(value), ResultProvenance::VALUE}; - case Operation::SQRT: - if (value < 0.0) - return {std::numeric_limits::quiet_NaN(), ResultProvenance::INVALID_DOMAIN}; - if (std::isinf(value)) - return {value, ResultProvenance::VALUE}; - return {std::sqrt(value), ResultProvenance::VALUE}; - } - return {std::numeric_limits::quiet_NaN(), ResultProvenance::VALUE}; -} - inline EvaluationResult apply_output_modifiers(EvaluationResult result, uint32_t omod, bool clamp) { const double unmodified_value = result.value; if (omod == 1) @@ -306,64 +263,54 @@ inline uint16_t round_f16_result(double value, uint32_t round_mode, uint32_t omo } /// @brief Execute a pseudo-scalar F32 transcendental operation. -/// @details Source absolute value and negation are applied before input-denormal handling and -/// operation evaluation. OMOD is then applied before CLAMP, result rounding, and output-denormal -/// handling. Round modes are 0 for nearest-even, 1 for positive infinity, 2 for negative infinity, -/// and 3 for zero. Denormal mode bit 0 allows input denormals and bit 1 allows output denormals. +/// @details Apply source modifiers and quiet NaNs before the hardware mapping, then apply +/// OMOD and CLAMP before nearest-even narrowing and unconditional output-denormal flushing. +/// All F32 operations ignore MODE rounding and denormal fields, matching their vector +/// equivalents. Output-modifier overflow rounds to infinity. /// @param operation Transcendental operation to execute. /// @param source Raw F32 source value. /// @param absolute Whether to clear the source sign bit before evaluation. /// @param negate Whether to toggle the source sign bit after applying absolute value. -/// @param round_mode Numeric MODE.FP_ROUND encoding for F32. -/// @param denorm_mode Numeric MODE.FP_DENORM encoding for F32. +/// @param round_mode Ignored F32 rounding mode; retained for the shared instruction interface. +/// @param denorm_mode Ignored F32 denormal mode; retained for the shared instruction interface. /// @param omod Numeric VOP3 OMOD encoding: 0 unchanged, 1 multiply by 2, 2 multiply by 4, and 3 /// multiply by 0.5. /// @param clamp Whether to clamp NaN and negative results to zero and results above one to one. /// @returns Raw 32-bit F32 result encoding. inline uint32_t execute_f32(Operation operation, float source, bool absolute, bool negate, - uint32_t round_mode, uint32_t denorm_mode, uint32_t omod, bool clamp) { + [[maybe_unused]] uint32_t round_mode, + [[maybe_unused]] uint32_t denorm_mode, uint32_t omod, bool clamp) { source = detail::apply_source_modifiers(source, absolute, negate); - source = detail::flush_input_f32(source, denorm_mode); source = detail::quiet_nan(source); - const detail::EvaluationResult value = detail::apply_output_modifiers( - detail::evaluate(operation, static_cast(source)), omod, clamp); - uint32_t result = detail::round_f64_to_f32(value, round_mode); - if (((denorm_mode & 2u) == 0 || omod != 0) && (result & 0x7f800000u) == 0 && - (result & 0x007fffffu) != 0) + // Physical GFX12 scalar and vector mappings agree in every FP MODE. + float mapped = 0.0f; + switch (operation) { + case Operation::EXP2: + mapped = util::amdgpu_exp_f32(source); + break; + case Operation::LOG2: + mapped = util::amdgpu_log_f32(source); + break; + case Operation::RCP: + mapped = util::amdgpu_rcp_f32(source); + break; + case Operation::RSQ: + mapped = util::amdgpu_rsq_f32(source); + break; + case Operation::SQRT: + mapped = util::amdgpu_sqrt_f32(source); + break; + } + const detail::EvaluationResult evaluated{mapped, detail::ResultProvenance::VALUE}; + detail::EvaluationResult value = detail::apply_output_modifiers(evaluated, omod, clamp); + // Output scaling is exact in F64. Detect F32 overflow before narrowing, + // so directed host rounding cannot replace the required infinity with a finite value. + if (std::isfinite(value.value) && std::abs(value.value) > std::numeric_limits::max()) + value.provenance = detail::ResultProvenance::FINITE_OVERFLOW; + uint32_t result = detail::round_f64_to_f32(value, 0); + if ((result & 0x7f800000u) == 0 && (result & 0x007fffffu) != 0) result &= 0x80000000u; return result; } -/// @brief Execute a pseudo-scalar F16 transcendental operation. -/// @details Source absolute value and negation are applied before input-denormal handling and -/// operation evaluation. OMOD is then applied before CLAMP, result rounding, and output-denormal -/// handling. Round modes are 0 for nearest-even, 1 for positive infinity, 2 for negative infinity, -/// and 3 for zero. Denormal mode bit 0 allows input denormals and bit 1 allows output denormals. -/// FP16_OVFL clamps finite overflow to signed maximum finite F16 regardless of round mode, but does -/// not clamp true infinity or divide-by-zero results. -/// @param operation Transcendental operation to execute. -/// @param source F16 source value represented exactly as an F32 value. -/// @param absolute Whether to clear the source sign bit before evaluation. -/// @param negate Whether to toggle the source sign bit after applying absolute value. -/// @param round_mode Numeric MODE.FP_ROUND encoding for F16. -/// @param denorm_mode Numeric MODE.FP_DENORM encoding for F16. -/// @param omod Numeric VOP3 OMOD encoding: 0 unchanged, 1 multiply by 2, 2 multiply by 4, and 3 -/// multiply by 0.5. -/// @param clamp Whether to clamp NaN and negative results to zero and results above one to one. -/// @param fp16_ovfl Whether MODE.FP16_OVFL finite-overflow saturation is enabled. -/// @returns Raw F16 encoding in bits 15:0 with bits 31:16 cleared. -inline uint32_t execute_f16(Operation operation, float source, bool absolute, bool negate, - uint32_t round_mode, uint32_t denorm_mode, uint32_t omod, bool clamp, - bool fp16_ovfl) { - source = detail::apply_source_modifiers(source, absolute, negate); - source = detail::flush_input_f16(source, denorm_mode); - source = detail::quiet_nan(source); - const detail::EvaluationResult value = detail::apply_output_modifiers( - detail::evaluate(operation, static_cast(source)), omod, clamp); - uint16_t result = detail::round_f64_to_f16(value, round_mode, fp16_ovfl); - if (((denorm_mode & 2u) == 0 || omod != 0) && (result & 0x7c00u) == 0 && (result & 0x03ffu) != 0) - result &= 0x8000u; - return result; -} - } // namespace rocjitsu::amdgpu::pseudo_scalar diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_coverage.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_coverage.h index 7b0e9965d941..8d35afa07d8b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_coverage.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_coverage.h @@ -136,6 +136,7 @@ template + requires(!(UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS && sizeof(Float) == 8)) inline auto div_scale_simd(util::native value, util::native denominator, util::native numerator, uint32_t rounding, uint32_t denorm) { using F = DivisionFormat; @@ -185,6 +186,23 @@ inline auto div_scale_simd(util::native value, util::native denomi return std::pair{std::bit_cast>(result), post_bits}; } +#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS +template + requires(sizeof(Float) == 8) +inline auto div_scale_simd(util::native value, util::native denominator, + util::native numerator, uint32_t rounding, uint32_t denorm) { + util::native result(0); + uint64_t post_bits = 0; + for (std::size_t lane = 0; lane < result.size(); ++lane) { + const auto scalar = + div_scale(value[lane], denominator[lane], numerator[lane], rounding, denorm); + result[lane] = scalar.value; + post_bits |= uint64_t{scalar.post_scale} << lane; + } + return std::pair{result, post_bits}; +} +#endif + template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_div_scale_simd(Inst &inst, Wavefront &wf, @@ -359,20 +377,20 @@ template U result; switch (slot.op) { case 0: - result = std::bit_cast(util::stdx::fma(af, bf, acc->template load_native(base))); + result = std::bit_cast(fma_f32_simd(af, bf, acc->template load_native(base), wf)); break; case 1: case 19: - result = std::bit_cast(util::stdx::fma(af, bf, cf)); + result = std::bit_cast(fma_f32_simd(af, bf, cf, wf)); break; case 2: - result = std::bit_cast(util::stdx::fma(af, cf, bf)); + result = std::bit_cast(fma_f32_simd(af, cf, bf, wf)); break; case 3: - result = std::bit_cast(af * bf); + result = std::bit_cast(binary_f32_simd(af, bf, wf)); break; case 4: - result = std::bit_cast(af + bf); + result = std::bit_cast(binary_f32_simd(af, bf, wf)); break; case 5: result = std::bit_cast(af - bf); @@ -525,7 +543,8 @@ template return fma_f16_mode_simd( (av >> shift) & U(0xffffu), (bv >> shift) & U(0xffffu), (cv >> shift) & U(0xffffu), abs & 1, abs & 2, false, neg & 1, neg & 2, false, wf.fp_round_mode_f16_f64(), - wf.fp_denorm_mode_f16_f64(), omod, clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), omod, clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf), + fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); }; U low = compute(0), high = compute(16); dst.template store_native(base, low | (high << 16), mask); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index ad1ba835b96c..661b198171e7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -15,10 +15,12 @@ #define ROCJITSU_ISA_AMDGPU_SHARED_SIMD_GLUE_H_ #include "rocjitsu/isa/arch/amdgpu/generated/shared/isa_properties.h" +#include "rocjitsu/isa/arch/amdgpu/shared/comparison.h" #include "rocjitsu/isa/arch/amdgpu/shared/division.h" #include "rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h" #include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" #include "rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h" +#include "rocjitsu/isa/arch/amdgpu/shared/mixed_fma_simd.h" #include "rocjitsu/isa/operand.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" #include "rocjitsu/vm/amdgpu/register_access.h" @@ -89,11 +91,15 @@ inline bool floating_clamp_nan_to_zero(const Wavefront &wf) { /// @brief Clamp one floating result with an explicit NaN conversion policy. template inline T clamp_floating_result(T value, bool clamp_nan_to_zero) { static_assert(std::is_floating_point_v); - if (std::isnan(value)) + using U = std::conditional_t; + constexpr U sign = U(1) << (sizeof(T) * 8 - 1); + const U bits = std::bit_cast(value); + const U magnitude = bits & ~sign; + if (magnitude > std::bit_cast(std::numeric_limits::infinity())) return clamp_nan_to_zero ? T(0) : value; - if (value <= T(0)) + if ((bits & sign) || magnitude == 0) return T(0); - if (value > T(1)) + if (magnitude > std::bit_cast(T(1))) return T(1); return value; } @@ -249,13 +255,13 @@ inline std::make_unsigned_t vop3_integer_sub(std::make_unsigned_t lhs, } /// @brief Return whether V_DOT4 integer instructions honor the encoded CLAMP bit. -/// @details GFX12 (RDNA4) and the CDNA5-backed gfx1250 profile explicitly ignore CLAMP -/// for the integer DOT4 forms; earlier CDNA/RDNA profiles retain the ordinary -/// integer saturation behavior. Keep this policy shared by scalar generation -/// and SIMD execution. +/// @details The RDNA4 ISA manual says V_DOT4_I32_IU8 and V_DOT4_U32_U8 ignore +/// CLAMP, but experiments with those instructions on gfx1201 hardware show +/// saturation when CLAMP is set. The CDNA5-backed gfx1250 profile still ignores +/// it. Keep this policy shared by scalar and SIMD execution. inline bool dot4_clamp_supported(const Wavefront &wf) { const rj_code_arch_t arch = wf.cu().arch(); - return arch != ROCJITSU_CODE_ARCH_RDNA4 && arch != ROCJITSU_CODE_ARCH_CDNA5; + return arch != ROCJITSU_CODE_ARCH_CDNA5; } inline uint32_t sign_extend_u32(uint32_t value, unsigned bits) { @@ -581,12 +587,31 @@ util::native apply_vop3_src_mod_f64(util::native v, uint32_t abs template util::native apply_vop3_dst_mod(util::native v, uint32_t omod, uint32_t clamp, bool clamp_nan_to_zero) { + if constexpr (std::is_same_v) { + if (omod != 0) { + using U = util::native; + U bits = std::bit_cast(v); + util::stdx::where((bits & U(0x7fffffffu)) < U(0x00800000u), bits) = U(0); + v = std::bit_cast>(bits); + } + } + const auto unscaled = v; if (omod == 1) v = v * T(2); else if (omod == 2) v = v * T(4); else if (omod == 3) v = v * T(0.5); + if constexpr (std::is_same_v) { + using U = util::native; + U bits = std::bit_cast(v); + const U original = std::bit_cast(unscaled); + util::stdx::where((original & U(0x7fffffffu)) > U(0x7f800000u), bits) = original; + if (omod == 3) + util::stdx::where((original & U(0x7fffffffu)) < U(0x01000000u), bits) = + original & U(0x80000000u); + v = std::bit_cast>(bits); + } if (clamp) { #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS if constexpr (std::is_same_v) { @@ -613,16 +638,27 @@ util::native apply_vop3_dst_mod(util::native v, uint32_t omod, uint32_t cl using U = util::native; U bits = std::bit_cast(v); const auto subnormal = ((bits & U(0x7f800000u)) == U(0)) && ((bits & U(0x007fffffu)) != U(0)); + // Existing zeros were canonicalized before scaling. Preserve the sign + // of a newly underflowed result. util::stdx::where(subnormal, bits) = bits & U(0x80000000u); - util::stdx::where((bits & U(0x7fffffffu)) == U(0), bits) = U(0); v = std::bit_cast>(bits); } else { using U = util::native; U bits = std::bit_cast(v); +#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS + bits = util::map_native64_scalar(bits, [](uint64_t lane) { + const bool subnormal = + (lane & 0x7ff0000000000000ULL) == 0 && (lane & 0x000fffffffffffffULL) != 0; + if (subnormal) + lane &= 0x8000000000000000ULL; + return (lane & 0x7fffffffffffffffULL) == 0 ? 0 : lane; + }); +#else const auto subnormal = ((bits & U(0x7ff0000000000000ULL)) == U(0)) && ((bits & U(0x000fffffffffffffULL)) != U(0)); util::stdx::where(subnormal, bits) = bits & U(0x8000000000000000ULL); util::stdx::where((bits & U(0x7fffffffffffffffULL)) == U(0), bits) = U(0); +#endif v = std::bit_cast>(bits); } } @@ -639,8 +675,10 @@ inline util::native apply_vop3_dst_mod_f32(util::native v, uint32_ return apply_vop3_dst_mod(v, omod, clamp, clamp_nan_to_zero); } -inline uint32_t effective_vop3_omod_f32(const Wavefront &wf, uint32_t omod) { - return fp_mode::effective_omod(wf.cu().arch(), wf.fp_denorm_mode_f32(), wf.ieee_mode(), omod); +inline uint32_t effective_vop3_omod_f32(const Wavefront &wf, uint32_t omod, + bool force_output_flush = false) { + return fp_mode::effective_omod(wf.cu().arch(), force_output_flush ? 0 : wf.fp_denorm_mode_f32(), + wf.ieee_mode(), omod); } inline uint32_t effective_vop3_omod_f16(const Wavefront &wf, uint32_t omod) { @@ -665,14 +703,15 @@ inline util::native finalize_omod_f16_bits_simd(util::native /// @brief Execute a native-width batch of architectural F16 fused multiply-adds. /// @details The raw F16 operands remain in 32-bit SIMD lanes. Each native-width -/// batch is split into double-width chunks so the multiply-add itself is fused -/// in SIMD without the erroneous F16-to-F32-to-F16 double rounding. The final +/// batch is split into double-width chunks and retains an exact-sum residual +/// without an intermediate F32 rounding. The final /// F16 rounding and output policy reuse the scalar architectural primitive. inline util::native fma_f16_mode_simd(util::native src0, util::native src1, util::native src2, bool abs0, bool abs1, bool abs2, bool neg0, bool neg1, bool neg2, uint32_t round_mode, uint32_t denorm_mode, uint32_t omod, - bool clamp, bool fp16_ovfl, bool clamp_nan_to_zero) { + bool clamp, bool fp16_ovfl, bool clamp_nan_to_zero, bool quiet_nan) { + fp_mode::ScopedEnvironment nearest_environment(0); constexpr std::size_t W32 = util::native_width_v; constexpr std::size_t W64 = util::native_width64; static_assert(W32 % W64 == 0); @@ -700,11 +739,15 @@ fma_f16_mode_simd(util::native src0, util::native src1, const util::native a(a_lanes, util::stdx::vector_aligned); const util::native b(b_lanes, util::stdx::vector_aligned); const util::native c(c_lanes, util::stdx::vector_aligned); - const util::native result = util::stdx::fma(a, b, c); + const util::native product = a * b; + const util::native result = product + c; + const util::native addend_virtual = result - product; + const util::native residual = + (product - (result - addend_virtual)) + (c - addend_virtual); alignas(util::native) double result_lanes[W64]; result.copy_to(result_lanes, util::stdx::vector_aligned); for (std::size_t i = 0; i < W64; ++i) { - double value = result_lanes[i]; + double value = fp_mode::detail::round_to_odd({result_lanes[i], residual[i]}); if (value == 0.0) { // F16 products cannot underflow in double, so zero means exact // cancellation or zero operands. Apply the guest sign policy. @@ -713,25 +756,111 @@ fma_f16_mode_simd(util::native src0, util::native src1, c_lanes[i] == 0.0 && negative_product == std::signbit(c_lanes[i]); value = (matching_zeros ? negative_product : round_mode == 2) ? -0.0 : 0.0; } - const bool nan_input = - std::isnan(a_lanes[i]) || std::isnan(b_lanes[i]) || std::isnan(c_lanes[i]); - uint16_t rounded = nan_input - ? fp_mode::fma_f16(static_cast(raw0[base + i]), - static_cast(raw1[base + i]), - static_cast(raw2[base + i]), abs0, abs1, - abs2, neg0, neg1, neg2, round_mode, denorm_mode, - omod, clamp, fp16_ovfl, clamp_nan_to_zero) - : pseudo_scalar::round_f16_result(value, round_mode, omod, clamp, - fp16_ovfl, clamp_nan_to_zero); - if ((denorm_mode & 2u) == 0 && (rounded & 0x7c00u) == 0 && (rounded & 0x03ffu) != 0) - rounded &= 0x8000u; - out[base + i] = fp_mode::finalize_omod_f16(rounded, omod); + const bool exceptional_input = + !std::isfinite(a_lanes[i]) || !std::isfinite(b_lanes[i]) || !std::isfinite(c_lanes[i]); + uint16_t rounded = + exceptional_input ? fp_mode::fma_f16(static_cast(raw0[base + i]), + static_cast(raw1[base + i]), + static_cast(raw2[base + i]), abs0, abs1, + abs2, neg0, neg1, neg2, round_mode, denorm_mode, + omod, clamp, fp16_ovfl, clamp_nan_to_zero, quiet_nan) + : fp_mode::finish_fma_f16(value, round_mode, denorm_mode, omod, clamp, + fp16_ovfl, clamp_nan_to_zero); + out[base + i] = rounded; } } return util::native(out, util::stdx::vector_aligned); } +/// F32 FMA runs in the caller's guest-rounding environment and applies MODE +/// denormal controls explicitly. Exceptional lanes use scalar NaN selection +/// and minimum-normal boundary handling. +inline util::native fma_f32_simd(util::native a, util::native b, + util::native c, const Wavefront &wf, + uint32_t omod = 0, bool force_flush = false) { + using U = util::native; + const uint32_t denorm_mode = force_flush ? 0 : wf.fp_denorm_mode_f32(); + if (!(denorm_mode & 1u)) { + a = util::flush_denorm_f32_simd(a); + b = util::flush_denorm_f32_simd(b); + c = util::flush_denorm_f32_simd(c); + } + const bool flush_output = omod || !(denorm_mode & 2u); + auto exceptional = (std::bit_cast(a) & U(0x7fffffffu)) >= U(0x7f800000u) || + (std::bit_cast(b) & U(0x7fffffffu)) >= U(0x7f800000u) || + (std::bit_cast(c) & U(0x7fffffffu)) >= U(0x7f800000u); + auto result = util::stdx::fma(a, b, c); + if (flush_output) + exceptional |= (std::bit_cast(result) & U(0x7fffffffu)) == U(0x00800000u); + if (util::stdx::any_of(exceptional)) + for (std::size_t i = 0; i < U::size(); ++i) + if (exceptional[i]) + result[i] = fp_mode::fma_f32(a[i], b[i], c[i], wf.cu().arch(), wf.ieee_mode(), denorm_mode, + omod != 0); + // FMA's subnormal intermediate is flushed before OMOD can scale it normal. + return flush_output ? util::flush_denorm_f32_simd(result) : result; +} + +/// ADD and MUL share FMA's NaN and pre-packing tininess policy. SUB retains +/// direct host subtraction's policy. The caller establishes guest rounding. +template +inline util::native binary_f32_simd(util::native a, util::native b, + const Wavefront &wf, uint32_t omod = 0) { + static_assert(operation == fp_mode::Arithmetic::ADD || operation == fp_mode::Arithmetic::SUB || + operation == fp_mode::Arithmetic::MUL); + if constexpr (operation == fp_mode::Arithmetic::SUB) { + // Preserve direct subtraction's existing host NaN policy. Rewriting this as + // architectural ADD with a negated input would select different NaN bits. + using U = util::native; + const uint32_t denorm_mode = wf.fp_denorm_mode_f32(); + if (!(denorm_mode & 1u)) { + a = util::flush_denorm_f32_simd(a); + b = util::flush_denorm_f32_simd(b); + } + auto result = a - b; + const auto nan_input = (std::bit_cast(a) & U(0x7fffffffu)) > U(0x7f800000u) || + (std::bit_cast(b) & U(0x7fffffffu)) > U(0x7f800000u); + if (util::stdx::any_of(nan_input)) + for (std::size_t i = 0; i < U::size(); ++i) + if (nan_input[i]) + result[i] = fp_mode::detail::evaluate_arithmetic( + a[i], b[i], 0.0f); + return (denorm_mode & 2u) ? result : util::flush_denorm_f32_simd(result); + } else if constexpr (operation == fp_mode::Arithmetic::ADD) + return fma_f32_simd(a, util::native(1.0f), b, wf, omod); + else { + using U = util::native; + auto zero = std::bit_cast>((std::bit_cast(a) ^ std::bit_cast(b)) & + U(0x80000000u)); + return fma_f32_simd(a, b, zero, wf, omod); + } +} + +/// DX9 accumulator and three-source forms share the scalar flushing, zero-product, +/// NaN and underflow policies. +inline util::native fma_dx9_zero_f32_simd(util::native a, util::native b, + util::native c, const Wavefront &wf) { + using U = util::native; + fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); + a = util::flush_denorm_f32_simd(a); + b = util::flush_denorm_f32_simd(b); + const auto zero_product = (std::bit_cast(a) & U(0x7fffffffu)) == U(0) || + (std::bit_cast(b) & U(0x7fffffffu)) == U(0); + // DX9 supplies a positive zero product even when the other operand is + // infinite or NaN. The addition still applies rounding and addend NaN rules. + U a_bits = std::bit_cast(a), b_bits = std::bit_cast(b); + util::stdx::where(zero_product, a_bits) = U(0); + util::stdx::where(zero_product, b_bits) = U(0x3f800000u); + a = std::bit_cast>(a_bits); + b = std::bit_cast>(b_bits); + return fma_f32_simd(a, b, c, wf, 0, true); +} + /// @brief Execute a MODE-aware native batch of architectural F64 FMAs. +#if defined(__GNUC__) && !defined(__clang__) +// Keep hardware FMA inside the guest rounding environment when GCC inlines it. +[[gnu::optimize("rounding-math")]] +#endif inline util::native fma_f64_mode_simd(util::native src0, util::native src1, util::native src2, uint32_t round_mode, uint32_t denorm_mode) { @@ -741,9 +870,17 @@ inline util::native fma_f64_mode_simd(util::native src0, util::n const U original2 = std::bit_cast(src2); auto flush = [](util::native value) { U bits = std::bit_cast(value); +#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS + bits = util::map_native64_scalar(bits, [](uint64_t lane) { + const bool denormal = + (lane & 0x7ff0000000000000ULL) == 0 && (lane & 0x000fffffffffffffULL) != 0; + return denormal ? lane & 0x8000000000000000ULL : lane; + }); +#else const auto denormal = (bits & 0x7ff0000000000000ULL) == 0 && (bits & 0x000fffffffffffffULL) != 0; util::stdx::where(denormal, bits) = bits & 0x8000000000000000ULL; +#endif return std::bit_cast>(bits); }; if ((denorm_mode & 1u) == 0) { @@ -1104,12 +1241,7 @@ template /// try_execute_ternary_vop2_acc_simd below so register observation follows the /// instruction-visible read set exactly. /// -/// `util::stdx::fma` is bit-identical to the scalar `std::fma` for all finite -/// and infinite inputs (including Inf*0 -> NaN). When an *input* is NaN the -/// packed and scalar FMA may propagate a different NaN operand (a toolchain- -/// dependent payload, observed on g++-13/AVX-512); that NaN-payload divergence -/// is accepted — the result is a NaN either way. The finite/Inf bit-exactness -/// the fast path relies on is guarded by UtilSimd.Fma_VectorMatchesScalar_*. +/// F32 operations use fma_f32_simd to preserve architectural NaN payloads. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_ternary_vop2_simd(Inst &inst, Wavefront &wf, @@ -1203,11 +1335,11 @@ template const uint64_t chunk = (exec >> base) & chunk_full; if (chunk == 0) continue; - const auto result = - fma_f16_mode_simd(src0.template load_native(base), src1.template load_native(base), - acc.template load_native(base), false, false, false, false, false, - false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, - false, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + const auto result = fma_f16_mode_simd( + src0.template load_native(base), src1.template load_native(base), + acc.template load_native(base), false, false, false, false, false, false, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, wf.fp16_ovfl(), + floating_clamp_nan_to_zero(wf), fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); acc.template store_native(base, result, chunk); } } else { @@ -1222,10 +1354,12 @@ template Shape == F16Vop2FmaShape::AddLiteral ? fma_f16_mode_simd(a, b, literal_value, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)) + wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf), + fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())) : fma_f16_mode_simd(a, literal_value, b, false, false, false, false, false, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, false, - wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf), + fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); dst.template store_native(base, result, chunk); } } @@ -1496,6 +1630,18 @@ template return false; } +/// @brief Apply V_CVT_F32_F16 policy before widening a SIMD batch. +inline util::native cvt_f32_f16_mode_simd(util::native raw, const Wavefront &wf) { + using U = util::native; + raw &= U(0xffffu); + const U magnitude = raw & U(0x7fffu); + if (!(wf.fp_denorm_mode_f16_f64() & 1u)) + util::stdx::where(magnitude < U(0x0400u), raw) = raw & U(0x8000u); + if (fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())) + util::stdx::where(magnitude > U(0x7c00u), raw) = raw | U(0x0200u); + return util::f16_to_f32_simd(raw); +} + /// VOP3 v_cvt_f32_f16 fast path. The generic form reads the low f16 half and /// writes the full f32 dword, matching non-true16 scalar write_lane semantics. /// The true16 form selects src0 with op_sel[0] before widening; the destination @@ -1510,6 +1656,11 @@ template const uint32_t abs = inst.inst_.abs; const uint32_t neg = inst.inst_.neg; const uint32_t opsel = vop3_opsel(inst.inst_); + const uint32_t omod = effective_vop3_omod_f32(wf, inst.inst_.omod); + const uint32_t clamp = inst.inst_.clamp; + std::optional environment; + if (omod || clamp) + environment.emplace(wf.fp_round_mode_f32()); constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); const uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -1525,8 +1676,9 @@ template raw = select_vop3_true16_src(raw, opsel, 0); else raw = raw & util::broadcast(0xffffu); - const auto src = apply_vop3_src_mod_f32<0>(util::f16_to_f32_simd(raw), abs, neg); - dst.template store_native(base, std::bit_cast>(src), chunk); + const auto src = apply_vop3_src_mod_f32<0>(cvt_f32_f16_mode_simd(raw, wf), abs, neg); + const auto result = apply_vop3_dst_mod_f32(src, omod, clamp, floating_clamp_nan_to_zero(wf)); + dst.template store_native(base, std::bit_cast>(result), chunk); } return true; } @@ -1662,15 +1814,17 @@ template [[nodiscard]] bool try_execute_cndmask_b16_vop3_simd(In /// inactive-lane VCC bits are preserved (mirroring the scalar body, which /// flips only active-lane bits). VOPC writes VCC only — there is no vdst /// operand and CDNA4 has no v_cmpx (EXEC-writing) form, so this single shape -/// covers every compare. `T` is the 32-bit lane read type (float32_t for the -/// f32 relations, int32_t/uint32_t for the integer ones); the f16 and 16-bit -/// integer relations also read as 32-bit lanes and narrow/convert inside the -/// functor. The VCC merge is identical to the carry path's. +/// covers every compare. `T` is the 32-bit lane read type (uint32_t raw +/// encodings for the f16/f32 relations, int32_t/uint32_t for the integer ones); +/// the f16 and 16-bit integer relations read as 32-bit lanes and narrow/convert +/// inside the functor. The VCC merge is identical to the carry path's. /// -/// Float comparison operators (and stdx::isnan, used by the ordered/unordered -/// relations) produce the same per-lane boolean as the scalar `<`/`==`/isnan, -/// for all inputs including NaN/Inf/±0 — so the compares are bit-exact with no -/// accepted-divergence carve-out (unlike fma / min-max). +/// Float relations never see host floats: their functors call +/// comparison::evaluate on the raw encodings with the captured MODE +/// input-denormal policy, the same evaluation the scalar body uses, so host +/// DAZ cannot alter a lane and the compares are bit-exact for every input +/// including NaN/Inf/±0/subnormals (no accepted-divergence carve-out, unlike +/// fma / min-max). template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_vopc_simd(Inst &inst, Wavefront &wf, CmpOp cmp_op, @@ -1710,9 +1864,11 @@ template } /// 64-bit-lane VOPC compare SIMD fast path (f64/i64/u64 relations). Identical to -/// try_execute_vopc_simd but reads each operand as `native` (T = double / -/// int64_t / uint64_t) through 64-bit RegisterAccess operand views, so -/// it processes `native_width64` lanes per chunk. Same VCC merge. +/// try_execute_vopc_simd but reads each operand as `native` (T = uint64_t +/// raw encodings for f64, int64_t / uint64_t for the integer relations) through +/// 64-bit RegisterAccess operand views, so it processes `native_width64` lanes +/// per chunk. f64 functors evaluate through comparison::evaluate as above. +/// Same VCC merge. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_vopc64_simd(Inst &inst, Wavefront &wf, CmpOp cmp_op, @@ -2084,14 +2240,14 @@ template return false; } -/// VOP3 integer/bitwise VOPC compare SIMD fast path (32-bit lane). The VOP3 form -/// of v_cmp__ reads src0/src1 (not src0/vsrc1) and writes -/// the per-lane compare result into an arbitrary wave-mask scalar destination -/// through a caller-provided architectural commit. The -/// integer/bitwise scalar bodies apply no source/result modifiers (abs/neg/omod -/// are float-only; clamp on integer is unused here), so the plain functor is -/// bit-identical to the scalar body on every input. The raw result contains -/// active EXEC lanes only. Returns true when the SIMD path executed. +/// VOP3 raw-lane VOPC compare SIMD fast path (32-bit lane). The VOP3 form +/// of v_cmp__ reads src0/src1 (not src0/vsrc1) and +/// writes the per-lane compare result into an arbitrary wave-mask scalar +/// destination through a caller-provided architectural commit. The functor +/// receives the raw source encodings: integer relations use no modifiers, and +/// float relations apply their captured abs/neg fields in shared/comparison.h. +/// The raw result contains active EXEC lanes only. Returns true when the SIMD +/// path executed. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_vopc_vop3_int_simd(Inst &inst, Wavefront &wf, CmpOp cmp_op, @@ -2128,11 +2284,11 @@ template ` (T = int64_t / uint64_t) through 64-bit RegisterAccess operand /// views, so it processes `native_width64` lanes per chunk. The caller performs -/// the result commit. No modifiers. +/// the result commit; any float modifiers are applied by the functor. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_vopc64_vop3_int_simd(Inst &inst, Wavefront &wf, CmpOp cmp_op, @@ -2165,141 +2321,6 @@ template return false; } -/// VOP3 f32 VOPC compare SIMD fast path. It reads src0/src1 as `native` -/// and applies the -/// per-source abs/neg VOP3 modifiers — bit-identical to the scalar body which -/// does `std::fabs` then unary minus per source before comparing. The compare -/// itself is the existing VOPC f32 functor (omod/clamp are not applied because -/// the compare result is a single bit, not an f32; the scalar bodies for these -/// kernels likewise ignore omod/clamp). NaN handling mirrors the scalar -/// `<`/`==`/etc. exactly. Returns true when the SIMD path executed. -template - requires(util::has_stdx_simd) -[[nodiscard]] inline bool try_execute_vopc_vop3_fp32_simd(Inst &inst, Wavefront &wf, CmpOp cmp_op, - WriteResult write_result) { - if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable()) - return false; - using T = float32_t; - const uint32_t abs = inst.inst_.abs; - const uint32_t neg = inst.inst_.neg; - constexpr std::size_t W = util::native_width_v; - const uint64_t chunk_full = util::mask(static_cast(W)); - const uint64_t exec = dpp::execution_lane_mask(inst, wf); - uint64_t dst = 0; - RegisterAccess regs(wf); - auto src0 = regs.read_operand(inst.src0, exec); - auto src1 = regs.read_operand(inst.src1, exec); - for (uint32_t base = 0; base < wf.wf_size(); base += static_cast(W)) { - const uint64_t chunk = (exec >> base) & chunk_full; - if (chunk == 0) - continue; - const auto a = apply_vop3_src_mod_f32<0>(src0.template load_native(base), abs, neg); - const auto b = apply_vop3_src_mod_f32<1>(src1.template load_native(base), abs, neg); - const uint64_t cmp_bits = util::simd_mask_to_bits(cmp_op(a, b)); - dst = (dst & ~(chunk << base)) | ((cmp_bits & chunk) << base); - } - write_result(dst); - return true; -} - -/// Unconstrained fallback for the VOP3 f32 VOPC path; see the binary-path note. -template -[[nodiscard]] bool try_execute_vopc_vop3_fp32_simd(Inst &, Wavefront &, CmpOp, WriteResult) { - return false; -} - -/// VOP3 f16 VOPC compare SIMD fast path. The generic form reads the low f16 -/// half; the true16 form selects source halves with VOP3 op_sel. Both then -/// widen each f16 src to f32 (`util::f16_to_f32`) and only then apply abs/neg -/// (std::fabs / unary minus on the f32), matching their scalar bodies. -template - requires(util::has_stdx_simd) -[[nodiscard]] inline bool try_execute_vopc_vop3_fp16_simd(Inst &inst, Wavefront &wf, CmpOp cmp_op, - WriteResult write_result) { - if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable()) - return false; - using T = uint32_t; - const uint32_t abs = inst.inst_.abs; - const uint32_t neg = inst.inst_.neg; - const uint32_t opsel = vop3_opsel(inst.inst_); - constexpr std::size_t W = util::native_width_v; - const uint64_t chunk_full = util::mask(static_cast(W)); - const uint64_t exec = dpp::execution_lane_mask(inst, wf); - uint64_t dst = 0; - RegisterAccess regs(wf); - auto src0 = regs.read_operand(inst.src0, exec); - auto src1 = regs.read_operand(inst.src1, exec); - for (uint32_t base = 0; base < wf.wf_size(); base += static_cast(W)) { - const uint64_t chunk = (exec >> base) & chunk_full; - if (chunk == 0) - continue; - auto a_raw = src0.template load_native(base); - auto b_raw = src1.template load_native(base); - if constexpr (True16) { - a_raw = select_vop3_true16_src(a_raw, opsel, 0); - b_raw = select_vop3_true16_src(b_raw, opsel, 1); - } else { - a_raw = a_raw & util::broadcast(0xffffu); - b_raw = b_raw & util::broadcast(0xffffu); - } - const auto a = apply_vop3_src_mod_f32<0>(util::f16_to_f32_simd(a_raw), abs, neg); - const auto b = apply_vop3_src_mod_f32<1>(util::f16_to_f32_simd(b_raw), abs, neg); - const uint64_t cmp_bits = util::simd_mask_to_bits(cmp_op(a, b)); - dst = (dst & ~(chunk << base)) | ((cmp_bits & chunk) << base); - } - write_result(dst); - return true; -} - -/// Unconstrained fallback for the VOP3 f16 VOPC path; see the binary-path note. -template -[[nodiscard]] bool try_execute_vopc_vop3_fp16_simd(Inst &, Wavefront &, CmpOp, WriteResult) { - return false; -} - -/// VOP3 f64 VOPC compare SIMD fast path. 64-bit-lane counterpart of the f32 -/// path: reads src0/src1 as `native` through 64-bit RegisterAccess -/// operand views, applies the per-source abs/neg modifiers in the f64 -/// domain (apply_vop3_src_mod_f64; sign-bit AND/XOR — bit-identical incl. NaN -/// payload), and calls the compare functor on `native` operands. The -/// packed lane-mask result is returned through `write_result`, which owns the -/// one architectural commit after any DPP masking. Lanes are processed -/// `native_width64` at a time. Bit-identical to the scalar body for every input. -template - requires(util::has_stdx_simd) -[[nodiscard]] inline bool try_execute_vopc64_vop3_fp64_simd(Inst &inst, Wavefront &wf, CmpOp cmp_op, - WriteResult write_result) { - if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable()) - return false; - using T = double; - const uint32_t abs = inst.inst_.abs; - const uint32_t neg = inst.inst_.neg; - constexpr std::size_t W = util::native_width64; - const uint64_t chunk_full = util::mask(static_cast(W)); - const uint64_t exec = dpp::execution_lane_mask(inst, wf); - uint64_t dst = 0; - RegisterAccess regs(wf); - auto src0 = regs.read_operand64(inst.src0, exec); - auto src1 = regs.read_operand64(inst.src1, exec); - for (uint32_t base = 0; base < wf.wf_size(); base += static_cast(W)) { - const uint64_t chunk = (exec >> base) & chunk_full; - if (chunk == 0) - continue; - const auto a = apply_vop3_src_mod_f64<0>(src0.template load_native(base), abs, neg); - const auto b = apply_vop3_src_mod_f64<1>(src1.template load_native(base), abs, neg); - const uint64_t cmp_bits = cmp_bits64(a, b, cmp_op); - dst = (dst & ~(chunk << base)) | ((cmp_bits & chunk) << base); - } - write_result(dst); - return true; -} - -/// Unconstrained fallback for the VOP3 f64 VOPC path; see the binary-path note. -template -[[nodiscard]] bool try_execute_vopc64_vop3_fp64_simd(Inst &, Wavefront &, CmpOp, WriteResult) { - return false; -} - /// VOP3 f64 binary SIMD fast path. 64-bit-lane counterpart of /// try_execute_binary_vop3_fp_simd: reads src0/src1 as `native` through /// 64-bit RegisterAccess operand views, applies the per-source abs/neg @@ -2388,11 +2409,14 @@ template /// form reads the low source half and zero-extends the full destination dword; /// the true16 form selects the source half and writes the selected destination /// half per the ISA's op_sel[3] policy. +/// With rounded_result, the operation supplies an architectural half and OMOD +/// acts on that half before CLAMP. Other operations retain promoted arithmetic. /// All steps bit-exact per the f16 VOP3 cmp slice's widening probe (f16_to_f32 /// + f32_to_f16_mode verified against the scalar helper incl. NaN payload). template requires(util::has_stdx_simd) -[[nodiscard]] inline bool try_execute_unary_vop3_fp16_simd(Inst &inst, Wavefront &wf, UnOp un_op) { +[[nodiscard]] inline bool try_execute_unary_vop3_fp16_simd(Inst &inst, Wavefront &wf, UnOp un_op, + bool rounded_result = false) { if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || !inst.vdst.simd_capable()) return false; @@ -2402,6 +2426,14 @@ template const uint32_t neg = inst.inst_.neg; const uint32_t omod = effective_vop3_omod_f16(wf, inst.inst_.omod); const uint32_t clamp = inst.inst_.clamp; + const auto modify_result = [&](util::native value) { + if (!rounded_result) + return apply_vop3_dst_mod_f32(value, omod, clamp, floating_clamp_nan_to_zero(wf)); + return util::map_native_convert_scalar(value, [&](float lane) { + lane = fp_mode::apply_omod_f16(lane, omod, wf.fp16_ovfl()); + return clamp ? clamp_floating_result(lane, wf) : lane; + }); + }; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); const uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -2419,7 +2451,7 @@ template raw = select_vop3_true16_src(raw, opsel, 0); const auto in = util::f16_to_f32_simd(raw); const auto a = apply_vop3_src_mod_f32<0>(in, abs, neg); - const auto r = apply_vop3_dst_mod_f32(un_op(a), omod, clamp, floating_clamp_nan_to_zero(wf)); + const auto r = modify_result(un_op(a)); const auto out_half = finalize_omod_f16_bits_simd(util::f32_to_f16_mode_simd(r, wf.fp16_ovfl()), omod); auto prev = dst.template load_native(base); @@ -2438,7 +2470,7 @@ template auto raw = src0.template load_native(base) & util::broadcast(0xffffu); const auto in = util::f16_to_f32_simd(raw); const auto a = apply_vop3_src_mod_f32<0>(in, abs, neg); - const auto r = apply_vop3_dst_mod_f32(un_op(a), omod, clamp, floating_clamp_nan_to_zero(wf)); + const auto r = modify_result(un_op(a)); const auto out = finalize_omod_f16_bits_simd(util::f32_to_f16_mode_simd(r, wf.fp16_ovfl()), omod) & util::broadcast(0xffffu); @@ -2450,7 +2482,7 @@ template /// Unconstrained fallback for the VOP3 f16 unary path; see the binary-path note. template -[[nodiscard]] bool try_execute_unary_vop3_fp16_simd(Inst &, Wavefront &, UnOp) { +[[nodiscard]] bool try_execute_unary_vop3_fp16_simd(Inst &, Wavefront &, UnOp, bool = false) { return false; } @@ -2581,20 +2613,21 @@ template /// VOP3 f32 ternary SIMD fast path (FMA / MAD family). Reads src0/src1/src2 as /// `native`, applies the per-source abs/neg VOP3 modifiers, runs /// `tern_op(a, b, c)`, applies CLAMP and optionally OMOD. FIXUP handles OMOD -/// inside its operation using guest rounding. NaN-payload divergence -/// between stdx::fma and std::fma is the standard accepted carve-out (the A/B -/// test skips NaN-input lanes), same as the existing VOP2 ternary path. +/// inside its operation using guest rounding. FMA uses the shared hardware +/// NaN selection policy in both scalar and SIMD execution. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_ternary_vop3_fp_simd(Inst &inst, Wavefront &wf, FmaOp tern_op, - bool apply_omod = true) { + bool apply_omod = true, + bool force_output_flush = false) { if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) return false; using T = float32_t; const uint32_t abs = inst.inst_.abs; const uint32_t neg = inst.inst_.neg; - const uint32_t omod = apply_omod ? effective_vop3_omod_f32(wf, inst.inst_.omod) : 0; + const uint32_t omod = + apply_omod ? effective_vop3_omod_f32(wf, inst.inst_.omod, force_output_flush) : 0; const uint32_t clamp = inst.inst_.clamp; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); @@ -2619,7 +2652,8 @@ template } template -[[nodiscard]] bool try_execute_ternary_vop3_fp_simd(Inst &, Wavefront &, FmaOp, bool = true) { +[[nodiscard]] bool try_execute_ternary_vop3_fp_simd(Inst &, Wavefront &, FmaOp, bool = true, + bool = false) { return false; } @@ -2739,11 +2773,11 @@ template auto a = select_vop3_true16_src(src0.template load_native(base), opsel, 0); auto b = select_vop3_true16_src(src1.template load_native(base), opsel, 1); auto c = select_vop3_true16_src(src2.template load_native(base), opsel, 2); - const auto out_half = - fma_f16_mode_simd(a, b, c, inst.inst_.abs & 1u, inst.inst_.abs & 2u, inst.inst_.abs & 4u, - inst.inst_.neg & 1u, inst.inst_.neg & 2u, inst.inst_.neg & 4u, - wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst.inst_.clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + const auto out_half = fma_f16_mode_simd( + a, b, c, inst.inst_.abs & 1u, inst.inst_.abs & 2u, inst.inst_.abs & 4u, + inst.inst_.neg & 1u, inst.inst_.neg & 2u, inst.inst_.neg & 4u, wf.fp_round_mode_f16_f64(), + wf.fp_denorm_mode_f16_f64(), omod, inst.inst_.clamp, wf.fp16_ovfl(), + floating_clamp_nan_to_zero(wf), fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); auto prev = dst.template load_native(base); auto out = (opsel & 0x8u) ? ((prev & 0x0000ffffu) | (out_half << 16)) : ((prev & 0xffff0000u) | out_half); @@ -2762,7 +2796,8 @@ template src2.template load_native(base), inst.inst_.abs & 1u, inst.inst_.abs & 2u, inst.inst_.abs & 4u, inst.inst_.neg & 1u, inst.inst_.neg & 2u, inst.inst_.neg & 4u, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, inst.inst_.clamp, - wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf), + fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); dst.template store_native(base, out, chunk); } } @@ -2868,14 +2903,15 @@ template /// result omod/clamp, masked-store back to inst.vdst (overwriting accumulator). template requires(util::has_stdx_simd) -[[nodiscard]] inline bool try_execute_fmac_vop3_fp_simd(Inst &inst, Wavefront &wf, FmaOp tern_op) { +[[nodiscard]] inline bool try_execute_fmac_vop3_fp_simd(Inst &inst, Wavefront &wf, FmaOp tern_op, + bool force_output_flush = false) { if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; using T = float32_t; const uint32_t abs = inst.inst_.abs; const uint32_t neg = inst.inst_.neg; - const uint32_t omod = effective_vop3_omod_f32(wf, inst.inst_.omod); + const uint32_t omod = effective_vop3_omod_f32(wf, inst.inst_.omod, force_output_flush); const uint32_t clamp = inst.inst_.clamp; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); @@ -2899,7 +2935,7 @@ template } template -[[nodiscard]] bool try_execute_fmac_vop3_fp_simd(Inst &, Wavefront &, FmaOp) { +[[nodiscard]] bool try_execute_fmac_vop3_fp_simd(Inst &, Wavefront &, FmaOp, bool = false) { return false; } @@ -3013,7 +3049,8 @@ template const auto out_half = fma_f16_mode_simd( a, b, c, inst.inst_.abs & 1u, inst.inst_.abs & 2u, false, inst.inst_.neg & 1u, inst.inst_.neg & 2u, false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), omod, - inst.inst_.clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + inst.inst_.clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf), + fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); auto out = out_half; if constexpr (True16) { if (opsel & 0x8u) @@ -3209,13 +3246,14 @@ template /// bit-exact, so the fast path stays correct with modifiers set; no bail. template requires(util::has_stdx_simd) -[[nodiscard]] inline bool try_execute_unary_vop3_fp_simd(Inst &inst, Wavefront &wf, UnOp un_op) { +[[nodiscard]] inline bool try_execute_unary_vop3_fp_simd(Inst &inst, Wavefront &wf, UnOp un_op, + bool force_output_flush = false) { if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || !inst.vdst.simd_capable()) return false; const uint32_t abs = inst.inst_.abs; const uint32_t neg = inst.inst_.neg; - const uint32_t omod = effective_vop3_omod_f32(wf, inst.inst_.omod); + const uint32_t omod = effective_vop3_omod_f32(wf, inst.inst_.omod, force_output_flush); const uint32_t clamp = inst.inst_.clamp; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); @@ -3236,7 +3274,7 @@ template /// Unconstrained fallback for the VOP3 f32 unary path. template -[[nodiscard]] bool try_execute_unary_vop3_fp_simd(Inst &, Wavefront &, UnOp) { +[[nodiscard]] bool try_execute_unary_vop3_fp_simd(Inst &, Wavefront &, UnOp, bool = false) { return false; } @@ -3644,11 +3682,12 @@ template !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) return false; -#if defined(__clang__) && defined(__FMA__) +#if defined(__FMA__) if constexpr (!Fused && DstMode == FmaMixDst::F32) { // Legacy scalar a*b+c and stdx SIMD expressions can contract differently - // under Clang. Keep the established fallback for those profiles; CDNA5 - // explicitly uses fused arithmetic in both execution paths. + // with hardware FMA, including GCC with UBSan. Preserve the scalar path + // for those profiles; CDNA5 explicitly uses fused arithmetic in both + // execution paths. return false; } #endif @@ -3696,7 +3735,7 @@ template (neg >> 2) & 1u); F r; if constexpr (Fused) - r = util::stdx::fma(a, b, c); + r = fma_f32_simd(a, b, c, wf); else r = fma_mix_mul_add(a, b, c); if (clamp) @@ -3704,6 +3743,7 @@ template return r; }; if constexpr (DstMode == FmaMixDst::F32) { + fp_mode::ScopedEnvironment environment(wf.fp_round_mode_f32()); auto dst = regs.write_operand(inst.vdst, exec); for (uint32_t base = 0; base < wf.wf_size(); base += static_cast(W)) { const uint64_t chunk = (exec >> base) & chunk_full; @@ -3717,7 +3757,20 @@ template const uint64_t chunk = (exec >> base) & chunk_full; if (chunk == 0) continue; - U h = util::f32_to_f16_mode_simd(compute_result(base), wf.fp16_ovfl()); + U h; + if constexpr (Fused) { + const F a = + load_src(src0, base, inst.inst_.src0, op_sel_hi & 1u, op_sel & 1u, abs & 1u, neg & 1u); + const F b = + load_src(src1, base, inst.inst_.src1, op_sel_hi & 2u, op_sel & 2u, abs & 2u, neg & 2u); + const F c = + load_src(src2, base, inst.inst_.src2, op_sel_hi_2, op_sel & 4u, abs & 4u, neg & 4u); + h = mixed_fma_f16_simd(a, b, c, wf.fp_round_mode_f16_f64(), clamp, wf.fp16_ovfl(), + floating_clamp_nan_to_zero(wf)); + } else { + h = mixed_fma_f16_simd(F(0), F(0), compute_result(base), wf.fp_round_mode_f16_f64(), + false, wf.fp16_ovfl(), false); + } U prev = dst.template load_native(base); U packed; if constexpr (DstMode == FmaMixDst::F16_LO) { @@ -3739,8 +3792,8 @@ template enum class PackedFloatOp { ADD, MUL, FMA, MIN, MAX, MINIMUM, MAXIMUM }; /// SIMD arithmetic in F64 avoids the F32 intermediate rounding of packed F16. -/// Scalar final rounding is shared with the architectural implementation. BF16 -/// uses an error-free sum because its exponent range can exceed F64 precision. +/// Scalar final rounding is shared with the architectural implementation. Fused +/// F16 and BF16 sums retain a residual when their terms exceed F64 precision. template inline U packed_float_half_simd(U a_bits, U b_bits, U c_bits, Wavefront &wf, bool clamp) { using D = util::native; @@ -3776,13 +3829,11 @@ inline U packed_float_half_simd(U a_bits, U b_bits, U c_bits, Wavefront &wf, boo else util::stdx::where(order(b_bits) > order(a_bits), selected) = b_bits; result = D([&](auto i) { return widen(selected[i]); }); - } else if constexpr (!Bf16) { + } else if constexpr (!Bf16 && Op != PackedFloatOp::FMA) { if constexpr (Op == PackedFloatOp::ADD) result = a + b; else if constexpr (Op == PackedFloatOp::MUL) result = a * b; - else - result = util::stdx::fma(a, b, c); } else { D product = Op == PackedFloatOp::ADD ? a : a * b; D addend = Op == PackedFloatOp::ADD ? b : c; @@ -3842,7 +3893,8 @@ inline U packed_float_half_simd(U a_bits, U b_bits, U c_bits, Wavefront &wf, boo : fp_mode::PackedBinaryOp::MAXIMUM; output[i] = fp_mode::packed_binary_f16( operation, a_bits[i], b_bits[i], wf.fp_round_mode_f16_f64(), - wf.fp_denorm_mode_f16_f64(), clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + wf.fp_denorm_mode_f16_f64(), clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); continue; } } @@ -3851,19 +3903,24 @@ inline U packed_float_half_simd(U a_bits, U b_bits, U c_bits, Wavefront &wf, boo value = wf.fp_round_mode_f16_f64() == 2 ? -0.0 : 0.0; } if constexpr (Op == PackedFloatOp::FMA) { - if (nan || std::isnan(c[i])) { - output[i] = - fp_mode::fma_f16(a_bits[i], b_bits[i], c_bits[i], false, false, false, false, false, - false, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, - clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + if (!std::isfinite(a[i]) || !std::isfinite(b[i]) || !std::isfinite(c[i])) { + output[i] = fp_mode::fma_f16( + a_bits[i], b_bits[i], c_bits[i], false, false, false, false, false, false, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, clamp, wf.fp16_ovfl(), + floating_clamp_nan_to_zero(wf), fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); continue; } + value = fp_mode::detail::round_to_odd({value, residual[i]}); if (value == 0.0) { const bool product_negative = ((a_bits[i] ^ b_bits[i]) & 0x8000u) != 0; const bool matching_zeros = c[i] == 0.0 && product_negative == std::signbit(c[i]); value = (matching_zeros ? product_negative : wf.fp_round_mode_f16_f64() == 2) ? -0.0 : 0.0; } + output[i] = + fp_mode::finish_fma_f16(value, wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), + 0, clamp, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + continue; } uint16_t rounded = pseudo_scalar::round_f16_result(value, wf.fp_round_mode_f16_f64(), 0, clamp, @@ -4106,8 +4163,7 @@ template /// same per-register native read/write as the binary form. OP_SEL /// chooses each source half independently; op_sel_hi_2 selects src2-hi. /// neg/neg_hi bits 0/1/2 sign-flip the respective half. MODE and CLAMP match -/// the scalar helper. NaN-input payload divergence between stdx::fma and -/// std::fma is accepted, as in the f16 packed ternary and fma_mix helpers. +/// the scalar helper, including FMA NaN payload selection. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_vop3p_pk_ternary_f32_simd(Inst &inst, Wavefront &wf, @@ -4247,17 +4303,19 @@ template if (packed_opsel(inst.inst_) != 0u || packed_opsel_hi(inst.inst_) != 3u) return false; } + const bool clamp = inst.inst_.clamp && (ElemBits != 8 || dot4_clamp_supported(wf)); +#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS + if (clamp) + return false; +#endif constexpr int N = 32 / ElemBits; constexpr uint32_t kElemMask = (ElemBits == 16) ? 0xFFFFu : (ElemBits == 8) ? 0xFFu : 0xFu; using T = uint32_t; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); const uint64_t exec = dpp::execution_lane_mask(inst, wf); - const bool clamp = inst.inst_.clamp && (ElemBits != 8 || dot4_clamp_supported(wf)); using U = util::native; using I = util::native; - using U64 = util::stdx::fixed_size_simd; - using I64 = util::stdx::fixed_size_simd; RegisterAccess regs(wf); auto src0 = regs.read_operand(inst.src0, exec); auto src1 = regs.read_operand(inst.src1, exec); @@ -4271,7 +4329,9 @@ template const U raw1 = src1.template load_native(base); const U acc = src2.template load_native(base); if constexpr (Signed) { +#if !UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS if (clamp) { + using I64 = util::stdx::fixed_size_simd; I64 sum = util::stdx::static_simd_cast(std::bit_cast(acc)); for (int i = 0; i < N; ++i) { const U ea = (raw0 >> (i * ElemBits)) & kElemMask; @@ -4285,19 +4345,22 @@ template util::stdx::where(sum > I64(std::numeric_limits::max()), sum) = I64(std::numeric_limits::max()); dst.template store_native(base, util::stdx::static_simd_cast(sum), chunk); - } else { - U sum = acc; - for (int i = 0; i < N; ++i) { - const U ea = (raw0 >> (i * ElemBits)) & kElemMask; - const U eb = (raw1 >> (i * ElemBits)) & kElemMask; - const I a = std::bit_cast(simd_sign_extend_u32(ea, ElemBits)); - const I b = std::bit_cast(simd_sign_extend_u32(eb, ElemBits)); - sum += std::bit_cast(a * b); - } - dst.template store_native(base, sum, chunk); + continue; + } +#endif + U sum = acc; + for (int i = 0; i < N; ++i) { + const U ea = (raw0 >> (i * ElemBits)) & kElemMask; + const U eb = (raw1 >> (i * ElemBits)) & kElemMask; + const I a = std::bit_cast(simd_sign_extend_u32(ea, ElemBits)); + const I b = std::bit_cast(simd_sign_extend_u32(eb, ElemBits)); + sum += std::bit_cast(a * b); } + dst.template store_native(base, sum, chunk); } else { +#if !UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS if (clamp) { + using U64 = util::stdx::fixed_size_simd; U64 sum = util::stdx::static_simd_cast(acc); for (int i = 0; i < N; ++i) { const U ea = (raw0 >> (i * ElemBits)) & kElemMask; @@ -4307,15 +4370,16 @@ template util::stdx::where(sum > U64(std::numeric_limits::max()), sum) = U64(std::numeric_limits::max()); dst.template store_native(base, util::stdx::static_simd_cast(sum), chunk); - } else { - U sum = acc; - for (int i = 0; i < N; ++i) { - const U ea = (raw0 >> (i * ElemBits)) & kElemMask; - const U eb = (raw1 >> (i * ElemBits)) & kElemMask; - sum += ea * eb; - } - dst.template store_native(base, sum, chunk); + continue; + } +#endif + U sum = acc; + for (int i = 0; i < N; ++i) { + const U ea = (raw0 >> (i * ElemBits)) & kElemMask; + const U eb = (raw1 >> (i * ElemBits)) & kElemMask; + sum += ea * eb; } + dst.template store_native(base, sum, chunk); } } return true; @@ -4433,11 +4497,14 @@ template const uint64_t chunk_full = util::mask(static_cast(W)); const uint64_t exec = dpp::execution_lane_mask(inst, wf); const bool clamp = inst.inst_.clamp && (ElemBits != 8 || dot4_clamp_supported(wf)); +#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS + if (clamp) + return false; +#endif const bool src0_signed = (inst.inst_.neg & 0x1u) != 0; const bool src1_signed = (inst.inst_.neg & 0x2u) != 0; using U = util::native; using I = util::native; - using I64 = util::stdx::fixed_size_simd; RegisterAccess regs(wf); auto src0 = regs.read_operand(inst.src0, exec); auto src1 = regs.read_operand(inst.src1, exec); @@ -4450,7 +4517,9 @@ template const U raw0 = src0.template load_native(base); const U raw1 = src1.template load_native(base); const U acc = src2.template load_native(base); +#if !UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS if (clamp) { + using I64 = util::stdx::fixed_size_simd; I64 sum = util::stdx::static_simd_cast(std::bit_cast(acc)); for (int i = 0; i < N; ++i) { const U ea = (raw0 >> (i * ElemBits)) & kElemMask; @@ -4466,19 +4535,20 @@ template util::stdx::where(sum > I64(std::numeric_limits::max()), sum) = I64(std::numeric_limits::max()); dst.template store_native(base, util::stdx::static_simd_cast(sum), chunk); - } else { - U sum = acc; - for (int i = 0; i < N; ++i) { - const U ea = (raw0 >> (i * ElemBits)) & kElemMask; - const U eb = (raw1 >> (i * ElemBits)) & kElemMask; - const I a = src0_signed ? std::bit_cast(simd_sign_extend_u32(ea, ElemBits)) - : util::stdx::static_simd_cast(ea); - const I b = src1_signed ? std::bit_cast(simd_sign_extend_u32(eb, ElemBits)) - : util::stdx::static_simd_cast(eb); - sum += std::bit_cast(a * b); - } - dst.template store_native(base, sum, chunk); + continue; + } +#endif + U sum = acc; + for (int i = 0; i < N; ++i) { + const U ea = (raw0 >> (i * ElemBits)) & kElemMask; + const U eb = (raw1 >> (i * ElemBits)) & kElemMask; + const I a = src0_signed ? std::bit_cast(simd_sign_extend_u32(ea, ElemBits)) + : util::stdx::static_simd_cast(ea); + const I b = src1_signed ? std::bit_cast(simd_sign_extend_u32(eb, ElemBits)) + : util::stdx::static_simd_cast(eb); + sum += std::bit_cast(a * b); } + dst.template store_native(base, sum, chunk); } return true; } @@ -4684,7 +4754,7 @@ template /// 64-bit-lane MODE-aware VOP2 FMA counterpart (v_fmac_f64). #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOP2_FMA_F64() +#define ROCJITSU_TRY_SIMD_VOP2_FMA_F64() static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOP2_FMA_F64() \ if (::rocjitsu::amdgpu::try_execute_ternary_vop2_f64_simd(inst, wf)) \ @@ -4695,7 +4765,7 @@ template /// type fixed to double, read/written through the split lo/hi VGPR-pair path /// (vsrc1 as the second source). Variadic in the functor. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOP2_BINARY_FP64(...) +#define ROCJITSU_TRY_SIMD_VOP2_BINARY_FP64(...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOP2_BINARY_FP64(...) \ if (::rocjitsu::amdgpu::try_execute_binary_vop2_f64_simd(inst, wf, __VA_ARGS__)) \ @@ -4706,7 +4776,7 @@ template /// for the f64 math ops, `uint64_t` for v_mov_b64). Variadic in the functor so /// its commas pass through as one token sequence. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOP1_UNARY_F64(T, ...) +#define ROCJITSU_TRY_SIMD_VOP1_UNARY_F64(T, ...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOP1_UNARY_F64(T, ...) \ if (::rocjitsu::amdgpu::try_execute_unary_vop1_f64_simd(inst, wf, __VA_ARGS__)) \ @@ -4717,7 +4787,7 @@ template /// result lane type; the functor (`native -> narrow32`) is variadic /// so its commas pass through as one token sequence. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_CVT_F64_TO_B32(Tout, ...) +#define ROCJITSU_TRY_SIMD_CVT_F64_TO_B32(Tout, ...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_CVT_F64_TO_B32(Tout, ...) \ if (::rocjitsu::amdgpu::try_execute_cvt_f64_to_b32_simd(inst, wf, __VA_ARGS__)) \ @@ -4727,7 +4797,7 @@ template /// Mixed-width cvt counterpart, 32-bit source -> f64 dst. `Tin` is the 32-bit /// source lane type; the functor (`narrow32 -> native`) is variadic. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_CVT_B32_TO_F64(Tin, ...) +#define ROCJITSU_TRY_SIMD_CVT_B32_TO_F64(Tin, ...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_CVT_B32_TO_F64(Tin, ...) \ if (::rocjitsu::amdgpu::try_execute_cvt_b32_to_f64_simd(inst, wf, __VA_ARGS__)) \ @@ -4737,7 +4807,7 @@ template /// VOP3 f64-source -> 32-bit-fp-dst cvt counterpart (src0 abs/neg + result /// omod/clamp; for v_frexp_exp_i32_f64). Functor: native -> narrow32. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_CVT_VOP3_F64_TO_B32_FP(...) +#define ROCJITSU_TRY_SIMD_CVT_VOP3_F64_TO_B32_FP(...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_CVT_VOP3_F64_TO_B32_FP(...) \ if (::rocjitsu::amdgpu::try_execute_cvt_vop3_f64_to_b32_fp_simd(inst, wf, __VA_ARGS__)) \ @@ -4764,7 +4834,7 @@ template /// 64-bit-lane VOPC compare counterpart (f64/i64/u64). `T` is the 64-bit lane /// read type; the comparison functor is variadic so its commas pass through. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOPC64(T, ...) +#define ROCJITSU_TRY_SIMD_VOPC64(T, ...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOPC64(T, ...) \ if (::rocjitsu::amdgpu::try_execute_vopc64_simd(inst, wf, __VA_ARGS__)) \ @@ -4775,7 +4845,7 @@ template /// argument; the class functor `(native bits, narrow32 mask) /// -> mask` is variadic so its commas pass through as one token sequence. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOPC_CLASS_F64(...) +#define ROCJITSU_TRY_SIMD_VOPC_CLASS_F64(...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOPC_CLASS_F64(...) \ if (::rocjitsu::amdgpu::try_execute_vopc_class_f64_simd(inst, wf, __VA_ARGS__)) \ @@ -4813,8 +4883,8 @@ template /// VOP3 v_cmp_class_f64 counterpart (64-bit value). `SM` is the f64 sign-bit mask /// (0x8000000000000000); the class functor is variadic. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOP3_CLASS_F64(SM, ...) -#define ROCJITSU_TRY_SIMD_VOP3_CLASS_F64_RESULT(WRITE_RESULT, SM, ...) +#define ROCJITSU_TRY_SIMD_VOP3_CLASS_F64(SM, ...) static_cast(inst) +#define ROCJITSU_TRY_SIMD_VOP3_CLASS_F64_RESULT(WRITE_RESULT, SM, ...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOP3_CLASS_F64(SM, ...) \ if (::rocjitsu::amdgpu::try_execute_vop3_class_f64_simd( \ @@ -4864,9 +4934,11 @@ template if (::rocjitsu::amdgpu::try_execute_unary_vop3_fp_simd(inst, wf, __VA_ARGS__)) \ return -/// VOP3 integer/bitwise VOPC compare counterpart (32-bit lane, no modifiers, -/// SGPR-pair dst). `T` is the 32-bit integer lane read type; variadic in the -/// functor so its commas pass through as one token sequence. +/// VOP3 raw-lane VOPC compare counterpart (32-bit lane, SGPR-pair dst). Despite +/// the _INT name, f16/f32 relations use it too: the glue applies no modifiers, +/// and a float functor applies the captured ABS/NEG fields and MODE input +/// flush itself via comparison::evaluate. `T` is the 32-bit raw lane read type; +/// variadic in the functor so its commas pass through as one token sequence. #define ROCJITSU_TRY_SIMD_VOPC_VOP3_INT(T, ...) \ if (::rocjitsu::amdgpu::try_execute_vopc_vop3_int_simd( \ inst, wf, __VA_ARGS__, [&](uint64_t result) { \ @@ -4877,81 +4949,36 @@ template if (::rocjitsu::amdgpu::try_execute_vopc_vop3_int_simd(inst, wf, __VA_ARGS__, WRITE_RESULT)) \ return -/// 64-bit-lane VOP3 integer/bitwise VOPC compare counterpart (i64/u64, no -/// modifiers, SGPR-pair dst). `T` is the 64-bit integer lane read type; -/// variadic in the functor. -#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT(T, ...) -#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT(WRITE_RESULT, T, ...) -#else -#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT(T, ...) \ - if (::rocjitsu::amdgpu::try_execute_vopc64_vop3_int_simd( \ - inst, wf, __VA_ARGS__, [&](uint64_t result) { \ - ::rocjitsu::amdgpu::write_explicit_lane_mask(inst.vdst, wf, result); \ - })) \ - return -#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT(WRITE_RESULT, T, ...) \ - if (::rocjitsu::amdgpu::try_execute_vopc64_vop3_int_simd(inst, wf, __VA_ARGS__, \ - WRITE_RESULT)) \ - return -#endif - -/// VOP3 f32 VOPC compare counterpart (per-source abs/neg modifiers, SGPR-pair -/// dst). Lane type is fixed to float32_t; the functor takes already-modified -/// `native` arguments and is variadic so its commas pass through. -#define ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32(...) \ - if (::rocjitsu::amdgpu::try_execute_vopc_vop3_fp32_simd( \ - inst, wf, __VA_ARGS__, [&](uint64_t result) { \ - ::rocjitsu::amdgpu::write_explicit_lane_mask(inst.vdst, wf, result); \ - })) \ - return -#define ROCJITSU_TRY_SIMD_VOPC_VOP3_FP32_RESULT(WRITE_RESULT, ...) \ - if (::rocjitsu::amdgpu::try_execute_vopc_vop3_fp32_simd(inst, wf, __VA_ARGS__, WRITE_RESULT)) \ - return - -/// VOP3 f16 VOPC compare counterpart. Lane type is fixed to uint32_t (raw f16 -/// bits in low 16); the glue widens to f32 then applies the abs/neg modifier. -/// The functor takes the same already-widened, already-modified `native` -/// arguments as the f32 path; variadic in the functor. -#define ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16(...) \ - if (::rocjitsu::amdgpu::try_execute_vopc_vop3_fp16_simd( \ - inst, wf, __VA_ARGS__, [&](uint64_t result) { \ - ::rocjitsu::amdgpu::write_explicit_lane_mask(inst.vdst, wf, result); \ - })) \ - return -#define ROCJITSU_TRY_SIMD_VOPC_VOP3_FP16_RESULT(WRITE_RESULT, ...) \ - if (::rocjitsu::amdgpu::try_execute_vopc_vop3_fp16_simd(inst, wf, __VA_ARGS__, \ - WRITE_RESULT)) \ - return - -/// VOP3 f16 VOPC compare counterpart for true16 OPSEL source halves. -#define ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16(...) \ - if (::rocjitsu::amdgpu::try_execute_vopc_vop3_fp16_simd( \ +/// VOP3 raw-lane compare counterpart whose 16-bit sources select their half +/// with true16 OPSEL. +#define ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT(T, ...) \ + if (::rocjitsu::amdgpu::try_execute_vopc_vop3_int_simd( \ inst, wf, __VA_ARGS__, [&](uint64_t result) { \ ::rocjitsu::amdgpu::write_explicit_lane_mask(inst.vdst, wf, result); \ })) \ return -#define ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_FP16_RESULT(WRITE_RESULT, ...) \ - if (::rocjitsu::amdgpu::try_execute_vopc_vop3_fp16_simd(inst, wf, __VA_ARGS__, \ - WRITE_RESULT)) \ +#define ROCJITSU_TRY_SIMD_VOPC_VOP3_TRUE16_INT_RESULT(WRITE_RESULT, T, ...) \ + if (::rocjitsu::amdgpu::try_execute_vopc_vop3_int_simd(inst, wf, __VA_ARGS__, \ + WRITE_RESULT)) \ return -/// VOP3 f64 VOPC compare counterpart (per-source abs/neg modifiers, 64-bit -/// lane via split lo/hi VGPR-pair, SGPR-pair dst). Lane type is fixed to -/// `double`; the functor takes already-modified `native` arguments -/// and is variadic so its commas pass through. +/// 64-bit-lane VOP3 raw-lane VOPC compare counterpart (i64/u64/f64, SGPR-pair +/// dst). As with the 32-bit macro, f64 callers use it too, with the functor +/// applying ABS/NEG and MODE input flush via comparison::evaluate. `T` is the +/// 64-bit raw lane read type; variadic in the functor. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64(...) -#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(WRITE_RESULT, ...) +#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT(T, ...) static_cast(inst) +#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT(WRITE_RESULT, T, ...) static_cast(inst) #else -#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64(...) \ - if (::rocjitsu::amdgpu::try_execute_vopc64_vop3_fp64_simd( \ +#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT(T, ...) \ + if (::rocjitsu::amdgpu::try_execute_vopc64_vop3_int_simd( \ inst, wf, __VA_ARGS__, [&](uint64_t result) { \ ::rocjitsu::amdgpu::write_explicit_lane_mask(inst.vdst, wf, result); \ })) \ return -#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_FP64_RESULT(WRITE_RESULT, ...) \ - if (::rocjitsu::amdgpu::try_execute_vopc64_vop3_fp64_simd(inst, wf, __VA_ARGS__, WRITE_RESULT)) \ +#define ROCJITSU_TRY_SIMD_VOPC64_VOP3_INT_RESULT(WRITE_RESULT, T, ...) \ + if (::rocjitsu::amdgpu::try_execute_vopc64_vop3_int_simd(inst, wf, __VA_ARGS__, \ + WRITE_RESULT)) \ return #endif @@ -4997,7 +5024,7 @@ template return #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_FMA_VOP3_FP64() +#define ROCJITSU_TRY_SIMD_FMA_VOP3_FP64() static_cast(inst) #else #define ROCJITSU_TRY_SIMD_FMA_VOP3_FP64() \ if (::rocjitsu::amdgpu::try_execute_fma_vop3_fp64_simd(inst, wf)) \ @@ -5008,7 +5035,7 @@ template /// abs/neg, omod/clamp). /// Variadic. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP64(...) +#define ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP64(...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOP3_TERNARY_FP64(...) \ if (::rocjitsu::amdgpu::try_execute_ternary_vop3_fp64_simd(inst, wf, __VA_ARGS__)) \ @@ -5041,7 +5068,7 @@ template return #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_FMAC_VOP3_MODE_FP64() +#define ROCJITSU_TRY_SIMD_FMAC_VOP3_MODE_FP64() static_cast(inst) #else #define ROCJITSU_TRY_SIMD_FMAC_VOP3_MODE_FP64() \ if (::rocjitsu::amdgpu::try_execute_fmac_vop3_fp64_mode_simd(inst, wf)) \ @@ -5050,7 +5077,7 @@ template /// VOP3 dst-accumulate FMA counterpart (f64). #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_FMAC_VOP3_FP64(...) +#define ROCJITSU_TRY_SIMD_FMAC_VOP3_FP64(...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_FMAC_VOP3_FP64(...) \ if (::rocjitsu::amdgpu::try_execute_fmac_vop3_fp64_simd(inst, wf, __VA_ARGS__)) \ @@ -5064,7 +5091,7 @@ template /// VOP3 ldexp counterpart (f64 src0 + int32 src1 exp). Variadic functor. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_LDEXP_VOP3_FP64(...) +#define ROCJITSU_TRY_SIMD_LDEXP_VOP3_FP64(...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_LDEXP_VOP3_FP64(...) \ if (::rocjitsu::amdgpu::try_execute_ldexp_vop3_fp64_simd(inst, wf, __VA_ARGS__)) \ @@ -5077,7 +5104,7 @@ template return #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_DIV_FMAS_VOP3_FP64() +#define ROCJITSU_TRY_SIMD_DIV_FMAS_VOP3_FP64() static_cast(inst) #else #define ROCJITSU_TRY_SIMD_DIV_FMAS_VOP3_FP64() \ if (::rocjitsu::amdgpu::try_execute_div_fmas_f64_simd(inst, wf)) \ @@ -5088,7 +5115,7 @@ template /// abs/neg, omod/clamp on the result). Variadic in the functor so its commas pass through as one /// token sequence. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOP3_BINARY_FP64(...) +#define ROCJITSU_TRY_SIMD_VOP3_BINARY_FP64(...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOP3_BINARY_FP64(...) \ if (::rocjitsu::amdgpu::try_execute_binary_vop3_fp64_simd(inst, wf, __VA_ARGS__)) \ @@ -5098,7 +5125,7 @@ template /// VOP3 f64 unary counterpart (64-bit RegisterAccess read, src0 abs/neg, /// omod/clamp on the result). Variadic in the functor. #if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS -#define ROCJITSU_TRY_SIMD_VOP3_UNARY_FP64(...) +#define ROCJITSU_TRY_SIMD_VOP3_UNARY_FP64(...) static_cast(inst) #else #define ROCJITSU_TRY_SIMD_VOP3_UNARY_FP64(...) \ if (::rocjitsu::amdgpu::try_execute_unary_vop3_fp64_simd(inst, wf, __VA_ARGS__)) \ @@ -5133,6 +5160,10 @@ template /// The functor takes `(narrow32 s0, narrow32 s1, /// native c)` and returns `SimdCarry, mask>`; /// variadic so its commas pass through. +#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS +#define ROCJITSU_TRY_SIMD_MAD_WIDE64_VOP3(...) static_cast(inst) +#define ROCJITSU_TRY_SIMD_MAD_WIDE64_VOP3_RESULT(WRITE_RESULT, ...) static_cast(inst) +#else #define ROCJITSU_TRY_SIMD_MAD_WIDE64_VOP3(...) \ if (::rocjitsu::amdgpu::try_execute_mad_wide64_vop3_simd(inst, wf, __VA_ARGS__)) \ return @@ -5140,6 +5171,7 @@ template if (::rocjitsu::amdgpu::try_execute_mad_wide64_vop3_result_simd(inst, wf, __VA_ARGS__, \ WRITE_RESULT)) \ return +#endif /// VOP3 carry-OUT counterpart (no carry-in; carry-out to SGPR sdst). Lane type /// fixed to uint32_t; variadic in the SimdCarry functor. diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/transcendental.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/transcendental.h index 077b9a6362a4..635181d1900d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/transcendental.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/transcendental.h @@ -9,12 +9,26 @@ /// These reference implementations produce results within the ULP accuracy /// specified by the ISA manuals (typically 1 ULP for f32, 2 ULP for f64). /// They are used by the simulator's execute() bodies for V_RCP_F32, -/// V_RSQ_F32, V_SQRT_F32, V_LOG_F32, V_EXP_F32, V_SIN_F32, V_COS_F32, +/// V_RSQ_F32, V_RSQ_F16, V_SQRT_F32, V_SQRT_F16, V_LOG_F32, V_EXP_F32, V_SIN_F32, V_COS_F32, /// V_RCP_F64, V_RSQ_F64, V_SQRT_F64. +/// F32 reciprocal, square root and F32/F16 reciprocal square root match the captured RDNA3/4 +/// mappings. F16 RSQ/SQRT apply the half input-denormal policy after promotion to F32. F32 LOG/EXP +/// and SIN/COS use staged integer arithmetic modeled from RDNA3/4 captures, including coordinate +/// truncation and intermediate product rounding. /// /// All functions handle special cases (NaN, Inf, denormals, ±0) per the /// AMD ISA specification. +#include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" +#include "util/amdgpu_exp.h" +#include "util/amdgpu_log.h" +#include "util/amdgpu_rcp.h" +#include "util/amdgpu_rsq.h" +#include "util/amdgpu_sqrt.h" +#include "util/amdgpu_trig.h" +#include "util/simd.h" + +#include #include #include #include @@ -24,100 +38,173 @@ namespace rocjitsu { namespace amdgpu { namespace transcendental { -/// @brief Flush f32 denormals to sign-preserving zero. -/// -/// @details AMD transcendental micro-ops always operate in FTZ mode -/// regardless of the shader's denorm mode. This helper reproduces -/// that behaviour for rcp, rsq, sqrt, exp, and log. -inline float flush_denorm_f32(float x) { - uint32_t bits = std::bit_cast(x); - if ((bits & 0x7F800000u) == 0 && (bits & 0x007FFFFFu) != 0) - return std::copysign(0.0f, x); - return x; +/// @brief AMD single-precision reciprocal matching physical RDNA3/4 (within 1 ULP). +inline float rcp_f32(float x, bool quiet_snan = true) { + return util::amdgpu_rcp_f32(x, quiet_snan); } -/// @brief 1.0 / x (single-precision reciprocal, ~0.5 ULP). -inline float rcp_f32(float x) { - x = flush_denorm_f32(x); - if (std::isnan(x)) - return std::bit_cast(std::bit_cast(x) | 0x00400000u); - if (x == 0.0f) - return std::copysign(std::numeric_limits::infinity(), x); - if (std::isinf(x)) - return std::copysign(0.0f, x); - return flush_denorm_f32(1.0f / x); +/// @brief AMD single-precision reciprocal square root matching physical RDNA3/4 (within 1 ULP). +inline float rsq_f32(float x, bool quiet_snan = true) { + return util::amdgpu_rsq_f32(x, quiet_snan); } -/// @brief 1.0 / sqrt(x) (single-precision reciprocal square root, ~1 ULP). -inline float rsq_f32(float x) { - x = flush_denorm_f32(x); - if (std::isnan(x)) - return std::bit_cast(std::bit_cast(x) | 0x00400000u); - if (x == 0.0f) - return std::copysign(std::numeric_limits::infinity(), x); - if (x < 0.0f) - return std::numeric_limits::quiet_NaN(); - if (std::isinf(x)) - return 0.0f; - return flush_denorm_f32(1.0f / std::sqrt(x)); +/// @brief Single-precision square root matching physical RDNA3/4 (within 1 ULP). +inline float sqrt_f32(float x, bool quiet_snan = true) { + return util::amdgpu_sqrt_f32(x, quiet_snan); } -/// @brief sqrt(x) (single-precision square root, correctly-rounded). -inline float sqrt_f32(float x) { - x = flush_denorm_f32(x); - if (std::isnan(x)) - return std::bit_cast(std::bit_cast(x) | 0x00400000u); - if (x < 0.0f) - return std::numeric_limits::quiet_NaN(); - return std::sqrt(x); +/// @brief log2(x) using the captured RDNA3/4 reduction and staged approximation. +inline float log_f32(float x, bool quiet_snan = true) { + return util::amdgpu_log_f32(x, quiet_snan); } -/// @brief log2(x) (single-precision base-2 logarithm, ~1 ULP). -inline float log_f32(float x) { - x = flush_denorm_f32(x); - if (std::isnan(x)) - return std::bit_cast(std::bit_cast(x) | 0x00400000u); - if (x == 0.0f) - return -std::numeric_limits::infinity(); - if (x < 0.0f) - return std::numeric_limits::quiet_NaN(); - if (std::isinf(x)) - return std::numeric_limits::infinity(); - return std::log2(x); +/// @brief 2^x using the captured RDNA3/4 reduction and staged approximation. +inline float exp_f32(float x, bool quiet_snan = true) { + return util::amdgpu_exp_f32(x, quiet_snan); } -/// @brief 2^x (single-precision base-2 exponential, ~1 ULP). -inline float exp_f32(float x) { - x = flush_denorm_f32(x); - if (std::isnan(x)) - return std::bit_cast(std::bit_cast(x) | 0x00400000u); - if (x == -std::numeric_limits::infinity()) - return 0.0f; - if (x == std::numeric_limits::infinity()) - return std::numeric_limits::infinity(); - return flush_denorm_f32(std::exp2(x)); +namespace detail { +// The caller establishes nearest rounding and preserves the host environment. +// The returned F32 value represents the already rounded architectural half. +template +inline float log_exp_f16_nearest(float x, uint32_t denorm_mode, bool fp16_ovfl, bool quiet_snan) { + uint32_t bits = std::bit_cast(x); + uint32_t magnitude = bits & 0x7fffffffu; + if (magnitude > 0x7f800000u) + return std::bit_cast(bits | (quiet_snan ? 0x00400000u : 0u)); + if (magnitude == 0x7f800000u) { + if constexpr (Logarithm) + return bits & 0x80000000u ? std::bit_cast(0xffc00000u) : x; + return bits & 0x80000000u ? 0.0f : x; + } + if (!(denorm_mode & 1u) && magnitude < 0x38800000u) { + bits &= 0x80000000u; + magnitude = 0; + x = std::bit_cast(bits); + } + double value; + if constexpr (Logarithm) { + if (magnitude == 0) + return fp16_ovfl ? -65504.0f : -std::numeric_limits::infinity(); + if (bits & 0x80000000u) + return std::bit_cast(0xffc00000u); + value = std::log2(static_cast(x)); + } else { + // Outside this interval every nearest half result is zero or overflows. + // Keep the finite-input provenance instead of overflowing host libm. + value = std::exp2(std::clamp(static_cast(x), -32.0, 32.0)); + } + // F32 evaluation can land on a half midpoint and round in the wrong direction. + uint16_t result = pseudo_scalar::round_f16_result(value, 0, 0, false, fp16_ovfl, false); + if (!(denorm_mode & 2u) && (result & 0x7c00u) == 0) + result &= 0x8000u; + return util::f16_to_f32(result); +} +} // namespace detail + +/// @brief Evaluate LOG/EXP with half input, output and finite-overflow policies. +/// @details Round once to F16 before any output scaling. Guest rounding is ignored. +template +inline float log_exp_f16(float x, uint32_t denorm_mode, bool fp16_ovfl, bool quiet_snan) { + fp_mode::ScopedEnvironment environment(0); + return detail::log_exp_f16_nearest(x, denorm_mode, fp16_ovfl, quiet_snan); +} + +/// @brief Evaluate a native batch with one saved host floating-point environment. +template +inline util::native log_exp_f16_simd(util::native x, uint32_t denorm_mode, + bool fp16_ovfl, bool quiet_snan) { + fp_mode::ScopedEnvironment environment(0); + return util::map_native_convert_scalar(x, [=](float lane) { + return detail::log_exp_f16_nearest(lane, denorm_mode, fp16_ovfl, quiet_snan); + }); } -/// @brief sin(2*pi*x) (single-precision, ~1 ULP). +/// @brief sin(2*pi*x) using full-range reduction and a captured RDNA3/4 approximation. /// /// @details The AMD ISA computes sin(2*pi*x), NOT sin(x). Input is in /// units of 2*pi radians. Output range is [-1.0, 1.0]. -inline float sin_f32(float x) { - if (std::isnan(x) || std::isinf(x)) - return std::numeric_limits::quiet_NaN(); - constexpr float TWO_PI = 6.283185307179586476925286766559f; - return std::sin(x * TWO_PI); +inline float sin_f32(float x, uint32_t denorm_mode = 3, bool quiet_snan = true) { + return util::amdgpu_trig_f32(x, false, denorm_mode, quiet_snan); } -/// @brief cos(2*pi*x) (single-precision, ~1 ULP). +/// @brief cos(2*pi*x) using full-range reduction and a captured RDNA3/4 approximation. /// /// @details The AMD ISA computes cos(2*pi*x), NOT cos(x). Input is in /// units of 2*pi radians. Output range is [-1.0, 1.0]. -inline float cos_f32(float x) { - if (std::isnan(x) || std::isinf(x)) - return std::numeric_limits::quiet_NaN(); - constexpr float TWO_PI = 6.283185307179586476925286766559f; - return std::cos(x * TWO_PI); +inline float cos_f32(float x, uint32_t denorm_mode = 3, bool quiet_snan = true) { + return util::amdgpu_trig_f32(x, true, denorm_mode, quiet_snan); +} + +/// @brief Half transcendental operations share input, rounding and output policies. +/// @details Evaluate in F32, then round to half before applying any modifiers. +/// FP16_OVFL also saturates infinity produced from zero or a flushed input. +enum class HalfOperation { RCP, RSQ, SQRT, SIN, COS }; + +template +inline float map_f16(float source, uint32_t denorm_mode, bool fp16_ovfl, bool quiet_snan) { + const float input = pseudo_scalar::detail::flush_input_f16(source, denorm_mode); + float value; + if constexpr (Op == HalfOperation::RCP) + value = util::amdgpu_rcp_f32(input, quiet_snan); + else if constexpr (Op == HalfOperation::RSQ) + value = util::amdgpu_rsq_f32(input, quiet_snan); + else if constexpr (Op == HalfOperation::SQRT) + value = util::amdgpu_sqrt_f32(input, quiet_snan); + else + value = util::amdgpu_trig_f32(input, Op == HalfOperation::COS, 3, quiet_snan); + const uint32_t bits = std::bit_cast(value); + if (fp16_ovfl && (bits & 0x7fffffffu) == 0x7f800000u && + (std::bit_cast(source) & 0x7fffffffu) < 0x7f800000u) + return std::bit_cast((bits & 0x80000000u) | 0x477fe000u); + uint16_t result = util::f32_to_f16_mode(value, fp16_ovfl); + if (!(denorm_mode & 2u) && (result & 0x7c00u) == 0) + result &= 0x8000u; + return util::f16_to_f32(result); +} + +template +inline util::native map_f16_simd(util::native source, uint32_t denorm_mode, + bool fp16_ovfl, bool quiet_snan) { + return util::map_native_convert_scalar( + source, [=](float lane) { return map_f16(lane, denorm_mode, fp16_ovfl, quiet_snan); }); +} + +/// @brief Apply the vector half mapping and result stages to a pseudo-scalar op. +/// @details Physical gfx1201 ignores guest rounding for every half transcendental. +/// Half rounding and output-denormal handling precede OMOD, just as for vectors. +inline uint32_t execute_pseudo_f16(pseudo_scalar::Operation operation, float source, bool absolute, + bool negate, [[maybe_unused]] uint32_t round_mode, + uint32_t denorm_mode, uint32_t omod, bool clamp, + bool fp16_ovfl) { + source = pseudo_scalar::detail::apply_source_modifiers(source, absolute, negate); + float value = 0.0f; + switch (operation) { + case pseudo_scalar::Operation::LOG2: + value = log_exp_f16(source, denorm_mode, fp16_ovfl, true); + break; + case pseudo_scalar::Operation::EXP2: + value = log_exp_f16(source, denorm_mode, fp16_ovfl, true); + break; + case pseudo_scalar::Operation::RCP: + value = map_f16(source, denorm_mode, fp16_ovfl, true); + break; + case pseudo_scalar::Operation::RSQ: + value = map_f16(source, denorm_mode, fp16_ovfl, true); + break; + case pseudo_scalar::Operation::SQRT: + value = map_f16(source, denorm_mode, fp16_ovfl, true); + break; + } + value = fp_mode::apply_omod_f16(value, omod, fp16_ovfl); + if (clamp) { + const uint32_t bits = std::bit_cast(value); + if ((bits & 0x80000000u) || (bits & 0x7fffffffu) > 0x7f800000u) + value = 0.0f; + else if (bits > 0x3f800000u) + value = 1.0f; + } + return fp_mode::finalize_omod_f16(util::f32_to_f16(value), omod); } /// @brief Hyperbolic tangent (single-precision, correctly-rounded libm reference). diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/decoder.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/decoder.h index b9ddc6dabcc7..1ae7121401fe 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/decoder.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/decoder.h @@ -30,8 +30,9 @@ struct IsaExecutionBackend; /// By default, decoded instructions are heap-allocated. Call /// ``enable_pool()`` to route Instruction::operator new/delete through /// the decoder's O(1) free-list pool. Only enable the pool when all -/// decoded instructions will be deleted before the decoder is destroyed -/// (e.g., the ComputeUnit simulation loop). +/// decoded instructions will be deleted on the bound thread before the decoder +/// is destroyed. CU execution uses heap storage because decoded instructions +/// can survive issue quanta and move between workers. class Decoder { public: using Pool = util::ArenaAlloc<512, 128>; @@ -97,8 +98,8 @@ class Decoder { /// /// When active, Instruction::operator new/delete route through the /// decoder's pool for O(1) alloc/free. Only enable when the caller - /// guarantees all instructions will be deleted before the decoder - /// is destroyed (e.g., the ComputeUnit hot path). + /// guarantees all instructions will be deleted on the bound thread + /// before the decoder is destroyed. CU execution uses heap storage. void enable_pool() { activate_pool([](void *p, size_t s) -> void * { return static_cast(p)->allocate(s); }, [](void *p, void *ptr) { static_cast(p)->deallocate(ptr); }, &pool_); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/instruction.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/instruction.h index 28b09770026c..9012ba1a56f1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/instruction.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/instruction.h @@ -72,7 +72,9 @@ enum InstFlags : uint64_t { /// @brief This execution skipped a conditional memory-counter register result. MEMORY_WAIT_RESULT_SUPPRESSED = (1ULL << 18), /// @brief Non-control-flow instruction that implicitly drains gfx1250 XCNT. - XCNT_DRAIN = (1ULL << 19) + XCNT_DRAIN = (1ULL << 19), + /// @brief This ISA suppresses issue of this instruction while MODE.VSKIP is set. + VSKIP_AFFECTED = (1ULL << 20) }; class BasicBlock; @@ -112,9 +114,9 @@ class Instruction : public util::IListNode d) { data_ = std::move(d); } @@ -296,6 +306,8 @@ class Instruction : public util::IListNode -#include - -namespace rocjitsu { - -namespace { - -// Mark a contiguous range of register indices in one register-class bitset. -template void set_range(std::bitset &bits, size_t base, size_t width) { - for (size_t i = 0; i < width && base + i < N; ++i) - bits.set(base + i); -} - -// Clear a contiguous range of register indices in one register-class bitset. -template void reset_range(std::bitset &bits, size_t base, size_t width) { - for (size_t i = 0; i < width && base + i < N; ++i) - bits.reset(base + i); -} - -template -[[nodiscard]] bool contains_range(const std::bitset &bits, size_t base, size_t width) { - for (size_t i = 0; i < width; ++i) { - if (base + i >= N || !bits.test(base + i)) - return false; - } - return true; -} - -template -[[nodiscard]] bool intersects_range(const std::bitset &bits, size_t base, size_t width) { - for (size_t i = 0; i < width; ++i) { - if (base + i < N && bits.test(base + i)) - return true; - } - return false; -} - -template void subtract(std::bitset &lhs, const std::bitset &rhs) { lhs &= ~rhs; } - -} // namespace - -void RegisterSet::expand(RegisterRef ref) { - const size_t width = std::max(1, ref.width); - switch (ref.cls) { - case RegClass::SGPR: - set_range(sgprs_, ref.index, width); - break; - case RegClass::VGPR: - set_range(vgprs_, ref.index, width); - break; - case RegClass::ACC_VGPR: - set_range(acc_vgprs_, ref.index, width); - break; - case RegClass::TTMP: - // Trap temporaries are known to the ISA but not tracked by this set: no - // bitset and no mask bit, so index/width are dropped (as they are on the - // general def/use path). Explicit arm keeps the switch exhaustive. - break; - case RegClass::EXEC: - case RegClass::VCC: - case RegClass::SCC: - case RegClass::M0: - case RegClass::FLAT_SCRATCH: - case RegClass::PC: - // Special singleton: index/width are meaningless, so just set its bit. - special_regs_ |= special_bit(ref.cls); - break; - } -} - -void RegisterSet::erase(RegisterRef ref) { - const size_t width = std::max(1, ref.width); - switch (ref.cls) { - case RegClass::SGPR: - reset_range(sgprs_, ref.index, width); - break; - case RegClass::VGPR: - reset_range(vgprs_, ref.index, width); - break; - case RegClass::ACC_VGPR: - reset_range(acc_vgprs_, ref.index, width); - break; - case RegClass::TTMP: // untracked: nothing to clear - break; - case RegClass::EXEC: - case RegClass::VCC: - case RegClass::SCC: - case RegClass::M0: - case RegClass::FLAT_SCRATCH: - case RegClass::PC: - special_regs_ &= static_cast(~special_bit(ref.cls)); - break; - } -} - -void RegisterSet::clear_class(RegClass cls) { - switch (cls) { - case RegClass::SGPR: - sgprs_.reset(); - break; - case RegClass::VGPR: - vgprs_.reset(); - break; - case RegClass::ACC_VGPR: - acc_vgprs_.reset(); - break; - case RegClass::TTMP: // untracked: nothing to clear - break; - case RegClass::EXEC: - case RegClass::VCC: - case RegClass::SCC: - case RegClass::M0: - case RegClass::FLAT_SCRATCH: - case RegClass::PC: - special_regs_ &= static_cast(~special_bit(cls)); - break; - } -} - -bool RegisterSet::contains(RegisterRef ref) const { - const size_t width = std::max(1, ref.width); - switch (ref.cls) { - case RegClass::SGPR: - return contains_range(sgprs_, ref.index, width); - case RegClass::VGPR: - return contains_range(vgprs_, ref.index, width); - case RegClass::ACC_VGPR: - return contains_range(acc_vgprs_, ref.index, width); - case RegClass::TTMP: // untracked: never present - return false; - case RegClass::EXEC: - case RegClass::VCC: - case RegClass::SCC: - case RegClass::M0: - case RegClass::FLAT_SCRATCH: - case RegClass::PC: - return (special_regs_ & special_bit(ref.cls)) != 0; - } - return false; // unreachable for a valid RegClass; a new class trips -Wswitch first -} - -bool RegisterSet::none() const { - return sgprs_.none() && vgprs_.none() && acc_vgprs_.none() && special_regs_ == 0; -} - -size_t RegisterSet::size() const { - return ordinary_size() + static_cast(std::popcount(special_regs_)); -} - -size_t RegisterSet::ordinary_size() const { - return sgprs_.count() + vgprs_.count() + acc_vgprs_.count(); -} - -bool RegisterSet::intersects(RegisterRef ref) const { - const size_t width = std::max(1, ref.width); - switch (ref.cls) { - case RegClass::SGPR: - return intersects_range(sgprs_, ref.index, width); - case RegClass::VGPR: - return intersects_range(vgprs_, ref.index, width); - case RegClass::ACC_VGPR: - return intersects_range(acc_vgprs_, ref.index, width); - default: - return false; - } -} - -bool RegisterSet::intersects(const RegisterSet &rhs) const { - return (sgprs_ & rhs.sgprs_).any() || (vgprs_ & rhs.vgprs_).any() || - (acc_vgprs_ & rhs.acc_vgprs_).any() || (special_regs_ & rhs.special_regs_) != 0; -} - -RegisterSet &RegisterSet::operator|=(const RegisterSet &rhs) { - sgprs_ |= rhs.sgprs_; - vgprs_ |= rhs.vgprs_; - acc_vgprs_ |= rhs.acc_vgprs_; - special_regs_ |= rhs.special_regs_; - return *this; -} - -RegisterSet &RegisterSet::operator&=(const RegisterSet &rhs) { - sgprs_ &= rhs.sgprs_; - vgprs_ &= rhs.vgprs_; - acc_vgprs_ &= rhs.acc_vgprs_; - special_regs_ &= rhs.special_regs_; - return *this; -} - -RegisterSet &RegisterSet::operator-=(const RegisterSet &rhs) { - subtract(sgprs_, rhs.sgprs_); - subtract(vgprs_, rhs.vgprs_); - subtract(acc_vgprs_, rhs.acc_vgprs_); - special_regs_ &= static_cast(~rhs.special_regs_); - return *this; -} - -} // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/register_set.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/register_set.h index ce010a8fff1e..df8761306512 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/register_set.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/register_set.h @@ -18,9 +18,14 @@ #include #include -#include #include #include +#include +#include + +#if defined(__AVX2__) +#include +#endif namespace rocjitsu { @@ -119,6 +124,257 @@ struct RegisterRef { constexpr bool operator==(const RegisterRef &) const = default; }; +/// @brief Storage implementation used by a register set. +enum class RegisterSetWordType { StandardUint64, Avx2M256 }; + +namespace register_set_detail { +template struct WordOps {}; + +template <> struct WordOps { + using Word = uint64_t; + static constexpr size_t kWordBits = std::numeric_limits::digits; + static Word word_all_bits() { return std::numeric_limits::max(); } + static Word word_or(Word a, Word b) { return a | b; } + static Word word_and(Word a, Word b) { return a & b; } + static Word word_and_not(Word a, Word b) { return a & ~b; } + static bool word_none(Word a) { return a == 0; } + static bool word_equal(Word a, Word b) { return a == b; } + static Word word_bit_mask(unsigned bitIndex) { return Word{1} << bitIndex; } + // A prefix of [0, end), including the empty and full-word cases. + static Word word_prefix(unsigned end) { + return end == kWordBits ? word_all_bits() : (Word{1} << end) - 1; + } + template static size_t count_bits(const Word (&words)[NumWords]) { + size_t count = 0; + for (Word word : words) + count += std::popcount(word); + return count; + } +}; + +#if defined(__AVX2__) +template <> struct WordOps { + using Word = __m256i; + static constexpr size_t kWordBits = 256; + static Word word_all_bits() { return _mm256_set1_epi64x(-1); } + static unsigned nonempty_lanes(Word word) { + const Word empty = _mm256_cmpeq_epi64(word, _mm256_setzero_si256()); + return ~static_cast(_mm256_movemask_pd(_mm256_castsi256_pd(empty))) & 15u; + } + static Word word_or(Word a, Word b) { return _mm256_or_si256(a, b); } + static Word word_and(Word a, Word b) { return _mm256_and_si256(a, b); } + static Word word_and_not(Word a, Word b) { + // Computes a & ~b; the intrinsic complements its first operand. + return _mm256_andnot_si256(b, a); + } + static bool word_none(Word a) { return _mm256_testz_si256(a, a) != 0; } + static bool word_equal(Word a, Word b) { + return _mm256_movemask_epi8(_mm256_cmpeq_epi8(a, b)) == -1; + } + static Word word_bit_mask(unsigned bitIndex) { + // XOR selects one lane. All other shifts are >= 64 and produce zero. + const Word shiftBits = + _mm256_xor_si256(_mm256_set1_epi64x(bitIndex), _mm256_setr_epi64x(0, 64, 128, 192)); + return _mm256_sllv_epi64(_mm256_set1_epi64x(1), shiftBits); + } + static Word word_prefix(unsigned end) { + const Word counts = + _mm256_sub_epi64(_mm256_set1_epi64x(end), _mm256_set_epi64x(192, 128, 64, 0)); + const Word positive = _mm256_cmpgt_epi64(counts, _mm256_setzero_si256()); + const Word high = _mm256_sllv_epi64(word_all_bits(), counts); + return _mm256_andnot_si256(high, positive); + } + static Word word_count_lanes(Word bits) { + const Word table = _mm256_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, + 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4); + const Word low_mask = _mm256_set1_epi8(15); + const Word low = _mm256_shuffle_epi8(table, _mm256_and_si256(bits, low_mask)); + const Word high = + _mm256_shuffle_epi8(table, _mm256_and_si256(_mm256_srli_epi16(bits, 4), low_mask)); + return _mm256_sad_epu8(_mm256_add_epi8(low, high), _mm256_setzero_si256()); + } + static size_t sum_count_lanes(Word sums) { + const __m128i halves = + _mm_add_epi64(_mm256_castsi256_si128(sums), _mm256_extracti128_si256(sums, 1)); + const __m128i total = _mm_add_epi64(halves, _mm_unpackhi_epi64(halves, halves)); + return static_cast(_mm_cvtsi128_si64(total)); + } + template static size_t count_bits(const Word (&words)[NumWords]) { + // Keep counts in SIMD lanes across words, reducing to a scalar only once. + Word numBitsPerLane = _mm256_setzero_si256(); + for (Word word : words) + numBitsPerLane = _mm256_add_epi64(numBitsPerLane, word_count_lanes(word)); + return sum_count_lanes(numBitsPerLane); + } +}; +#endif + +// Range and set operations shared by scalar and SIMD words. +template struct WordArrayOps final : WordOps { + using Word = typename WordOps::Word; + using WordOps::kWordBits; + using WordOps::word_all_bits; + using WordOps::word_or; + using WordOps::word_and; + using WordOps::word_and_not; + using WordOps::word_none; + using WordOps::word_equal; + using WordOps::word_bit_mask; + using WordOps::word_prefix; + // Single-bit helpers require bitIndex < kWordBits. + static void word_set_bit(Word &word, unsigned bitIndex) { + word = word_or(word, word_bit_mask(bitIndex)); + } + static void word_reset_bit(Word &word, unsigned bitIndex) { + word = word_and_not(word, word_bit_mask(bitIndex)); + } + static bool word_test_bit(Word word, unsigned bitIndex) { + return !word_none(word_and(word, word_bit_mask(bitIndex))); + } + static Word word_range(unsigned begin, unsigned end) { + return word_and_not(word_prefix(end), word_prefix(begin)); + } + + template static bool words_equal(const Word (&a)[Words], const Word (&b)[Words]) { + for (size_t i = 0; i < Words; ++i) + if (!word_equal(a[i], b[i])) + return false; + return true; + } + // Visit a nonempty, in-bounds bit range. Stop when an operation returns a + // value other than kContinue; otherwise return kContinue after the last word. + // Full words need no range mask. Inline to retain the caller's capacity bounds. + template + [[gnu::always_inline]] static inline bool + visit_range(WordType (&words)[NumWords], size_t startBit, size_t endBit, Operation operation) { + // Prologue: initial partial word. + size_t word = startBit / kWordBits; + const size_t firstBitInWord = startBit % kWordBits; + if (firstBitInWord != 0) { + const size_t endBitInWord = std::min(kWordBits, endBit - word * kWordBits); + const Word mask = word_range(firstBitInWord, endBitInWord); + if (operation.partial_word(words[word], mask) != Operation::kContinue) + return !Operation::kContinue; + if (endBitInWord < kWordBits) + return Operation::kContinue; + ++word; + } + + // Middle loop: full words. + const size_t endWord = endBit / kWordBits; + for (; word < endWord; ++word) + if (operation.full_word(words[word]) != Operation::kContinue) + return !Operation::kContinue; + + // Epilogue: final partial word. + const size_t endBitInWord = endBit % kWordBits; + if (endBitInWord != 0) { + const Word mask = word_prefix(endBitInWord); + return operation.partial_word(words[word], mask); + } + return Operation::kContinue; + } + + struct ContainsWords { + static constexpr bool kContinue = true; + bool partial_word(const Word &word, Word mask) const { + return word_none(word_and_not(mask, word)); + } + bool full_word(const Word &word) const { return word_equal(word, word_all_bits()); } + }; + struct IntersectsWords { + static constexpr bool kContinue = false; + bool partial_word(const Word &word, Word mask) const { + return !word_none(word_and(word, mask)); + } + bool full_word(const Word &word) const { return !word_none(word); } + }; + + template struct UpdateWords { + static constexpr bool kContinue = true; + Update update; + bool partial_word(Word &word, Word mask) const { + word = update(word, mask); + return true; + } + bool full_word(Word &word) const { + word = update(word, word_all_bits()); + return true; + } + }; + + template + static void update_range(Word (&words)[NumWords], size_t startBit, size_t numBits, + Update update) { + if (numBits == 0) + return; + if (startBit >= CapacityBits) + return; + const size_t endBit = startBit + std::min(numBits, CapacityBits - startBit); + visit_range(words, startBit, endBit, UpdateWords{update}); + } + template + static void set_range(Word (&words)[NumWords], size_t startBit, size_t numBits) { + if (numBits == 1) { + if (startBit < CapacityBits) + word_set_bit(words[startBit / kWordBits], startBit % kWordBits); + return; + } + update_range(words, startBit, numBits, word_or); + } + template + static void reset_range(Word (&words)[NumWords], size_t startBit, size_t numBits) { + if (numBits == 1) { + if (startBit < CapacityBits) + word_reset_bit(words[startBit / kWordBits], startBit % kWordBits); + return; + } + update_range(words, startBit, numBits, word_and_not); + } + template + static bool contains_range(const Word (&words)[NumWords], size_t startBit, size_t numBits) { + if (startBit >= CapacityBits || numBits > CapacityBits - startBit) + return false; + if (numBits == 1) + return word_test_bit(words[startBit / kWordBits], startBit % kWordBits); + if (numBits == 0) + return true; + const size_t endBit = startBit + numBits; + + return visit_range(words, startBit, endBit, ContainsWords{}); + } + template + static bool intersects_range(const Word (&words)[NumWords], size_t startBit, size_t numBits) { + if (numBits == 1) + return startBit < CapacityBits && + word_test_bit(words[startBit / kWordBits], startBit % kWordBits); + if (numBits == 0 || startBit >= CapacityBits) + return false; + const size_t endBit = startBit + std::min(numBits, CapacityBits - startBit); + + return visit_range(words, startBit, endBit, IntersectsWords{}); + } + template + static void combine(Word (&lhs)[NumWords], const Word (&rhs)[NumWords], Operation operation) { + for (size_t word = 0; word < NumWords; ++word) + lhs[word] = operation(lhs[word], rhs[word]); + } + template static bool bits_none(const Word (&words)[NumWords]) { + for (const Word word : words) + if (!word_none(word)) + return false; + return true; + } + template + static bool intersects_bits(const Word (&lhs)[NumWords], const Word (&rhs)[NumWords]) { + for (size_t word = 0; word < NumWords; ++word) + if (!word_none(word_and(lhs[word], rhs[word]))) + return true; + return false; + } +}; +} // namespace register_set_detail + /// @brief Per-class register set used for def/use and liveness dataflow. /// /// @details A RegisterSet can represent an instruction's use set, def set, @@ -135,11 +391,20 @@ struct RegisterRef { /// describe the full set; consumers that reason only about allocatable/indexed /// registers use the ordinary views (`ordinary_size`, `for_each_ordinary`, /// `ordinary_only`) so singleton special state never looks spillable or indexed. -class RegisterSet { +/// @tparam wordType The storage implementation. Scalar instantiations remain +/// available in AVX2 builds for testing. +template class RegisterSetT { + using Ops = register_set_detail::WordArrayOps; + using Word = typename Ops::Word; + public: + using WordType = Word; + static constexpr RegisterSetWordType kWordType = wordType; + /// @brief Add `ref`. For ordinary classes this marks every 32-bit lane it /// covers; for special classes it marks the singleton (index/width ignored). - void expand(RegisterRef ref); + // Keep constant register classes and widths visible in iteration callbacks. + [[gnu::always_inline]] inline void expand(RegisterRef ref); /// @brief Remove `ref`. For ordinary classes this clears every lane it /// covers; for special classes it clears the singleton (index/width ignored). @@ -150,13 +415,15 @@ class RegisterSet { /// @brief Return true if `ref` is present. For ordinary classes every covered /// lane must be present; for special classes only membership is checked. - [[nodiscard]] bool contains(RegisterRef ref) const; + // Inline the class/width dispatch and expose the read-only contract so + // iteration can omit mutation checks around callbacks using contains(). + [[nodiscard, gnu::pure, gnu::always_inline]] inline bool contains(RegisterRef ref) const; /// @brief Return true if any lane covered by `ref` is present. /// /// @details The any-of counterpart to contains(). Register allocation asks /// this to reject a candidate tuple: a run is usable only when none of its - /// lanes is in the unavailable set. Classes RegisterSet does not track answer + /// lanes is in the unavailable set. Classes RegisterSetT does not track answer /// false, matching contains(). [[nodiscard]] bool intersects(RegisterRef ref) const; @@ -176,31 +443,34 @@ class RegisterSet { [[nodiscard]] bool has_specials() const { return special_regs_ != 0; } /// @brief Return true if any member is present in both sets (ordinary or special). - [[nodiscard]] bool intersects(const RegisterSet &rhs) const; + [[nodiscard]] bool intersects(const RegisterSetT &rhs) const; - RegisterSet &operator|=(const RegisterSet &rhs); - RegisterSet &operator&=(const RegisterSet &rhs); - RegisterSet &operator-=(const RegisterSet &rhs); + RegisterSetT &operator|=(const RegisterSetT &rhs); + RegisterSetT &operator&=(const RegisterSetT &rhs); + RegisterSetT &operator-=(const RegisterSetT &rhs); - friend RegisterSet operator|(RegisterSet lhs, const RegisterSet &rhs) { + friend RegisterSetT operator|(RegisterSetT lhs, const RegisterSetT &rhs) { lhs |= rhs; return lhs; } - friend RegisterSet operator&(RegisterSet lhs, const RegisterSet &rhs) { + friend RegisterSetT operator&(RegisterSetT lhs, const RegisterSetT &rhs) { lhs &= rhs; return lhs; } - friend RegisterSet operator-(RegisterSet lhs, const RegisterSet &rhs) { + friend RegisterSetT operator-(RegisterSetT lhs, const RegisterSetT &rhs) { lhs -= rhs; return lhs; } - friend bool operator==(const RegisterSet &, const RegisterSet &) = default; + friend bool operator==(const RegisterSetT &a, const RegisterSetT &b) { + return Ops::words_equal(a.sgprs_, b.sgprs_) && Ops::words_equal(a.vgprs_, b.vgprs_) && + Ops::words_equal(a.acc_vgprs_, b.acc_vgprs_) && a.special_regs_ == b.special_regs_; + } /// @brief Return a copy holding only the ordinary members; special singletons /// are dropped. This is the projection scratch/liveness/spill consumers use. - [[nodiscard]] RegisterSet ordinary_only() const { - RegisterSet copy = *this; + [[nodiscard]] RegisterSetT ordinary_only() const { + RegisterSetT copy = *this; copy.special_regs_ = 0; return copy; } @@ -218,18 +488,9 @@ class RegisterSet { /// @brief Invoke @p f with each ordinary single-lane register (SGPR, VGPR, /// AccVGPR) in ascending index order. Special singletons are not visited. template void for_each_ordinary(F &&f) const { - for (size_t i = 0; i < sgprs_.size(); ++i) { - if (sgprs_.test(i)) - f(RegisterRef{RegClass::SGPR, static_cast(i), 1}); - } - for (size_t i = 0; i < vgprs_.size(); ++i) { - if (vgprs_.test(i)) - f(RegisterRef{RegClass::VGPR, static_cast(i), 1}); - } - for (size_t i = 0; i < acc_vgprs_.size(); ++i) { - if (acc_vgprs_.test(i)) - f(RegisterRef{RegClass::ACC_VGPR, static_cast(i), 1}); - } + for_each_bits(sgprs_, f); + for_each_bits(vgprs_, f); + for_each_bits(acc_vgprs_, f); } /// @brief Invoke @p f with each present special class, in ascending @@ -256,13 +517,273 @@ class RegisterSet { static_assert(widest_special_reg_class() < 16, "special_regs_ must hold a bit for every special RegClass"); - std::bitset sgprs_; - std::bitset vgprs_; - std::bitset acc_vgprs_; + // SIMD words support set algebra and lane-occupancy scans on AVX2; + // other targets use scalar words. expand() clips to the class capacity, so + // the unused tail bits stay zero under every set operation. + template + using RegisterBits = Word[(Capacity + Ops::kWordBits - 1) / Ops::kWordBits]; + + // Keep construction and the callback together so the class stays constant. + // Clang needs a call-site hint for larger callbacks, such as string formatting. + // Their callees still follow the compiler's normal inlining decisions. + template + [[gnu::always_inline]] static inline void emit_register(F &f, uint16_t index) { +#if defined(__clang__) + [[clang::always_inline]] +#endif + f(RegisterRef{cls, index, 1}); + } + + // Keep the class constant even when this helper is not inlined. + template + static void for_each_bits(const Word (&words)[Words], F &f) { +#if defined(__AVX2__) + if constexpr (wordType == RegisterSetWordType::Avx2M256) { + for (size_t word = 0; word < Words; ++word) { + unsigned lanes = Ops::nonempty_lanes(words[word]); + while (lanes) { + unsigned lane = std::countr_zero(lanes); + // Scalar membership cursors avoid carrying a SIMD snapshot across + // callbacks. memcpy reads the vector's representation without aliasing + // violations and lowers to a scalar load for this fixed-size copy. + const auto *address = reinterpret_cast(&words[word]) + lane * 8; + const auto load = [&] { + uint64_t value; + std::memcpy(&value, address, sizeof(value)); + return value; + }; + uint64_t members = load(), remaining = members; + // Full lanes emit consecutive indices. After an edit, resume set-bit + // iteration strictly after the last emitted index, including at bit 63. + unsigned cursor = 0; + if (members == ~uint64_t{0}) { +#pragma GCC unroll 8 + for (; cursor < 64; ++cursor) { + emit_register(f, static_cast(word * 256 + lane * 64 + cursor)); + members = load(); + if (members != ~uint64_t{0}) { + remaining = members & (~uint64_t{1} << cursor); + break; + } + } + if (cursor == 64) + remaining = 0; + } + while (remaining) { + unsigned bit = std::countr_zero(remaining); + emit_register(f, static_cast(word * 256 + lane * 64 + bit)); + // Keep edit recovery on a cold path. In particular, this prevents Clang + // from putting its shift and mask on the cursor's dependency chain. + const uint64_t updated = load(); + if (updated != members) [[unlikely]] { + remaining = updated & (~uint64_t{1} << bit); + members = updated; + } else { + remaining &= remaining - 1; + } + } + // Observe additions to later, previously empty lanes as well as removals. + lanes = Ops::nonempty_lanes(words[word]) & (~1u << lane); + } + } + } else +#endif + { + for (size_t word = 0; word < Words; ++word) { + Word members = words[word]; + Word remaining = members; + while (remaining) { + unsigned bit = std::countr_zero(remaining); + emit_register(f, static_cast(word * Ops::kWordBits + bit)); + Word updated = words[word]; + if (updated == members) + remaining &= remaining - 1; + else + remaining = + bit == Ops::kWordBits - 1 ? 0 : updated & (Ops::word_all_bits() << (bit + 1)); + members = updated; + } + } + } + } + + RegisterBits sgprs_{}; + RegisterBits vgprs_{}; + RegisterBits acc_vgprs_{}; /// @brief Bit `static_cast(cls)` set iff special class `cls` is /// present. Only special-class bits are ever set (see `expand`). uint16_t special_regs_ = 0; }; +template void RegisterSetT::expand(RegisterRef ref) { + const size_t numBits = std::max(1, ref.width); + switch (ref.cls) { + case RegClass::SGPR: + Ops::template set_range(sgprs_, ref.index, numBits); + break; + case RegClass::VGPR: + Ops::template set_range(vgprs_, ref.index, numBits); + break; + case RegClass::ACC_VGPR: + Ops::template set_range(acc_vgprs_, ref.index, numBits); + break; + case RegClass::TTMP: + // Trap temporaries are known to the ISA but not tracked by this set: no + // bitset and no mask bit, so index/width are dropped (as they are on the + // general def/use path). Explicit arm keeps the switch exhaustive. + break; + case RegClass::EXEC: + case RegClass::VCC: + case RegClass::SCC: + case RegClass::M0: + case RegClass::FLAT_SCRATCH: + case RegClass::PC: + // Special singleton: index/width are meaningless, so just set its bit. + special_regs_ |= special_bit(ref.cls); + break; + } +} + +template void RegisterSetT::erase(RegisterRef ref) { + const size_t numBits = std::max(1, ref.width); + switch (ref.cls) { + case RegClass::SGPR: + Ops::template reset_range(sgprs_, ref.index, numBits); + break; + case RegClass::VGPR: + Ops::template reset_range(vgprs_, ref.index, numBits); + break; + case RegClass::ACC_VGPR: + Ops::template reset_range(acc_vgprs_, ref.index, numBits); + break; + case RegClass::TTMP: // untracked: nothing to clear + break; + case RegClass::EXEC: + case RegClass::VCC: + case RegClass::SCC: + case RegClass::M0: + case RegClass::FLAT_SCRATCH: + case RegClass::PC: + special_regs_ &= static_cast(~special_bit(ref.cls)); + break; + } +} + +template void RegisterSetT::clear_class(RegClass cls) { + switch (cls) { + case RegClass::SGPR: + std::fill(std::begin(sgprs_), std::end(sgprs_), Word{}); + break; + case RegClass::VGPR: + std::fill(std::begin(vgprs_), std::end(vgprs_), Word{}); + break; + case RegClass::ACC_VGPR: + std::fill(std::begin(acc_vgprs_), std::end(acc_vgprs_), Word{}); + break; + case RegClass::TTMP: // untracked: nothing to clear + break; + case RegClass::EXEC: + case RegClass::VCC: + case RegClass::SCC: + case RegClass::M0: + case RegClass::FLAT_SCRATCH: + case RegClass::PC: + special_regs_ &= static_cast(~special_bit(cls)); + break; + } +} + +template +bool RegisterSetT::contains(RegisterRef ref) const { + const size_t numBits = std::max(1, ref.width); + switch (ref.cls) { + case RegClass::SGPR: + return Ops::template contains_range(sgprs_, ref.index, numBits); + case RegClass::VGPR: + return Ops::template contains_range(vgprs_, ref.index, numBits); + case RegClass::ACC_VGPR: + return Ops::template contains_range(acc_vgprs_, ref.index, numBits); + case RegClass::TTMP: // untracked: never present + return false; + case RegClass::EXEC: + case RegClass::VCC: + case RegClass::SCC: + case RegClass::M0: + case RegClass::FLAT_SCRATCH: + case RegClass::PC: + return (special_regs_ & special_bit(ref.cls)) != 0; + } + return false; // unreachable for a valid RegClass; a new class trips -Wswitch first +} + +template bool RegisterSetT::none() const { + return Ops::bits_none(sgprs_) && Ops::bits_none(vgprs_) && Ops::bits_none(acc_vgprs_) && + special_regs_ == 0; +} + +template size_t RegisterSetT::size() const { + return ordinary_size() + static_cast(std::popcount(special_regs_)); +} + +template size_t RegisterSetT::ordinary_size() const { + return Ops::count_bits(sgprs_) + Ops::count_bits(vgprs_) + Ops::count_bits(acc_vgprs_); +} + +template +bool RegisterSetT::intersects(RegisterRef ref) const { + const size_t numBits = std::max(1, ref.width); + switch (ref.cls) { + case RegClass::SGPR: + return Ops::template intersects_range(sgprs_, ref.index, numBits); + case RegClass::VGPR: + return Ops::template intersects_range(vgprs_, ref.index, numBits); + case RegClass::ACC_VGPR: + return Ops::template intersects_range(acc_vgprs_, ref.index, + numBits); + default: + return false; + } +} + +template +bool RegisterSetT::intersects(const RegisterSetT &rhs) const { + return Ops::intersects_bits(sgprs_, rhs.sgprs_) || Ops::intersects_bits(vgprs_, rhs.vgprs_) || + Ops::intersects_bits(acc_vgprs_, rhs.acc_vgprs_) || + (special_regs_ & rhs.special_regs_) != 0; +} + +template +RegisterSetT &RegisterSetT::operator|=(const RegisterSetT &rhs) { + Ops::combine(sgprs_, rhs.sgprs_, Ops::word_or); + Ops::combine(vgprs_, rhs.vgprs_, Ops::word_or); + Ops::combine(acc_vgprs_, rhs.acc_vgprs_, Ops::word_or); + special_regs_ |= rhs.special_regs_; + return *this; +} + +template +RegisterSetT &RegisterSetT::operator&=(const RegisterSetT &rhs) { + Ops::combine(sgprs_, rhs.sgprs_, Ops::word_and); + Ops::combine(vgprs_, rhs.vgprs_, Ops::word_and); + Ops::combine(acc_vgprs_, rhs.acc_vgprs_, Ops::word_and); + special_regs_ &= rhs.special_regs_; + return *this; +} + +template +RegisterSetT &RegisterSetT::operator-=(const RegisterSetT &rhs) { + Ops::combine(sgprs_, rhs.sgprs_, Ops::word_and_not); + Ops::combine(vgprs_, rhs.vgprs_, Ops::word_and_not); + Ops::combine(acc_vgprs_, rhs.acc_vgprs_, Ops::word_and_not); + special_regs_ &= static_cast(~rhs.special_regs_); + return *this; +} + +#if defined(__AVX2__) +using RegisterSet = RegisterSetT; +#else +using RegisterSet = RegisterSetT; +#endif + } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/events.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/events.cpp index 5cf799293026..c20c5708090a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/events.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/events.cpp @@ -59,6 +59,23 @@ bool EventState::release_page(void *ptr) { return true; } +void EventState::activate_waiters(GpuEvent &event) { + // KFD delivers a signal to every waiter registered for signals. In particular, + // auto-reset clears the shared flag, not these per-wait activations: otherwise + // legacy waits lose the wake, and wait-all forgets signals received earlier. + for (auto *waiter : event.waiters) { + if (waiter->listen_for_signals) { + waiter->activated = true; + waiter->cv->notify_one(); + } + } +} + +bool EventState::has_signal_waiters(const GpuEvent &event) { + return std::any_of(event.waiters.begin(), event.waiters.end(), + [](const EventWaiter *waiter) { return waiter->listen_for_signals; }); +} + /// @brief Signal event(s) from the CP's interrupt callback. /// @details When event_id is non-zero, signals that specific event. When /// event_id is zero, broadcasts to all type-0 events — matching real @@ -68,28 +85,26 @@ void EventState::signal_interrupt(uint32_t event_id) { if (event_id == 0) { for (auto &[id, ev] : events_) { if (ev.event_type == 0) { - ev.signaled = !ev.auto_reset || ev.waiters.empty(); + ev.signaled = !ev.auto_reset || !has_signal_waiters(ev); if (!(++ev.event_age)) ev.event_age = 2; write_event_slot(page_, page_size_, id, ev.event_age); util::Logger::cp("SIGNAL_BROADCAST: event_id=", id, " age=", ev.event_age, " waiters=", ev.waiters.size()); - for (auto *cv : ev.waiters) - cv->notify_one(); + activate_waiters(ev); } } return; } auto it = events_.find(event_id); if (it != events_.end() && it->second.event_type == 0) { - it->second.signaled = !it->second.auto_reset || it->second.waiters.empty(); + it->second.signaled = !it->second.auto_reset || !has_signal_waiters(it->second); if (!(++it->second.event_age)) it->second.event_age = 2; write_event_slot(page_, page_size_, event_id, it->second.event_age); util::Logger::cp("SIGNAL_INTERRUPT: event_id=", event_id, " age=", it->second.event_age, " waiters=", it->second.waiters.size(), " page=", page_ ? "valid" : "null"); - for (auto *cv : it->second.waiters) - cv->notify_one(); + activate_waiters(it->second); } else { util::Logger::cp("SIGNAL_INTERRUPT_MISS: event_id=", event_id, " NOT FOUND or wrong type, events_.size()=", events_.size()); @@ -106,9 +121,8 @@ bool EventState::signal_memory_fault(const MemoryFault &fault) { // the shared page and the runtime does not poll it, it parks a thread in // WAIT_EVENTS. Record the payload for that thread to collect and wake it. ev.fault = fault; - ev.signaled = true; - for (auto *cv : ev.waiters) - cv->notify_one(); + ev.signaled = !ev.auto_reset || !has_signal_waiters(ev); + activate_waiters(ev); delivered = true; } return delivered; @@ -119,8 +133,8 @@ void EventState::notify_closing() { std::lock_guard lock(mutex_); closing_.store(true, std::memory_order_release); for (auto &[id, ev] : events_) { - for (auto *cv : ev.waiters) - cv->notify_one(); + for (auto *waiter : ev.waiters) + waiter->cv->notify_one(); } } @@ -145,8 +159,8 @@ void EventState::begin_wait_cancel() { std::lock_guard lock(mutex_); wait_cancelled_.store(true, std::memory_order_release); for (auto &[id, ev] : events_) { - for (auto *cv : ev.waiters) - cv->notify_one(); + for (auto *waiter : ev.waiters) + waiter->cv->notify_one(); } } @@ -224,15 +238,15 @@ int EventState::destroy_event(void *arg) { std::lock_guard lock(mutex_); auto it = events_.find(args->event_id); if (it != events_.end()) { - for (auto *cv : it->second.waiters) - cv->notify_one(); + for (auto *waiter : it->second.waiters) + waiter->cv->notify_one(); events_.erase(it); } write_event_slot(page_, page_size_, args->event_id, KFD_SIGNAL_EVENT_LIMIT); return 0; } -/// @brief Signal an event: set signaled flag, increment age, wake waiters. +/// @brief Advance the event age and deliver to waiting or future callers. int EventState::set_event(void *arg) { assert(arg && "set_event called with null arg"); auto *args = static_cast(arg); @@ -243,18 +257,17 @@ int EventState::set_event(void *arg) { " events_.size()=", events_.size()); return -EINVAL; } - it->second.signaled = !it->second.auto_reset || it->second.waiters.empty(); + it->second.signaled = !it->second.auto_reset || !has_signal_waiters(it->second); if (!(++it->second.event_age)) it->second.event_age = 2; write_event_slot(page_, page_size_, args->event_id, it->second.event_age); util::Logger::cp("SET_EVENT: event_id=", args->event_id, " age=", it->second.event_age, " waiters=", it->second.waiters.size()); - for (auto *cv : it->second.waiters) - cv->notify_one(); + activate_waiters(it->second); return 0; } -/// @brief Reset an event's age to 0 (unsignaled). +/// @brief Clear an event's signaled flag without revoking waiter activations. int EventState::reset_event(void *arg) { assert(arg && "reset_event called with null arg"); auto *args = static_cast(arg); @@ -296,29 +309,46 @@ int EventState::wait_events(void *arg, uint32_t process_id) { }); auto satisfied = [](const GpuEvent &ev, const kfd_event_data &ed) -> bool { - if (ev.event_type == 0) { - uint64_t caller_age = ed.signal_event_data.last_event_age; - if (caller_age == 0) - return ev.signaled; - return ev.event_age != caller_age; - } - return ev.signaled; + return ev.signaled || + (ev.event_type == KFD_IOC_EVENT_SIGNAL && ed.signal_event_data.last_event_age != 0 && + ev.event_age != ed.signal_event_data.last_event_age); }; std::condition_variable my_cv; + std::vector waiters(args->num_events); std::unique_lock lock(mutex_); + // A cancelled/closing call must not consume pending auto-reset signals. + if (closing_) + return -EBADF; + if (wait_cancelled_) { + args->wait_result = KFD_IOC_WAIT_RESULT_TIMEOUT; + return 0; + } + for (uint32_t i = 0; i < args->num_events; ++i) { auto it = events_.find(ev_data[i].event_id); - if (it != events_.end()) - it->second.waiters.push_back(&my_cv); + if (it == events_.end()) + continue; + auto &ev = it->second; + const bool activated = satisfied(ev, ev_data[i]); + waiters[i] = {&my_cv, activated, !activated, + ev.event_type == KFD_IOC_EVENT_SIGNAL && + ev_data[i].signal_event_data.last_event_age != 0}; + // Consume an already-pending auto-reset signal when this waiter takes it, + // so a later legacy wait cannot steal it while a wait-all is still parked. + if (ev.auto_reset) + ev.signaled = false; + // Even an initially satisfied entry must wake if its event is destroyed + // while a wait-all call is still blocked on a different event. + ev.waiters.push_back(&waiters[i]); } auto unregister_waiters = [&]() { for (uint32_t i = 0; i < args->num_events; ++i) { auto it = events_.find(ev_data[i].event_id); if (it != events_.end()) - std::erase(it->second.waiters, &my_cv); + std::erase(it->second.waiters, &waiters[i]); } }; @@ -331,7 +361,7 @@ int EventState::wait_events(void *arg, uint32_t process_id) { auto it = events_.find(ev_data[i].event_id); if (it == events_.end()) return true; - if (satisfied(it->second, ev_data[i])) + if (waiters[i].activated) any_satisfied = true; else all_satisfied = false; @@ -372,34 +402,9 @@ int EventState::wait_events(void *arg, uint32_t process_id) { bool any_destroyed = false; bool all_ready = true; for (uint32_t i = 0; i < args->num_events; ++i) { - auto it = events_.find(ev_data[i].event_id); - if (it == events_.end()) { - any_destroyed = true; - all_ready = false; - continue; - } - if (satisfied(it->second, ev_data[i])) { - any_ready = true; - if (it->second.event_type == 0) - ev_data[i].signal_event_data.last_event_age = it->second.event_age; - if (it->second.event_type == KFD_IOC_EVENT_MEMORY) { - // The union member the runtime reads for this event type. Report the - // address that faulted so the failure names its own cause. - auto &exception = ev_data[i].memory_exception_data; - exception = {}; - exception.va = it->second.fault.va; - exception.gpu_id = it->second.fault.gpu_id; - exception.failure.NotPresent = it->second.fault.not_present ? 1u : 0u; - exception.failure.ReadOnly = it->second.fault.read_only ? 1u : 0u; - } - if (it->second.auto_reset) { - it->second.signaled = false; - if (it->second.event_type == 0) - write_event_slot(page_, page_size_, it->second.event_id, KFD_SIGNAL_EVENT_LIMIT); - } - } else { - all_ready = false; - } + any_destroyed |= !events_.contains(ev_data[i].event_id); + any_ready |= waiters[i].activated; + all_ready &= waiters[i].activated; } if (any_destroyed) @@ -409,6 +414,30 @@ int EventState::wait_events(void *arg, uint32_t process_id) { else args->wait_result = KFD_IOC_WAIT_RESULT_TIMEOUT; + // KFD copies event data only on completion. Advancing ages on a partial + // wait-all timeout would make a retry forget signals it already observed. + if (args->wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) { + for (uint32_t i = 0; i < args->num_events; ++i) { + if (!waiters[i].activated) + continue; + auto &ev = events_.find(ev_data[i].event_id)->second; + if (waiters[i].event_age_enabled) + ev_data[i].signal_event_data.last_event_age = ev.event_age; + if (ev.event_type == KFD_IOC_EVENT_MEMORY) { + // The union member the runtime reads for this event type. Report the + // address that faulted so the failure names its own cause. + auto &exception = ev_data[i].memory_exception_data; + exception = {}; + exception.va = ev.fault.va; + exception.gpu_id = ev.fault.gpu_id; + exception.failure.NotPresent = ev.fault.not_present ? 1u : 0u; + exception.failure.ReadOnly = ev.fault.read_only ? 1u : 0u; + } + if (ev.auto_reset && !ev.signaled && ev.event_type == KFD_IOC_EVENT_SIGNAL) + write_event_slot(page_, page_size_, ev.event_id, KFD_SIGNAL_EVENT_LIMIT); + } + } + static thread_local uint32_t wait_log_counter = 0; if (args->wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) { for (uint32_t i = 0; i < args->num_events && i < 4; ++i) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/events.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/events.h index 7c31d4ba2a89..c55ca731540f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/events.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/events.h @@ -12,10 +12,9 @@ /// per-event waiter list matching the kernel's wait_queue model — when an event /// fires, only threads registered on that specific event are woken. /// -/// Event age semantics follow the real ROCR ↔ KFD protocol: ROCR passes -/// last_event_age=1 on every WAIT_EVENTS call. set_event/interrupt sets -/// event_age to 1; auto-reset clears it to 0. The wait predicate checks -/// event_age >= last_event_age for signal events only. +/// Signal events maintain an age counter starting at 1. Nonzero caller ages +/// detect a changed counter; age zero uses the legacy signaled flag. Each +/// registered waiter latches activation independently of the auto-reset flag. #include #include @@ -69,22 +68,22 @@ class EventState { /// @brief Handle KFD SET_EVENT ioctl. /// @param arg Pointer to kfd_ioctl_set_event_args. - /// @details Sets event_age to 1, writes to the event page slot, and wakes + /// @details Increments event_age, writes to the event page slot, and activates /// all registered waiters. /// @returns 0 on success, -EINVAL if event_id not found. int set_event(void *arg); /// @brief Handle KFD RESET_EVENT ioctl. /// @param arg Pointer to kfd_ioctl_reset_event_args. - /// @details Clears event_age to 0. + /// @details Clears signaled without changing the age or registered activations. /// @returns 0 on success, -EINVAL if event_id not found. int reset_event(void *arg); /// @brief Handle KFD WAIT_EVENTS ioctl. /// @param arg Pointer to kfd_ioctl_wait_events_args. /// @details Blocks the caller until waited-on signal events satisfy the - /// age predicate, respecting wait_for_all (AND vs OR semantics). - /// Creates a thread-local CV and registers it with each waited event. + /// legacy/age predicate, respecting wait_for_all (AND vs OR semantics). + /// Registers per-event activation records sharing a call-local CV. /// @returns 0 on success, -EBADF if the driver is closing. int wait_events(void *arg, uint32_t process_id = 0); @@ -107,7 +106,7 @@ class EventState { /// @brief Deliver a memory violation to this process's memory-exception event. /// @details Finds the process's KFD_IOC_EVENT_MEMORY event -- the runtime /// creates exactly one and parks a handler thread on it -- records @p fault as - /// its payload for the next WAIT_EVENTS to collect, and wakes its waiters. + /// its payload for the next WAIT_EVENTS to collect, and activates its waiters. /// @returns false when the process registered no such event, in which case the /// violation has nowhere to go and the caller should say so itself. bool signal_memory_fault(const MemoryFault &fault); @@ -169,18 +168,33 @@ class EventState { void *page_ = nullptr; ///< Mapped signal page (libhsakmt polls slots here). size_t page_size_ = 0; ///< Size of the mapped event page in bytes. + /// One WAIT_EVENTS entry. All fields are protected by mutex_; storage belongs + /// to the wait call and outlives its registration in the event's waiter list. + struct EventWaiter { + std::condition_variable *cv = nullptr; ///< Shared by the entries of one wait call. + bool activated = false; ///< Latched independently of the shared signaled flag. + // Initially satisfied entries only need destruction notifications. Keeping + // them in the list must not consume a later signal intended for a new wait. + bool listen_for_signals = false; + bool event_age_enabled = false; ///< Caller opted in to signal-event age copyback. + }; + /// @brief Internal event representation. struct GpuEvent { uint32_t event_id = 0; ///< KFD event ID (1-based, matches slot index). uint32_t event_type = 0; ///< HSA event type (0 = signal, others = system). - bool auto_reset = false; ///< If true, signaled clears after wakeup. - bool signaled = false; ///< True when the event has been signaled. + bool auto_reset = false; ///< Consume pending state when delivered to a waiter. + bool signaled = false; ///< Pending state for new waits, separate from waiter activation. uint64_t event_age = 1; ///< Monotonic age counter (starts at 1, matching real KFD). /// Payload for the most recent violation on a memory-exception event. MemoryFault fault; - std::vector waiters; ///< Per-event waiter list (kernel wait_queue). + std::vector waiters; ///< Per-event waiter list (kernel wait_queue). }; + /// Latch activation before notification, with mutex_ held by the caller. + static void activate_waiters(GpuEvent &event); + static bool has_signal_waiters(const GpuEvent &event); + mutable std::mutex mutex_; ///< Protects all mutable event state. std::unordered_map events_; ///< Event table keyed by event_id. uint32_t next_event_id_ = 1; ///< Next event ID to allocate. diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/interposer.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/interposer.cpp index 2c96b4dc2fa4..40effb905557 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/interposer.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/interposer.cpp @@ -119,8 +119,11 @@ static_assert(std::atomic::is_always_lock_free, /// @brief Sleep while @p word still equals @p expected, until @p deadline or a signal. long futex_wait_until(std::atomic &word, uint32_t expected, const timespec *deadline) { - return syscall(SYS_futex, &word, FUTEX_WAIT_BITSET_PRIVATE, expected, deadline, nullptr, - FUTEX_BITSET_MATCH_ANY); + // Wine may call pthread_exit from a signal handler during this wait. Use the + // resolved pointer to avoid libc's noexcept declaration of syscall, which + // would terminate forced unwinding instead of releasing the waiter's state. + return rocjitsu::libc_passthrough().syscall(SYS_futex, &word, FUTEX_WAIT_BITSET_PRIVATE, expected, + deadline, nullptr, FUTEX_BITSET_MATCH_ANY); } /// @brief Wake every waiter sleeping on @p word. @@ -1437,9 +1440,13 @@ class InterposerContext { std::unordered_set contexts; bool vmid_reserved = false; std::mutex submission_mutex; - std::unordered_map - pm4_queue_keys; ///< Context/ring queues until context or file release. - uint64_t next_submission = 1; ///< Next file-local CS sequence number. + /// Submission identity and retained fences for one context/engine/ring namespace. + struct Pm4Queue { + uint64_t key = 0; ///< SimulatedKfd queue identity, retained until release. + uint64_t next_sequence = 1; ///< CS sequence local to this namespace. + std::map> fences; ///< Bounded WAIT_CS history. + }; + std::unordered_map pm4_queues; ///< Context/engine/ring namespaces. }; using DrmFileToken = std::shared_ptr; @@ -1468,7 +1475,7 @@ class InterposerContext { struct DrmFinalRelease { std::optional file_id; DrmBackendLease lease; - std::unordered_map queue_keys; + std::unordered_map queues; }; /// @brief Drop one fd or reservation reference while fd_mutex_ is held. @@ -1483,7 +1490,7 @@ class InterposerContext { --state->open_fds; if (state->open_fds != 0) return {}; - return {state->id, std::move(state->backend_lease), std::move(state->pm4_queue_keys)}; + return {state->id, std::move(state->backend_lease), std::move(state->pm4_queues)}; } struct DrmUntrackResult { @@ -1495,8 +1502,8 @@ class InterposerContext { /// @details Must run with NO interposer lock held. void complete_drm_release(DrmFinalRelease release) { if (auto *driver = dynamic_cast(release.lease.local().get())) - for (const auto &[context_ring, key] : release.queue_keys) - driver->retire_pm4_queue(key); + for (const auto &[context_ring, queue] : release.queues) + driver->retire_pm4_queue(queue.key); if (release.file_id) reap_gem_for_drm_file(*release.file_id); // release.lease destructs here, dropping the open reference. For a local @@ -1719,10 +1726,10 @@ class InterposerContext { std::lock_guard lock(fd_mutex_); if (!file->contexts.erase(request.ctx_id)) return -EINVAL; - std::erase_if(file->pm4_queue_keys, [&](const auto &entry) { + std::erase_if(file->pm4_queues, [&](const auto &entry) { if ((entry.first >> 32) != request.ctx_id) return false; - retired.push_back(entry.second); + retired.push_back(entry.second.key); return true; }); } @@ -1863,20 +1870,40 @@ class InterposerContext { entry.has_fence = true; } - int transfer_syncobj(const DrmFileToken &file, const drm_syncobj_transfer &request) { + int transfer_syncobj(const DrmFileToken &file, const drm_syncobj_transfer request) { if (!file) return -EBADF; - if (request.flags || request.pad) + if ((request.flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) || request.pad) return -EINVAL; { - std::lock_guard lock(fd_mutex_); + std::unique_lock lock(fd_mutex_); auto source = lookup_syncobj_locked(file, request.src_handle); auto target = lookup_syncobj_locked(file, request.dst_handle); if (!source || !target) return -ENOENT; - auto fence = syncobj_fence_locked(*source, request.src_point); - if (!fence) - return -EINVAL; + // Linux bounds WAIT_FOR_SUBMIT transfer waits to five seconds. Wait for + // the payload to exist, not for its GPU work to finish; retain the source + // and destination objects across handle deletion and replacement. + timespec deadline{}; + if (clock_gettime(CLOCK_MONOTONIC, &deadline) != 0) + return -errno; + deadline.tv_sec += 5; + std::shared_ptr fence; + while (true) { + const uint32_t generation = file->syncobj_generation.load(std::memory_order_acquire); + fence = syncobj_fence_locked(*source, request.src_point); + if (fence) + break; + if (!(request.flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) + return -EINVAL; + lock.unlock(); + const long rc = futex_wait_until(file->syncobj_generation, generation, &deadline); + const int wait_errno = errno; + lock.lock(); + if (rc == 0 || wait_errno == EAGAIN) + continue; + return wait_errno == ETIMEDOUT ? -ETIME : -wait_errno; + } if (request.dst_point) { append_timeline_fence_locked(*target, request.dst_point, std::move(fence)); } else { @@ -2382,6 +2409,7 @@ class InterposerContext { uint64_t mmap_offset = 0; uint32_t alloc_flags = 0; void *cpu_ptr = nullptr; + bool sealed_ram = false; SimulatedKfd *owner = nullptr; std::vector installed_vas; }; @@ -2422,7 +2450,7 @@ class InterposerContext { (request.domains & ~domains)) return -EINVAL; uint64_t size = (request.bo_size + 4095) & ~uint64_t{4095}; - int fd = real().memfd_create("rocjitsu_gem", MFD_CLOEXEC); + int fd = real().memfd_create("rocjitsu_gem", MFD_CLOEXEC | MFD_ALLOW_SEALING); if (fd < 0) return -errno; if (ftruncate(fd, static_cast(size)) != 0) { @@ -2430,6 +2458,7 @@ class InterposerContext { real().close(fd); return -error; } + const bool sealed_ram = real().fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK) == 0; auto backing = PrivateDrmFd::duplicate(fd); const int error = errno; real().close(fd); @@ -2443,6 +2472,7 @@ class InterposerContext { candidate = next_gem_handle_++; GemEntry entry{}; entry.dmabuf_fd = std::move(backing); + entry.sealed_ram = sealed_ram; entry.drm_file_id = file->id; entry.size = size; entry.mmap_offset = next_gem_mmap_offset_; @@ -2460,12 +2490,15 @@ class InterposerContext { if (!driver) return -ENODEV; std::shared_ptr finished; + // Keep queue identity stable through acceptance and exception cleanup. + std::lock_guard submission_lock(file->submission_mutex); + std::optional pending_queue; + uint64_t sequence = 0; try { const auto request = argument->in; if (request.flags || request.bo_list_handle || !request.num_chunks || request.num_chunks > 1024) return -EINVAL; - std::lock_guard submission_lock(file->submission_mutex); std::vector pointers; if (int rc = snapshot_user_array(request.chunks, request.num_chunks, pointers); rc) return rc; @@ -2568,7 +2601,7 @@ class InterposerContext { retained->handles.erase(std::ranges::unique(retained->handles).begin(), retained->handles.end()); std::shared_ptr fence_fd; - uint64_t sequence, queue_key; + uint64_t queue_key; { std::lock_guard lock(fd_mutex_); if (!file->contexts.count(request.ctx_id)) @@ -2616,11 +2649,16 @@ class InterposerContext { } const uint64_t key = (uint64_t{request.ctx_id} << 32) | (engine << 16) | ring; static std::atomic next_queue_key{1}; - auto [it, inserted] = file->pm4_queue_keys.try_emplace(key, 0); + auto [it, inserted] = file->pm4_queues.try_emplace(key); + auto &queue = it->second; if (inserted) - it->second = next_queue_key.fetch_add(1, std::memory_order_relaxed); - queue_key = it->second; - sequence = file->next_submission++; + queue.key = next_queue_key.fetch_add(1, std::memory_order_relaxed); + queue_key = queue.key; + // Reserve storage before acceptance, but publish the sequence and + // retire history only after the command processor accepts the job. + sequence = queue.next_sequence; + queue.fences.emplace(sequence, finished); + pending_queue = key; } submission.ready = [this, dependencies = std::move(dependencies)]() mutable { std::lock_guard lock(fd_mutex_); @@ -2650,6 +2688,26 @@ class InterposerContext { notify_syncobj_waiters(file); }; int result = driver->submit_pm4(file->render_minor, queue_key, std::move(submission)); + { + std::lock_guard lock(fd_mutex_); + // A concurrent final close may already have retired this file's queues. + auto it = file->pm4_queues.find(*pending_queue); + if (it != file->pm4_queues.end() && result) { + it->second.fences.erase(sequence); + } else if (it != file->pm4_queues.end()) { + auto &queue = it->second; + ++queue.next_sequence; + // Rejected submissions must not evict accepted fence errors from + // the bounded history or change WAIT_CS's latest sequence. + while (queue.fences.size() > 64) { + const auto &oldest = queue.fences.begin()->second; + if (!oldest->is_signaled() && !oldest->is_failed()) + break; + queue.fences.erase(queue.fences.begin()); + } + } + pending_queue.reset(); + } if (result) { finished->failed.store(true, std::memory_order_release); notify_syncobj_waiters(file); @@ -2659,6 +2717,11 @@ class InterposerContext { notify_syncobj_waiters(file); return 0; } catch (const std::exception &error) { + if (pending_queue) { + std::lock_guard lock(fd_mutex_); + if (auto it = file->pm4_queues.find(*pending_queue); it != file->pm4_queues.end()) + it->second.fences.erase(sequence); + } if (finished) { finished->failed.store(true, std::memory_order_release); notify_syncobj_waiters(file); @@ -2668,6 +2731,59 @@ class InterposerContext { } } + /// @brief Query a context/engine/ring submission fence using an absolute deadline. + int wait_drm_cs(const DrmFileToken &file, drm_amdgpu_wait_cs *argument) { + if (!file || !argument) + return -EINVAL; + const auto request = argument->in; + if ((request.ip_type != AMDGPU_HW_IP_GFX && request.ip_type != AMDGPU_HW_IP_COMPUTE) || + request.ip_instance || request.ring >= (request.ip_type == AMDGPU_HW_IP_GFX ? 1u : 4u)) + return -EINVAL; + std::unique_lock lock(fd_mutex_); + if (!file->contexts.count(request.ctx_id)) + return -EINVAL; + const uint64_t key = (uint64_t{request.ctx_id} << 32) | (request.ip_type << 16) | request.ring; + const auto queue = file->pm4_queues.find(key); + const uint64_t next = queue == file->pm4_queues.end() ? 1 : queue->second.next_sequence; + const uint64_t sequence = request.handle == UINT64_MAX ? next - 1 : request.handle; + if (sequence >= next) + return -EINVAL; + std::shared_ptr fence; + if (sequence && queue != file->pm4_queues.end()) { + const auto it = queue->second.fences.find(sequence); + if (it != queue->second.fences.end()) + fence = it->second; + } + const auto finish = [&](bool busy) { + std::memset(argument, 0, sizeof(*argument)); + argument->out.status = busy; + return 0; + }; + while (true) { + const uint32_t generation = file->syncobj_generation.load(std::memory_order_acquire); + if (fence && fence->is_failed()) + return -EIO; + if (!fence || fence->is_signaled()) + return finish(false); + timespec now{}; + if (clock_gettime(CLOCK_MONOTONIC, &now) != 0) + return -errno; + const uint64_t now_ns = static_cast(now.tv_sec) * 1'000'000'000 + now.tv_nsec; + if (request.timeout <= now_ns) + return finish(true); + const timespec deadline{static_cast(request.timeout / 1'000'000'000), + static_cast(request.timeout % 1'000'000'000)}; + lock.unlock(); + const long rc = futex_wait_until(file->syncobj_generation, generation, + request.timeout == UINT64_MAX ? nullptr : &deadline); + const int wait_errno = errno; + lock.lock(); + if (rc == 0 || wait_errno == EAGAIN || wait_errno == ETIMEDOUT) + continue; + return -wait_errno; + } + } + int gem_mmap_offset(const DrmFileToken &file, uint32_t handle, uint64_t *offset) { std::lock_guard lock(fd_mutex_); auto it = gem_entries_.find(handle); @@ -2759,37 +2875,31 @@ class InterposerContext { GemEntry &gem = gem_entries_[handle]; gem = {}; gem.dmabuf_fd = std::move(backing_fd); + // Repeated imports can map the same file bytes at different host addresses. + // Only fresh GEM allocations qualify for disjoint parallel RAM accesses. gem.drm_file_id = drm_file->id; gem.size = size; gem.alloc_flags = alloc_flags; return handle; } - /// @brief Install (or replace) a GEM_VA range in the GPU page table for @p handle. - /// @details Runs entirely under fd_mutex_ and performs BOTH the bookkeeping AND - /// the page-table install (drv->gem_va_map) and output timeline signal atomically, - /// so a concurrent GEM_CLOSE - /// (untrack_gem, also under fd_mutex_) can never interleave between recording the - /// range and installing the PTEs — which would otherwise leave PTEs pointing into - /// a munmapped cpu_ptr with no entry left to tear them down. It lazily mmaps the - /// dmabuf fd's backing pages the first time (the fd is still open at GEM_VA time), - /// bounds-checks the request against the BO size, records the range, installs - /// the PTEs, and publishes the output timeline point. The lock order - /// fd_mutex_ -> driver page-table lock matches - /// teardown_gem_entry_locked, so there is no inversion. - /// @param replace When true (AMDGPU_VA_OP_REPLACE), reject overlap with another - /// DRM file before evicting an existing range in the calling DRM-file namespace. - /// The namespaces share one simulated GPU page table, so neither operation may - /// overwrite another file's PTEs. When false (AMDGPU_VA_OP_MAP), any overlapping - /// pre-existing range is a conflict. - /// @param publish_timeline Whether this update publishes its output timeline. + /// @brief Install or replace a resident mapping or sparse VA reservation. + /// @details Serializes range bookkeeping, PTE updates and output timeline signals + /// under fd_mutex_, so GEM_CLOSE cannot remove backing during an update. Resident + /// mappings validate BO bounds and lazily mmap the retained dmabuf. The lock order + /// fd_mutex_ -> driver page-table lock matches teardown_gem_entry_locked. + /// @param replace Preserve existing tails outside the updated range. Reject + /// overlap with other DRM files; without replacement, reject any overlap. + /// @param publish_timeline Whether to publish the output timeline point. /// AMDGPU_VM_DELAY_UPDATE suppresses publication, matching the kernel contract. - /// @returns Zero when the range was installed, `-ENOENT` for an unknown handle in - /// this DRM-file namespace, or another negative errno for invalid requests. + /// @param prt Ignore the GEM handle and reserve VA in this DRM file without + /// allocating backing or installing PTEs. + /// @returns Zero on success, -ENOENT for an unknown resident handle, or another + /// negative errno for an invalid request. [[nodiscard]] int gem_map(const DrmFileToken &file, uint32_t handle, uint64_t va_address, uint64_t offset_in_bo, uint64_t map_size, bool replace, bool publish_timeline = true, uint32_t timeline_handle = 0, - uint64_t timeline_point = 0) { + uint64_t timeline_point = 0, bool prt = false) { std::unique_lock lock(fd_mutex_); if (!file) return -EBADF; @@ -2800,15 +2910,15 @@ class InterposerContext { if (!drv) return -ENODEV; auto it = gem_entries_.find(handle); - if (it == gem_entries_.end() || it->second.handle_closed || it->second.drm_file_id != file->id) + if (!prt && (it == gem_entries_.end() || it->second.handle_closed || + it->second.drm_file_id != file->id)) return -ENOENT; - GemEntry &gem = it->second; - // Bound the request within the BO without letting offset_in_bo + map_size - // overflow (both are caller-controlled __u64 from the UAPI struct): a wrap - // would defeat a naive sum-vs-size check and install PTEs pointing outside - // the mmap. Reject a zero-size BO, a zero-size map, and any range past the end. - if (gem.size == 0 || map_size == 0 || offset_in_bo > gem.size || - map_size > gem.size - offset_in_bo) + if (map_size == 0 || va_address > UINT64_MAX - map_size || offset_in_bo > UINT64_MAX - map_size) + return -EINVAL; + if ((va_address | offset_in_bo | map_size) & 4095) + return -EINVAL; + GemEntry &gem = prt ? prt_entries_[file->id] : it->second; + if (!prt && (gem.size == 0 || offset_in_bo > gem.size || map_size > gem.size - offset_in_bo)) return -EINVAL; const GemMapping range{va_address, map_size}; // Handle the target VA range's current occupant. Independent DRM files have @@ -2826,13 +2936,10 @@ class InterposerContext { va_address); return -EINVAL; } - if (replace) { - if (!evict_range_locked(drv, file->id, range, /*allow_missing=*/true)) - return -EINVAL; - } else if (range_is_mapped_locked(range)) { + if (!replace && range_is_mapped_locked(range)) { return -EINVAL; } - if (!gem.cpu_ptr) { + if (!prt && !gem.cpu_ptr) { void *p = gem.dmabuf_fd->use([&](int fd) { return real().mmap(nullptr, gem.size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); }); @@ -2840,6 +2947,8 @@ class InterposerContext { return -EINVAL; gem.cpu_ptr = p; } + if (replace && !evict_range_locked(drv, file->id, range, /*allow_missing=*/true)) + return -EINVAL; // Record which SimulatedKfd's page table receives these PTEs so GEM_CLOSE (or a // DRM-file-close reap) unmaps through the driver that installed them, never a // replacement one. Set the owner on the first mapping and keep it: all ranges of @@ -2851,13 +2960,13 @@ class InterposerContext { gem.owner = drv; else assert(gem.owner == drv && "GEM ranges of one BO must share one owning driver"); - void *host = static_cast(gem.cpu_ptr) + offset_in_bo; + void *host = prt ? nullptr : static_cast(gem.cpu_ptr) + offset_in_bo; // Install the PTEs while still holding fd_mutex_ so the range record and the // page-table state stay consistent against a concurrent teardown. gem_va_map // only returns false if the local process vanished mid-call; treat that as a // failed map (do not record the range) so GEM_VA reports the error rather than a // phantom success. - if (!drv->gem_va_map(va_address, host, map_size, gem.alloc_flags)) + if (!prt && !drv->gem_va_map(va_address, host, map_size, gem.alloc_flags, gem.sealed_ram)) return -EINVAL; gem.installed_vas.push_back(range); if (publish_timeline) @@ -2868,17 +2977,16 @@ class InterposerContext { return 0; } - /// @brief Remove a GEM_VA range from the GPU page table for @p handle (UNMAP). - /// @details Performs the page-table unmap AND the bookkeeping erase atomically - /// under fd_mutex_ (same rationale as gem_map). Validates that @p handle actually - /// owns the exact {va_address, map_size} range before mutating the page table, so - /// an UNMAP with a wrong handle or range cannot tear down PTEs the handle does not - /// own and cannot report success for a no-op. - /// @returns Zero when the range was unmapped, `-ENOENT` for an unknown handle in - /// this DRM-file namespace, or another negative errno for invalid requests. + /// @brief Remove an exact resident mapping or sparse VA reservation. + /// @details Checks range ownership and updates bookkeeping and resident PTEs + /// under fd_mutex_, using the same lock order as gem_map. + /// @param prt Ignore the GEM handle and remove this DRM file's sparse reservation. + /// @returns Zero on success, -ENOENT for an unknown handle or reservation owner, + /// or another negative errno for an invalid request. [[nodiscard]] int gem_unmap(const DrmFileToken &file, uint32_t handle, uint64_t va_address, uint64_t map_size, bool publish_timeline = true, - uint32_t timeline_handle = 0, uint64_t timeline_point = 0) { + uint32_t timeline_handle = 0, uint64_t timeline_point = 0, + bool prt = false) { std::unique_lock lock(fd_mutex_); if (!file) return -EBADF; @@ -2889,13 +2997,16 @@ class InterposerContext { if (!drv) return -ENODEV; auto it = gem_entries_.find(handle); - if (it == gem_entries_.end() || it->second.handle_closed || it->second.drm_file_id != file->id) + auto prt_it = prt_entries_.find(file->id); + if (prt ? prt_it == prt_entries_.end() + : it == gem_entries_.end() || it->second.handle_closed || + it->second.drm_file_id != file->id) return -ENOENT; - GemEntry &gem = it->second; + GemEntry &gem = prt ? prt_it->second : it->second; const GemMapping range{va_address, map_size}; if (std::ranges::find(gem.installed_vas, range) == gem.installed_vas.end()) return -EINVAL; // This handle does not own the exact range — do not touch PTEs. - if (!drv->gem_va_unmap(va_address, map_size)) + if (!prt && !drv->gem_va_unmap(va_address, map_size)) return -EINVAL; std::erase(gem.installed_vas, range); if (publish_timeline) @@ -2921,6 +3032,8 @@ class InterposerContext { std::unique_lock lock(fd_mutex_); if (!file) return -EBADF; + if (map_size == 0 || va_address > UINT64_MAX - map_size || ((va_address | map_size) & 4095)) + return -EINVAL; auto timeline = lookup_syncobj_locked(file, timeline_handle); if (timeline_handle != 0 && !timeline) return -ENOENT; @@ -2938,13 +3051,14 @@ class InterposerContext { return 0; } - /// @brief Reap every GEM handle owned by a closing DRM file (at its close()). + /// @brief Reap GEM handles and sparse reservations owned by a closing DRM file. /// @details A well-behaved caller GEM_CLOSEs each handle, but a crash or leak can /// leave handles live; the DRM file close is their backstop, mirroring the kernel /// dropping a drm_file's GEM objects. Tears each entry's PTEs + host mmap down /// under fd_mutex_ before erasing, so no state escapes the lock. void reap_gem_for_drm_file(uint64_t drm_file_id) { std::lock_guard lock(fd_mutex_); + prt_entries_.erase(drm_file_id); for (auto it = gem_entries_.begin(); it != gem_entries_.end();) { if (it->second.drm_file_id == drm_file_id) it->second.handle_closed = true; @@ -3327,6 +3441,9 @@ class InterposerContext { /// closes (reap_gem_for_drm_file). Because handles are not recycled with fd numbers, /// a reused dmabuf fd can never collide with a still-live BO. std::unordered_map gem_entries_; + // Sparse VA reservations have no GEM handle or resident backing. Keep their + // intervals per DRM file; allocating storage proportional to VA size defeats PRT. + std::unordered_map prt_entries_; /// @brief GEM mmap cursor, outside the KFD doorbell/queue offset namespace. uint64_t next_gem_mmap_offset_ = kGemMmapOffsetBase; /// @brief Next stable GEM handle to mint. Starts at 1 so 0 means "no handle". @@ -3366,48 +3483,45 @@ class InterposerContext { gem.dmabuf_fd.reset(); } - /// @brief Evict every recorded range that OVERLAPS @p range in one DRM file. - /// @details Caller holds fd_mutex_ and passes the live simulated @p drv and stable - /// DRM-file identity. Used by GEM_VA REPLACE (evict prior mappings before installing - /// the new one) and CLEAR (handle-agnostic teardown). Matching is by interval - /// intersection, not exact equality: a REPLACE at a VA previously mapped with a - /// DIFFERENT size (or a sub/super-range) must still evict the old mapping, otherwise - /// its stale bookkeeping would later double-unmap or leak the new PTEs. Each - /// overlapping range is unmapped by its OWN {va_address, map_size} extent (not @p - /// range's) so the page-table removal matches what was installed, then dropped from - /// its entry's bookkeeping. The host mmap is left intact — the owning handle still - /// exists and other ranges may reference it; it is munmapped only at GEM_CLOSE / - /// reap. - /// @param allow_missing When true, no overlap is a success (a MAP onto a free VA has - /// nothing to evict); when false (REPLACE/CLEAR), no overlap is a failure so the - /// ioctl reports EINVAL. - /// @retval true nothing overlapped (allow_missing) or all overlaps were evicted. - /// @retval false nothing overlapped (only when !allow_missing) or an unmap failed. - /// @note Not rolled back on a mid-loop unmap failure: already-evicted ranges stay - /// evicted. This is safe because gem_va_unmap() only fails when the local process - /// has already vanished (SimulatedKfd::gem_va_unmap), i.e. its page table is being - /// torn down anyway, so a partially-evicted state is never observed by a live GPU. + /// @brief Remove only the intersection with a VA update, preserving both tails. + /// @details PRT intervals carry no resident PTEs. Buffer intervals retain their + /// existing host translations outside the update, including the original BO offset. + /// Caller holds fd_mutex_. Return false if no range overlaps unless allow_missing, + /// or if a resident unmap fails. Completed removals are not rolled back on failure. [[nodiscard]] bool evict_range_locked(SimulatedKfd *drv, uint64_t drm_file_id, const GemMapping &range, bool allow_missing) { bool evicted_any = false; - for (auto &[handle, gem] : gem_entries_) { - if (gem.drm_file_id != drm_file_id) - continue; - for (auto vit = gem.installed_vas.begin(); vit != gem.installed_vas.end();) { - if (!vit->overlaps(range)) { - ++vit; + auto evict = [&](std::vector &ranges, bool resident) { + for (size_t i = 0; i < ranges.size();) { + const auto old = ranges[i]; + if (!old.overlaps(range)) { + ++i; continue; } - if (!drv->gem_va_unmap(vit->va_address, vit->map_size)) - return false; // process gone; page table already being destroyed (see @note) - vit = gem.installed_vas.erase(vit); + const uint64_t begin = std::max(old.va_address, range.va_address); + const uint64_t end = + std::min(old.va_address + old.map_size, range.va_address + range.map_size); + if (resident && !drv->gem_va_unmap(begin, end - begin)) + return false; + ranges.erase(ranges.begin() + i); + if (old.va_address < begin) + ranges.push_back({old.va_address, begin - old.va_address}); + if (end < old.va_address + old.map_size) + ranges.push_back({end, old.va_address + old.map_size - end}); evicted_any = true; } - } + return true; + }; + for (auto &[handle, gem] : gem_entries_) + if (gem.drm_file_id == drm_file_id && !evict(gem.installed_vas, true)) + return false; + auto prt = prt_entries_.find(drm_file_id); + if (prt != prt_entries_.end() && !evict(prt->second.installed_vas, false)) + return false; return evicted_any || allow_missing; } - /// @brief Whether any GEM entry owns a range OVERLAPPING @p range in the shared + /// @brief Whether any GEM mapping or sparse reservation overlaps @p range in the shared /// simulated GPU page table. Caller holds fd_mutex_. Used by plain MAP to reject /// a map that would collide with an existing range's PTEs, regardless of which /// DRM file owns it. @@ -3417,10 +3531,14 @@ class InterposerContext { if (existing.overlaps(range)) return true; } + for (const auto &[file_id, gem] : prt_entries_) + for (const auto &existing : gem.installed_vas) + if (existing.overlaps(range)) + return true; return false; } - /// @brief Whether a GEM entry owned by a DRM file other than @p drm_file_id + /// @brief Whether a GEM mapping or sparse reservation owned by a file other than @p drm_file_id /// overlaps @p range in the shared simulated GPU page table. Caller holds /// fd_mutex_. Used by REPLACE to reject a foreign collision before evicting any /// mapping owned by the caller. @@ -3433,6 +3551,11 @@ class InterposerContext { if (existing.overlaps(range)) return true; } + for (const auto &[file_id, gem] : prt_entries_) + if (file_id != drm_file_id) + for (const auto &existing : gem.installed_vas) + if (existing.overlaps(range)) + return true; return false; } @@ -3479,6 +3602,7 @@ extern "C" { static std::string redirect_sysfs_path(const char *path); static std::string redirect_sys_dev_char(const char *path); static std::optional interposer_gpu_info(uint32_t render_minor); +static std::optional synthetic_alias_for_host_drm(dev_t device); static std::optional interposer_gpu_info_for(const InterposerContext::DrmFileToken &file); @@ -4108,7 +4232,10 @@ __attribute__((destructor)) void rj_interposer_shutdown() { InterposerContext::ctx.request_local_vm_shutdown(); } -RJ_INTERPOSER_EXPORT int ioctl(int fd, unsigned long request, ...) { +// Use the public symbol without inheriting libc's noexcept declaration: a signal +// handler may terminate a thread blocked in a simulated ioctl via pthread_exit. +RJ_INTERPOSER_EXPORT int rj_ioctl(int fd, unsigned long request, ...) asm("ioctl"); +RJ_INTERPOSER_EXPORT int rj_ioctl(int fd, unsigned long request, ...) { assert(InterposerContext::real().ready()); va_list ap; va_start(ap, request); @@ -4146,6 +4273,9 @@ RJ_INTERPOSER_EXPORT int ioctl(int fd, unsigned long request, ...) { if (type == kDrmIoctlType && nr == DRM_COMMAND_BASE + DRM_AMDGPU_CS && arg) return kfd_ioctl_ret( InterposerContext::ctx.submit_drm_cs(drm_file, static_cast(arg))); + if (type == kDrmIoctlType && nr == DRM_COMMAND_BASE + DRM_AMDGPU_WAIT_CS && arg) + return kfd_ioctl_ret( + InterposerContext::ctx.wait_drm_cs(drm_file, static_cast(arg))); if (type == kDrmIoctlType && nr == _IOC_NR(DRM_IOCTL_GET_CAP) && arg) { auto *cap = static_cast(arg); cap->value = 0; @@ -4487,11 +4617,9 @@ RJ_INTERPOSER_EXPORT int ioctl(int fd, unsigned long request, ...) { return kfd_ioctl_ret(InterposerContext::ctx.untrack_gem(drm_file, gc->handle)); } if (type == kDrmIoctlType && nr == kDrmIoctlNrGemVa && arg) { - // GEM_VA installs (or tears down) a GPU virtual mapping for a prime- - // imported buffer. HSA's vmem path (hsa_amd_vmem_map) lowers to this via - // amdgpu_bo_va_op; IREE's ring allocator triple-maps one BO at adjacent - // VAs. We map by GEM handle, lazily mmap the backing pages, and - // install/remove them in the GPU page table. + // GEM_VA updates resident mappings or handle-less sparse VA reservations. + // Resident mappings lazily mmap the imported BO and update GPU PTEs; sparse + // reservations only record intervals in the calling DRM-file namespace. drm_amdgpu_gem_va request_va{}; std::memcpy(&request_va, arg, std::min(_IOC_SIZE(request), sizeof(request_va))); auto *va = &request_va; @@ -4511,6 +4639,9 @@ RJ_INTERPOSER_EXPORT int ioctl(int fd, unsigned long request, ...) { return -1; } const bool publish_timeline = (va->flags & AMDGPU_VM_DELAY_UPDATE) == 0; + const bool prt = (va->flags & AMDGPU_VM_PAGE_PRT) != 0; + if (prt && (va->flags & ~(AMDGPU_VM_PAGE_PRT | AMDGPU_VM_DELAY_UPDATE))) + return kfd_ioctl_ret(-EINVAL); int result = 0; switch (va->operation) { case AMDGPU_VA_OP_MAP: @@ -4521,13 +4652,13 @@ RJ_INTERPOSER_EXPORT int ioctl(int fd, unsigned long request, ...) { result = InterposerContext::ctx.gem_map( drm_file, va->handle, va->va_address, va->offset_in_bo, va->map_size, /*replace=*/va->operation == AMDGPU_VA_OP_REPLACE, publish_timeline, - va->vm_timeline_syncobj_out, va->vm_timeline_point); + va->vm_timeline_syncobj_out, va->vm_timeline_point, prt); break; case AMDGPU_VA_OP_UNMAP: // UNMAP requires the supplied handle to own the exact range. result = InterposerContext::ctx.gem_unmap( drm_file, va->handle, va->va_address, va->map_size, publish_timeline, - va->vm_timeline_syncobj_out, va->vm_timeline_point); + va->vm_timeline_syncobj_out, va->vm_timeline_point, prt); break; case AMDGPU_VA_OP_CLEAR: // CLEAR is handle-agnostic within the calling DRM-file namespace. @@ -4644,6 +4775,17 @@ RJ_INTERPOSER_EXPORT int ioctl(int fd, unsigned long request, ...) { } } + // A display server can lend us a real render fd through SCM_RIGHTS. Its + // discovery identity is aliased below, but it must never submit to hardware. + if (_IOC_TYPE(request) == kDrmIoctlType) { + struct stat info {}; + if (InterposerContext::real().fstat_fn(fd, &info) == 0 && S_ISCHR(info.st_mode) && + synthetic_alias_for_host_drm(info.st_rdev)) { + errno = ENODEV; + return -1; + } + } + // Every tracked KFD primary/dup is routed above by its recorded backend // (remote_lookup / kfd_backend_of). Deliberately NO local-driver fallback for // tracked dups here: a Remote-tagged dup observed during a remote teardown @@ -5462,19 +5604,31 @@ static int finish_drm_node_stat(const char *path, int result, mode_t *mode, dev_ return result; std::string_view view(path); if (!view.starts_with("/dev/dri/renderD") && - view.find("/dev_dri/renderD") == std::string_view::npos) + view.find("/dev_dri/renderD") == std::string_view::npos && + view.find("/dev_dri/card") == std::string_view::npos) return result; std::string drm_base; if (auto driver = InterposerContext::ctx.driver()) drm_base = driver->drm_path(); else drm_base = InterposerContext::ctx.remote_drm_path(); - uint32_t render_minor = 0; - if (!drm_base.empty() && render_minor_from_drm_node_path(path, drm_base.c_str(), &render_minor) && - interposer_gpu_info(render_minor)) { - // libdrm enumerates nodes by path as well as by fd. Both must describe the - // same character device even though the synthetic tree uses regular files. - *device = makedev(226, render_minor); + if (drm_base.empty()) + return result; + + uint32_t node_minor = 0; + bool synthetic_node = render_minor_from_drm_node_path(path, drm_base.c_str(), &node_minor) && + interposer_gpu_info(node_minor).has_value(); + const std::string primary_prefix = drm_base + "/dev_dri/card"; + if (view.starts_with(primary_prefix)) { + // The existing file belongs to the generated tree. Its primary index is + // independent of the configured render minor (card0 may pair with renderD129). + synthetic_node = parse_render_minor_suffix(view.data() + primary_prefix.size(), + view.data() + view.size(), &node_minor); + } + if (synthetic_node) { + // libdrm groups primary/render nodes using their character-device metadata. + // Backing files are regular files, but their public identities follow DRM. + *device = makedev(226, node_minor); *mode = (*mode & ~S_IFMT) | S_IFCHR; } return result; @@ -5557,7 +5711,58 @@ RJ_INTERPOSER_EXPORT DIR *opendir(const char *name) { return InterposerContext::real().opendir(name); } -// -- fstat interposition (DRM memfd → synthetic st_rdev) -- +// -- fstat interposition (DRM descriptors → synthetic device identity) -- + +// Keep borrowed display render descriptors in the same device namespace as +// /dev/dri and sysfs. libdrm matches fstat(fd).st_rdev against enumerated nodes; +// the host display GPU's minor need not exist in the simulated topology. +// Primary nodes retain their real identity for host display control, as do +// hardware-backed guests, which have no synthetic DRM enumeration tree. +static std::optional synthetic_alias_for_host_drm(dev_t device) { + if (major(device) != 226 || minor(device) < 128 || InterposerContext::in_construction) + return std::nullopt; + if (!InterposerContext::ctx.initialized()) + InterposerContext::ctx.get_or_create(); + std::string drm_base; + if (auto driver = InterposerContext::ctx.driver()) + drm_base = driver->drm_path(); + else + drm_base = InterposerContext::ctx.remote_drm_path(); + if (drm_base.empty()) + return std::nullopt; + // Use the render node published for card0, independently of host numbering + // and of which simulated GPU the caller will eventually use for rendering. + DIR *nodes = InterposerContext::real().opendir((drm_base + "/card0/device/drm").c_str()); + if (!nodes) + return std::nullopt; + std::optional render_minor; + while (const dirent *entry = InterposerContext::real().readdir(nodes)) { + const std::string_view name(entry->d_name); + uint32_t parsed = 0; + if (name.starts_with("renderD") && + parse_render_minor_suffix(name.data() + 7, name.data() + name.size(), &parsed)) { + render_minor = parsed; + break; + } + } + InterposerContext::real().closedir(nodes); + if (!render_minor) + return std::nullopt; + const std::string node = drm_base + "/dev_dri/renderD" + std::to_string(*render_minor); + if (InterposerContext::real().access(node.c_str(), F_OK) != 0) + return std::nullopt; + return makedev(226, *render_minor); +} + +static void finish_drm_fd_stat(int fd, mode_t *mode, dev_t *device) { + if (InterposerContext::ctx.is_drm(fd)) { + *device = makedev(226, InterposerContext::ctx.drm_render_minor(fd)); + *mode = (*mode & ~S_IFMT) | S_IFCHR; + } else if (S_ISCHR(*mode)) { + if (auto alias = synthetic_alias_for_host_drm(*device)) + *device = *alias; + } +} RJ_INTERPOSER_EXPORT int fstat(int fd, struct stat *buf) { if (!InterposerContext::real().ready()) { @@ -5567,11 +5772,8 @@ RJ_INTERPOSER_EXPORT int fstat(int fd, struct stat *buf) { if (!rj_owns_interposer_state()) return InterposerContext::real().fstat_fn(fd, buf); int rc = InterposerContext::real().fstat_fn(fd, buf); - if (rc == 0 && InterposerContext::ctx.is_drm(fd)) { - uint32_t render_minor = InterposerContext::ctx.drm_render_minor(fd); - buf->st_rdev = makedev(226, render_minor); - buf->st_mode = (buf->st_mode & ~S_IFMT) | S_IFCHR; - } + if (rc == 0) + finish_drm_fd_stat(fd, &buf->st_mode, &buf->st_rdev); return rc; } @@ -5584,16 +5786,12 @@ RJ_INTERPOSER_EXPORT int fstat64(int fd, struct stat64 *buf) { errno = ENOSYS; return -1; } - int rc = real.fstat64_fn(fd, reinterpret_cast(buf)); + int rc = real.fstat64_fn(fd, buf); // Gate BEFORE is_drm(), which takes fd_mutex_. No lazy static above this point: // its initialization guard could be inherited mid-init by a forked child. if (rc != 0 || !real.ready() || !rj_owns_interposer_state()) return rc; - if (InterposerContext::ctx.is_drm(fd)) { - uint32_t render_minor = InterposerContext::ctx.drm_render_minor(fd); - buf->st_rdev = makedev(226, render_minor); - buf->st_mode = (buf->st_mode & ~S_IFMT) | S_IFCHR; - } + finish_drm_fd_stat(fd, &buf->st_mode, &buf->st_rdev); return rc; } @@ -5611,11 +5809,7 @@ RJ_INTERPOSER_EXPORT int __fxstat(int ver, int fd, struct stat *buf) { // its initialization guard could be inherited mid-init by a forked child. if (rc != 0 || !real.ready() || !rj_owns_interposer_state()) return rc; - if (InterposerContext::ctx.is_drm(fd)) { - uint32_t render_minor = InterposerContext::ctx.drm_render_minor(fd); - buf->st_rdev = makedev(226, render_minor); - buf->st_mode = (buf->st_mode & ~S_IFMT) | S_IFCHR; - } + finish_drm_fd_stat(fd, &buf->st_mode, &buf->st_rdev); return rc; } @@ -5628,19 +5822,56 @@ RJ_INTERPOSER_EXPORT int __fxstat64(int ver, int fd, struct stat64 *buf) { errno = ENOSYS; return -1; } - int rc = real.fxstat64_fn(ver, fd, reinterpret_cast(buf)); + int rc = real.fxstat64_fn(ver, fd, buf); // Gate BEFORE is_drm(), which takes fd_mutex_. No lazy static above this point: // its initialization guard could be inherited mid-init by a forked child. if (rc != 0 || !real.ready() || !rj_owns_interposer_state()) return rc; - if (InterposerContext::ctx.is_drm(fd)) { - uint32_t render_minor = InterposerContext::ctx.drm_render_minor(fd); - buf->st_rdev = makedev(226, render_minor); - buf->st_mode = (buf->st_mode & ~S_IFMT) | S_IFCHR; - } + finish_drm_fd_stat(fd, &buf->st_mode, &buf->st_rdev); return rc; } +// libc canonicalization uses internal filesystem calls that bypass the ordinary +// stat/readlink wrappers. Redirect the input before resolving symlinks so libdrm +// cannot escape from a simulated node to the host GPU's PCI sysfs directory. +static std::string redirect_canonical_path(const char *path) { + auto redirected = redirect_sysfs_path(path); + if (redirected.empty()) + redirected = redirect_sys_dev_char(path); + if (redirected.empty()) + redirected = redirect_dev_dri(path); + return redirected; +} + +RJ_INTERPOSER_EXPORT char *realpath(const char *path, char *resolved_path) { + auto &real = InterposerContext::real(); + if (!real.ready()) { + auto fn = util::lookup_symbol(RTLD_NEXT, "realpath"); + return fn ? fn(path, resolved_path) : nullptr; + } + if (!rj_owns_interposer_state() || InterposerContext::in_construction) + return real.realpath_fn(path, resolved_path); + auto redirected = redirect_canonical_path(path); + return real.realpath_fn(redirected.empty() ? path : redirected.c_str(), resolved_path); +} + +RJ_INTERPOSER_EXPORT char *__realpath_chk(const char *path, char *resolved_path, size_t size) { + auto &real = InterposerContext::real(); + if (!real.ready()) { + auto fn = + util::lookup_symbol(RTLD_NEXT, "__realpath_chk"); + return fn ? fn(path, resolved_path, size) : nullptr; + } + if (!real.realpath_chk_fn) { + errno = ENOSYS; + return nullptr; + } + if (!rj_owns_interposer_state() || InterposerContext::in_construction) + return real.realpath_chk_fn(path, resolved_path, size); + auto redirected = redirect_canonical_path(path); + return real.realpath_chk_fn(redirected.empty() ? path : redirected.c_str(), resolved_path, size); +} + // -- readlink interposition (redirect /sys/dev/char/) -- RJ_INTERPOSER_EXPORT ssize_t readlink(const char *path, char *buf, size_t bufsiz) { @@ -5671,14 +5902,14 @@ RJ_INTERPOSER_EXPORT int stat64(const char *path, struct stat64 *buf) { // Gate BEFORE redirect_sysfs_path(), which reaches published driver and fd state. // No lazy static above this point (see LibcPassthrough for why). if (!real.ready() || !rj_owns_interposer_state()) - return real.stat64_fn(path, reinterpret_cast(buf)); + return real.stat64_fn(path, buf); auto redirected = redirect_sysfs_path(path); if (redirected.empty()) redirected = redirect_sys_dev_char(path); if (redirected.empty()) redirected = redirect_dev_dri(path); const char *actual = redirected.empty() ? path : redirected.c_str(); - int result = real.stat64_fn(actual, reinterpret_cast(buf)); + int result = real.stat64_fn(actual, buf); return result == 0 ? finish_drm_node_stat(actual, result, &buf->st_mode, &buf->st_rdev) : result; } @@ -5694,14 +5925,14 @@ RJ_INTERPOSER_EXPORT int lstat64(const char *path, struct stat64 *buf) { // Gate BEFORE redirect_sysfs_path(), which reaches published driver and fd state. // No lazy static above this point (see LibcPassthrough for why). if (!real.ready() || !rj_owns_interposer_state()) - return real.lstat64_fn(path, reinterpret_cast(buf)); + return real.lstat64_fn(path, buf); auto redirected = redirect_sysfs_path(path); if (redirected.empty()) redirected = redirect_sys_dev_char(path); if (redirected.empty()) redirected = redirect_dev_dri(path); const char *actual = redirected.empty() ? path : redirected.c_str(); - int result = real.lstat64_fn(actual, reinterpret_cast(buf)); + int result = real.lstat64_fn(actual, buf); return result == 0 ? finish_drm_node_stat(actual, result, &buf->st_mode, &buf->st_rdev) : result; } @@ -5740,14 +5971,14 @@ RJ_INTERPOSER_EXPORT int __xstat64(int ver, const char *path, struct stat64 *buf // Gate BEFORE redirect_sysfs_path(), which reaches published driver and fd state. // No lazy static above this point (see LibcPassthrough for why). if (!real.ready() || !rj_owns_interposer_state()) - return real.xstat64_fn(ver, path, reinterpret_cast(buf)); + return real.xstat64_fn(ver, path, buf); auto redirected = redirect_sysfs_path(path); if (redirected.empty()) redirected = redirect_sys_dev_char(path); if (redirected.empty()) redirected = redirect_dev_dri(path); const char *actual = redirected.empty() ? path : redirected.c_str(); - int result = real.xstat64_fn(ver, actual, reinterpret_cast(buf)); + int result = real.xstat64_fn(ver, actual, buf); return result == 0 ? finish_drm_node_stat(actual, result, &buf->st_mode, &buf->st_rdev) : result; } @@ -5786,14 +6017,14 @@ RJ_INTERPOSER_EXPORT int __lxstat64(int ver, const char *path, struct stat64 *bu // Gate BEFORE redirect_sysfs_path(), which reaches published driver and fd state. // No lazy static above this point (see LibcPassthrough for why). if (!real.ready() || !rj_owns_interposer_state()) - return real.lxstat64_fn(ver, path, reinterpret_cast(buf)); + return real.lxstat64_fn(ver, path, buf); auto redirected = redirect_sysfs_path(path); if (redirected.empty()) redirected = redirect_sys_dev_char(path); if (redirected.empty()) redirected = redirect_dev_dri(path); const char *actual = redirected.empty() ? path : redirected.c_str(); - int result = real.lxstat64_fn(ver, actual, reinterpret_cast(buf)); + int result = real.lxstat64_fn(ver, actual, buf); return result == 0 ? finish_drm_node_stat(actual, result, &buf->st_mode, &buf->st_rdev) : result; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/kfd_process.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/kfd_process.h index 4916e81634cc..1fe3f3e9b1aa 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/kfd_process.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/kfd_process.h @@ -488,6 +488,11 @@ class KfdProcess { return page_table_mutation_epoch_; } + /// @brief Retained admission state for immutable copied PTE snapshots. + std::shared_ptr page_table_cache_state() const { + return page_table_cache_state_; + } + /// @brief Return the lease shared by page-table readers and mutations. std::shared_ptr page_table_request_mutex() const { return page_table_request_mutex_; @@ -540,6 +545,7 @@ class KfdProcess { uint32_t gpu_ordinal; uint32_t doorbell_offset; amdgpu::QueueHandle queue_handle; + std::optional pm4_target_xcc; }; std::unordered_map queue_doorbell_map_; @@ -772,7 +778,8 @@ class KfdProcess { continue; if (out > 0) { auto &previous = extents[out - 1]; - if (previous.gpu_page_offset + previous.host_backed_bytes == extent.gpu_page_offset && + if (previous.owner == extent.owner && + previous.gpu_page_offset + previous.host_backed_bytes == extent.gpu_page_offset && previous.host_ptr + previous.host_backed_bytes == extent.host_ptr) { previous.host_backed_bytes += extent.host_backed_bytes; continue; @@ -800,10 +807,11 @@ class KfdProcess { continue; } if (extent_begin < replacement_begin) - updated.push_back({extent.host_ptr, replacement_begin - extent_begin, extent_begin}); + updated.push_back( + {extent.host_ptr, replacement_begin - extent_begin, extent_begin, extent.owner}); if (replacement_end < extent_end) updated.push_back({extent.host_ptr + (replacement_end - extent_begin), - extent_end - replacement_end, replacement_end}); + extent_end - replacement_end, replacement_end, extent.owner}); } updated.push_back(replacement); page.host_extents = std::move(updated); @@ -822,10 +830,11 @@ class KfdProcess { continue; } if (extent_begin < erased_begin) - updated.push_back({extent.host_ptr, erased_begin - extent_begin, extent_begin}); - if (erased_end < extent_end) updated.push_back( - {extent.host_ptr + (erased_end - extent_begin), extent_end - erased_end, erased_end}); + {extent.host_ptr, erased_begin - extent_begin, extent_begin, extent.owner}); + if (erased_end < extent_end) + updated.push_back({extent.host_ptr + (erased_end - extent_begin), extent_end - erased_end, + erased_end, extent.owner}); } page.host_extents = std::move(updated); normalize_host_extents(page); @@ -835,6 +844,7 @@ class KfdProcess { // that throws before publishing the ordinary translation generation. void invalidate_page_policies_locked() { page_table_mutation_epoch_->fetch_add(1, std::memory_order_release); + page_table_cache_state_->invalidate_and_wait(); } void publish_page_table_mutation_locked() { ++page_table_generation_; } @@ -852,6 +862,8 @@ class KfdProcess { /// @details Invalidate before mutation, including partially throwing updates. std::shared_ptr> page_table_mutation_epoch_ = std::make_shared>(1); + std::shared_ptr page_table_cache_state_ = + std::make_shared(); }; } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/legacy_gpu_vm.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/legacy_gpu_vm.cpp index dbf0b4ad9211..309195d938f0 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/legacy_gpu_vm.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/legacy_gpu_vm.cpp @@ -30,29 +30,45 @@ class LegacyGpuVmAdapter::Binding final : public AddressSpaceTranslator, Binding(std::shared_ptr memory, uint32_t vmid, LegacyAddressSpaceRegistration registration, std::shared_ptr frontend_lifetime) : memory_(std::move(memory)), address_space_(std::make_shared(*memory_)), - vmid_(vmid), mutation_epoch_(std::move(registration.mutation_epoch)), + vmid_(vmid), original_request_mutex_(registration.request_mutex), + mutation_epoch_(std::move(registration.mutation_epoch)), frontend_lifetime_(std::move(frontend_lifetime)) { address_space_->register_process(vmid_, registration.page_table, registration.page_table_mutex, registration.page_table_generation, - std::move(registration.request_mutex)); + std::move(registration.request_mutex), + std::move(registration.page_table_cache_state)); address_space_->set_process_client_pid(vmid_, registration.client_pid); address_space_->set_process_mem_fd(vmid_, registration.client_mem_fd); address_space_->set_process_passthrough(vmid_, registration.passthrough); address_space_->set_process_fault_reporter(vmid_, registration.fault_reporter); } + [[nodiscard]] bool try_write_private_dwords(PhysicalMemoryAccess &physical, + std::span stores, + Mtype instruction_mtype, + Mtype expected_mtype) const override { + return &physical == static_cast(this) && + address_space_->try_write_private_dwords(stores, vmid_, instruction_mtype, + expected_mtype, original_request_mutex_); + } + + [[nodiscard]] bool try_read_uncached_ram(PhysicalMemoryAccess &physical, uint64_t address, + std::span bytes) const override { + return &physical == static_cast(this) && + address_space_->try_read_uncached_ram(address, bytes, vmid_, original_request_mutex_); + } + [[nodiscard]] VmTranslationResult translate(uint64_t address, std::size_t size, VmAccessKind access) const override { if (size == 0 || size - 1 > std::numeric_limits::max() - address) return {.outcome = VmAccessOutcome::Malformed, .translation = {}}; const uint64_t page_bytes = kLegacyPageSize - (address & (kLegacyPageSize - 1)); - const std::size_t contiguous_bytes = - address_space_->translation_contiguous_bytes(address, vmid_); + const auto policy = address_space_->translation_policy(address, vmid_); // A host extent ending inside the current GPU page is a hard boundary, not // another transfer unit. Reject a request that crosses it before copying a // prefix so the non-resumable read/write APIs retain their all-or-nothing // physical-request contract. Ordinary page boundaries remain resumable. - if (contiguous_bytes < std::min(size, page_bytes)) + if (policy.contiguous_bytes < std::min(size, page_bytes)) return {.outcome = VmAccessOutcome::Faulted, .translation = {}}; return { .outcome = VmAccessOutcome::Complete, @@ -60,8 +76,8 @@ class LegacyGpuVmAdapter::Binding final : public AddressSpaceTranslator, { .domain = VmMemoryDomain::Compatibility, .address = address, - .contiguous_bytes = contiguous_bytes, - .mtype = address_space_->pte_mtype(address, vmid_), + .contiguous_bytes = policy.contiguous_bytes, + .mtype = policy.mtype, .permissions = { .readable = access == VmAccessKind::Read || access == VmAccessKind::Atomic, @@ -72,6 +88,37 @@ class LegacyGpuVmAdapter::Binding final : public AddressSpaceTranslator, }; } + [[nodiscard]] VmTransferStep read_step(PhysicalMemoryAccess &physical, uint64_t address, + std::span bytes, + VmAccessKind access) const override { + if (&physical != static_cast(this) || + access != VmAccessKind::Read) + return AddressSpaceTranslator::read_step(physical, address, bytes, access); + if (bytes.empty() || bytes.size() - 1 > std::numeric_limits::max() - address) + return {.outcome = VmAccessOutcome::Malformed, .report_translation_fault = true}; + const auto step = address_space_->read_step( + address, std::span(reinterpret_cast(bytes.data()), bytes.size()), + vmid_); + return {.completed_bytes = step.completed_bytes, + .outcome = vm_access_outcome(step.outcome), + .report_translation_fault = step.policy_fault}; + } + + [[nodiscard]] VmTransferStep write_step(PhysicalMemoryAccess &physical, uint64_t address, + std::span bytes) const override { + if (&physical != static_cast(this)) + return AddressSpaceTranslator::write_step(physical, address, bytes); + if (bytes.empty() || bytes.size() - 1 > std::numeric_limits::max() - address) + return {.outcome = VmAccessOutcome::Malformed, .report_translation_fault = true}; + const auto step = address_space_->write_step( + address, + std::span(reinterpret_cast(bytes.data()), bytes.size()), + vmid_); + return {.completed_bytes = step.completed_bytes, + .outcome = vm_access_outcome(step.outcome), + .report_translation_fault = step.policy_fault}; + } + [[nodiscard]] std::optional query_mtype(uint64_t address) const override { return address_space_->pte_mtype(address, vmid_); } @@ -81,7 +128,12 @@ class LegacyGpuVmAdapter::Binding final : public AddressSpaceTranslator, if (!mutation_epoch_ || !guard.cacheable()) return {.mtype = guard.owns_lock() ? address_space_->pte_mtype(address, guard) : query_mtype(address)}; - return {.mtype = address_space_->pte_mtype(address, guard), + bool may_batch_private_uc = false, may_batch_private_ram = false; + const auto mtype = + address_space_->pte_mtype(address, guard, &may_batch_private_uc, &may_batch_private_ram); + return {.mtype = mtype, + .may_batch_private_uc = may_batch_private_uc, + .may_batch_private_ram = may_batch_private_ram, .begin = address & ~(kLegacyPageSize - 1), .size = kLegacyPageSize, .mutation_epoch = mutation_epoch_, @@ -232,6 +284,9 @@ class LegacyGpuVmAdapter::Binding final : public AddressSpaceTranslator, std::shared_ptr memory_; std::shared_ptr address_space_; uint32_t vmid_ = 0; + // Synchronous word reads and stores must not destroy the final request-mutex owner + // under the caller's VM-state lock, even after raw registry replacement. + const std::shared_ptr original_request_mutex_; std::shared_ptr> mutation_epoch_; std::shared_ptr frontend_lifetime_; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/libc_passthrough.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/libc_passthrough.cpp index c005dbc1220e..c742b39fca6d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/libc_passthrough.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/libc_passthrough.cpp @@ -23,6 +23,7 @@ void LibcPassthrough::resolve() { read = util::lookup_symbol(handle, "read"); write = util::lookup_symbol(handle, "write"); ioctl = util::lookup_symbol(handle, "ioctl"); + syscall = util::lookup_symbol(handle, "syscall"); mmap = util::lookup_symbol(handle, "mmap"); munmap = util::lookup_symbol(handle, "munmap"); mremap = util::lookup_symbol(handle, "mremap"); @@ -54,15 +55,17 @@ void LibcPassthrough::resolve() { lxstat_fn = util::lookup_symbol(handle, "__lxstat"); lxstat64_fn = util::lookup_symbol(handle, "__lxstat64"); readlink_fn = util::lookup_symbol(handle, "readlink"); + realpath_fn = util::lookup_symbol(handle, "realpath"); + realpath_chk_fn = util::lookup_symbol(handle, "__realpath_chk"); fork = util::lookup_symbol(handle, "fork"); // Keep ready() false unless every required interposed libc entry point was // resolved. In release builds the asserts disappear, so the boolean must not // claim readiness while any later call would dereference a null function // pointer. - initialized_ = openat && close && read && write && ioctl && mmap && munmap && mprotect && - madvise && memfd_create && dup && dup2 && dup3 && fcntl && fopen && freopen && - opendir && readdir && closedir && stat && lstat && access && fstat_fn && - readlink_fn && fork; + initialized_ = openat && close && read && write && ioctl && syscall && mmap && munmap && + mprotect && madvise && memfd_create && dup && dup2 && dup3 && fcntl && fopen && + freopen && opendir && readdir && closedir && stat && lstat && access && fstat_fn && + readlink_fn && realpath_fn && fork; assert(initialized_); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/libc_passthrough.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/libc_passthrough.h index b05e1bbacbc2..39b92ae9374a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/libc_passthrough.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/libc_passthrough.h @@ -16,6 +16,9 @@ #include #include +// Forward declaration keeps legacy aliases independent of feature-macro order. +struct stat64; + namespace rocjitsu { /// @brief Real libc function pointers resolved with dlsym(RTLD_NEXT). @@ -31,6 +34,7 @@ class LibcPassthrough { ssize_t (*read)(int, void *, size_t) = nullptr; ssize_t (*write)(int, const void *, size_t) = nullptr; int (*ioctl)(int, unsigned long, ...) = nullptr; + long (*syscall)(long, ...) = nullptr; void *(*mmap)(void *, size_t, int, int, int, off_t) = nullptr; int (*munmap)(void *, size_t) = nullptr; int (*mprotect)(void *, size_t, int) = nullptr; @@ -51,23 +55,24 @@ class LibcPassthrough { int (*access)(const char *, int) = nullptr; int (*fstat_fn)(int, struct stat *) = nullptr; ssize_t (*readlink_fn)(const char *, char *, size_t) = nullptr; + char *(*realpath_fn)(const char *, char *) = nullptr; + char *(*realpath_chk_fn)(const char *, char *, size_t) = nullptr; /// @brief The nine legacy stat aliases rocJITsu also exports. /// @details Resolved EAGERLY here rather than through a function-local static in /// each wrapper. A lazy static's C++ initialization guard can be inherited /// mid-initialization by a forked child whose initializing thread no longer /// exists, deadlocking the child before it can even reach the owner-PID gate. The /// interposer constructor is single-threaded, so resolving here has no such - /// window. struct stat64 is spelled void* to keep this header free of - /// _LARGEFILE64_SOURCE ordering constraints; the wrappers cast back. - int (*fstat64_fn)(int, void *) = nullptr; + /// window. Keep exact function types when calling another instrumented preload. + int (*fstat64_fn)(int, struct stat64 *) = nullptr; int (*fxstat_fn)(int, int, struct stat *) = nullptr; - int (*fxstat64_fn)(int, int, void *) = nullptr; - int (*stat64_fn)(const char *, void *) = nullptr; - int (*lstat64_fn)(const char *, void *) = nullptr; + int (*fxstat64_fn)(int, int, struct stat64 *) = nullptr; + int (*stat64_fn)(const char *, struct stat64 *) = nullptr; + int (*lstat64_fn)(const char *, struct stat64 *) = nullptr; int (*xstat_fn)(int, const char *, struct stat *) = nullptr; - int (*xstat64_fn)(int, const char *, void *) = nullptr; + int (*xstat64_fn)(int, const char *, struct stat64 *) = nullptr; int (*lxstat_fn)(int, const char *, struct stat *) = nullptr; - int (*lxstat64_fn)(int, const char *, void *) = nullptr; + int (*lxstat64_fn)(int, const char *, struct stat64 *) = nullptr; pid_t (*fork)() = nullptr; /// @brief Return true after all required symbols have been resolved. diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/remote_driver.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/remote_driver.cpp index ce540b43ee9e..14304b0950ae 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/remote_driver.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/remote_driver.cpp @@ -170,10 +170,9 @@ PtracerGrantVerdict authorize_daemon_address_space_access(int socket) { return verdict; } if (verdict == PtracerGrantVerdict::GrantUnverifiedPeer) { - // Said out loud, every time. This is the grant that rests on nothing but the - // peer having answered this socket, so which process received it belongs in - // the log rather than only in the kernel's unreadable exception slot. - util::Logger::warn( + // Attach mode can legitimately lack a launcher-provided PID. Keep this + // accepted trust decision in opt-in tracing so normal launches stay quiet. + util::Logger::driver( std::format("daemon: authorizing pid {} to read this address space; no launcher named a " "daemon for this client, so nothing here identifies it further", peer.pid)); @@ -411,7 +410,7 @@ PtracerGrantVerdict ptracer_grant_verdict(std::optional launched_daemon_p // daemon. Nothing here can identify the peer, and no check could be added that // would: a secret shared with the daemon authenticates nothing against an // attacker of this UID, who reads whatever this client can. The peer is - // trusted, and every such grant says so. + // trusted, and each such grant is logged only when DRIVER tracing is enabled. return PtracerGrantVerdict::GrantUnverifiedPeer; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/simulated_kfd.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/simulated_kfd.cpp index a0b02dcf472c..650768f4007c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/simulated_kfd.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/simulated_kfd.cpp @@ -8,8 +8,8 @@ #include "rocjitsu/kmd/linux/kfd_ioctl_utils.h" #include "rocjitsu/kmd/linux/kfd_topology.h" #include "rocjitsu/kmd/linux/libc_passthrough.h" -#include "rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h" #include "rocjitsu/vm/amdgpu/command_processor.h" +#include "rocjitsu/vm/amdgpu/compute_queue_binding_factory.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" #include "rocjitsu/vm/amdgpu/hwreg.h" @@ -162,14 +162,16 @@ amdgpu::Mtype SimulatedKfd::pte_mtype_for_flags(uint32_t flags) { return amdgpu::Mtype::RW; } -bool SimulatedKfd::gem_va_map(uint64_t gpu_va, void *host_ptr, size_t size, uint32_t alloc_flags) { +bool SimulatedKfd::gem_va_map(uint64_t gpu_va, void *host_ptr, size_t size, uint32_t alloc_flags, + bool sealed_ram) { auto proc = find_process(local_process_id_); if (!proc) return false; // GEM_VA uses the interposer's private read-write dmabuf mapping, whose // lifetime is tied to the GEM entry. It is not the application's CPU alias. map_to_gpu(*proc, gpu_va, host_ptr, size, pte_mtype_for_flags(alloc_flags), - KfdProcess::HostExtentOwner::Driver); + sealed_ram ? KfdProcess::HostExtentOwner::DriverSealedRam + : KfdProcess::HostExtentOwner::Driver); return true; } @@ -198,9 +200,8 @@ int SimulatedKfd::submit_pm4(uint32_t render_minor, uint64_t queue_key, auto *cp = gpus_[ordinal].soc->assign_queue_owner_cp(0); if (!cp) return -ENODEV; - amdgpu::Pm4SubmitQueue queue; + amdgpu::ComputeQueueConfig queue; queue.address_space = process->gpu(ordinal).address_space; - queue.pm4 = std::make_shared(); queue.process_id = local_process_id_; queue.queue_id = next_pm4_queue_id_++; const uint32_t queue_id = queue.queue_id; @@ -1166,6 +1167,7 @@ bool SimulatedKfd::register_process_address_spaces(const std::shared_ptrpage_table_generation(), .request_mutex = proc->page_table_request_mutex(), .mutation_epoch = proc->page_table_mutation_epoch(), + .page_table_cache_state = proc->page_table_cache_state(), .client_pid = client_pid, .client_mem_fd = -1, .passthrough = passthrough, @@ -2172,7 +2174,9 @@ void *SimulatedKfd::dispatch_mmap(KfdProcess &proc, void *addr, size_t length, i alloc.gpu_va, reinterpret_cast(host_ptr), length, alloc.flags, bool(flags & MAP_FIXED), alloc.user_va, alloc.memfd); }); - map_to_gpu(proc, alloc.gpu_va, host_ptr, length, pte_mtype_for_flags(alloc.flags)); + // mmap and KFD BOs cover whole pages, including the last partial page. + const size_t mapped_bytes = std::min(alloc.size, (uint64_t(length) + 0xFFF) & ~uint64_t(0xFFF)); + map_to_gpu(proc, alloc.gpu_va, host_ptr, mapped_bytes, pte_mtype_for_flags(alloc.flags)); return host_ptr; } @@ -2413,6 +2417,11 @@ int SimulatedKfd::unmap_memory_ioctl(void *arg) { int SimulatedKfd::alloc_memory_ioctl(KfdProcess &proc, void *arg) { auto *args = static_cast(arg); + if (args->size == 0 || args->size > UINT64_MAX - 0xFFF) + return -EINVAL; + // The kernel creates a PAGE_ALIGN(size) BO. LLVM can legally widen a scalar + // load within its mapped tail page even when the logical tensor is smaller. + const uint64_t allocation_size = (args->size + 0xFFF) & ~uint64_t(0xFFF); std::lock_guard lock(proc.alloc_mutex_); @@ -2420,12 +2429,12 @@ int SimulatedKfd::alloc_memory_ioctl(KfdProcess &proc, void *arg) { uint64_t va = args->va_addr; if (va == 0) { va = proc.next_gpu_va_; - proc.next_gpu_va_ += (args->size + 0xFFF) & ~0xFFFULL; + proc.next_gpu_va_ += allocation_size; } KfdProcess::GpuAllocation alloc{}; alloc.gpu_va = va; - alloc.size = args->size; + alloc.size = allocation_size; alloc.flags = args->flags; alloc.handle = proc.next_handle_++; alloc.host_ptr = nullptr; @@ -2437,7 +2446,7 @@ int SimulatedKfd::alloc_memory_ioctl(KfdProcess &proc, void *arg) { bool is_doorbell = (args->flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) != 0; if (is_userptr && !daemon_mode_) { alloc.host_ptr = reinterpret_cast(va); - map_to_gpu(proc, va, reinterpret_cast(va), args->size, alloc_mtype); + map_to_gpu(proc, va, reinterpret_cast(va), alloc.size, alloc_mtype); } else if (daemon_mode_ || !user_provided_va) { auto raw_fd = memfd_create("rocjitsu_alloc", MFD_CLOEXEC | MFD_ALLOW_SEALING); if (raw_fd >= 0) { @@ -2694,6 +2703,9 @@ int SimulatedKfd::unmap_memory_ioctl(KfdProcess &proc, void *arg) { int SimulatedKfd::create_queue_ioctl(KfdProcess &proc, void *arg) { auto *args = static_cast(arg); + const uint32_t scheduling_percentage = args->queue_percentage & 0xff; + if (scheduling_percentage > 100) + return -EINVAL; auto *gpu = find_gpu(args->gpu_id); if (!gpu || !gpu->soc) return -EINVAL; @@ -2703,9 +2715,7 @@ int SimulatedKfd::create_queue_ioctl(KfdProcess &proc, void *arg) { const bool is_sdma = args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA || args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI || args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_BY_ENG_ID; - if (is_pm4_compute) - return -ENOTSUP; - if (!is_aql_compute && !is_sdma) + if (!is_aql_compute && !is_pm4_compute && !is_sdma) return -ENOTSUP; const std::optional ring_size = normalize_queue_ring_size(args->ring_size); @@ -2717,12 +2727,18 @@ int SimulatedKfd::create_queue_ioctl(KfdProcess &proc, void *arg) { // different processes therefore share XCD resources while each process still // distributes additional queues across the device. const uint32_t queue_ordinal = proc.next_queue_id_ - 1; + // KFD reserves percentage bits 8..15 for a native PM4 queue's target XCC. + // An AQL vendor IB inherits the AQL queue's replicas instead of this placement. + const uint32_t owner_ordinal = + is_pm4_compute ? (args->queue_percentage >> 8) & 0xff : queue_ordinal; + if (is_pm4_compute && owner_ordinal >= gpu->soc->num_xcds()) + return -EINVAL; amdgpu::CommandProcessor *target_cp = - is_sdma ? nullptr : gpu->soc->assign_queue_owner_cp(queue_ordinal); + is_sdma ? nullptr : gpu->soc->assign_queue_owner_cp(owner_ordinal); if (!is_sdma && target_cp == nullptr) return -EINVAL; const uint32_t target_xcc_id = - is_sdma ? args->sdma_engine_id : gpu->soc->queue_xcd_id(queue_ordinal); + is_sdma ? args->sdma_engine_id : gpu->soc->queue_xcd_id(owner_ordinal); // Build the HW queue and reserve all per-process state under alloc_mutex_, then // register it with the CommandProcessor with the lock RELEASED. The CP thread @@ -2796,8 +2812,9 @@ int SimulatedKfd::create_queue_ioctl(KfdProcess &proc, void *arg) { .producer_pointer_address = args->write_pointer_address}; queue_request.binding_factory = is_sdma ? amdgpu::make_sdma_queue_binding_factory(gpu->soc->sdma_queue_scheduler()) - : amdgpu::make_aql_queue_binding_factory(*target_cp); + : amdgpu::make_compute_queue_binding_factory(*target_cp); queue_request.engine_id = args->sdma_engine_id; + queue_request.scheduling_percentage = scheduling_percentage; // doorbell_base is captured here under alloc_mutex_ but register_queue() runs // after the lock is released. This is stable because ROCr maps the doorbell // page before creating queues, and queue creation for a process is single- @@ -2809,13 +2826,13 @@ int SimulatedKfd::create_queue_ioctl(KfdProcess &proc, void *arg) { .host_base = gs.doorbell_monitor_page, .last_value = ~uint64_t(0)}; queue_request.type = is_sdma ? amdgpu::QueueType::Sdma : amdgpu::QueueType::Compute; - queue_request.packet_format = - is_sdma ? amdgpu::QueuePacketFormat::Sdma : amdgpu::QueuePacketFormat::Aql; - queue_request.abi = is_sdma ? amdgpu::QueueAbi::Generic : amdgpu::QueueAbi::KfdAql; - // Queue creation initializes both SDMA pointers to zero below. Preserve that - // device-side cursor explicitly so execution does not depend on reading the - // writeback destination before the first packet can retire. - if (is_sdma) + queue_request.packet_format = is_sdma ? amdgpu::QueuePacketFormat::Sdma + : is_pm4_compute ? amdgpu::QueuePacketFormat::Pm4 + : amdgpu::QueuePacketFormat::Aql; + queue_request.abi = is_aql_compute ? amdgpu::QueueAbi::KfdAql : amdgpu::QueueAbi::Generic; + // Fresh SDMA and native PM4 MQDs start at zero; the writeback destination + // need not contain the initial hardware cursor before the first retirement. + if (is_sdma || is_pm4_compute) queue_request.initial_consumer_cursor = 0; // The topology advertises every XCD's compute units as one agent, so a // compute dispatch must be able to reach all of them. Without this a @@ -2828,10 +2845,10 @@ int SimulatedKfd::create_queue_ioctl(KfdProcess &proc, void *arg) { // unsupported value silently acquires. queue_request.xcd_fanout = is_aql_compute; // amd_queue_t base: write_pointer_address points to write_dispatch_id. - if (!is_sdma) + if (is_aql_compute) queue_request.queue_descriptor_address = args->write_pointer_address - offsetof(amd_queue_t, write_dispatch_id); - if (!is_sdma && args->ctx_save_restore_address != 0) { + if (is_aql_compute && args->ctx_save_restore_address != 0) { constexpr uint32_t kErrorReasonOffset = 6 * sizeof(uint32_t); const std::optional access = gpu->soc->gpu_vm().snapshot(gs.address_space); @@ -2896,6 +2913,7 @@ int SimulatedKfd::create_queue_ioctl(KfdProcess &proc, void *arg) { .gpu_ordinal = gpu_ordinal(args->gpu_id), .doorbell_offset = queue_request.doorbell.offset, .queue_handle = queue_handle, + .pm4_target_xcc = is_pm4_compute ? std::optional(owner_ordinal) : std::nullopt, }; proc.queue_snapshot_map_[queue_id] = { .ring_base_address = args->ring_base_address, @@ -3009,6 +3027,10 @@ int SimulatedKfd::set_cu_mask_ioctl(KfdProcess &proc, void *arg) { int SimulatedKfd::update_queue_ioctl(KfdProcess &proc, void *arg) { auto *args = static_cast(arg); + // Decode scheduling independently of the native PM4 placement field. + const uint32_t scheduling_percentage = args->queue_percentage & 0xff; + if (scheduling_percentage > 100) + return -EINVAL; const std::optional ring_size = normalize_queue_ring_size(args->ring_size); if (!ring_size) return -EINVAL; @@ -3026,12 +3048,21 @@ int SimulatedKfd::update_queue_ioctl(KfdProcess &proc, void *arg) { if (!queue || queue->gpu_ordinal >= gpus_.size() || !gpus_[queue->gpu_ordinal].soc) return -EFAULT; + if (queue->pm4_target_xcc) { + const uint32_t target_xcc = (args->queue_percentage >> 8) & 0xff; + if (target_xcc >= gpus_[queue->gpu_ordinal].soc->num_xcds()) + return -EINVAL; + // Moving a live CP record requires migrating its registers and pending streams. + if (target_xcc != *queue->pm4_target_xcc) + return -EOPNOTSUPP; + } + amdgpu::QueueReconfigureResult update; try { update = gpus_[queue->gpu_ordinal].soc->queue_registry().reconfigure_queue( queue->queue_handle, {.ring_base_address = args->ring_base_address, .ring_size_bytes = *ring_size, - .scheduling_percentage = args->queue_percentage}); + .scheduling_percentage = scheduling_percentage}); } catch (const std::bad_alloc &) { return -ENOMEM; } catch (...) { @@ -4168,7 +4199,8 @@ void SimulatedKfd::complete_runtime_queue_exception(uint32_t process_id, uint32_ } bool SimulatedKfd::signal_runtime_queue_exception(uint32_t gpu_id, uint32_t queue_id, - uint32_t process_id, uint64_t exception_mask) { + uint32_t process_id, uint64_t exception_mask, + bool wait_for_ack) { auto *gpu = find_gpu(gpu_id); if (!gpu || !gpu->soc || exception_mask == 0) return false; @@ -4203,7 +4235,8 @@ bool SimulatedKfd::signal_runtime_queue_exception(uint32_t gpu_id, uint32_t queu bool published = false; { std::lock_guard lock(publication_lock->publication_mutex); - published = publisher->publish_queue_exception(queue_id, process_id, exception_mask); + published = + publisher->publish_queue_exception(queue_id, process_id, exception_mask, wait_for_ack); } if (queue_exception_cleanup_hook_for_testing_) queue_exception_cleanup_hook_for_testing_(false); @@ -6068,18 +6101,24 @@ int SimulatedKfd::debug_trap_ioctl(KfdProcess &caller, void *arg, int *target_me auto *gpu = find_gpu(event.gpu_id); if (!gpu || !gpu->soc) return -ENODEV; - // Drop debug_sessions_mutex_ first: signal_queue_exception() takes the CU - // wave-state lock and waits up to a second for the target to observe the - // exception word, while the engine thread runs its issue loop under that - // same wave-state lock and calls back into the trap/watchpoint handlers, - // which take debug_sessions_mutex_. Holding it across the call inverts - // that order -- the inversion DISABLE and SUSPEND_QUEUES both avoid -- and - // would additionally stall every trap callback for the duration of the - // wait. Nothing below this point reads debug_sessions_ or session_it. + // Drop debug_sessions_mutex_ before entering the CU wave-state locks: + // trap callbacks acquire these in the opposite order. Nothing below + // this point reads debug_sessions_ or session_it. lk.unlock(); - const bool delivered = signal_runtime_queue_exception( - event.gpu_id, event.queue_id, target_proc->process_id(), event.exception_mask); - return delivered ? 0 : -ENOENT; + // KFD's SEND_RUNTIME_EVENT publishes the status and event without + // waiting for ROCr to acknowledge them. ROCr removes its async exception + // handler after a fatal error, so later waves forwarding their aborts + // cannot satisfy such a wait. A debugger can also have the host stopped. + const bool published = signal_runtime_queue_exception( + event.gpu_id, event.queue_id, target_proc->process_id(), event.exception_mask, + /*wait_for_ack=*/false); + if (published) + return 0; + // The runtime may already have destroyed this queue after the first + // forwarded abort. KFD accepts that stale request too. Keep reporting + // actual publication failures for queues that still exist. + std::lock_guard alloc_lock(target_proc->alloc_mutex_); + return target_proc->queue_snapshot_map_.contains(event.queue_id) ? -ENOENT : 0; } std::lock_guard runtime_lock(runtime_handshake_mutex_); runtime_acked_.insert(target_pid); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/simulated_kfd.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/simulated_kfd.h index 3a9e85db9121..f4af1ad8d2fc 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/simulated_kfd.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/simulated_kfd.h @@ -10,7 +10,7 @@ #include "rocjitsu/kmd/linux/linux_kfd.h" #include "rocjitsu/kmd/linux/sysfs.h" #include "rocjitsu/vm/amdgpu/interrupt_sink.h" -#include "rocjitsu/vm/amdgpu/pm4.h" +#include "rocjitsu/vm/amdgpu/pm4/pm4.h" #include "rocjitsu/vm/soc.h" #include "simdojo/sim/simulation.h" @@ -449,7 +449,8 @@ class SimulatedKfd : public LinuxKfd { /// @retval true the range was installed. /// @retval false the local process is gone, so nothing was mapped (the caller /// must surface an error rather than report a phantom success). - [[nodiscard]] bool gem_va_map(uint64_t gpu_va, void *host_ptr, size_t size, uint32_t alloc_flags); + [[nodiscard]] bool gem_va_map(uint64_t gpu_va, void *host_ptr, size_t size, uint32_t alloc_flags, + bool sealed_ram = false); /// @brief Remove a GPU page-table range installed by gem_va_map (GEM_VA UNMAP). /// @retval true the range was unmapped. @@ -590,7 +591,7 @@ class SimulatedKfd : public LinuxKfd { int duplicate_debug_notifier(int fd); int retry_debug_notifications(pid_t target_pid, bool invoke_result_hook = false); bool signal_runtime_queue_exception(uint32_t gpu_id, uint32_t queue_id, uint32_t process_id, - uint64_t exception_mask); + uint64_t exception_mask, bool wait_for_ack = true); bool on_wave_single_step_complete(amdgpu::Wavefront &wf); void apply_debug_event_publication_hook_for_testing(const std::shared_ptr &proc); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/sysfs.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/sysfs.cpp index bfcdb52c68b7..eef7a3e50c7c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/sysfs.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/sysfs.cpp @@ -485,6 +485,14 @@ void Sysfs::write_drm_tree(const std::vector &gpus) { write_file(device_dir + "/revision", revision_hex.str()); write_file(device_dir + "/subsystem_vendor", vendor_hex.str()); write_file(device_dir + "/subsystem_device", device_hex.str()); + for (const auto &block : gpu.ip_versions) { + std::string instance_dir = + device_dir + "/ip_discovery/die/0/" + std::to_string(block.hardware_id) + "/0"; + make_dir(instance_dir); + write_file(instance_dir + "/major", std::to_string(block.major) + "\n"); + write_file(instance_dir + "/minor", std::to_string(block.minor) + "\n"); + write_file(instance_dir + "/revision", std::to_string(block.revision) + "\n"); + } } } @@ -546,6 +554,7 @@ Sysfs::GpuInfo gpu_info_from_config(const config::KfdDeviceConfig &dev, uint32_t gpu.debug_prop = dev.debug_prop; gpu.drm_render_minor = dev.drm_render_minor; gpu.marketing_name = dev.marketing_name; + gpu.ip_versions = dev.ip_versions; gpu.revision_id = dev.revision_id; gpu.pci_revision_id = dev.pci_revision_id; gpu.simd_count = dev.simd_count; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/sysfs.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/sysfs.h index df8aebb46621..3fb62cb57be4 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/sysfs.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/kmd/linux/sysfs.h @@ -41,6 +41,7 @@ class Sysfs { uint32_t revision_id = 0; uint32_t pci_revision_id = 0; std::string marketing_name; + std::vector ip_versions; // Compute unit organization uint32_t simd_count = 0; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/CMakeLists.txt b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/CMakeLists.txt index 8465cb5474c8..111db2b980bc 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/CMakeLists.txt +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/CMakeLists.txt @@ -3,7 +3,7 @@ rj_add_object_library(rocjitsu_vm_amdgpu aql/aql_packet_processor.cpp - aql/aql_queue_binding_factory.cpp + compute_queue_binding_factory.cpp circular_ring_reader.cpp buffer_format.cpp cluster_lds_multicast.cpp @@ -26,9 +26,6 @@ rj_add_object_library(rocjitsu_vm_amdgpu mes_engine.cpp memory_wait_scoreboard.cpp pm4/pm4_packet_processor.cpp - pm4/pm4_queue_binding_factory.cpp - pm4/pm4_queue_controller.cpp - pm4/pm4_ring_consumer.cpp sdma_packet_processor.cpp sdma_queue_binding_factory.cpp sdma_queue_scheduler.cpp diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.cpp index caec5b9e2d63..952708bd09e1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.cpp @@ -9,6 +9,7 @@ RJ_DIAGNOSTIC_IGNORE_PEDANTIC #include "hsa/amd_ext_aql_packet.h" RJ_DIAGNOSTIC_POP +#include #include #include #include @@ -228,10 +229,19 @@ AqlPacketProcessor::process_vendor(const Request &request, } if (extension.amd_format == kAmdAqlFormatPm4Ib) { - return admit(request, {.kind = AqlPreparedPacketKind::NonKernel, + // ROCr's amd_aql_pm4_ib: four-word INDIRECT_BUFFER, ten remaining words, + // eight reserved words, then the completion signal. Copy before ring retirement. + std::array words{}; + std::memcpy(words.data(), &packet, sizeof(packet)); + if (words[1] != 0xc0023f00 || (words[2] & 3) || words[3] > 0xffff || + (words[4] & ~0x000fffffu) != (1u << 23) || !(words[4] & 0xfffff) || words[5] != 10) + return terminal(PacketProcessStatus::Malformed, AqlPacketDiagnostic::MalformedPm4Ib); + return admit(request, {.kind = AqlPreparedPacketKind::Pm4Ib, + .pm4_ib_address = uint64_t{words[2]} | (uint64_t{words[3]} << 32), + .pm4_ib_dwords = words[4] & 0xfffff, .completion_signal = extension.completion_signal.handle, .barrier_bit = ((packet.header >> HSA_PACKET_HEADER_BARRIER) & 1) != 0, - .blocks_following = false}); + .blocks_following = true}); } return terminal(PacketProcessStatus::Unsupported, AqlPacketDiagnostic::UnsupportedVendorFormat); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.h index 79fbac4adb4d..868f8ef992de 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.h @@ -48,11 +48,13 @@ enum class AqlPacketDiagnostic : uint8_t { InvalidSignalAddress, MissingSignalReader, MissingAdmissionCallback, + MalformedPm4Ib, }; enum class AqlPreparedPacketKind : uint8_t { KernelDispatch, NonKernel, + Pm4Ib, }; /// @brief Outcome of attempting to commit a decoded AQL action to its CP queue. @@ -74,6 +76,8 @@ struct AqlPreparedPacket { AqlPreparedPacketKind kind = AqlPreparedPacketKind::NonKernel; hsa_kernel_dispatch_packet_t kernel_dispatch{}; ClusterDispatchShape cluster_shape{}; + uint64_t pm4_ib_address = 0; + uint32_t pm4_ib_dwords = 0; uint64_t completion_signal = 0; bool barrier_bit = false; bool blocks_following = false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h deleted file mode 100644 index af7040eab878..000000000000 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -/// @file aql/aql_queue_binding_factory.h -/// @brief Queue binding factory for the shared AQL command processor. - -#pragma once - -#include "rocjitsu/vm/amdgpu/gpu_queue_registry.h" - -#include - -namespace rocjitsu::amdgpu { - -class CommandProcessor; - -/// @brief Create a reusable binding factory for AQL queues owned by @p command_processor. -/// @details The factory and bindings borrow the command processor. Its owner must -/// outlive the factory and every queue binding created from it. -[[nodiscard]] std::shared_ptr -make_aql_queue_binding_factory(CommandProcessor &command_processor); - -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/async_scoreboard.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/async_scoreboard.h index 32367b4d2e54..dc3df00e3525 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/async_scoreboard.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/async_scoreboard.h @@ -119,8 +119,8 @@ class AsyncInstructionQueue { }; /// @brief A bounded same-wave execution window. -/// @details Draining before CU rescheduling keeps decoder pools, wave storage -/// and data caches owned by the issuing thread. +/// @details Draining before CU rescheduling keeps wave storage and data caches +/// stable until asynchronous work finishes. class AsyncInstructionWindow { using Access = async_execution::Access; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/atomic_op.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/atomic_op.h index 4136aee70b70..1b1b2e7cdaa5 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/atomic_op.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/atomic_op.h @@ -2,12 +2,14 @@ // SPDX-License-Identifier: MIT /// @file atomic_op.h -/// @brief The atomic read-modify-write operations a memory instruction can name. +/// @brief Atomic operations and shared integer read-modify-write semantics. #ifndef ROCJITSU_VM_AMDGPU_ATOMIC_OP_H_ #define ROCJITSU_VM_AMDGPU_ATOMIC_OP_H_ +#include #include +#include namespace rocjitsu { namespace amdgpu { @@ -45,6 +47,62 @@ enum class AtomicOp : uint8_t { BARRIER_ARRIVE, ///< LDS barrier-arrive state update. }; +/// @brief Apply an integer atomic RMW operation (32-bit or 64-bit). +template T apply_int_atomic(AtomicOp op, T old_val, T src_val, T cmp_val = 0) { + using S = std::make_signed_t; + switch (op) { + case AtomicOp::SWAP: + return src_val; + case AtomicOp::CONDXCHG32: { + static_assert(sizeof(T) == 4 || sizeof(T) == 8); + T result = old_val; + for (uint32_t shift = 0; shift < sizeof(T) * 8; shift += 32) { + const T half_mask = T{0xffffffffu} << shift; + const T store_mask = T{0x7fffffffu} << shift; + if ((src_val >> shift) & 0x80000000u) + result = (result & ~half_mask) | (src_val & store_mask); + } + return result; + } + case AtomicOp::CMPSWAP: + return (old_val == cmp_val) ? src_val : old_val; + case AtomicOp::MSKOR: + return (old_val & ~src_val) | cmp_val; + case AtomicOp::ADD: + return old_val + src_val; + case AtomicOp::SUB: + return old_val - src_val; + case AtomicOp::SUB_CLAMP: + return old_val >= src_val ? old_val - src_val : T{0}; + case AtomicOp::COND_SUB: + return old_val >= src_val ? old_val - src_val : old_val; + case AtomicOp::WRAP: + return old_val >= src_val ? old_val - src_val : old_val + cmp_val; + case AtomicOp::RSUB: + return src_val - old_val; + case AtomicOp::SMIN: + return static_cast(std::min(static_cast(old_val), static_cast(src_val))); + case AtomicOp::UMIN: + return std::min(old_val, src_val); + case AtomicOp::SMAX: + return static_cast(std::max(static_cast(old_val), static_cast(src_val))); + case AtomicOp::UMAX: + return std::max(old_val, src_val); + case AtomicOp::AND: + return old_val & src_val; + case AtomicOp::OR: + return old_val | src_val; + case AtomicOp::XOR: + return old_val ^ src_val; + case AtomicOp::INC: + return (old_val >= src_val) ? T{0} : old_val + 1; + case AtomicOp::DEC: + return (old_val == 0 || old_val > src_val) ? src_val : old_val - 1; + default: + return old_val; + } +} + } // namespace amdgpu } // namespace rocjitsu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.cpp index 21b1ad87966d..224cd904e4d1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.cpp @@ -8,8 +8,7 @@ #include "rocjitsu/vm/amdgpu/gpu_vm.h" #include "rocjitsu/vm/amdgpu/hsa_clock.h" #include "rocjitsu/vm/amdgpu/mem_state.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h" +#include "rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h" #include "rocjitsu/base/rj_compiler.h" RJ_DIAGNOSTIC_PUSH @@ -23,6 +22,7 @@ RJ_DIAGNOSTIC_POP #include "util/log.h" #include +#include #include #include #include @@ -62,6 +62,13 @@ CommandProcessor::CommandProcessor(std::string name, simdojo::ExecMode exec_mode retry_event_pending_.store(false, std::memory_order_release); handle_doorbell(ts); }); + stall_recheck_event_.set_handler([this](simdojo::Tick ts, simdojo::Message *message) { + if (!message || message->payload() != stall_recheck_generation_) + return; + stall_recheck_pending_ = false; + stall_recheck_tick_ = simdojo::TICK_MAX; + handle_doorbell(ts); + }); dispatch_continuation_event_.set_handler([this](simdojo::Tick ts, simdojo::Message *message) { if (!message || message->payload() != dispatch_continuation_generation_) return; @@ -91,89 +98,101 @@ void CommandProcessor::set_gpu_vm(GpuVm *gpu_vm, AddressSpaceHandle default_addr throw std::invalid_argument("command processor default address space is not registered"); if (gpu_vm_ != gpu_vm || default_address_space_ != default_address_space) { std::lock_guard lock(hw_queue_mutex_); - if (!aql_queues_.empty() || !drm_queues_.empty() || active_queue_registrations_ != 0) + if (!compute_queues_.empty() || active_queue_registrations_ != 0) throw std::logic_error("cannot replace the command processor VM while queues exist"); - if (pm4_queue_controller_ && pm4_queue_controller_->active_queues() != 0) { - throw std::logic_error("cannot replace the command processor VM while PM4 queues exist"); - } gpu_vm_ = gpu_vm; default_address_space_ = gpu_vm_ ? default_address_space : AddressSpaceHandle{}; - pm4_queue_controller_ = gpu_vm_ ? std::make_unique(*gpu_vm_) : nullptr; } for (ComputeUnitCore *cu : cus_) cu->set_gpu_vm(gpu_vm_); } -std::shared_ptr -CommandProcessor::make_pm4_queue_binding_factory(Pm4PacketCallbacks callbacks) { - return amdgpu::make_pm4_queue_binding_factory(*this, std::move(callbacks)); -} - uint64_t CommandProcessor::register_pm4_queue(Pm4QueueConfig config) { - std::lock_guard lock(hw_queue_mutex_); - if (pm4_queue_controller_ == nullptr) - return 0; - return pm4_queue_controller_->attach(std::move(config)); + // Compatibility entry point: PM4 is initial state of the same compute queue. + return register_queue({.address_space = config.address_space, + .ring_base_va = config.ring_base, + .ring_size = config.ring_size_bytes, + .read_ptr_va = config.consumer_pointer_address, + .last_doorbell = config.initial_consumer_cursor.value_or(~uint64_t(0)), + .packet_format = QueuePacketFormat::Pm4, + .initial_consumer_cursor = config.initial_consumer_cursor, + .packet_callbacks = std::move(config.packet_callbacks)}); } -bool CommandProcessor::unregister_pm4_queue_registration(uint64_t registration_id) noexcept { - std::lock_guard lock(hw_queue_mutex_); - return pm4_queue_controller_ != nullptr && pm4_queue_controller_->detach(registration_id); +bool CommandProcessor::unregister_pm4_queue_registration(uint64_t id) noexcept { + return unregister_queue_registration(id); } QueuePrepareCloseStatus -CommandProcessor::prepare_unregister_pm4_queue_registration(uint64_t registration_id) noexcept { - std::lock_guard lock(hw_queue_mutex_); - if (pm4_queue_controller_ == nullptr) - return QueuePrepareCloseStatus::Faulted; - const QueuePrepareCloseStatus status = pm4_queue_controller_->prepare_detach(registration_id); - if (status == QueuePrepareCloseStatus::Busy) { - if (engine()) - engine()->schedule_event_now(&doorbell_event_); - else - (void)pm4_queue_controller_->service(); - } - return status; +CommandProcessor::prepare_unregister_pm4_queue_registration(uint64_t id) noexcept { + return prepare_unregister_queue_registration(id); } QueueReconfigureStatus -CommandProcessor::update_pm4_queue_registration(uint64_t registration_id, +CommandProcessor::update_pm4_queue_registration(uint64_t id, const QueueReconfigureRequest &request) { - std::lock_guard lock(hw_queue_mutex_); - if (pm4_queue_controller_ == nullptr) + std::lock_guard lock(hw_queue_mutex_); + const auto queue = std::ranges::find(compute_queues_, id, &ComputeQueueRecord::registration_id); + if (queue == compute_queues_.end()) return QueueReconfigureStatus::Stale; - return pm4_queue_controller_->update(registration_id, request); + if (queue->faulted || queue->command_fault_pending || request.scheduling_percentage > 100) + return QueueReconfigureStatus::Invalid; + const bool disabled = !request.ring_base_address || !request.scheduling_percentage; + const uint64_t base = disabled ? queue->ring_base_va : request.ring_base_address; + const uint32_t bytes = disabled ? queue->ring_size : request.ring_size_bytes; + if (!disabled && (!base || base % 4 || bytes < 4 || bytes % 4)) + return QueueReconfigureStatus::Invalid; + if ((base != queue->ring_base_va || bytes != queue->ring_size) && queue->has_pending_commands()) + return QueueReconfigureStatus::Busy; + // Ring reconfiguration and runtime suspension share the same wave gates as AQL. + if (!update_queue_registration(id, base, bytes, disabled ? 0 : request.scheduling_percentage)) + return QueueReconfigureStatus::Invalid; + return disabled ? QueueReconfigureStatus::Disabled : QueueReconfigureStatus::Applied; } -QueueSubmissionStatus CommandProcessor::notify_pm4_queue_doorbell(uint64_t registration_id, - uint64_t producer_cursor) { - std::lock_guard lock(hw_queue_mutex_); - if (pm4_queue_controller_ == nullptr) +QueueSubmissionStatus CommandProcessor::notify_pm4_queue_doorbell(uint64_t id, uint64_t producer) { + std::lock_guard lock(hw_queue_mutex_); + const auto queue = std::ranges::find(compute_queues_, id, &ComputeQueueRecord::registration_id); + if (queue == compute_queues_.end() || queue->faulted || queue->command_fault_pending || + queue->runtime_suspended || queue->packet_format != QueuePacketFormat::Pm4 || + queue->submission_queue) + return QueueSubmissionStatus::Faulted; + const uint64_t reference = queue->last_doorbell == ~uint64_t(0) + ? queue->read_pointer_journal.cursor() + : queue->last_doorbell; + const auto normalized = normalize_pm4_producer_cursor(producer, reference, queue->ring_size / 4); + if (!normalized) { + // CU cleanup belongs to the event thread, even for a rejected notification. + queue->command_fault_pending = true; + if (engine()) + engine()->schedule_event_now(doorbell_event()); + else + fail_pm4_queue(*queue, queue->dispatches); return QueueSubmissionStatus::Faulted; - const QueueSubmissionStatus status = - pm4_queue_controller_->notify(registration_id, producer_cursor); - if (status != QueueSubmissionStatus::Accepted) - return status; - if (engine()) { - engine()->schedule_event_now(&doorbell_event_); - } else { - // Unit-level CP fixtures without a SimulationEngine still exercise the - // same controller. Production frontends always take the event path above. - if (pm4_queue_controller_->service()) - return QueueSubmissionStatus::Retry; } - return status; + queue->last_doorbell = *normalized; + if (engine()) { + // Validate synchronously; packet effects remain on the CP event context. + engine()->schedule_event_now(doorbell_event()); + return QueueSubmissionStatus::Accepted; + } + for (auto &record : compute_queues_) + record.command_retry_pending = false; + service_command_streams(0); + return queue->faulted ? QueueSubmissionStatus::Faulted + : queue->has_pending_commands() ? QueueSubmissionStatus::Retry + : QueueSubmissionStatus::Accepted; } size_t CommandProcessor::registered_pm4_queue_count_for_test() const { - std::lock_guard lock(hw_queue_mutex_); - return pm4_queue_controller_ ? pm4_queue_controller_->active_queues() : 0; + std::lock_guard lock(hw_queue_mutex_); + return std::ranges::count_if( + compute_queues_, [](const auto &q) { return q.packet_format == QueuePacketFormat::Pm4; }); } bool CommandProcessor::has_registered_queues() const { - std::lock_guard lock(hw_queue_mutex_); - return active_queue_registrations_ != 0 || !aql_queues_.empty() || !drm_queues_.empty() || - (pm4_queue_controller_ && pm4_queue_controller_->active_queues() != 0); + std::lock_guard lock(hw_queue_mutex_); + return active_queue_registrations_ != 0 || !compute_queues_.empty(); } void CommandProcessor::configure_for_arch(rj_code_arch_t arch) { @@ -427,7 +446,7 @@ void CommandProcessor::set_dispatch_threads(uint32_t threads) { pooled_due_ticks_[cu] = tick; } } - const simdojo::Tick next = next_pooled_due_tick(now); + const simdojo::Tick next = next_pooled_due_tick(); if (next != simdojo::TICK_MAX) arm_dispatch_continuation(next); return; @@ -813,22 +832,32 @@ void CommandProcessor::startup() { completion_ = std::make_unique(*gpu_vm_, cus_, l2_caches_); completion_->set_plugin_group(plugin_group_); completion_->set_dispatch_retirement_gate( - [this](AqlQueueRecord &queue, const DispatchEntry &entry) { + [this](ComputeQueueRecord &queue, const DispatchEntry &entry) { return gate_dispatch_retirement(queue, entry); }); completion_->set_dispatch_retired_callback([this](const DispatchEntry &entry) { erase_cluster_workgroups(entry.dispatch_id); dispatch_launch_metadata_.erase(entry.dispatch_id); // Resume the ring after the blocking packet's completion is durable. - if (entry.blocks_following && engine()) + if (entry.blocks_following && engine()) { + stall_recheck_backoff_ = 1; arm_stall_recheck(engine()->context(partition_id()).current_tick()); + } }); completion_->set_grid_retired_callback([this](const DispatchEntry &) { wake_all_xcds(); }); + // Waves admitted before engine attachment could not notify the pool driver. + // Seed those CUs once; admission and resume callbacks maintain the set later. + if (dispatch_threads_ > 1) + for (auto *cu : cus_) + on_cu_pool_ready(cu); } void CommandProcessor::shutdown() { stop_doorbell_monitor(); retry_event_pending_.store(false, std::memory_order_release); + stall_recheck_pending_ = false; + stall_recheck_tick_ = simdojo::TICK_MAX; + ++stall_recheck_generation_; if (is_primary_ && engine()) { engine()->primary_release(); is_primary_ = false; @@ -871,8 +900,8 @@ void CommandProcessor::set_scratch_xcc_layout_for_test(uint32_t xcc_id, uint32_t scratch_xcc_count_ = xcc_count; } -AqlQueueRecord *CommandProcessor::find_aql_queue(uint32_t queue_id, uint32_t process_id) { - for (AqlQueueRecord &queue : aql_queues_) { +ComputeQueueRecord *CommandProcessor::find_compute_queue(uint32_t queue_id, uint32_t process_id) { + for (ComputeQueueRecord &queue : compute_queues_) { if (queue.queue_id == queue_id && queue.process_id == process_id) return &queue; } @@ -948,11 +977,11 @@ void CommandProcessor::drain_fanout_inbox() { std::lock_guard lock(hw_queue_mutex_); for (DispatchEntry &shard : inbox) { - const std::vector::iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + const std::vector::iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == shard.queue_id && candidate.process_id == shard.process_id; }); - if (queue == aql_queues_.end()) { + if (queue == compute_queues_.end()) { // The replica was destroyed between the owner handing this shard over and // this drain. Drop it, exactly as unregister_queue drops a share it never // published: these workgroups have not run and this XCD's caches have not @@ -1072,11 +1101,11 @@ void CommandProcessor::replicate_non_kernel_entry(const DispatchEntry &dp) { } } -uint64_t CommandProcessor::register_queue(AqlQueueConfig config) { +uint64_t CommandProcessor::register_queue(ComputeQueueConfig config) { return register_queue(std::move(config), false); } -uint64_t CommandProcessor::register_queue(AqlQueueConfig config, bool fanout_replica) { +uint64_t CommandProcessor::register_queue(ComputeQueueConfig config, bool fanout_replica) { QueueRegistrationTransaction registration(*this); GpuVm *registration_vm = nullptr; { @@ -1087,10 +1116,21 @@ uint64_t CommandProcessor::register_queue(AqlQueueConfig config, bool fanout_rep } if (registration_vm == nullptr || !config.address_space) return 0; - if (!valid_aql_queue_layout(config.ring_base_va, config.ring_size, config.read_ptr_va, - config.write_ptr_va)) { - return 0; + if (!config.submission_queue) { + const bool valid = config.packet_format == QueuePacketFormat::Aql + ? valid_aql_queue_layout(config.ring_base_va, config.ring_size, + config.read_ptr_va, config.write_ptr_va) + : config.packet_format == QueuePacketFormat::Pm4 && + config.ring_base_va != 0 && config.ring_base_va % 4 == 0 && + config.ring_size >= 4 && config.ring_size % 4 == 0 && + config.read_ptr_va != 0 && config.read_ptr_va % 4 == 0; + if (!valid) + return 0; } + // Native PM4 has no VM-doorbell poller. Reject it until polling is implemented. + if (config.packet_format == QueuePacketFormat::Pm4 && + config.doorbell_mode == QueueDoorbellMode::VmPolled) + return 0; // A host-polled queue may be registered before its doorbell page is mapped; // set_process_doorbell_base() publishes that mapping later. VM polling has no // equivalent deferred binding and therefore requires an address up front. @@ -1101,7 +1141,7 @@ uint64_t CommandProcessor::register_queue(AqlQueueConfig config, bool fanout_rep registration_vm->retain_binding(config.address_space); if (!address_space_lease) return 0; - AqlQueueRecord queue(std::move(config)); + ComputeQueueRecord queue(std::move(config)); queue.address_space_lease = std::move(*address_space_lease); queue.fanout_replica = fanout_replica; if (!fanout_replica && queue.uses_kfd_queue_abi && queue.queue_desc_va != 0 && !cus_.empty() && @@ -1137,7 +1177,7 @@ uint64_t CommandProcessor::register_queue(AqlQueueConfig config, bool fanout_rep class OwnerRegistrationRollback { public: - explicit OwnerRegistrationRollback(std::vector &queues) + explicit OwnerRegistrationRollback(std::vector &queues) : queues_(queues), initial_queue_count_(queues.size()) {} OwnerRegistrationRollback(const OwnerRegistrationRollback &) = delete; OwnerRegistrationRollback &operator=(const OwnerRegistrationRollback &) = delete; @@ -1151,7 +1191,7 @@ uint64_t CommandProcessor::register_queue(AqlQueueConfig config, bool fanout_rep void commit() { committed_ = true; } private: - std::vector &queues_; + std::vector &queues_; std::size_t initial_queue_count_ = 0; bool committed_ = false; }; @@ -1179,7 +1219,7 @@ uint64_t CommandProcessor::register_queue(AqlQueueConfig config, bool fanout_rep // bug rather than a crash, which is precisely what an assert compiled out of a // release build would let through. Enforced in every build for that reason. std::lock_guard lock(hw_queue_mutex_); - if (find_aql_queue(queue.queue_id, queue.process_id) != nullptr) + if (find_compute_queue(queue.queue_id, queue.process_id) != nullptr) return 0; queue.registration_id = next_queue_registration_id_++; if (next_queue_registration_id_ == 0) @@ -1196,7 +1236,7 @@ uint64_t CommandProcessor::register_queue(AqlQueueConfig config, bool fanout_rep for (uint32_t rank = 0; rank < num_xcds; ++rank) { if (rank == xcd_rank_) continue; - AqlQueueConfig replica = queue; + ComputeQueueConfig replica = queue; replica.xcd_fanout = false; if (xcd_peers_[rank]->register_queue(std::move(replica), true) == 0) return 0; @@ -1206,10 +1246,10 @@ uint64_t CommandProcessor::register_queue(AqlQueueConfig config, bool fanout_rep { std::unique_lock structure_lock(queue_structure_mutex_); std::lock_guard lock(hw_queue_mutex_); - if (find_aql_queue(queue.queue_id, queue.process_id) != nullptr) + if (find_compute_queue(queue.queue_id, queue.process_id) != nullptr) return 0; - OwnerRegistrationRollback owner_rollback(aql_queues_); - aql_queues_.push_back(std::move(queue)); + OwnerRegistrationRollback owner_rollback(compute_queues_); + compute_queues_.push_back(std::move(queue)); // KFD queues rely on the VM-level primary (rj_vm.cpp); only internal test // queues (no host-accessible queue anywhere on this CP) need the CP to own the // primary lifecycle. Gate on the same aggregate predicate as the teardown @@ -1264,9 +1304,9 @@ void CommandProcessor::drain_doorbell_inbox() { std::lock_guard lock(hw_queue_mutex_); for (const DoorbellNotification ¬ification : notifications) { - const std::vector::iterator queue = std::ranges::find( - aql_queues_, notification.registration_id, &AqlQueueRecord::registration_id); - if (queue != aql_queues_.end()) + const std::vector::iterator queue = std::ranges::find( + compute_queues_, notification.registration_id, &ComputeQueueRecord::registration_id); + if (queue != compute_queues_.end()) queue->last_doorbell = notification.value; } } @@ -1275,11 +1315,11 @@ bool CommandProcessor::signal_queue_exception(uint32_t queue_id, uint32_t proces uint64_t status, bool publish_interrupt) { { std::lock_guard lk(hw_queue_mutex_); - auto queue = - std::find_if(aql_queues_.begin(), aql_queues_.end(), [&](const AqlQueueRecord &candidate) { + auto queue = std::find_if( + compute_queues_.begin(), compute_queues_.end(), [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == queue_id && candidate.process_id == process_id; }); - if (queue == aql_queues_.end() || queue->exception_status_va == 0) + if (queue == compute_queues_.end() || queue->exception_status_va == 0) return false; queue->exception_suspended = true; } @@ -1302,7 +1342,7 @@ bool CommandProcessor::signal_queue_exception(uint32_t queue_id, uint32_t proces } bool CommandProcessor::publish_queue_exception(uint32_t queue_id, uint32_t process_id, - uint64_t status) { + uint64_t status, bool wait_for_ack) { if (!gpu_vm_) return false; @@ -1313,10 +1353,10 @@ bool CommandProcessor::publish_queue_exception(uint32_t queue_id, uint32_t proce std::chrono::milliseconds ack_timeout; { std::lock_guard lk(hw_queue_mutex_); - auto queue = std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + auto queue = std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == queue_id && candidate.process_id == process_id; }); - if (queue == aql_queues_.end() || queue->exception_status_va == 0) + if (queue == compute_queues_.end() || queue->exception_status_va == 0) return false; exception_status_va = queue->exception_status_va; exception_event_id = queue->exception_event_id; @@ -1334,6 +1374,8 @@ bool CommandProcessor::publish_queue_exception(uint32_t queue_id, uint32_t proce combined_status) != VmAccessOutcome::Complete) return false; interrupt_sink.deliver(process_id, exception_event_id); + if (!wait_for_ack) + return true; const auto deadline = std::chrono::steady_clock::now() + ack_timeout; AtomicLoadResult exception_status = gpu_vm_->atomic_load(address_space, exception_status_va, sizeof(uint64_t)); @@ -1348,6 +1390,12 @@ bool CommandProcessor::publish_queue_exception(uint32_t queue_id, uint32_t proce QueuePrepareCloseStatus CommandProcessor::prepare_unregister_queue_registration(uint64_t registration_id) noexcept { + if (!engine()) { + std::lock_guard lock(hw_queue_mutex_); + for (auto &queue : compute_queues_) + queue.command_retry_pending = false; + service_command_streams(0); + } return close_queue_registration(registration_id, false); } @@ -1374,12 +1422,13 @@ QueuePrepareCloseStatus CommandProcessor::close_queue_registration(uint64_t regi // way -- a CU-side call back into the CP while the wave-state lock is held // would deadlock a DESTROY_QUEUE against the engine worker. std::lock_guard lock(hw_queue_mutex_); - const std::vector::iterator queue = - std::ranges::find(aql_queues_, registration_id, &AqlQueueRecord::registration_id); - if (queue == aql_queues_.end()) + const std::vector::iterator queue = + std::ranges::find(compute_queues_, registration_id, &ComputeQueueRecord::registration_id); + if (queue == compute_queues_.end()) return QueuePrepareCloseStatus::Faulted; if (!force) { - if (queue->publication_faulted) + if (queue->publication_faulted || queue->command_fault_pending || + (queue->faulted && (!queue->entries.empty() || !queue->dispatches.entries.empty()))) return QueuePrepareCloseStatus::Faulted; retry_close = queue->read_pointer_journal.publication_pending() || queue->publication_retry_pending || queue->idle_publication.active() || @@ -1387,7 +1436,7 @@ QueuePrepareCloseStatus CommandProcessor::close_queue_registration(uint64_t regi // while allocating backing. Only an unfinished notification publication // must delay that removal. queue->scratch_request.publication_pending() || - queue->scratch_reclaim.active() || !queue->entries.empty(); + queue->scratch_reclaim.active() || queue->has_pending_commands(); } if (!retry_close) { queue_id = queue->queue_id; @@ -1405,6 +1454,9 @@ QueuePrepareCloseStatus CommandProcessor::close_queue_registration(uint64_t regi drop_replicas = queue->xcd_fanout; for (const DispatchEntry &entry : queue->entries) dispatch_launch_metadata_.erase(entry.dispatch_id); + // Cancel every child stream before releasing the queue's VM lease. + if (!queue->commands.submissions.empty() || !queue->dispatches.entries.empty()) + fail_pm4_queue(*queue, queue->dispatches); // Any shares still unpublished here are simply dropped. They cannot be // credited to the grid from this thread: publish_share is the release edge // that must follow this XCD's cache write-back, and flushing walks cus_, @@ -1416,7 +1468,7 @@ QueuePrepareCloseStatus CommandProcessor::close_queue_registration(uint64_t regi // every XCD at once: the KFD paths sweep all command processors, and an // owner cascades to its replicas below. No XCD is left holding a grid that // can no longer retire. Unregistering a lone replica is not supported. - aql_queues_.erase(queue); + compute_queues_.erase(queue); } } if (retry_close) { @@ -1453,7 +1505,7 @@ void CommandProcessor::unregister_queue(uint32_t queue_id, uint32_t process_id) uint64_t registration_id = 0; { std::lock_guard lock(hw_queue_mutex_); - const AqlQueueRecord *queue = find_aql_queue(queue_id, process_id); + const ComputeQueueRecord *queue = find_compute_queue(queue_id, process_id); if (queue != nullptr) registration_id = queue->registration_id; } @@ -1464,7 +1516,7 @@ void CommandProcessor::set_queue_cu_selection(uint32_t queue_id, uint32_t proces const QueueCuSelection &enabled_cus) { { std::lock_guard lock(hw_queue_mutex_); - for (auto &queue : aql_queues_) { + for (auto &queue : compute_queues_) { if (queue.queue_id == queue_id && queue.process_id == process_id) queue.enabled_cus = enabled_cus; } @@ -1478,7 +1530,7 @@ bool CommandProcessor::update_queue(uint32_t queue_id, uint32_t process_id, uint uint64_t registration_id = 0; { std::lock_guard lock(hw_queue_mutex_); - const AqlQueueRecord *queue = find_aql_queue(queue_id, process_id); + const ComputeQueueRecord *queue = find_compute_queue(queue_id, process_id); if (queue != nullptr) registration_id = queue->registration_id; } @@ -1489,7 +1541,7 @@ bool CommandProcessor::update_queue_registration(uint64_t registration_id, uint6 uint32_t ring_size, uint32_t queue_percentage) { if (registration_id == 0) return false; - if (!valid_aql_packet_ring(ring_base_va, ring_size)) + if (ring_base_va == 0 || ring_base_va % 4 || ring_size < 4 || ring_size % 4) return false; const bool suspended = queue_percentage == 0; @@ -1501,8 +1553,19 @@ bool CommandProcessor::update_queue_registration(uint64_t registration_id, uint6 uint32_t process_id = 0; { std::lock_guard lock(hw_queue_mutex_); - for (auto &q : aql_queues_) { + for (auto &q : compute_queues_) { if (q.registration_id == registration_id) { + if (q.packet_format == QueuePacketFormat::Aql && + !valid_aql_packet_ring(ring_base_va, ring_size)) + return false; + if (q.packet_format == QueuePacketFormat::Pm4 && + (q.ring_base_va != ring_base_va || q.ring_size != ring_size)) { + if (q.has_pending_commands()) + return false; + q.read_pointer_journal.reset(); + q.command_access.reset(); + q.last_doorbell = ~uint64_t(0); + } found = true; queue_id = q.queue_id; process_id = q.process_id; @@ -1549,7 +1612,7 @@ bool CommandProcessor::update_queue_registration(uint64_t registration_id, uint6 } // Runtime resume has to release deferred queue work the same way a debugger // resume does, or already-fetched work sits until the next doorbell. - if (wake_command_processor && engine()) + if ((wake_command_processor || (changed && !suspended)) && engine()) engine()->schedule_event_now(&doorbell_event_); return found; } @@ -1559,7 +1622,7 @@ void CommandProcessor::set_queue_debug_suspended(uint32_t queue_id, uint32_t pro bool wake_command_processor = false; { std::lock_guard lock(hw_queue_mutex_); - for (AqlQueueRecord &q : aql_queues_) { + for (ComputeQueueRecord &q : compute_queues_) { if (q.queue_id == queue_id && q.process_id == process_id) { const bool exception_resolved = resolve_exception && q.exception_suspended; if (q.debug_suspended == suspended && !exception_resolved) @@ -1594,7 +1657,7 @@ void CommandProcessor::set_queue_debug_suspended(uint32_t queue_id, uint32_t pro void CommandProcessor::set_doorbell_base(uint32_t process_id, void *base) { std::lock_guard lock(hw_queue_mutex_); - for (auto &q : aql_queues_) { + for (auto &q : compute_queues_) { if (q.process_id == process_id) q.doorbell_base = base; } @@ -1716,7 +1779,7 @@ VmAccessOutcome CommandProcessor::write_gpu_block(const GpuVmAccess &access, uin bool CommandProcessor::scan_doorbells() { bool found = false; std::lock_guard lock(hw_queue_mutex_); - for (auto &q : aql_queues_) { + for (auto &q : compute_queues_) { // A replica shares the owner's ring and doorbell. Only the owning XCD may // consume them, or every XCD would dispatch the whole grid. if (q.fanout_replica) @@ -1818,16 +1881,16 @@ void CommandProcessor::doorbell_poll_loop(std::stop_token stop) { std::vector> idle_queues; { std::lock_guard lock(hw_queue_mutex_); - for (size_t queue_index = 0; queue_index < aql_queues_.size(); ++queue_index) { + for (size_t queue_index = 0; queue_index < compute_queues_.size(); ++queue_index) { // A replica does not own the queue, so it must not report it idle: its // shards drain before the owner's and the same KFD queue would otherwise // raise this from several CPs at once. - if (aql_queues_[queue_index].fanout_replica) + if (compute_queues_[queue_index].fanout_replica) continue; - if (aql_queues_[queue_index].entries.empty() && - aql_queues_[queue_index].process_id != 0) { - idle_queues.emplace_back(aql_queues_[queue_index].interrupt_sink, - aql_queues_[queue_index].process_id); + if (!compute_queues_[queue_index].has_pending_commands() && + compute_queues_[queue_index].process_id != 0) { + idle_queues.emplace_back(compute_queues_[queue_index].interrupt_sink, + compute_queues_[queue_index].process_id); } } } @@ -1837,7 +1900,7 @@ void CommandProcessor::doorbell_poll_loop(std::stop_token stop) { if (poll_count % 5000 == 1) { std::lock_guard lock(hw_queue_mutex_); - for (auto &q : aql_queues_) { + for (auto &q : compute_queues_) { uint64_t current = q.last_doorbell; if (q.doorbell_mode == QueueDoorbellMode::HostPolled && q.doorbell_base) { current = std::atomic_ref( @@ -1863,24 +1926,24 @@ void CommandProcessor::doorbell_poll_loop(std::stop_token stop) { } } -AqlQueueRecord *CommandProcessor::schedule_next_queue() { - if (aql_queues_.empty()) +ComputeQueueRecord *CommandProcessor::schedule_next_queue() { + if (compute_queues_.empty()) return nullptr; size_t start = next_queue_idx_; - for (size_t i = 0; i < aql_queues_.size(); ++i) { - size_t idx = (start + i) % aql_queues_.size(); - auto &qs = aql_queues_[idx]; + for (size_t i = 0; i < compute_queues_.size(); ++i) { + size_t idx = (start + i) % compute_queues_.size(); + auto &qs = compute_queues_[idx]; if (qs.faulted || qs.suspended()) continue; if (qs.next_dispatch_idx < qs.entries.size()) { - next_queue_idx_ = (idx + 1) % aql_queues_.size(); + next_queue_idx_ = (idx + 1) % compute_queues_.size(); return &qs; } } return nullptr; } -bool CommandProcessor::barrier_satisfied(const AqlQueueRecord &qs, size_t idx) const { +bool CommandProcessor::barrier_satisfied(const ComputeQueueRecord &qs, size_t idx) const { if (idx == 0 && !qs.implicit_barrier_next) return true; @@ -1903,8 +1966,8 @@ void CommandProcessor::drain_pending_wg_completions() { break; mark_cluster_workgroup_complete(completion.dispatch_id, completion.wg_id); if (completion_) - completion_->notify_wg_complete(completion.dispatch_id, completion.wg_id, aql_queues_); - for (auto &queue : drm_queues_) + completion_->notify_wg_complete(completion.dispatch_id, completion.wg_id, compute_queues_); + for (auto &queue : compute_queues_) for (auto &entry : queue.dispatches.entries) if (entry.dispatch_id == completion.dispatch_id) { ++entry.completed_wgs; @@ -2119,9 +2182,9 @@ void CommandProcessor::mark_cluster_workgroup_complete(uint32_t dispatch_id, uin } // The waves halted (and freed) before their pin was released, so reclaim each -// peer CU's LDS once the whole cluster is done. Runs with -// cluster_placements_mutex_ released: maybe_reset_lds_alloc() reaches the CU's -// wave-state lock, which is ordered ahead of it. +// peer CU's LDS once the whole cluster is done. Keep CU callbacks outside +// cluster_placements_mutex_. maybe_reset_lds_alloc() now checks the atomic +// activity count and LDS pin flag without taking the wave-state lock. void CommandProcessor::release_cluster_lds_pins( const std::vector> &unpin) { for (const auto &[cu, cluster_key] : unpin) { @@ -2131,8 +2194,8 @@ void CommandProcessor::release_cluster_lds_pins( } void CommandProcessor::erase_cluster_workgroup(uint32_t dispatch_id, uint32_t wg_id) { - // maybe_reset_lds_alloc() takes the CU's wave-state lock, so it runs after the - // placements lock is dropped -- see cluster_placements_mutex_. + // Collect pins under the placements lock, then release them and check for + // LDS reclamation outside it, preserving the CU callback ordering. std::vector> unpin; { std::lock_guard lock(cluster_placements_mutex_); @@ -2264,10 +2327,20 @@ CommandProcessor::dispatch_workgroups(DispatchEntry &entry) { // maybe_reset_lds_alloc() cannot reach it; release_wgp_workgroup() is the // matching release (the same call notify_wg_complete uses on the normal path). // Without the WGP release a failed WGP dispatch would permanently pin that WGP. - std::vector wg_wavefronts; - wg_wavefronts.reserve(entry.wfs_per_workgroup); + // Most workgroups need at most 32 waves. Keep their temporary reservation + // list local, with dynamic storage for larger internal workgroups. + std::array local_wavefronts; + std::vector large_wavefronts; + std::span wg_wavefronts; + if (entry.wfs_per_workgroup <= local_wavefronts.size()) { + wg_wavefronts = std::span(local_wavefronts).first(entry.wfs_per_workgroup); + } else { + large_wavefronts.resize(entry.wfs_per_workgroup); + wg_wavefronts = large_wavefronts; + } + uint32_t reserved_wavefronts = 0; const auto free_reserved = [&]() { - for (auto *claimed : wg_wavefronts) + for (auto *claimed : wg_wavefronts.first(reserved_wavefronts)) cu->free_wavefront_resources(*claimed); if (entry.wgp_mode) { for (auto *spi : spis_) @@ -2285,7 +2358,7 @@ CommandProcessor::dispatch_workgroups(DispatchEntry &entry) { free_reserved(); return VmAccessOutcome::Malformed; } - wg_wavefronts.push_back(wf); + wg_wavefronts[reserved_wavefronts++] = wf; } for (uint32_t w = 0; w < entry.wfs_per_workgroup; ++w) { Wavefront *wf = wg_wavefronts[w]; @@ -2321,9 +2394,9 @@ CommandProcessor::dispatch_workgroups(DispatchEntry &entry) { entry.num_named_barriers); register_cluster_workgroup(entry, local_wg_id, global_wg_id, cu, lds_base); - plugin_group_->onAmdgpuWorkgroupDispatched( - entry.dispatch_id, global_wg_id, cu->vgpr_allocation_block_size(), - cu->sgpr_allocation_block_size(), std::span(wg_wavefronts)); + plugin_group_->onAmdgpuWorkgroupDispatched(entry.dispatch_id, global_wg_id, + cu->vgpr_allocation_block_size(), + cu->sgpr_allocation_block_size(), wg_wavefronts); for (auto *wf : wg_wavefronts) plugin_group_->onAmdgpuWavefrontDispatched(*wf); @@ -2472,7 +2545,7 @@ bool CommandProcessor::drain_completions() { bool stop_processing = false; bool retry_pending = false; for (;;) { - const CompletionDrainResult result = completion_->drain_completions(aql_queues_); + const CompletionDrainResult result = completion_->drain_completions(compute_queues_); retry_pending |= result.retry_pending; if (!result.terminal_fault) break; @@ -2480,21 +2553,21 @@ bool CommandProcessor::drain_completions() { stop_processing = true; const CompletionDrainFault &fault = *result.terminal_fault; if (!fault.queue_idle) { - const std::vector::iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + const std::vector::iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == fault.queue_id && candidate.process_id == fault.process_id; }); - if (queue != aql_queues_.end()) + if (queue != compute_queues_.end()) queue->publication_faulted = true; notify_dispatch_vm_fault(fault.queue_id, fault.process_id, fault.dispatch_id, fault.outcome); continue; } - const std::vector::iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + const std::vector::iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == fault.queue_id && candidate.process_id == fault.process_id; }); - if (queue != aql_queues_.end()) { + if (queue != compute_queues_.end()) { queue->publication_faulted = true; queue->faulted = true; } @@ -2529,13 +2602,13 @@ bool CommandProcessor::fault_dispatch_local(uint32_t queue_id, uint32_t process_ } std::lock_guard lock(hw_queue_mutex_); - const std::vector::iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + const std::vector::iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == queue_id && candidate.process_id == process_id; }); - if (queue == aql_queues_.end()) + if (queue == compute_queues_.end()) return false; - AqlQueueRecord &state = *queue; + ComputeQueueRecord &state = *queue; size_t index = 0; for (std::deque::iterator entry = state.entries.begin(); @@ -2558,6 +2631,10 @@ bool CommandProcessor::fault_dispatch_local(uint32_t queue_id, uint32_t process_ dispatch_launch_metadata_.erase(static_cast(dispatch_id)); queue->faulted = true; + // A peer fault can cancel an AQL entry while its vendor PM4 stream is blocked. + // Release that stream and its VM snapshot along with the owning dispatch. + if (state.packet_format == QueuePacketFormat::Aql && !state.commands.submissions.empty()) + fail_pm4_queue(state, state.dispatches); util::Logger::cp([&](auto &os) { os << std::format("{}: terminal VM fault pid={} qid={} dispatch={} outcome={}", name(), process_id, queue_id, dispatch_id, static_cast(outcome)); @@ -2605,9 +2682,9 @@ void CommandProcessor::on_cu_idle() { if (!drain_completions()) return; - // Retire any non-kernel entries (barrier-kind packets) that are now at - // the head, then drain again so a dependent kernel behind them can proceed. - for (AqlQueueRecord &qs : aql_queues_) { + // Execute and retire eligible non-kernel entries at the head, then drain + // again so a dependent kernel behind them can proceed. + for (ComputeQueueRecord &qs : compute_queues_) { if (qs.faulted || qs.suspended() || qs.publication_retry_pending) continue; while (qs.next_dispatch_idx < qs.entries.size()) { @@ -2616,6 +2693,8 @@ void CommandProcessor::on_cu_idle() { break; if (!e.is_non_kernel()) break; + if (!execute_aql_pm4(qs, e, engine()->context(partition_id()).current_tick())) + break; const uint32_t queue_id = e.queue_id; const uint32_t process_id = e.process_id; const uint32_t dispatch_id = e.dispatch_id; @@ -2649,7 +2728,7 @@ void CommandProcessor::on_cu_idle() { std::vector was_idle(cus_.size()); for (size_t i = 0; i < cus_.size(); ++i) was_idle[i] = cus_[i]->is_idle(); - for (AqlQueueRecord &qs : aql_queues_) { + for (ComputeQueueRecord &qs : compute_queues_) { if (qs.faulted || qs.suspended() || qs.publication_retry_pending) continue; if (qs.next_dispatch_idx < qs.entries.size()) { @@ -2701,8 +2780,8 @@ bool CommandProcessor::step() { void CommandProcessor::process_queues() { if (engine()) - service_drm_queues(engine()->context(partition_id()).current_tick()); - for (AqlQueueRecord &qs : aql_queues_) { + service_command_streams(engine()->context(partition_id()).current_tick()); + for (ComputeQueueRecord &qs : compute_queues_) { if (qs.faulted || qs.suspended() || qs.publication_retry_pending) continue; while (qs.next_dispatch_idx < qs.entries.size()) { @@ -2712,10 +2791,12 @@ void CommandProcessor::process_queues() { break; // Stalled on barrier bit. if (entry.is_non_kernel()) { + if (!execute_aql_pm4(qs, entry, engine()->context(partition_id()).current_tick())) + break; const uint32_t queue_id = entry.queue_id; const uint32_t process_id = entry.process_id; const uint32_t dispatch_id = entry.dispatch_id; - entry.completed_wgs = entry.total_wgs; // 0 == 0, immediately complete. + entry.completed_wgs = entry.total_wgs; const VmAccessOutcome outcome = completion_ ? completion_->complete_non_kernel(entry) : VmAccessOutcome::Complete; if (outcome == VmAccessOutcome::Unavailable) { @@ -2748,28 +2829,15 @@ void CommandProcessor::process_queues() { } } -bool CommandProcessor::has_runnable_cus() const { - for (auto *cu : cus_) { - if (cu->has_runnable_wfs()) - return true; - } - return false; -} - -void CommandProcessor::refresh_pooled_due_ticks(simdojo::Tick now) { - for (auto it = pooled_due_ticks_.begin(); it != pooled_due_ticks_.end();) { - if (!it->first->has_runnable_wfs()) - it = pooled_due_ticks_.erase(it); - else - ++it; - } - for (auto *cu : cus_) - if (cu->has_runnable_wfs()) - pooled_due_ticks_.try_emplace(cu, now + 1); +void CommandProcessor::prune_pooled_due_ticks() { + // Admission and resume notify on_cu_pool_ready(), which inserts runnable + // CUs. Only previously scheduled CUs can need pruning after a pause or + // cancellation; idle CUs need no wave-state lock or slot scan. + std::erase_if(pooled_due_ticks_, [](const auto &due) { return !due.first->has_runnable_wfs(); }); } -simdojo::Tick CommandProcessor::next_pooled_due_tick(simdojo::Tick now) { - refresh_pooled_due_ticks(now); +simdojo::Tick CommandProcessor::next_pooled_due_tick() { + prune_pooled_due_ticks(); simdojo::Tick next = simdojo::TICK_MAX; for (const auto &[_, tick] : pooled_due_ticks_) next = std::min(next, tick); @@ -2777,21 +2845,23 @@ simdojo::Tick CommandProcessor::next_pooled_due_tick(simdojo::Tick now) { } FunctionalQuantumResult CommandProcessor::run_active_cus_once(simdojo::Tick now) { - refresh_pooled_due_ticks(now); + prune_pooled_due_ticks(); active_cu_scratch_.clear(); + // Pruning already checked each CU's runnable-wave count. Keep SPI order + // while selecting due work without repeating those checks. + const auto append_due = [&](const auto &cus) { + for (auto *cu : cus) { + const auto due = pooled_due_ticks_.find(cu); + if (due != pooled_due_ticks_.end() && due->second <= now) + active_cu_scratch_.push_back(cu); + } + }; if (!spis_.empty()) { for (auto *spi : spis_) - spi->append_active_cus(active_cu_scratch_); + append_due(spi->compute_units()); } else { - for (auto *cu : cus_) { - if (cu->has_runnable_wfs()) - active_cu_scratch_.push_back(cu); - } + append_due(cus_); } - std::erase_if(active_cu_scratch_, [&](ComputeUnitCore *cu) { - auto due = pooled_due_ticks_.find(cu); - return !cu->has_runnable_wfs() || due == pooled_due_ticks_.end() || due->second > now; - }); if (active_cu_scratch_.empty()) return {}; @@ -2823,54 +2893,112 @@ FunctionalQuantumResult CommandProcessor::run_active_cus_once(simdojo::Tick now) return result; } -bool CommandProcessor::register_drm_queue(Pm4SubmitQueue queue) { - std::unique_lock structure_lock(queue_structure_mutex_); - std::lock_guard lock(hw_queue_mutex_); - if (!queue.address_space) - queue.address_space = default_address_space_; - if (!gpu_vm_ || !queue.pm4 || !queue.address_space) - return false; - queue.binding = gpu_vm_->retain_binding(queue.address_space); - if (!queue.binding || std::ranges::any_of(drm_queues_, [&](const auto &existing) { - return existing.queue_id == queue.queue_id && existing.process_id == queue.process_id; - })) - return false; - drm_queues_.push_back(std::move(queue)); - return true; +bool CommandProcessor::register_drm_queue(ComputeQueueConfig config) { + config.packet_format = QueuePacketFormat::Pm4; + config.submission_queue = true; + return register_queue(std::move(config)) != 0; } void CommandProcessor::unregister_drm_queues(uint32_t process_id) { - std::unique_lock structure_lock(queue_structure_mutex_); - std::lock_guard lock(hw_queue_mutex_); - for (auto &queue : drm_queues_) - if (queue.process_id == process_id) - fail_pm4_queue(queue, queue.dispatches); - std::erase_if(drm_queues_, [&](const auto &queue) { return queue.process_id == process_id; }); - if (engine()) - engine()->schedule_event_now(doorbell_event()); + std::vector ids; + { + std::lock_guard lock(hw_queue_mutex_); + for (const auto &queue : compute_queues_) + if (queue.submission_queue && queue.process_id == process_id) + ids.push_back(queue.registration_id); + } + for (const auto id : ids) + (void)unregister_queue_registration(id); } void CommandProcessor::unregister_drm_queue(uint32_t queue_id, uint32_t process_id) { - std::unique_lock structure_lock(queue_structure_mutex_); - std::lock_guard lock(hw_queue_mutex_); - const auto it = std::ranges::find_if(drm_queues_, [&](const auto &queue) { - return queue.queue_id == queue_id && queue.process_id == process_id; - }); - if (it == drm_queues_.end()) + unregister_queue(queue_id, process_id); +} + +void CommandProcessor::service_pm4_ring(ComputeQueueRecord &queue, simdojo::Tick now) { + if (queue.packet_format != QueuePacketFormat::Pm4 || queue.submission_queue || + queue.fanout_replica || queue.faulted || queue.command_retry_pending) return; - fail_pm4_queue(*it, it->dispatches); - drm_queues_.erase(it); - if (engine()) - engine()->schedule_event_now(doorbell_event()); + auto outcome = queue.read_pointer_journal.publish(); + if (outcome == VmAccessOutcome::Unavailable) { + queue.command_retry_pending = true; + arm_stall_recheck(now); + return; + } + if (outcome != VmAccessOutcome::Complete) { + queue.publication_faulted = true; + fail_pm4_queue(queue, queue.dispatches); + return; + } + if (queue.suspended()) { + queue.debug_work_deferred = true; + return; + } + if (!queue.commands.submissions.empty()) + return; + if (queue.last_doorbell == ~uint64_t(0)) + return; + if (!queue.command_access) + queue.command_access = gpu_vm_->snapshot_pinned(queue.address_space); + const auto &access = queue.command_access; + if (!access) { + fail_pm4_queue(queue, queue.dispatches); + return; + } + // Publication replaces the GART binding; an unready pinned snapshot cannot observe it. + if (!access->info().ready) { + queue.command_access.reset(); + queue.command_retry_pending = true; + arm_stall_recheck(now); + return; + } + outcome = queue.read_pointer_journal.initialize(*access); + if (outcome == VmAccessOutcome::Unavailable) { + queue.command_retry_pending = true; + arm_stall_recheck(now); + return; + } + const auto consumer = queue.read_pointer_journal.cursor(); + const auto producer = + normalize_pm4_producer_cursor(queue.last_doorbell, consumer, queue.ring_size / 4); + if (outcome != VmAccessOutcome::Complete || !producer) { + fail_pm4_queue(queue, queue.dispatches); + return; + } + if (*producer == consumer) { + queue.command_access.reset(); + return; + } + Pm4Submission submission; + submission.buffers.push_back({.address = consumer * 4, + .dwords = static_cast(*producer - consumer), + .ring_base = queue.ring_base_va, + .ring_bytes = queue.ring_size}); + submission.failure->wake = [this] { + if (engine()) + engine()->schedule_event_now(doorbell_event()); + }; + queue.commands.submissions.push_back(std::move(submission)); } -void CommandProcessor::service_drm_queues(simdojo::Tick now) { - for (auto &queue : drm_queues_) { +void CommandProcessor::service_command_streams(simdojo::Tick now) { + for (auto &queue : compute_queues_) { + // AQL vendor streams advance only while their owning entry is eligible. + if (queue.packet_format == QueuePacketFormat::Aql) + continue; auto &state = queue.dispatches; - if (queue.faulted) + if (queue.command_fault_pending) { + queue.command_fault_pending = false; + fail_pm4_queue(queue, state); + } + // Publication of a committed packet must finish even while execution is paused. + service_pm4_ring(queue, now); + if (queue.suspended() && !queue.commands.submissions.empty()) + queue.debug_work_deferred = true; + if (queue.faulted || queue.suspended() || queue.command_retry_pending) continue; - if (!queue.pm4->submissions.empty() && - queue.pm4->submissions.front().failure->failed.load(std::memory_order_acquire)) { + if (!queue.commands.submissions.empty() && + queue.commands.submissions.front().failure->failed.load(std::memory_order_acquire)) { fail_pm4_queue(queue, state); continue; } @@ -2883,6 +3011,9 @@ void CommandProcessor::service_drm_queues(simdojo::Tick now) { state.entries.pop_front(); } fetch_pm4(queue, state, now); + service_pm4_ring(queue, now); + if (!queue.commands.submissions.empty() && state.entries.empty() && !queue.faulted) + arm_stall_recheck(now); if (queue.faulted || state.entries.empty()) continue; auto &entry = state.entries.front(); @@ -2894,7 +3025,7 @@ void CommandProcessor::service_drm_queues(simdojo::Tick now) { std::max(scratch_wave_divisor_, scratch_shader_engine_count_) * dispatch.private_segment_fixed_size * dispatch.kernel_wave_size; }; - const bool busy = std::ranges::any_of(drm_queues_, [&](const auto &other) { + const bool busy = std::ranges::any_of(compute_queues_, [&](const auto &other) { if (&other == &queue || other.address_space != queue.address_space) return false; return std::ranges::any_of(other.dispatches.entries, [&](const auto &active) { @@ -2927,10 +3058,10 @@ bool CommandProcessor::submit_pm4(uint32_t queue_id, uint32_t process_id, Pm4Submission submission) { { std::lock_guard lock(hw_queue_mutex_); - auto it = std::ranges::find_if(drm_queues_, [&](const Pm4SubmitQueue &queue) { + auto it = std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &queue) { return queue.queue_id == queue_id && queue.process_id == process_id; }); - if (it == drm_queues_.end() || !it->pm4) + if (it == compute_queues_.end() || !it->submission_queue) throw std::runtime_error("PM4 submission has no registered queue"); if (it->faulted) return false; @@ -2938,7 +3069,7 @@ bool CommandProcessor::submit_pm4(uint32_t queue_id, uint32_t process_id, if (engine()) engine()->schedule_event_now(doorbell_event()); }; - it->pm4->submissions.push_back(std::move(submission)); + it->commands.submissions.push_back(std::move(submission)); if (!is_primary_ && engine() && !has_kfd_queues()) { engine()->register_as_primary(); is_primary_ = true; @@ -2949,15 +3080,18 @@ bool CommandProcessor::submit_pm4(uint32_t queue_id, uint32_t process_id, return true; } -void CommandProcessor::dispatch_pm4(const Pm4SubmitQueue &queue, Pm4DispatchState &qs, +void CommandProcessor::dispatch_pm4(const ComputeQueueRecord &queue, Pm4DispatchState &qs, const std::array &dimensions) { + if (!queue.commands.submissions.front().allow_dispatch) + throw std::runtime_error("shader dispatch inside an AQL PM4 IB is unsupported"); using namespace rocr::llvm::amdhsa; - const auto ®s = queue.pm4->sh_registers; + const auto ®s = queue.commands.sh_registers; const uint32_t initiator = dimensions[3]; if (!(initiator & 1)) return; - if (cus_.empty() || (initiator & (1u << 5))) - throw std::runtime_error("unsupported PM4 dispatch dimensions"); + if (cus_.empty()) + throw std::runtime_error("PM4 dispatch requires a compute unit"); + const bool thread_dimensions = initiator & (1u << 5); const auto arch = cus_[0]->config().arch; const uint32_t rsrc1 = regs[kPm4ComputePgmRsrc1], rsrc2 = regs[kPm4ComputePgmRsrc2]; DispatchEntry dp; @@ -2966,6 +3100,8 @@ void CommandProcessor::dispatch_pm4(const Pm4SubmitQueue &queue, Pm4DispatchStat dp.address_space = queue.address_space; dp.process_id = queue.process_id; dp.queue_id = queue.queue_id; + dp.interrupt_sink = queue.interrupt_sink; + dp.enabled_cus = queue.enabled_cus; dp.kernel_wave_size = (initiator & (1u << 15)) ? 32 : 64; if (AMDHSA_BITS_GET(rsrc2, COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT)) { dp.pm4_scratch_waves_per_se = regs[kPm4ComputeTmpringSize] & 0xfff; @@ -2990,7 +3126,7 @@ void CommandProcessor::dispatch_pm4(const Pm4SubmitQueue &queue, Pm4DispatchStat if (dp.num_user_sgprs > dp.user_sgprs.size()) throw std::runtime_error("PM4 launch exceeds compute user-data registers"); dp.pm4_abi = true; - dp.pm4_failure = queue.pm4->submissions.front().failure; + dp.pm4_failure = queue.commands.submissions.front().failure; std::ranges::copy_n(regs.begin() + kPm4ComputeUserData0, dp.num_user_sgprs, dp.user_sgprs.begin()); dp.sgprs_per_wf = cus_[0]->config().sgprs_per_wf; @@ -3009,10 +3145,26 @@ void CommandProcessor::dispatch_pm4(const Pm4SubmitQueue &queue, Pm4DispatchStat dp.enable_vgpr_workitem_id = AMDHSA_BITS_GET(rsrc2, COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID); dp.wgp_mode = AMDHSA_BITS_GET(rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE); dp.group_segment_fixed_size = AMDHSA_BITS_GET(rsrc2, COMPUTE_PGM_RSRC2_GRANULATED_LDS_SIZE) * 512; + for (uint32_t i = 0; i < 3; ++i) + if (!(regs[kPm4ComputeNumThreadX + i] & 0xffff) || + (regs[kPm4ComputeNumThreadX + i] & 0xffff) > 1024) + throw std::runtime_error("invalid PM4 workgroup dimension"); + dp.workgroup_size_x = regs[kPm4ComputeNumThreadX] & 0xffff; + dp.workgroup_size_y = regs[kPm4ComputeNumThreadY] & 0xffff; + dp.workgroup_size_z = regs[kPm4ComputeNumThreadZ] & 0xffff; + uint64_t threads = uint64_t{dp.workgroup_size_x} * dp.workgroup_size_y * dp.workgroup_size_z; + if (threads > 1024) + throw std::runtime_error("PM4 workgroup exceeds 1024 threads"); + dp.wfs_per_workgroup = (threads + dp.kernel_wave_size - 1) / dp.kernel_wave_size; std::array counts{}; for (uint32_t i = 0; i < 3; ++i) { dp.workgroup_origin[i] = (initiator & (1u << 2)) ? 0 : regs[kPm4ComputeStartX + i]; - counts[i] = dimensions[i] > dp.workgroup_origin[i] ? dimensions[i] - dp.workgroup_origin[i] : 0; + if (thread_dimensions && dp.workgroup_origin[i]) + throw std::runtime_error("thread-dimension PM4 dispatch requires a zero origin"); + const uint32_t full = regs[kPm4ComputeNumThreadX + i] & 0xffff; + const uint32_t end = + thread_dimensions ? dimensions[i] / full + (dimensions[i] % full != 0) : dimensions[i]; + counts[i] = end > dp.workgroup_origin[i] ? end - dp.workgroup_origin[i] : 0; } dp.grid_wgs_x = counts[0]; dp.grid_wgs_y = counts[1]; @@ -3025,17 +3177,6 @@ void CommandProcessor::dispatch_pm4(const Pm4SubmitQueue &queue, Pm4DispatchStat total *= counts[i]; } dp.total_wgs = total; - for (uint32_t i = 0; i < 3; ++i) - if (!(regs[kPm4ComputeNumThreadX + i] & 0xffff) || - (regs[kPm4ComputeNumThreadX + i] & 0xffff) > 1024) - throw std::runtime_error("invalid PM4 workgroup dimension"); - dp.workgroup_size_x = regs[kPm4ComputeNumThreadX] & 0xffff; - dp.workgroup_size_y = regs[kPm4ComputeNumThreadY] & 0xffff; - dp.workgroup_size_z = regs[kPm4ComputeNumThreadZ] & 0xffff; - uint64_t threads = uint64_t{dp.workgroup_size_x} * dp.workgroup_size_y * dp.workgroup_size_z; - if (threads > 1024) - throw std::runtime_error("PM4 workgroup exceeds 1024 threads"); - dp.wfs_per_workgroup = (threads + dp.kernel_wave_size - 1) / dp.kernel_wave_size; if (dp.pm4_scratch_pool && dp.total_wgs) { if (!dp.pm4_scratch_pool->available(dp.wfs_per_workgroup)) throw std::runtime_error("PM4 scratch cannot accommodate one workgroup"); @@ -3047,14 +3188,14 @@ void CommandProcessor::dispatch_pm4(const Pm4SubmitQueue &queue, Pm4DispatchStat VmAccessOutcome::Complete) throw std::runtime_error("PM4 scratch descriptor exceeds its mapped buffer"); } - if ((dp.grid_wgs_x && dp.workgroup_size_x > UINT32_MAX / dp.grid_wgs_x) || - (dp.grid_wgs_y && dp.workgroup_size_y > UINT32_MAX / dp.grid_wgs_y) || - (dp.grid_wgs_z && dp.workgroup_size_z > UINT32_MAX / dp.grid_wgs_z)) + if (!thread_dimensions && ((dp.grid_wgs_x && dp.workgroup_size_x > UINT32_MAX / dp.grid_wgs_x) || + (dp.grid_wgs_y && dp.workgroup_size_y > UINT32_MAX / dp.grid_wgs_y) || + (dp.grid_wgs_z && dp.workgroup_size_z > UINT32_MAX / dp.grid_wgs_z))) throw std::runtime_error("PM4 grid exceeds supported invocation count"); - dp.grid_size_x = dp.grid_wgs_x * dp.workgroup_size_x; - dp.grid_size_y = dp.grid_wgs_y * dp.workgroup_size_y; - dp.grid_size_z = dp.grid_wgs_z * dp.workgroup_size_z; - if (initiator & 2) { + dp.grid_size_x = thread_dimensions ? dimensions[0] : dp.grid_wgs_x * dp.workgroup_size_x; + dp.grid_size_y = thread_dimensions ? dimensions[1] : dp.grid_wgs_y * dp.workgroup_size_y; + dp.grid_size_z = thread_dimensions ? dimensions[2] : dp.grid_wgs_z * dp.workgroup_size_z; + if (!thread_dimensions && (initiator & 2)) { uint32_t *sizes[] = {&dp.grid_size_x, &dp.grid_size_y, &dp.grid_size_z}; for (uint32_t i = 0; i < 3; ++i) { uint32_t partial = regs[kPm4ComputeNumThreadX + i] >> 16; @@ -3091,8 +3232,11 @@ void CommandProcessor::dispatch_pm4(const Pm4SubmitQueue &queue, Pm4DispatchStat qs.push_entry(std::move(dp)); } -void CommandProcessor::fail_pm4_queue(Pm4SubmitQueue &queue, Pm4DispatchState &qs) { +void CommandProcessor::fail_pm4_queue(ComputeQueueRecord &queue, Pm4DispatchState &qs) { queue.faulted = true; + for (auto &entry : queue.entries) + if (entry.grid_completion) + entry.grid_completion->mark_faulted(); // The CP owns the queue lock and CU workers have rejoined. Stop all resident // waves before releasing the submission's BO references or publishing failure. for (auto *cu : cus_) { @@ -3107,322 +3251,37 @@ void CommandProcessor::fail_pm4_queue(Pm4SubmitQueue &queue, Pm4DispatchState &q } flush_gpu_caches(); qs.entries.clear(); - for (auto &submission : queue.pm4->submissions) + for (auto &submission : queue.commands.submissions) if (submission.complete) submission.complete(false); - queue.pm4->submissions.clear(); + queue.commands.submissions.clear(); + queue.command_access.reset(); } -void CommandProcessor::fetch_pm4(Pm4SubmitQueue &queue, Pm4DispatchState &qs, simdojo::Tick now) { - if (queue.faulted) - return; - auto &state = *queue.pm4; - if (!state.submissions.empty() && - state.submissions.front().failure->failed.load(std::memory_order_acquire)) { - fail_pm4_queue(queue, qs); - return; - } - if (!qs.entries.empty()) - return; - try { - const auto access = snapshot_gpu_access(queue.address_space); - if (!access) - throw std::runtime_error("PM4 queue has no GPU address space"); - // Bound one event's packet work, including IB chains. - for (uint32_t budget = 0; budget < 4096 && !state.submissions.empty(); ++budget) { - auto &submission = state.submissions.front(); - if (submission.ready && !submission.ready()) { - arm_stall_recheck(now); - return; - } - if (submission.buffers.empty()) { - flush_gpu_caches(); - if (submission.complete) - submission.complete(true); - state.submissions.pop_front(); - continue; - } - auto &ib = submission.buffers.front(); - if (!ib.dwords) { - submission.buffers.pop_front(); - continue; - } - uint32_t header = 0; - if (access->read(ib.address, {reinterpret_cast(&header), sizeof(header)}) != - VmAccessOutcome::Complete) - throw std::runtime_error("unmapped PM4 command buffer"); - const uint32_t type = header >> 30; - const uint32_t count = type == 2 || header == 0xffff1000 ? 1 : ((header >> 16) & 0x3fff) + 2; - if (count > ib.dwords || (type != 2 && type != 3)) - throw std::runtime_error("invalid PM4 packet size or type"); - std::vector words(count - 1); - if (!words.empty() && - access->read(ib.address + 4, {reinterpret_cast(words.data()), - words.size() * 4}) != VmAccessOutcome::Complete) - throw std::runtime_error("unmapped PM4 packet payload"); - ib.address += count * 4; - ib.dwords -= count; - if (type == 2 || header == 0xffff1000) - continue; - const auto require = [&](size_t size) { - if (words.size() != size) - throw std::runtime_error(std::format("PM4 opcode {:#x} expects {} payload words, got {}", - (header >> 8) & 0xff, size, words.size())); - }; - const auto address = [&](size_t index) { - return uint64_t{words[index]} | (uint64_t{words[index + 1]} << 32); - }; - const uint32_t opcode = (header >> 8) & 0xff; - util::Logger::vm("PM4 packet ", std::hex, opcode, std::dec, " words=", count); - switch (static_cast(opcode)) { - case Pm4Opcode::Nop: - break; - case Pm4Opcode::ContextControl: - case Pm4Opcode::ClearState: - case Pm4Opcode::PfpSyncMe: - case Pm4Opcode::SetContextReg: - case Pm4Opcode::SetContextRegPairs: - case Pm4Opcode::SetContextRegPairsPacked: - if (!submission.graphics_engine) - throw std::runtime_error("graphics state packet on compute engine"); - // Graphics context state is unused by compute shaders. The single CP - // retires preceding work before these packets, including PFP_SYNC_ME. - if (opcode == uint32_t(Pm4Opcode::ContextControl)) - require(2); - else if (opcode == uint32_t(Pm4Opcode::ClearState) || - opcode == uint32_t(Pm4Opcode::PfpSyncMe)) - require(1); - else if (words.size() < 2) - throw std::runtime_error("invalid graphics register payload"); - break; - case Pm4Opcode::SetBase: - require(3); - if (!submission.graphics_engine || words[0] != 1) - throw std::runtime_error("unsupported SET_BASE index"); - state.indirect_base = address(1); - break; - case Pm4Opcode::WriteData: { - if (words.size() < 4) - throw std::runtime_error("invalid WRITE_DATA payload"); - const uint32_t destination = (words[0] >> 8) & 15; - if ((destination != 1 && destination != 2 && destination != 5) || (words[0] & (1u << 16))) - throw std::runtime_error("unsupported WRITE_DATA destination"); - flush_gpu_caches(); - if (access->write(address(1), {reinterpret_cast(words.data() + 3), - (words.size() - 3) * 4}) != VmAccessOutcome::Complete) - throw std::runtime_error("PM4 WRITE_DATA failed"); - break; - } - case Pm4Opcode::ReleaseMem: { - require(7); - flush_gpu_caches(); - const uint32_t selection = words[1] >> 29; - uint64_t value = address(4); - if (selection == 3) - value = hsa_system_timestamp(); - else if (selection != 0 && selection != 1 && selection != 2) - throw std::runtime_error("unsupported RELEASE_MEM data source"); - const size_t bytes = selection == 0 ? 0 : selection == 1 ? 4 : 8; - if (bytes && access->write(address(2), {reinterpret_cast(&value), - bytes}) != VmAccessOutcome::Complete) - throw std::runtime_error("PM4 RELEASE_MEM failed"); - break; - } - case Pm4Opcode::CopyData: { - require(5); - const uint32_t source = words[0] & 15, destination = (words[0] >> 8) & 15; - const size_t bytes = (words[0] & (1u << 16)) ? 8 : 4; - if (destination != 1 && destination != 2 && destination != 5) - throw std::runtime_error("unsupported COPY_DATA destination"); - flush_gpu_caches(); - uint64_t value = 0; - if (source == 5) - value = address(1); - else if (source == 9) - value = hsa_system_timestamp(); - else if (source == 1 || source == 2) { - if (access->read(address(1), {reinterpret_cast(&value), bytes}) != - VmAccessOutcome::Complete) - throw std::runtime_error("PM4 COPY_DATA read failed"); - } else { - throw std::runtime_error("unsupported COPY_DATA source"); - } - if (access->write(address(3), {reinterpret_cast(&value), bytes}) != - VmAccessOutcome::Complete) - throw std::runtime_error("PM4 COPY_DATA write failed"); - break; - } - case Pm4Opcode::WaitRegMem: { - require(6); - if (((words[0] >> 4) & 3) != 1) - throw std::runtime_error("unsupported WAIT_REG_MEM register space"); - flush_gpu_caches(); - uint32_t value = 0; - if (access->read(address(1), {reinterpret_cast(&value), sizeof(value)}) != - VmAccessOutcome::Complete) - throw std::runtime_error("PM4 WAIT_REG_MEM read failed"); - value &= words[4]; - uint32_t reference = words[3] & words[4]; - bool ready; - switch (words[0] & 7) { - case 0: - ready = true; - break; - case 1: - ready = value < reference; - break; - case 2: - ready = value <= reference; - break; - case 3: - ready = value == reference; - break; - case 4: - ready = value != reference; - break; - case 5: - ready = value >= reference; - break; - case 6: - ready = value > reference; - break; - default: - throw std::runtime_error("invalid WAIT_REG_MEM comparison"); - } - if (!ready) { - ib.address -= count * 4; - ib.dwords += count; - arm_stall_recheck(now); - return; - } - break; - } - case Pm4Opcode::LoadShRegIndex: { - require(4); - const uint32_t first = words[2], count = words[3]; - // Direct-address mode, contiguous values (no register/value pairs). - if ((words[0] & 3) || !count || first >= state.sh_registers.size() || - count > state.sh_registers.size() - first) - throw std::runtime_error("unsupported LOAD_SH_REG_INDEX range or mode"); - flush_gpu_caches(); - if (access->read(address(0), - {reinterpret_cast(state.sh_registers.data() + first), - count * 4}) != VmAccessOutcome::Complete) - throw std::runtime_error("PM4 LOAD_SH_REG_INDEX read failed"); - break; - } - case Pm4Opcode::SetShReg: { // SET_SH_REG - if (words.size() < 2 || words[0] >= state.sh_registers.size() || - words.size() - 1 > state.sh_registers.size() - words[0]) - throw std::runtime_error("invalid SET_SH_REG range"); - std::ranges::copy(words.begin() + 1, words.end(), state.sh_registers.begin() + words[0]); - break; - } - case Pm4Opcode::SetShRegPairs: // SET_SH_REG_PAIRS - if (words.size() % 2) - throw std::runtime_error("invalid SET_SH_REG_PAIRS payload"); - for (size_t i = 0; i < words.size(); i += 2) { - if (words[i] >= state.sh_registers.size()) - throw std::runtime_error("invalid SET_SH_REG_PAIRS register"); - state.sh_registers[words[i]] = words[i + 1]; - } - break; - case Pm4Opcode::SetUconfigReg: // SET_UCONFIG_REG: graphics index selection, irrelevant to - // compute. - case Pm4Opcode::SetUconfigRegPairs: // SET_UCONFIG_REG_PAIRS - break; - case Pm4Opcode::AcquireMem: // ACQUIRE_MEM: earlier dispatches and DMA are already retired. - require(7); - flush_gpu_caches(); - break; - case Pm4Opcode::EventWrite: { - const uint32_t event = words[0] & 0x3f; - if (submission.graphics_engine && event == 56) { - // PIXEL_PIPE_STAT_CONTROL configures graphics counters, not a memory write. - require(3); - break; - } - require(1); - if (event != 7 && !(submission.graphics_engine && - (event == 15 || event == 16 || event == 25 || event == 26 || - event == 36 || event == 44 || event == 46))) - throw std::runtime_error(std::format("unsupported PM4 EVENT_WRITE event {}", event)); - flush_gpu_caches(); - break; - } - case Pm4Opcode::DmaData: { - require(6); - const uint32_t src_select = (words[0] >> 29) & 3; - const uint32_t dst_select = (words[0] >> 20) & 3; - const uint32_t bytes = words[5] & 0x3ffffff; - if (!bytes || dst_select == 2) // DMA drain or prefetch only. - break; - if (dst_select != 0 && dst_select != 3) - throw std::runtime_error("unsupported PM4 DMA destination"); - flush_gpu_caches(); - if (src_select == 2) { - std::vector data(bytes); - for (uint32_t i = 0; i < bytes; ++i) - data[i] = words[1] >> ((i % 4) * 8); - if (access->write(address(3), std::as_bytes(std::span(data))) != - VmAccessOutcome::Complete) - throw std::runtime_error("PM4 DMA fill failed"); - } else if (src_select == 0 || src_select == 3) { - std::vector data(bytes); - if (access->read(address(1), data) != VmAccessOutcome::Complete || - access->write(address(3), data) != VmAccessOutcome::Complete) - throw std::runtime_error("PM4 DMA copy failed"); - } else { - throw std::runtime_error("unsupported PM4 DMA source"); - } - break; - } - case Pm4Opcode::IndirectBuffer: { - require(3); - const bool chained = words[2] & (1u << 20); - const uint32_t depth = ib.depth + (chained ? 0 : 1); - if (++submission.indirect_expansions > Pm4Submission::kMaxIndirectExpansions || - depth >= Pm4Submission::kMaxIndirectDepth) { - fail_pm4_queue(queue, qs); - return; - } - // CHAIN jumps at the current IB level; a non-chained IB returns here. - // Other root IBs supplied by the same CS remain queued behind this one. - if (words[2] & (1u << 20)) - submission.buffers.pop_front(); - submission.buffers.push_front({address(0), words[2] & 0xfffff, depth}); - break; - } - case Pm4Opcode::DispatchDirect: - require(4); - dispatch_pm4(queue, qs, {words[0], words[1], words[2], words[3]}); - if (!qs.entries.empty()) - return; - break; - case Pm4Opcode::DispatchIndirect: { - require(submission.graphics_engine ? 2 : 3); - flush_gpu_caches(); - const uint64_t arguments = - submission.graphics_engine ? state.indirect_base + words[0] : address(0); - std::array dimensions{0, 0, 0, words.back()}; - if (access->read(arguments, {reinterpret_cast(dimensions.data()), 12}) != - VmAccessOutcome::Complete) - throw std::runtime_error("PM4 DISPATCH_INDIRECT read failed"); - dispatch_pm4(queue, qs, dimensions); - if (!qs.entries.empty()) - return; - break; - } - default: - throw std::runtime_error(std::format("unsupported PM4 opcode {:#x}", opcode)); - } - } - if (!state.submissions.empty()) - arm_stall_recheck(now); - } catch (const std::exception &error) { - util::Logger::warn("PM4 queue failed: ", error.what()); - fail_pm4_queue(queue, qs); - } +void CommandProcessor::fetch_pm4(ComputeQueueRecord &queue, Pm4DispatchState &qs, + simdojo::Tick now) { + process_pm4_packets( + queue, gpu_vm_, + { + .arch = cus_.empty() ? ROCJITSU_CODE_ARCH_INVALID : cus_[0]->config().arch, + .xcc_id = scratch_xcc_id_, + .flush_caches = [this] { flush_gpu_caches(); }, + .dispatch = + [this, &queue, &qs](const std::array &dimensions) { + dispatch_pm4(queue, qs, dimensions); + }, + .retry = + [this, &queue, now] { + queue.command_retry_pending = true; + arm_stall_recheck(now); + }, + .wake = + [this, now] { + stall_recheck_backoff_ = 1; + arm_stall_recheck(now); + }, + .fault_queue = [this, &queue, &qs] { fail_pm4_queue(queue, qs); }, + }); } CommandProcessor::KernelDescriptorReadResult @@ -3436,7 +3295,7 @@ CommandProcessor::read_kernel_descriptor(const GpuVmAccess &transaction_access, } VmAccessOutcome -CommandProcessor::publish_async_scratch_capability(const AqlQueueRecord &queue) const { +CommandProcessor::publish_async_scratch_capability(const ComputeQueueRecord &queue) const { if (!gpu_vm_ || !queue.address_space) return VmAccessOutcome::Unavailable; std::optional access = snapshot_gpu_access(queue.address_space); @@ -3462,7 +3321,7 @@ CommandProcessor::publish_async_scratch_capability(const AqlQueueRecord &queue) } } -VmAccessOutcome CommandProcessor::record_async_scratch_use(const AqlQueueRecord &queue, +VmAccessOutcome CommandProcessor::record_async_scratch_use(const ComputeQueueRecord &queue, const GpuVmAccess &access, uint64_t packet_index, bool alternate) const { @@ -3485,7 +3344,7 @@ VmAccessOutcome CommandProcessor::record_async_scratch_use(const AqlQueueRecord return VmAccessOutcome::Complete; } -AqlAdmissionResult CommandProcessor::request_dynamic_scratch(AqlQueueRecord &queue, +AqlAdmissionResult CommandProcessor::request_dynamic_scratch(ComputeQueueRecord &queue, const GpuVmAccess &transaction_access, uint64_t packet_index, uint64_t status) { @@ -3597,7 +3456,7 @@ AqlAdmissionResult CommandProcessor::request_dynamic_scratch(AqlQueueRecord &que } } -VmAccessOutcome CommandProcessor::advance_scratch_reclaim(AqlQueueRecord &queue, +VmAccessOutcome CommandProcessor::advance_scratch_reclaim(ComputeQueueRecord &queue, const DispatchEntry &entry) { constexpr uint64_t kQueueInactiveSignalOffset = offsetof(amd_queue_t, queue_inactive_signal); constexpr uint64_t kQueuePropertiesOffset = offsetof(amd_queue_t, queue_properties); @@ -3739,7 +3598,7 @@ VmAccessOutcome CommandProcessor::advance_scratch_reclaim(AqlQueueRecord &queue, } } -VmAccessOutcome CommandProcessor::gate_dispatch_retirement(AqlQueueRecord &queue, +VmAccessOutcome CommandProcessor::gate_dispatch_retirement(ComputeQueueRecord &queue, const DispatchEntry &entry) { if (entry.scratch_use_once && !entry.fanout_peer) return advance_scratch_reclaim(queue, entry); @@ -3758,7 +3617,7 @@ static const uint8_t *find_elf_base(const uint8_t *ptr, const uint8_t *limit) { } AqlAdmissionResult CommandProcessor::admit_kernel_dispatch( - const hsa_kernel_dispatch_packet_t &pkt, AqlQueueRecord &queue, + const hsa_kernel_dispatch_packet_t &pkt, ComputeQueueRecord &queue, const GpuVmAccess &transaction_access, uint64_t pkt_addr, uint32_t queue_packet_id, uint64_t aql_packet_id, ClusterDispatchShape cluster_shape) { const bool uses_kfd_queue_abi = queue.uses_kfd_queue_abi; @@ -4253,11 +4112,35 @@ AqlAdmissionResult CommandProcessor::admit_kernel_dispatch( return {.status = AqlAdmissionStatus::Complete}; } +bool CommandProcessor::execute_aql_pm4(ComputeQueueRecord &queue, DispatchEntry &entry, + simdojo::Tick now) { + // A peer can fault the grid before this CP drains its fault inbox. Leave + // cancellation to the inbox without starting or resuming the command buffer. + if (entry.grid_faulted()) + return false; + if (!entry.aql_pm4_ib_dwords || entry.completed_wgs == entry.total_wgs) + return true; + if (!entry.execution_begun) { + Pm4Submission submission; + submission.allow_dispatch = false; + submission.buffers.push_back({entry.aql_pm4_ib_address, entry.aql_pm4_ib_dwords}); + queue.commands.submissions.push_back(std::move(submission)); + entry.execution_begun = true; + } + fetch_pm4(queue, queue.dispatches, now); + if (queue.faulted) { + notify_dispatch_vm_fault(entry.queue_id, entry.process_id, entry.dispatch_id, + VmAccessOutcome::Faulted); + return false; + } + return queue.commands.submissions.empty(); +} + AqlAdmissionResult CommandProcessor::admit_aql_packet(const AqlPacketProcessRequest &request, AqlPreparedPacket prepared) { - const std::vector::iterator queue = - std::ranges::find(aql_queues_, request.registration_id, &AqlQueueRecord::registration_id); - if (queue == aql_queues_.end() || queue->process_id != request.process_id || + const std::vector::iterator queue = std::ranges::find( + compute_queues_, request.registration_id, &ComputeQueueRecord::registration_id); + if (queue == compute_queues_.end() || queue->process_id != request.process_id || queue->queue_id != request.queue_id || queue->faulted || queue->fanout_replica) { return {.status = AqlAdmissionStatus::Malformed}; } @@ -4278,7 +4161,18 @@ AqlAdmissionResult CommandProcessor::admit_aql_packet(const AqlPacketProcessRequ .kind = DispatchPacketKind::NonKernel, .wait_for_predecessors = prepared.barrier_bit, .blocks_following = prepared.blocks_following, + .aql_pm4_ib_address = prepared.pm4_ib_address, + .aql_pm4_ib_dwords = prepared.pm4_ib_dwords, }; + if (prepared.kind == AqlPreparedPacketKind::Pm4Ib) { + // One completion token per XCD prevents automatic retirement of an unexecuted + // non-kernel packet and keeps its successor blocked until every IB finishes. + entry.total_wgs = entry.dispatched_wgs = 1; + if (queue->xcd_fanout) { + entry.grid_completion = std::make_shared(); + entry.grid_completion->grid_wgs = xcd_peers_.size(); + } + } if (queue->xcd_fanout) replicate_non_kernel_entry(entry); queue->push_entry(std::move(entry)); @@ -4302,7 +4196,7 @@ void CommandProcessor::arm_grid_wait_recheck() { // // Caller must hold hw_queue_mutex_ and must be on this CP's own partition // thread, which is where the re-check is enqueued. - for (const auto &qs : aql_queues_) { + for (const auto &qs : compute_queues_) { if (qs.entries.empty()) continue; const auto &head = qs.entries.front(); @@ -4314,6 +4208,8 @@ void CommandProcessor::arm_grid_wait_recheck() { } void CommandProcessor::arm_stall_recheck(simdojo::Tick now) { + if (!engine()) + return; // A doorbell poll thread runs only for queues this CP polls; it re-checks // stall_pending_ at its 100us cadence, so the engine can idle instead of spinning. // Internal test queues have no poll thread — they are driven by engine->run()/ @@ -4336,11 +4232,24 @@ void CommandProcessor::arm_stall_recheck(simdojo::Tick now) { // which is where a 12x slowdown came from. Backing off is nearly free in // simulated time: with no other event pending the engine jumps straight to the // re-check, so a longer interval skips idle ticks rather than adding latency. - schedule_event(&doorbell_event_, now + stall_recheck_backoff_); + // Several queues or retry sites can remain blocked on the same pass. Keep + // one live re-check; otherwise each retry can enqueue several more. Fresh + // work can reset the backoff and pull the deadline earlier. Only that case + // leaves a stale event, which the generation check ignores when it fires. + const simdojo::Tick tick = now + stall_recheck_backoff_; + if (stall_recheck_pending_ && stall_recheck_tick_ <= tick) + return; + stall_recheck_pending_ = true; + stall_recheck_tick_ = tick; + schedule_event( + &stall_recheck_event_, tick, + std::make_unique(simdojo::MessageHeader{}, ++stall_recheck_generation_)); stall_recheck_backoff_ = std::min(stall_recheck_backoff_ * 2, kMaxStallRecheckBackoff); } -void CommandProcessor::fetch_from_queue(AqlQueueRecord &queue, simdojo::Tick now) { +void CommandProcessor::fetch_from_queue(ComputeQueueRecord &queue, simdojo::Tick now) { + if (queue.packet_format != QueuePacketFormat::Aql || queue.submission_queue) + return; // A replica's work arrives as dispatch shards, not from the ring. Reading the // ring here would also advance a read pointer the owning XCD owns, and its // suspension flags are the owner's copy rather than state this CP maintains. @@ -4625,32 +4534,30 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { drain_fanout_inbox(); drain_doorbell_inbox(); - // PM4 owns its queue lock and executes VM/register callbacks without the AQL - // lock, so service it before serializing AQL admission and completion. - const bool pm4_needs_retry = pm4_queue_controller_ && pm4_queue_controller_->service(); - + // AQL admission and PM4 execution share the queue lock and lifetime. std::unique_lock lock(hw_queue_mutex_); - if (pm4_needs_retry) - arm_stall_recheck(now); - util::Logger::cp( - [&](auto &os) { os << std::format("{}: DOORBELL queues={}", name(), aql_queues_.size()); }); + util::Logger::cp([&](auto &os) { + os << std::format("{}: DOORBELL queues={}", name(), compute_queues_.size()); + }); // Finish a prior durable publication before admitting a new queue generation. if (!drain_completions()) return; - service_drm_queues(now); + for (auto &queue : compute_queues_) + queue.command_retry_pending = false; + service_command_streams(now); size_t entries_before = 0; - for (const AqlQueueRecord &queue : aql_queues_) + for (const ComputeQueueRecord &queue : compute_queues_) entries_before += queue.entries.size(); - for (AqlQueueRecord &queue : aql_queues_) + for (ComputeQueueRecord &queue : compute_queues_) if (!queue.publication_retry_pending) fetch_from_queue(queue, now); size_t entries_after = 0; - for (const AqlQueueRecord &queue : aql_queues_) + for (const ComputeQueueRecord &queue : compute_queues_) entries_after += queue.entries.size(); if (entries_after != entries_before) stall_recheck_backoff_ = 1; @@ -4659,7 +4566,9 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { entries_after - entries_before, entries_after); }); - auto complete_non_kernel = [&](AqlQueueRecord &queue, DispatchEntry &entry) { + auto complete_non_kernel = [&](ComputeQueueRecord &queue, DispatchEntry &entry) { + if (!execute_aql_pm4(queue, entry, now)) + return false; const uint32_t queue_id = entry.queue_id; const uint32_t process_id = entry.process_id; const uint32_t dispatch_id = entry.dispatch_id; @@ -4682,7 +4591,7 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { bool completion_drain_failed = false; auto run_dispatch_workers = [&]() { if (exec_mode_ != simdojo::ExecMode::FUNCTIONAL || dispatch_threads_ <= 1 || - !has_runnable_cus()) + pooled_due_ticks_.empty()) return FunctionalQuantumResult{}; lock.unlock(); @@ -4706,7 +4615,7 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { while (progress && !yield_to_event_loop && !completion_drain_failed) { progress = false; - for (AqlQueueRecord &queue : aql_queues_) { + for (ComputeQueueRecord &queue : compute_queues_) { if (queue.faulted || queue.suspended() || queue.publication_retry_pending) continue; @@ -4797,7 +4706,7 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { util::Logger::cp([&](auto &os) { size_t remaining = 0; - for (const AqlQueueRecord &queue : aql_queues_) + for (const ComputeQueueRecord &queue : compute_queues_) remaining += queue.entries.size(); uint32_t active_cus = 0; for (const ComputeUnitCore *cu : cus_) @@ -4805,8 +4714,8 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { ++active_cus; os << std::format("{}: PHASE1_DONE remaining={} active_cus={}/{}", name(), remaining, active_cus, cus_.size()); - for (size_t queue_index = 0; queue_index < aql_queues_.size(); ++queue_index) { - AqlQueueRecord &queue = aql_queues_[queue_index]; + for (size_t queue_index = 0; queue_index < compute_queues_.size(); ++queue_index) { + ComputeQueueRecord &queue = compute_queues_[queue_index]; if (queue.entries.empty()) continue; os << std::format("\\n queue[{}] entries={} next_disp={} implicit_barrier={}", queue_index, @@ -4829,13 +4738,13 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { const uint32_t dispatch_id_before_refetch = next_dispatch_id_; if (!process_refetched_entries) { - for (AqlQueueRecord &queue : aql_queues_) + for (ComputeQueueRecord &queue : compute_queues_) if (!queue.publication_retry_pending) fetch_from_queue(queue, now); } process_refetched_entries = false; - for (AqlQueueRecord &queue : aql_queues_) { + for (ComputeQueueRecord &queue : compute_queues_) { if (queue.faulted || queue.suspended() || queue.publication_retry_pending) continue; while (queue.next_dispatch_idx < queue.entries.size()) { @@ -4867,7 +4776,7 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { } if (dispatch_threads_ > 1) { - const simdojo::Tick next = next_pooled_due_tick(now); + const simdojo::Tick next = next_pooled_due_tick(); if (next != simdojo::TICK_MAX) arm_dispatch_continuation(next); else @@ -4883,10 +4792,10 @@ void CommandProcessor::handle_doorbell_sync(simdojo::Tick now) { } } - const bool drm_done = std::ranges::all_of(drm_queues_, [](const auto &queue) { - return queue.dispatches.entries.empty() && queue.pm4->submissions.empty(); + const bool drm_done = std::ranges::all_of(compute_queues_, [](const auto &queue) { + return queue.dispatches.entries.empty() && queue.commands.submissions.empty(); }); - const bool all_done = drm_done && completion_ && completion_->all_complete(aql_queues_); + const bool all_done = drm_done && completion_ && completion_->all_complete(compute_queues_); const bool should_release = all_done && is_primary_ && !kfd; util::Logger::cp([&](auto &os) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.h index 0e0fa49772ba..34178bd3f0b1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.h @@ -13,8 +13,8 @@ /// Architecture: the CP directly owns queue state and doorbell monitoring /// (CP hardware functions). Four sub-blocks handle distinct pipeline stages: /// - AqlPacketProcessor: AQL framing, classification, and dependency decoding -/// - Pm4PacketProcessor: the supported PM4 compute-queue packet subset -/// - DispatchController: SPI+ADC WG iteration, CU resource check, WF creation +/// - PM4 packet processing: command-stream decoding and execution in pm4/ +/// - CP dispatch admission: SPI+ADC WG iteration, CU resource check, WF creation /// - CompletionTracker: per-dispatch WG counting, in-order signal retirement /// /// @see entries; - void push_entry(DispatchEntry entry) { entries.push_back(std::move(entry)); } -}; - -/// @brief CP-owned DRM queue, independent of AQL rings and PM4 transport rings. -struct Pm4SubmitQueue { +/// @brief PM4 queue configuration admitted by one CommandProcessor. +struct Pm4QueueConfig { AddressSpaceHandle address_space; - std::optional binding; - uint32_t process_id = 0; - uint32_t queue_id = 0; - bool faulted = false; - std::shared_ptr pm4; - Pm4DispatchState dispatches; + uint64_t ring_base = 0; + uint32_t ring_size_bytes = 0; + uint64_t consumer_pointer_address = 0; + std::optional initial_consumer_cursor = std::nullopt; + Pm4PacketCallbacks packet_callbacks{}; }; +struct QueueReconfigureRequest; + /// @brief AMDGPU command processor that dispatches wavefronts to compute units. /// /// @details Distributes AQL dispatch packets across the registered compute units in @@ -176,13 +166,6 @@ class CommandProcessor : public simdojo::Component { /// @param peers All XCD command processors of the SoC, in XCD index order. void set_xcd_topology(uint32_t rank, std::vector peers); - /// @brief Create a PM4 queue binding factory backed by this CP's queue controller. - /// @details The returned factory is a lifetime/notification adapter. This CP - /// owns PM4 ring, packet, retry, and cursor-publication state so semantics do - /// not migrate into MES or a PCI/VFIO transport adapter. - [[nodiscard]] std::shared_ptr - make_pm4_queue_binding_factory(Pm4PacketCallbacks callbacks); - [[nodiscard]] uint64_t register_pm4_queue(Pm4QueueConfig config); [[nodiscard]] QueuePrepareCloseStatus prepare_unregister_pm4_queue_registration(uint64_t registration_id) noexcept; @@ -195,7 +178,7 @@ class CommandProcessor : public simdojo::Component { /// @brief Whether this CP currently owns any AQL or PM4 queue state. [[nodiscard]] bool has_registered_queues() const; - /// @brief PM4 queues currently owned by this CP's controller. + /// @brief Compute queues whose root ring initially processes PM4. /// @details Test-only visibility for cross-layer teardown assertions. [[nodiscard]] size_t registered_pm4_queue_count_for_test() const; @@ -203,12 +186,12 @@ class CommandProcessor : public simdojo::Component { void set_scratch_xcc_layout_for_test(uint32_t xcc_id, uint32_t xcc_count); /// @brief Register a queue and return its CP-local lifetime identity. - uint64_t register_queue(AqlQueueConfig queue); + uint64_t register_queue(ComputeQueueConfig queue); /// @brief Remove only the queue incarnation identified by @p registration_id. [[nodiscard]] bool unregister_queue_registration(uint64_t registration_id); - /// @brief Gracefully remove one AQL registration without losing publication state. + /// @brief Gracefully remove one compute registration without losing publication state. /// @details Busy leaves the registration live so its owner thread can resume /// retryable cursor or completion publication. Faulted preserves terminal /// publication state for explicit force-cancel/reset handling. @@ -216,7 +199,7 @@ class CommandProcessor : public simdojo::Component { prepare_unregister_queue_registration(uint64_t registration_id) noexcept; /// @brief Register one DRM indirect-buffer submission queue. - [[nodiscard]] bool register_drm_queue(Pm4SubmitQueue queue); + [[nodiscard]] bool register_drm_queue(ComputeQueueConfig config); /// @brief Cancel DRM work before its frontend revokes the process VM binding. void unregister_drm_queues(uint32_t process_id); void unregister_drm_queue(uint32_t queue_id, uint32_t process_id); @@ -252,8 +235,10 @@ class CommandProcessor : public simdojo::Component { /// @details The caller must first stop every queue replica with /// signal_queue_exception(..., false). This operation is safe to serialize /// under a driver status-publication mutex because it cannot flush CU - /// notifications back into the driver. - bool publish_queue_exception(uint32_t queue_id, uint32_t process_id, uint64_t status); + /// notifications back into the driver. With wait_for_ack=false, success + /// means the status and interrupt were published, not consumed by ROCr. + bool publish_queue_exception(uint32_t queue_id, uint32_t process_id, uint64_t status, + bool wait_for_ack = true); void set_plugin_group(std::shared_ptr pg) { plugin_group_ = pg ? pg : ExecutionPluginGroup::empty_group(); @@ -282,6 +267,7 @@ class CommandProcessor : public simdojo::Component { cu->set_gpu_vm(gpu_vm_); cu->set_on_idle([this]() { on_cu_idle(); }); cu->set_on_pool_ready([this, cu]() { on_cu_pool_ready(cu); }); + on_cu_pool_ready(cu); } void startup() override; @@ -326,7 +312,7 @@ class CommandProcessor : public simdojo::Component { /// next one arrives. [[nodiscard]] size_t accepted_entry_count_for_test(uint32_t queue_id, uint32_t process_id) { std::lock_guard lock(hw_queue_mutex_); - const AqlQueueRecord *queue_state = find_aql_queue(queue_id, process_id); + const ComputeQueueRecord *queue_state = find_compute_queue(queue_id, process_id); return queue_state == nullptr ? 0 : queue_state->accepted_entries; } @@ -335,30 +321,30 @@ class CommandProcessor : public simdojo::Component { [[nodiscard]] std::array first_accepted_entry_kinds_for_test(uint32_t queue_id, uint32_t process_id) { std::lock_guard lock(hw_queue_mutex_); - const AqlQueueRecord *queue_state = find_aql_queue(queue_id, process_id); + const ComputeQueueRecord *queue_state = find_compute_queue(queue_id, process_id); return queue_state == nullptr ? std::array{} : queue_state->first_accepted_entry_kinds; } - /// @brief AQL queues registered with this CP, including fan-out replicas. + /// @brief Compute queues registered with this CP, including fan-out replicas. /// /// @details Test-only. Whether a queue is present here as an owner or as a /// replica is an internal placement detail, not something production code /// should branch on. [[nodiscard]] size_t registered_queue_count_for_test() const { std::lock_guard lock(hw_queue_mutex_); - return aql_queues_.size(); + return compute_queues_.size(); } [[nodiscard]] std::optional queue_uses_kfd_abi_for_test(uint32_t queue_id, uint32_t process_id) const { std::lock_guard lock(hw_queue_mutex_); - const std::vector::const_iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + const std::vector::const_iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == queue_id && candidate.process_id == process_id; }); - return queue == aql_queues_.end() ? std::nullopt - : std::optional(queue->uses_kfd_queue_abi); + return queue == compute_queues_.end() ? std::nullopt + : std::optional(queue->uses_kfd_queue_abi); } /// @brief Address-space identity retained by one registered queue. @@ -367,20 +353,20 @@ class CommandProcessor : public simdojo::Component { [[nodiscard]] AddressSpaceHandle queue_address_space_for_test(uint32_t queue_id, uint32_t process_id) const { std::lock_guard lock(hw_queue_mutex_); - const std::vector::const_iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + const std::vector::const_iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == queue_id && candidate.process_id == process_id; }); - return queue == aql_queues_.end() ? AddressSpaceHandle{} : queue->address_space; + return queue == compute_queues_.end() ? AddressSpaceHandle{} : queue->address_space; } /// @brief Address-space identity carried by the first accepted queue entry. - /// @details Test-only. The bounded history in AqlQueueRecord lets fan-out tests + /// @details Test-only. The bounded history in ComputeQueueRecord lets fan-out tests /// inspect an entry after it has retired without retaining production work. [[nodiscard]] AddressSpaceHandle first_accepted_address_space_for_test(uint32_t queue_id, uint32_t process_id) { std::lock_guard lock(hw_queue_mutex_); - const AqlQueueRecord *queue_state = find_aql_queue(queue_id, process_id); + const ComputeQueueRecord *queue_state = find_compute_queue(queue_id, process_id); return queue_state == nullptr ? AddressSpaceHandle{} : queue_state->first_accepted_address_space; } @@ -395,10 +381,10 @@ class CommandProcessor : public simdojo::Component { [[nodiscard]] std::optional queue_last_doorbell_for_test(uint64_t registration_id) const { std::lock_guard lock(hw_queue_mutex_); - const std::vector::const_iterator queue = - std::ranges::find(aql_queues_, registration_id, &AqlQueueRecord::registration_id); - return queue == aql_queues_.end() ? std::nullopt - : std::optional(queue->last_doorbell); + const std::vector::const_iterator queue = + std::ranges::find(compute_queues_, registration_id, &ComputeQueueRecord::registration_id); + return queue == compute_queues_.end() ? std::nullopt + : std::optional(queue->last_doorbell); } /// @brief Host-accessible queues this CP polls, excluding fan-out replicas. @@ -412,7 +398,7 @@ class CommandProcessor : public simdojo::Component { [[nodiscard]] size_t polled_kfd_queue_count_for_test() const { std::lock_guard lock(hw_queue_mutex_); size_t polled = 0; - for (const AqlQueueRecord &queue : aql_queues_) + for (const ComputeQueueRecord &queue : compute_queues_) polled += (queue.doorbell_mode == QueueDoorbellMode::HostPolled && !queue.fanout_replica) ? 1 : 0; return polled; @@ -462,35 +448,35 @@ class CommandProcessor : public simdojo::Component { /// @brief Test-only view of one queue's debugger suspension gate. [[nodiscard]] bool queue_debug_suspended_for_test(uint32_t queue_id, uint32_t process_id) { std::lock_guard lock(hw_queue_mutex_); - std::vector::iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + std::vector::iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == queue_id && candidate.process_id == process_id; }); - return queue != aql_queues_.end() && queue->debug_suspended; + return queue != compute_queues_.end() && queue->debug_suspended; } [[nodiscard]] bool queue_runtime_suspended_for_test(uint32_t queue_id, uint32_t process_id) { std::lock_guard lock(hw_queue_mutex_); - std::vector::iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + std::vector::iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == queue_id && candidate.process_id == process_id; }); - return queue != aql_queues_.end() && queue->runtime_suspended; + return queue != compute_queues_.end() && queue->runtime_suspended; } [[nodiscard]] bool queue_faulted_for_test(uint32_t queue_id, uint32_t process_id) const { std::lock_guard lock(hw_queue_mutex_); - const std::vector::const_iterator queue = - std::ranges::find_if(aql_queues_, [&](const AqlQueueRecord &candidate) { + const std::vector::const_iterator queue = + std::ranges::find_if(compute_queues_, [&](const ComputeQueueRecord &candidate) { return candidate.queue_id == queue_id && candidate.process_id == process_id; }); - return queue != aql_queues_.end() && queue->faulted; + return queue != compute_queues_.end() && queue->faulted; } [[nodiscard]] bool has_dispatch_for_test(uint32_t queue_id, uint32_t process_id, uint64_t dispatch_id) { std::lock_guard lock(hw_queue_mutex_); - const AqlQueueRecord *state = find_aql_queue(queue_id, process_id); + const ComputeQueueRecord *state = find_compute_queue(queue_id, process_id); return state != nullptr && std::ranges::any_of(state->entries, [dispatch_id](const DispatchEntry &entry) { return entry.dispatch_id == dispatch_id; @@ -504,10 +490,11 @@ class CommandProcessor : public simdojo::Component { [[nodiscard]] bool queue_exception_suspended_for_test(uint32_t queue_id, uint32_t process_id) { std::lock_guard lock(hw_queue_mutex_); - auto queue = std::find_if(aql_queues_.begin(), aql_queues_.end(), [&](const auto &candidate) { - return candidate.queue_id == queue_id && candidate.process_id == process_id; - }); - return queue != aql_queues_.end() && queue->exception_suspended; + auto queue = + std::find_if(compute_queues_.begin(), compute_queues_.end(), [&](const auto &candidate) { + return candidate.queue_id == queue_id && candidate.process_id == process_id; + }); + return queue != compute_queues_.end() && queue->exception_suspended; } /// @brief Test-only count of executed command-processor doorbell passes. @@ -524,6 +511,7 @@ class CommandProcessor : public simdojo::Component { private: friend class CommandProcessorCloseTestAccess; + friend class CommandProcessorPlacementTestAccess; class QueueRegistrationTransaction { public: @@ -536,7 +524,7 @@ class CommandProcessor : public simdojo::Component { CommandProcessor *owner_ = nullptr; }; - uint64_t register_queue(AqlQueueConfig queue, bool fanout_replica); + uint64_t register_queue(ComputeQueueConfig queue, bool fanout_replica); [[nodiscard]] QueuePrepareCloseStatus close_queue_registration(uint64_t registration_id, bool force) noexcept; @@ -574,7 +562,9 @@ class CommandProcessor : public simdojo::Component { /// notifications and schedules them for the following tick so they cannot starve /// device work already queued there. Internal test queues have no poll thread and /// are driven by engine->run()/step(), so there the re-check must be kept alive by - /// rescheduling the doorbell event at @p now + 1. + /// one engine-owned stall event at @p now plus the current backoff. A shorter + /// deadline supersedes that event; repeated later requests are coalesced. Real + /// doorbells remain independent so they can wake the CP immediately. void arm_stall_recheck(simdojo::Tick now); /// @brief Re-arm a re-check while this CP holds a shard whose grid is still @@ -585,7 +575,7 @@ class CommandProcessor : public simdojo::Component { void arm_grid_wait_recheck(); /// @brief Fetch AQL packets from a single AQL queue. - void fetch_from_queue(AqlQueueRecord &queue, simdojo::Tick now); + void fetch_from_queue(ComputeQueueRecord &queue, simdojo::Tick now); /// @brief Coarse writeback+invalidate of the GPU data caches (L1 K$/V$ + L2). /// @details Scalar and vector L1 are write-through and only need @@ -593,27 +583,28 @@ class CommandProcessor : public simdojo::Component { void flush_gpu_caches(); /// @brief Ask ROCr to provision scratch while leaving the head packet unconsumed. - [[nodiscard]] AqlAdmissionResult request_dynamic_scratch(AqlQueueRecord &queue, + [[nodiscard]] AqlAdmissionResult request_dynamic_scratch(ComputeQueueRecord &queue, const GpuVmAccess &transaction_access, uint64_t packet_index, uint64_t status); /// @brief Advertise and maintain ROCr's amd_queue_v2_t scratch-reclaim contract. - [[nodiscard]] VmAccessOutcome publish_async_scratch_capability(const AqlQueueRecord &queue) const; - [[nodiscard]] VmAccessOutcome record_async_scratch_use(const AqlQueueRecord &queue, + [[nodiscard]] VmAccessOutcome + publish_async_scratch_capability(const ComputeQueueRecord &queue) const; + [[nodiscard]] VmAccessOutcome record_async_scratch_use(const ComputeQueueRecord &queue, const GpuVmAccess &access, uint64_t packet_index, bool alternate) const; /// @brief Hold retirement until ROCr acknowledges a use-once scratch reclaim. - [[nodiscard]] VmAccessOutcome gate_dispatch_retirement(AqlQueueRecord &queue, + [[nodiscard]] VmAccessOutcome gate_dispatch_retirement(ComputeQueueRecord &queue, const DispatchEntry &entry); - [[nodiscard]] VmAccessOutcome advance_scratch_reclaim(AqlQueueRecord &queue, + [[nodiscard]] VmAccessOutcome advance_scratch_reclaim(ComputeQueueRecord &queue, const DispatchEntry &entry); /// @brief Build and admit one normalized AQL kernel-dispatch packet. /// @param packet_index Absolute AQL ring index for debugger correlation. [[nodiscard]] AqlAdmissionResult - admit_kernel_dispatch(const hsa_kernel_dispatch_packet_t &packet, AqlQueueRecord &queue, + admit_kernel_dispatch(const hsa_kernel_dispatch_packet_t &packet, ComputeQueueRecord &queue, const GpuVmAccess &transaction_access, uint64_t packet_address, uint32_t ring_slot, uint64_t packet_index = 0, ClusterDispatchShape cluster_shape = {}); @@ -629,11 +620,14 @@ class CommandProcessor : public simdojo::Component { /// is returned. The caller must then fault the dispatch without retaining a /// reference into the queue entry container across that operation. [[nodiscard]] DispatchWorkgroupResult dispatch_workgroups(DispatchEntry &entry); - void fail_pm4_queue(Pm4SubmitQueue &queue, Pm4DispatchState &qs); - void fetch_pm4(Pm4SubmitQueue &queue, Pm4DispatchState &qs, simdojo::Tick now); - void dispatch_pm4(const Pm4SubmitQueue &queue, Pm4DispatchState &qs, + [[nodiscard]] bool execute_aql_pm4(ComputeQueueRecord &queue, DispatchEntry &entry, + simdojo::Tick now); + void fail_pm4_queue(ComputeQueueRecord &queue, Pm4DispatchState &qs); + void fetch_pm4(ComputeQueueRecord &queue, Pm4DispatchState &qs, simdojo::Tick now); + void dispatch_pm4(const ComputeQueueRecord &queue, Pm4DispatchState &qs, const std::array &dimensions); - void service_drm_queues(simdojo::Tick now); + void service_command_streams(simdojo::Tick now); + void service_pm4_ring(ComputeQueueRecord &queue, simdojo::Tick now); /// @brief Split a dispatch across the SoC's XCDs, keeping this XCD's share. /// @@ -714,10 +708,10 @@ class CommandProcessor : public simdojo::Component { } /// @brief Locate the queue state for a (queue_id, process_id) pair. - /// @returns Pointer into aql_queues_, or null when not registered. Caller + /// @returns Pointer into compute_queues_, or null when not registered. Caller /// must hold hw_queue_mutex_ and must not use the result across a registration /// change. - AqlQueueRecord *find_aql_queue(uint32_t queue_id, uint32_t process_id); + ComputeQueueRecord *find_compute_queue(uint32_t queue_id, uint32_t process_id); void register_cluster_workgroup(const DispatchEntry &entry, uint32_t local_wg_id, uint32_t global_wg_id, ComputeUnitCore *cu, uint32_t lds_base); @@ -739,7 +733,7 @@ class CommandProcessor : public simdojo::Component { /// @brief Drain completed entries and preserve retry/terminal VM outcomes. /// @details Caller must hold @ref hw_queue_mutex_. /// @returns False when terminal state must stop this processing pass. A - /// transient retry is recorded on only the affected AqlQueueRecord so unrelated + /// transient retry is recorded on only the affected ComputeQueueRecord so unrelated /// queues remain runnable. [[nodiscard]] bool drain_completions(); bool fault_dispatch_local(uint32_t queue_id, uint32_t process_id, uint64_t dispatch_id, @@ -755,10 +749,9 @@ class CommandProcessor : public simdojo::Component { /// @brief Process all queues: dispatch undispatched entries, handle non-kernel entries. void process_queues(); - bool has_runnable_cus() const; FunctionalQuantumResult run_active_cus_once(simdojo::Tick now); - void refresh_pooled_due_ticks(simdojo::Tick now); - simdojo::Tick next_pooled_due_tick(simdojo::Tick now); + void prune_pooled_due_ticks(); + simdojo::Tick next_pooled_due_tick(); void arm_dispatch_continuation(simdojo::Tick tick); void cancel_dispatch_continuation(); @@ -770,20 +763,19 @@ class CommandProcessor : public simdojo::Component { void on_cu_pool_ready(ComputeUnitCore *cu); /// @brief Queue scheduling: select next queue with undispatched entries. - AqlQueueRecord *schedule_next_queue(); + ComputeQueueRecord *schedule_next_queue(); void handle_doorbell_sync(simdojo::Tick timestamp); /// @brief Check if barrier is satisfied for an entry. - bool barrier_satisfied(const AqlQueueRecord &qs, size_t idx) const; + bool barrier_satisfied(const ComputeQueueRecord &qs, size_t idx) const; /// @brief Return total pending entries across all queues. size_t pending_entries() const { size_t total = 0; - for (auto &qs : aql_queues_) - total += qs.entries.size(); - for (const auto &queue : drm_queues_) - total += queue.dispatches.entries.size() + queue.pm4->submissions.size(); + for (const auto &queue : compute_queues_) + total += queue.entries.size() + queue.dispatches.entries.size() + + queue.commands.submissions.size(); return total; } @@ -792,8 +784,8 @@ class CommandProcessor : public simdojo::Component { /// @details Answers whether this CP's lifecycle is anchored by the VM-level /// primary, which a fan-out replica does anchor just as its owner does. bool has_kfd_queues() const { - for (const auto &q : aql_queues_) - if (q.uses_kfd_queue_abi) + for (const auto &q : compute_queues_) + if (q.uses_kfd_queue_abi || q.doorbell_mode == QueueDoorbellMode::HostPolled) return true; return false; } @@ -805,7 +797,7 @@ class CommandProcessor : public simdojo::Component { /// to the doorbell monitor must ask this rather than has_kfd_queues(), or a CP /// left holding only replicas keeps a monitor alive for a ring it never reads. bool polls_kfd_queues() const { - for (const auto &q : aql_queues_) + for (const auto &q : compute_queues_) if (q.doorbell_mode == QueueDoorbellMode::HostPolled && !q.fanout_replica) return true; return false; @@ -816,9 +808,7 @@ class CommandProcessor : public simdojo::Component { std::vector spis_; std::vector l2_caches_; AqlPacketProcessor aql_packet_processor_; - std::unique_ptr pm4_queue_controller_; - std::vector aql_queues_; - std::vector drm_queues_; + std::vector compute_queues_; std::unordered_map dispatch_launch_metadata_; std::vector cus_; std::vector dispatch_ports_; @@ -1005,6 +995,12 @@ class CommandProcessor : public simdojo::Component { /// re-checks on its poll thread's own cadence instead. simdojo::Tick stall_recheck_backoff_ = 1; static constexpr simdojo::Tick kMaxStallRecheckBackoff = 4096; + // Owned by the CP's engine thread, unlike the external poller retry below. + // The generation rejects superseded deadlines without duplicating live retries. + bool stall_recheck_pending_ = false; + simdojo::Tick stall_recheck_tick_ = simdojo::TICK_MAX; + uintptr_t stall_recheck_generation_ = 0; + simdojo::Event stall_recheck_event_{this, simdojo::EventType::TIMER_CALLBACK}; // Set when a queue stalls on a barrier or another unsatisfied dependency -- // progress external to the current engine pass (a peer rank's kernel completion diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.cpp index abc71d3222b4..6a8f5a293e2f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.cpp @@ -36,7 +36,7 @@ namespace rocjitsu { namespace amdgpu { void CompletionTracker::notify_wg_complete(uint32_t dispatch_id, uint32_t wg_id, - std::vector &queues) { + std::vector &queues) { for (auto &qs : queues) { for (auto &entry : qs.entries) { if (entry.dispatch_id == dispatch_id) { @@ -54,7 +54,8 @@ void CompletionTracker::notify_wg_complete(uint32_t dispatch_id, uint32_t wg_id, } } -CompletionDrainResult CompletionTracker::drain_completions(std::vector &queues) { +CompletionDrainResult +CompletionTracker::drain_completions(std::vector &queues) { CompletionDrainResult result; for (auto &qs : queues) qs.publication_retry_pending = false; @@ -348,7 +349,7 @@ void CompletionTracker::flush_caches(uint32_t vmid) { } } -bool CompletionTracker::all_complete(const std::vector &queues) const { +bool CompletionTracker::all_complete(const std::vector &queues) const { for (const auto &qs : queues) { if (!qs.entries.empty() || qs.idle_publication.active() || qs.scratch_reclaim.active()) return false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.h index b20ff5d1729b..f3f92fbaeba3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.h @@ -45,7 +45,7 @@ class CompletionTracker { /// Unavailable keeps the completed entry queued so a durable external /// publication can resume without firing the dispatch completion twice. using DispatchRetirementGate = - std::function; + std::function; /// Called by the one shard whose publish completes a fanned-out dispatch, so /// every participating XCD can be woken: the owner to fire the completion /// signal, and any peer parked behind a barrier bit waiting on this dispatch. @@ -69,10 +69,10 @@ class CompletionTracker { /// @brief Notify that a workgroup has completed all its wavefronts. void notify_wg_complete(uint32_t dispatch_id, uint32_t wg_id, - std::vector &queues); + std::vector &queues); /// @brief Scan all queues and fire completion signals for retired dispatches. - [[nodiscard]] CompletionDrainResult drain_completions(std::vector &queues); + [[nodiscard]] CompletionDrainResult drain_completions(std::vector &queues); /// @brief Deliver an out-of-order-ready non-kernel packet's completion. [[nodiscard]] VmAccessOutcome complete_non_kernel(DispatchEntry &entry); @@ -81,7 +81,7 @@ class CompletionTracker { void flush_caches(uint32_t vmid = 0); /// @brief Check if all queues have no pending entries. - bool all_complete(const std::vector &queues) const; + bool all_complete(const std::vector &queues) const; private: [[nodiscard]] VmAccessOutcome deliver_completion(DispatchEntry &entry); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_queue_binding_factory.cpp similarity index 65% rename from emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.cpp rename to emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_queue_binding_factory.cpp index e8a2fdcbc321..2592e272a217 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_queue_binding_factory.cpp @@ -1,7 +1,7 @@ // Copyright (c) 2026 Advanced Micro Devices, Inc. // SPDX-License-Identifier: MIT -#include "rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h" +#include "rocjitsu/vm/amdgpu/compute_queue_binding_factory.h" #include "rocjitsu/vm/amdgpu/command_processor.h" @@ -9,12 +9,13 @@ namespace rocjitsu::amdgpu { namespace { -class AqlQueueBinding final : public QueueBinding { +class ComputeQueueBinding final : public QueueBinding { public: - AqlQueueBinding(CommandProcessor &command_processor, uint64_t registration_id) - : command_processor_(command_processor), registration_id_(registration_id) {} + ComputeQueueBinding(CommandProcessor &command_processor, uint64_t registration_id, + QueuePacketFormat format) + : command_processor_(command_processor), registration_id_(registration_id), format_(format) {} - ~AqlQueueBinding() override { + ~ComputeQueueBinding() override { if (registration_id_) (void)command_processor_.unregister_queue_registration(registration_id_); } @@ -30,6 +31,8 @@ class AqlQueueBinding final : public QueueBinding { } QueueReconfigureStatus reconfigure(const QueueReconfigureRequest &request) override { + if (format_ == QueuePacketFormat::Pm4) + return command_processor_.update_pm4_queue_registration(registration_id_, request); if (!valid_aql_packet_ring(request.ring_base_address, request.ring_size_bytes)) return QueueReconfigureStatus::Invalid; if (!registration_id_ || !command_processor_.update_queue_registration( @@ -44,6 +47,8 @@ class AqlQueueBinding final : public QueueBinding { QueueSubmissionStatus submit_producer(uint64_t producer_cursor) override { if (!registration_id_) return QueueSubmissionStatus::Faulted; + if (format_ == QueuePacketFormat::Pm4) + return command_processor_.notify_pm4_queue_doorbell(registration_id_, producer_cursor); command_processor_.notify_queue_doorbell(registration_id_, producer_cursor); return QueueSubmissionStatus::Accepted; } @@ -51,21 +56,19 @@ class AqlQueueBinding final : public QueueBinding { private: CommandProcessor &command_processor_; uint64_t registration_id_ = 0; + QueuePacketFormat format_; }; -class AqlQueueBindingFactory final : public QueueBindingFactory { +class ComputeQueueBindingFactory final : public QueueBindingFactory { public: - explicit AqlQueueBindingFactory(CommandProcessor &command_processor) - : command_processor_(command_processor) {} + explicit ComputeQueueBindingFactory(CommandProcessor &command_processor, + Pm4PacketCallbacks callbacks) + : command_processor_(command_processor), callbacks_(std::move(callbacks)) {} QueueBindingCreateResult create_binding(const QueueRegistrationRequest &request) override { - if (request.type != QueueType::Compute || request.packet_format != QueuePacketFormat::Aql) + if (request.type != QueueType::Compute || (request.packet_format != QueuePacketFormat::Aql && + request.packet_format != QueuePacketFormat::Pm4)) return {}; - if (!valid_aql_queue_layout(request.ring.base_address, request.ring.size_bytes, - request.ring.consumer_pointer_address, - request.ring.producer_pointer_address)) { - return {}; - } const uint64_t registration_id = command_processor_.register_queue({ .address_space = request.identity.address_space, .interrupt_sink = request.identity.interrupt_sink, @@ -85,12 +88,17 @@ class AqlQueueBindingFactory final : public QueueBindingFactory { .exception_status_va = request.exception_status_address, .exception_event_id = request.exception_event_id, .xcd_fanout = request.xcd_fanout, + .packet_format = request.packet_format, + .initial_consumer_cursor = request.initial_consumer_cursor, + .packet_callbacks = callbacks_, + .scheduling_percentage = request.scheduling_percentage, }); if (registration_id == 0) return {}; try { return {.status = QueueBindingCreateStatus::Bound, - .binding = std::make_unique(command_processor_, registration_id)}; + .binding = std::make_unique(command_processor_, registration_id, + request.packet_format)}; } catch (...) { (void)command_processor_.unregister_queue_registration(registration_id); throw; @@ -99,13 +107,15 @@ class AqlQueueBindingFactory final : public QueueBindingFactory { private: CommandProcessor &command_processor_; + Pm4PacketCallbacks callbacks_; }; } // namespace std::shared_ptr -make_aql_queue_binding_factory(CommandProcessor &command_processor) { - return std::make_shared(command_processor); +make_compute_queue_binding_factory(CommandProcessor &command_processor, + Pm4PacketCallbacks callbacks) { + return std::make_shared(command_processor, std::move(callbacks)); } } // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_queue_binding_factory.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_queue_binding_factory.h new file mode 100644 index 000000000000..71a9bc89f5d6 --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_queue_binding_factory.h @@ -0,0 +1,29 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file compute_queue_binding_factory.h +/// @brief Queue binding factory for the compute command processor. + +#pragma once + +#include "rocjitsu/vm/amdgpu/gpu_queue_registry.h" +#include "rocjitsu/vm/amdgpu/pm4/pm4_packet_types.h" + +#include + +namespace rocjitsu::amdgpu { + +class CommandProcessor; + +/// @brief Create a reusable binding factory for compute queues owned by @p command_processor. +/// @details The factory and bindings borrow the command processor. Its owner must +/// outlive the factory and every queue binding created from it. Packet callbacks +/// are copied into each CP-owned PM4 queue and must obey their own captured +/// lifetime contracts. The factory is a lifetime/notification adapter; CP owns +/// ring, packet, retry, and cursor-publication state so semantics do not migrate +/// into MES or a PCI/VFIO transport adapter. +[[nodiscard]] std::shared_ptr +make_compute_queue_binding_factory(CommandProcessor &command_processor, + Pm4PacketCallbacks callbacks = {}); + +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.cpp index ebea8894f5bc..3644bf88e2f4 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.cpp @@ -165,10 +165,9 @@ ComputeUnitCore::ComputeUnitCore(std::string name, const Config &config, GpuMemo inst_cache_.set_l2(l2_); - // Enable pool allocation for the hot decode-execute path. - // Instructions decoded during step() are always deleted before the CU - // (and its decoder) are destroyed, so pool allocation is safe here. - decoder_->enable_pool(); + // Decoded instructions use heap storage: cache entries and asynchronous + // work can outlive an issue quantum and move between execution workers. + // Do not install this decoder's pool on the constructing thread. wfs_.resize(config.num_wf_slots); sgpr_file_.init(config.num_wf_slots * config.sgprs_per_wf, config.sgprs_per_wf); @@ -305,6 +304,7 @@ Wavefront *ComputeUnitCore::dispatch_wf_at(uint32_t wf_id, uint32_t wg_id, uint6 if (!wfs_[wf_id]) wfs_[wf_id] = create_wavefront(wf_id); auto *wf = wfs_[wf_id].get(); + wf->activity_tracked_ = true; const uint32_t dispatched_wave_size = wave_size == 0 ? wf->default_wf_size_ : wave_size; if ((dispatched_wave_size != 32 && dispatched_wave_size != 64) || dispatched_wave_size > wf->max_wf_size_) @@ -344,7 +344,7 @@ Wavefront *ComputeUnitCore::dispatch_wf_at(uint32_t wf_id, uint32_t wg_id, uint6 ++wf->dispatch_generation_; if (wf->dispatch_generation_ == 0) ++wf->dispatch_generation_; - wf->state_ = WfState::RUNNING; + wf->set_state(WfState::RUNNING); wf->set_ready_cycle(cycle_counter_); wf->trace_inst_count_ = 0; @@ -738,23 +738,7 @@ void ComputeUnitCore::abort_dispatch(uint32_t dispatch_id) { bool ComputeUnitCore::can_accept_workgroup(uint32_t num_wfs, uint32_t lds_bytes, uint32_t scratch_wave_limit_per_se) const { - // Count free wavefront slots. - uint32_t free_slots = 0; - const size_t slot_limit = scratch_wave_limit_per_se == UINT32_MAX - ? wfs_.size() - : std::min(wfs_.size(), scratch_slots_per_cu_); - for (size_t slot = 0; slot < slot_limit; ++slot) { - const uint64_t scratch_scoreboard_id = static_cast(scratch_scoreboard_base_) + slot; - if (scratch_scoreboard_id < scratch_wave_limit_per_se && - (!wfs_[slot] || wfs_[slot]->is_halted())) - ++free_slots; - } - if (free_slots < num_wfs) { - util::Logger::vm("CU ", this->name(), " can_accept_wg: REJECT free_slots=", free_slots, - " < num_wfs=", num_wfs); - return false; - } - + // Reject exhausted register allocations before scanning the wave slots. // Check SGPR register blocks. uint32_t free_sgpr = sgpr_file_.free_block_count(); if (free_sgpr < num_wfs) { @@ -771,6 +755,31 @@ bool ComputeUnitCore::can_accept_workgroup(uint32_t num_wfs, uint32_t lds_bytes, return false; } + // Unrestricted placement can reuse the resident count. Paused waves still + // occupy slots; scratch-limited placement must inspect the eligible prefix. + uint32_t free_slots = 0; + if (scratch_wave_limit_per_se == UINT32_MAX && + uint64_t{scratch_scoreboard_base_} + wfs_.size() <= UINT32_MAX) { + const uint32_t active = static_cast(wave_activity_.load(std::memory_order_acquire)); + assert(active <= wfs_.size()); + free_slots = static_cast(wfs_.size()) - active; + } else { + const size_t slot_limit = scratch_wave_limit_per_se == UINT32_MAX + ? wfs_.size() + : std::min(wfs_.size(), scratch_slots_per_cu_); + for (size_t slot = 0; slot < slot_limit && free_slots < num_wfs; ++slot) { + const uint64_t scratch_scoreboard_id = static_cast(scratch_scoreboard_base_) + slot; + if (scratch_scoreboard_id < scratch_wave_limit_per_se && + (!wfs_[slot] || wfs_[slot]->is_halted())) + ++free_slots; + } + } + if (free_slots < num_wfs) { + util::Logger::vm("CU ", this->name(), " can_accept_wg: REJECT free_slots=", free_slots, + " < num_wfs=", num_wfs); + return false; + } + if (lds_bytes > 0) { uint32_t aligned = util::align_up(lds_bytes, 256u); uint32_t lds_capacity_bytes = config_.lds_size_kb * 1024u; @@ -822,9 +831,8 @@ VmAccessOutcome ComputeUnitCore::route_memory_inst(Instruction *inst, Wavefront } // Under the current uniform-address-space assumption, FLAT operations // targeting the shared aperture use the LDS pipeline. Scratch-targeting - // FLATs stay on the global path. Architectural wait-counter obligations - // remain properties of the decoded instruction; this route selects only - // the memory path used by the emulator. + // FLATs stay on the global path. Counter participation is selected from all + // requesting lanes below, independently of the first-lane functional route. const uint64_t request_lanes = transpose_request_lane_mask(d, wf_size); const uint32_t first_lane = request_lanes == 0 ? wf_size : static_cast(std::countr_zero(request_lanes)); @@ -853,6 +861,15 @@ VmAccessOutcome ComputeUnitCore::route_memory_inst(Instruction *inst, Wavefront } } + if (decoded_route_tag == GLOBAL_MEM && inst->mnemonic().starts_with("flat_")) { + auto &d = *inst->data_as(); + if (const auto *issue = inst->amdgpu_memory_issue_info()) { + const uint64_t requests = transpose_request_lane_mask(d, wf.wf_size()); + const uint64_t shared = (flat_local_lane_mask | flat_dds_lane_mask) & requests; + d.routed_issue_info = issue->for_flat_memory_domains((requests & ~shared) != 0, shared != 0); + } + } + const uint8_t route_tag = inst->data()->tag(); // After the aperture rewrite, before the pipeline takes the instruction: // this is the one point at which the space, the counter, and the addresses @@ -864,8 +881,7 @@ VmAccessOutcome ComputeUnitCore::route_memory_inst(Instruction *inst, Wavefront report_routed_access(*inst, wf, route_tag, decoded_route_tag, normalized_to_local, pre_routing_addresses, flat_local_lane_mask, flat_dds_lane_mask); - // Resolved FLAT lanes determine which pipeline produces each register result. - // Keep both architectural counter entries, including the counter-only one. + // Resolved FLAT lanes determine counter participation and result readiness. if (config_.memory_wait_diagnostics != MemoryWaitDiagnostics::Off && inst->is_memory_wait_producer()) track_memory_wait(*inst, wf, flat_local_lane_mask | flat_dds_lane_mask); @@ -984,6 +1000,30 @@ void ComputeUnitCore::track_memory_wait(Instruction &inst, Wavefront &wf, ? inst.data_as()->exec_mask : wf.exec(); + // FLAT counter admission depends on addresses. It cannot prove operands + // ready before those addresses are read; apply it here, before writeback. + const bool flat = inst.mnemonic().starts_with("flat_"); + const uint64_t flat_requests = + flat && inst.data() && (inst.data()->tag() == GLOBAL_MEM || inst.data()->tag() == LOCAL_MEM) + ? transpose_request_lane_mask(*inst.data_as(), wf.wf_size()) + : 0; + const auto flat_counter_lanes = [&](WaitCounterKind counter) { + return counter == WaitCounterKind::Ds ? flat_requests & flat_shared_lanes + : flat_requests & ~flat_shared_lanes; + }; + const auto is_flat_memory_counter = [&](WaitCounterKind counter) { + return flat && (counter == WaitCounterKind::Load || counter == WaitCounterKind::Store || + counter == WaitCounterKind::Ds); + }; + for (const auto &event : classified.value()) { + if (!is_flat_memory_counter(event.counter) || !flat_counter_lanes(event.counter)) + continue; + const auto maximum = WaitcheckTarget::maximum_dependency_wait(config_.arch, event.counter); + if (maximum.succeeded()) + scoreboard.backpressure(event.counter, maximum.value() + 1, + scoreboard.issue_units(inst, event, config_.arch)); + } + struct Destination { RegisterRef reg; uint64_t lanes; @@ -1097,6 +1137,8 @@ void ComputeUnitCore::track_memory_wait(Instruction &inst, Wavefront &wf, std::optional xcnt_completion; for (const auto &event : classified.value()) { const auto counter = event.counter; + if (is_flat_memory_counter(counter) && !flat_counter_lanes(counter)) + continue; if (counter == WaitCounterKind::X) { if (track_xcnt && (xscalar || vector_lanes || scoreboard.outstanding(counter))) { const auto sequence = scoreboard.issue_xcnt(xcnt_completion, xscalar); @@ -1394,6 +1436,9 @@ void ComputeUnitCore::issue_async_instruction(Wavefront *active, MmaAdmissionCac async_execution::stats.flush(); } +thread_local ComputeUnitCore::InstructionVmSnapshot *ComputeUnitCore::instruction_vm_snapshot_ = + nullptr; + template [[gnu::always_inline]] inline void ComputeUnitCore::issue_instruction_impl( Wavefront *active, @@ -1407,7 +1452,12 @@ template } }; - std::optional vm_access; + std::optional fresh_vm_access; + const GpuVmAccess *vm_access = nullptr; + InstructionVmSnapshot *snapshot = instruction_vm_snapshot_; + // A callback may recursively issue another wave. Keep its fetch from + // replacing the snapshot borrowed by this instruction's later debug probes. + ScopedInstructionVmSnapshot nested_scope(nullptr); if (active->address_space() || vmid != 0) { if (gpu_vm_ == nullptr) { drain_async_window(); @@ -1416,8 +1466,25 @@ template handle_terminal_vm_fault(*active, VmAccessOutcome::Faulted); return; } - vm_access = active->address_space() ? gpu_vm_->snapshot(active->address_space()) - : gpu_vm_->snapshot_vmid(vmid); + const AddressSpaceHandle address_space = active->address_space(); + if (snapshot && snapshot->compute_unit == this) { + auto &cached = *snapshot; + if (cached.owner != gpu_vm_ || cached.address_space != address_space || cached.vmid != vmid || + !cached.access || !cached.access->is_current()) { + cached.access = + address_space ? gpu_vm_->snapshot(address_space) : gpu_vm_->snapshot_vmid(vmid); + cached.owner = gpu_vm_; + cached.address_space = address_space; + cached.vmid = vmid; + } + if (cached.access) + vm_access = &*cached.access; + } else { + fresh_vm_access = + address_space ? gpu_vm_->snapshot(address_space) : gpu_vm_->snapshot_vmid(vmid); + if (fresh_vm_access) + vm_access = &*fresh_vm_access; + } if (!vm_access) { drain_async_window(); util::Logger::vm("CU ", this->name(), ": wf", active->wf_id(), @@ -1427,7 +1494,7 @@ template return; } } - const bool vm_address_space = vm_access.has_value(); + const bool vm_address_space = vm_access != nullptr; rj_code_binary_inst_t words[4]; static_assert(sizeof(words) == InstructionCache::kFetchBytes, @@ -1475,7 +1542,9 @@ template active->trace_inst_count_++; util::StringDiagnostic decode_error; - DecodeResult decoded = decoder_->decode(words, decode_error.emitter()); + const bool reuse_decoded = !debug_active(); + DecodeResult decoded = decoded_inst_cache_.decode(*decoder_, active->pc, vmid, words, + decode_error.emitter(), reuse_decoded); if (decoded.failed()) { drain_async_window(); util::Logger::vm("CU ", this->name(), ": wf", active->wf_id(), " HALT(decode rejection) pc=0x", @@ -1491,10 +1560,18 @@ template return; } Instruction *inst = decoded.value().get(); + DecodedInstructionCache::ScopedReturn return_decoded(decoded_inst_cache_, decoded.value(), + active->pc, vmid, words, reuse_decoded); int inst_size_signed = inst->size(); assert(inst_size_signed > 0 && "instruction size must be positive"); auto inst_size = static_cast(inst_size_signed); + // Skipped vector instructions never issue: no register/plugin effects, wait + // counter changes, or async matrix submission. Scalar instructions still run. + if ((active->mode_raw() & Wavefront::VSKIP_BIT) && inst->is_vskip_affected()) { + active->pc += inst_size; + return; + } auto *wait_state = config_.memory_wait_diagnostics == MemoryWaitDiagnostics::Off ? nullptr : active->memory_wait_scoreboard(); @@ -1521,9 +1598,8 @@ template if (async_pool().available()) { MmaAdmissionCache::Words first; std::ranges::copy_n(words, first.size(), first.begin()); - issuer = - admission->inspect(*decoder_, inst_cache_, *memory_, vm_access ? &*vm_access : nullptr, - active->pc, vmid, active->num_vgprs(), storage->has_accvgprs, first); + issuer = admission->inspect(*decoder_, inst_cache_, *memory_, vm_access, active->pc, vmid, + active->num_vgprs(), storage->has_accvgprs, first); may_submit = issuer.has_value(); } } @@ -1692,6 +1768,7 @@ template }(); if (execution_result.failed()) [[unlikely]] { + return_decoded.discard(); if constexpr (EnableAsync) { if (window) window->drain(); @@ -1714,7 +1791,7 @@ template // pc, and allocations are now zeroed, so the after-execute hook, result logging, // and pc-advance below must not run on the dead slot. The dedicated // onAmdgpuWavefrontHalted hook already fired (with live state) from halt(). - // s_endpgm is never a memory op, so just reclaim the decoded instruction. + // The completed s_endpgm can return to the decoded cache on scope exit. // // Note the intentional asymmetry: an s_endpgm that defers to ENDING (pending // memory waits) is NOT halted here, so it DOES fire onAmdgpuAfterExecuteInstruction @@ -1896,8 +1973,6 @@ template return; } } - } else { - decoded.value().reset(); } active->pc += inst_size; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.h index cab9b2f9b80e..cb4661cadf1f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.h @@ -11,7 +11,9 @@ #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" #include "rocjitsu/vm/amdgpu/cluster_lds_multicast.h" +#include "rocjitsu/vm/amdgpu/decoded_instruction_cache.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" +#include "rocjitsu/vm/amdgpu/gpu_vm.h" #include "rocjitsu/vm/amdgpu/instruction_cache.h" #include "rocjitsu/vm/amdgpu/l1_scalar_cache.h" #include "rocjitsu/vm/amdgpu/l1_vector_cache.h" @@ -247,6 +249,12 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { /// @brief Execute up to one functional quantum of step() iterations on this CU. /// @returns Whether wavefronts ran and whether one requested an event-loop yield. FunctionalQuantumResult run_quantum() { + const GpuVmAccessBatchGuard vm_access_batch; + // Reuse instruction-fetch snapshots only within this execution quantum. + // Restore the outer scope on exceptions and nested quantum execution too. + InstructionVmSnapshot snapshot; + snapshot.compute_unit = this; + ScopedInstructionVmSnapshot scope(&snapshot); // A request left by direct step() execution must not shorten this quantum. functional_yield_requested_ = false; FunctionalQuantumResult result; @@ -284,7 +292,6 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { /// @brief Check whether this CU has no runnable wavefronts. /// @retval true No wavefront can currently execute. /// @retval false At least one wavefront can execute. - /// @warning NOT thread-safe (see has_runnable_wfs()): engine-thread only. virtual bool is_idle() const { return !has_runnable_wfs(); } /// @brief Register a callback invoked when this CU becomes idle. @@ -369,6 +376,7 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { /// @brief Set the device VM service shared by legacy and PCI/VFIO queues. void set_gpu_vm(GpuVm *gpu_vm) { + inst_cache_.invalidate_all(); gpu_vm_ = gpu_vm; l1_vector_.set_gpu_vm(gpu_vm); l1_scalar_.set_gpu_vm(gpu_vm); @@ -439,8 +447,25 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { void abort_dispatch(uint32_t dispatch_id); /// @brief Set the execution plugin group (shared ownership). + /// @details Replacement refreshes resident waves' hot-hook subscriptions but + /// does not replay dispatch callbacks or migrate or clear wave-local plugin + /// state. Stateful plugins must tolerate missing initialization and state + /// left in a reused slot when attached to an already-resident wave. void set_plugin_group(std::shared_ptr pg) { - plugin_group_ = pg ? std::move(pg) : ExecutionPluginGroup::empty_group(); + std::lock_guard lock(wave_state_mutex_); + auto replacement = pg ? std::move(pg) : ExecutionPluginGroup::empty_group(); + if (plugin_group_.get() != replacement.get()) { + // A resident wave's cached decisions belong to the group that observed + // its dispatch. A replacement group may have the same plugin count but + // different per-wave subscriptions, so force it onto the live-query path. + for (const auto &wf : wfs_) { + if (!wf) + continue; + wf->hot_hook_subscriptions_valid_ = false; + wf->hot_hook_observer_count_ = 0; + } + } + plugin_group_ = std::move(replacement); observes_before_execute_instruction_ = plugin_group_->observes_before_execute_instruction(); observes_after_execute_instruction_ = plugin_group_->observes_after_execute_instruction(); observes_async_instruction_issued_ = plugin_group_->observes_async_instruction_issued(); @@ -486,15 +511,25 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { /// @brief Record this CU's physical location within its XCC. /// @param shader_engine_id Zero-based shader-engine index within the XCC. /// @param cu_index Zero-based CU index within the shader engine. - void set_shader_engine_location(uint32_t shader_engine_id, uint32_t cu_index) { + /// @param cus_per_shader_array Width of each shader array in CU order; zero means unknown. + void set_shader_engine_location(uint32_t shader_engine_id, uint32_t cu_index, + uint32_t cus_per_shader_array = 0) { shader_engine_id_ = shader_engine_id; shader_engine_cu_index_ = cu_index; scratch_scoreboard_base_ = shader_engine_cu_index_ * scratch_slots_per_cu_; + cus_per_shader_array_ = cus_per_shader_array; + shader_array_cu_id_ = cus_per_shader_array ? cu_index % cus_per_shader_array : 0; } /// @brief Return this CU's physical shader-engine index. uint32_t shader_engine_id() const { return shader_engine_id_; } + /// @brief Return the shader-array width, or zero when the geometry is unknown. + uint32_t cus_per_shader_array() const { return cus_per_shader_array_; } + + /// @brief Return this CU's index within its shader array when the width is known. + uint32_t shader_array_cu_id() const { return shader_array_cu_id_; } + /// @brief Return the first scratch scoreboard slot owned by this CU. uint32_t scratch_scoreboard_base() const { return scratch_scoreboard_base_; } @@ -656,29 +691,19 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { /// @brief Check whether any wavefront slot is actively executing. /// @retval true At least one resident wavefront is not halted. /// @retval false All slots are unallocated or halted. - /// @warning NOT thread-safe: reads the non-atomic per-wave state_. Safe only on the - /// shared partition engine thread (CP and its CUs share one partition, asserted in - /// CommandProcessor::startup()); callers on any other thread would race a halt(). bool has_active_wfs() const { - std::lock_guard lock(wave_state_mutex_); - for (const auto &w : wfs_) - if (w && !w->is_halted()) - return true; - return false; + return static_cast(wave_activity_.load(std::memory_order_acquire)) != 0; } /// @brief Check whether any wavefront can currently make forward progress. - /// @details A debug-halted wave occupies its slot (so @ref has_active_wfs + /// @details A wave for which debug_paused() is true occupies its slot (so @ref has_active_wfs /// stays true and the wave is not retired) but cannot run, so it must not /// keep the CU's event loop spinning. Idle detection uses this instead of /// @ref has_active_wfs so the engine can quiesce while a wave is stopped at - /// a breakpoint. @retval true At least one non-halted, non-debug-halted wave. + /// a breakpoint or suspended by the debugger or runtime. + /// @retval true At least one resident wave is neither halted nor paused. bool has_runnable_wfs() const { - std::lock_guard lock(wave_state_mutex_); - for (const auto &w : wfs_) - if (w && !w->is_halted() && !w->debug_paused()) - return true; - return false; + return (wave_activity_.load(std::memory_order_acquire) >> 32) != 0; } template decltype(auto) with_wave_state_locked(F &&fn) { @@ -1060,7 +1085,7 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { void replace_decoder_for_test(std::unique_ptr decoder) { assert(decoder != nullptr); assert(!has_active_wfs()); - decoder->enable_pool(); + decoded_inst_cache_.clear(); decoder_ = std::move(decoder); } @@ -1208,6 +1233,8 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { uint32_t shader_engine_id_ = 0; uint32_t shader_engine_cu_index_ = 0; uint32_t scratch_slots_per_cu_ = 1; + uint32_t cus_per_shader_array_ = 0; + uint32_t shader_array_cu_id_ = 0; uint32_t scratch_scoreboard_base_ = 0; bool sram_ecc_ = false; const bool setreg_vgpr_msb_fixup_ = false; @@ -1215,6 +1242,10 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { SgprFile sgpr_file_{"sgpr"}; /// Null slots are idle; materialized waves persist across dispatches. std::vector> wfs_; + friend class Wavefront; + // Low/high halves count active/runnable waves. Publish both in one update + // so a pause or retirement cannot expose half of an activity transition. + std::atomic wave_activity_{0}; /// @brief Hold the wave-state lock, then notify the CP once it is released. /// @details The CP takes hw_queue_mutex_ and then this lock when it dispatches /// (handle_doorbell -> dispatch_workgroups -> dispatch_wf), so anything running @@ -1291,6 +1322,7 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { L1ScalarCache l1_scalar_; L1VectorCache l1_vector_; InstructionCache inst_cache_; + DecodedInstructionCache decoded_inst_cache_; /// @brief Debug attach/detach transitions seen by set_debug_active(). std::atomic inst_cache_debug_epoch_{0}; /// @brief The epoch this CU's thread has already invalidated the I$ for. @@ -1324,6 +1356,32 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { CommandProcessor *cp_ = nullptr; GpuVm *gpu_vm_ = nullptr; + /// @brief Instruction-fetch access retained for one CU's functional quantum. + /// @details CU, VM and address-space identity prevent reuse across nested dispatches. + struct InstructionVmSnapshot { + ComputeUnitCore *compute_unit = nullptr; + GpuVm *owner = nullptr; + AddressSpaceHandle address_space; + uint32_t vmid = 0; + std::optional access; + }; + /// Only the scope pointer is thread-local; the snapshot itself lives on the + /// quantum's stack and releases all backing references before returning. + static thread_local InstructionVmSnapshot *instruction_vm_snapshot_; + /// @brief Restore the enclosing fetch scope after a quantum or nested issue. + /// @details A null scope prevents nested memory requests from borrowing fetch access. + class ScopedInstructionVmSnapshot { + public: + explicit ScopedInstructionVmSnapshot(InstructionVmSnapshot *snapshot) + : previous_(std::exchange(instruction_vm_snapshot_, snapshot)) {} + ScopedInstructionVmSnapshot(const ScopedInstructionVmSnapshot &) = delete; + ScopedInstructionVmSnapshot &operator=(const ScopedInstructionVmSnapshot &) = delete; + ~ScopedInstructionVmSnapshot() { instruction_vm_snapshot_ = previous_; } + + private: + InstructionVmSnapshot *previous_; + }; + std::unordered_map active_wgs_; struct BarrierCounter { @@ -1407,6 +1465,16 @@ inline bool InstructionComputeUnitView::setreg_vgpr_msb_fixup() const { return raw_cu().setreg_vgpr_msb_fixup(); } inline rj_code_arch_t InstructionComputeUnitView::arch() const { return raw_cu().arch(); } +inline bool InstructionComputeUnitView::observes_register_access() const { + return raw_cu().observes_register_access(); +} +inline bool InstructionComputeUnitView::debug_active() const { return raw_cu().debug_active(); } +inline uint32_t InstructionComputeUnitView::cus_per_shader_array() const { + return raw_cu().cus_per_shader_array(); +} +inline uint32_t InstructionComputeUnitView::shader_array_cu_id() const { + return raw_cu().shader_array_cu_id(); +} inline uint32_t InstructionComputeUnitView::wf_size() const { return raw_cu().wf_size(); } inline uint32_t InstructionComputeUnitView::sgprs_per_wf() const { return raw_cu().config().sgprs_per_wf; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/cpu_dispatch_pool.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/cpu_dispatch_pool.h index ac3c3fe22093..cf0ac4698bb3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/cpu_dispatch_pool.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/cpu_dispatch_pool.h @@ -40,9 +40,10 @@ class CpuDispatchPoolTestAccess; /// /// Workers take submission assignments under a short shared lock, then claim CUs /// with a submission-local atomic counter. Callers always drain their own work. -/// Each submission joins only its assigned workers, so an unrelated slow batch -/// cannot hold up its completion. The pool retains N-1 workers total, independent -/// of the number of callers; no additional worker pool is created per XCD. +/// The first caller or worker to exhaust the task index cancels pending worker +/// assignments. Each submission joins only its assigned workers, so an unrelated +/// slow batch cannot hold up its completion. The pool retains N-1 workers total, +/// independent of the number of callers; no additional worker pool is created per XCD. class CpuDispatchPool { public: explicit CpuDispatchPool(uint32_t threads) : CpuDispatchPool(threads, std::nullopt) {} @@ -94,9 +95,7 @@ class CpuDispatchPool { // Cancel unused worker assignments and join only workers holding this batch. // The final worker notifies while holding mutex_, before this stack object // can be destroyed by its caller. - if (submission.queued) - unlink(submission); - submission.worker_tickets = 0; + cancel_pending_assignments(submission); submission.done_cv.wait(lock, [&] { return submission.active_workers == 0; }); auto first_exception = submission.first_exception; lock.unlock(); @@ -167,6 +166,15 @@ class CpuDispatchPool { submission.queued = false; } + // Called under mutex_ after a drain has claimed every CU. Active workers + // retain their assignments and must still finish before the caller returns. + void cancel_pending_assignments(Submission &submission) { + assert(submission.next_task.load(std::memory_order_relaxed) >= submission.tasks.size()); + if (submission.queued) + unlink(submission); + submission.worker_tickets = 0; + } + void drain_tasks(Submission &submission) { while (true) { const size_t i = submission.next_task.fetch_add(1, std::memory_order_relaxed); @@ -198,6 +206,9 @@ class CpuDispatchPool { lock.unlock(); drain_tasks(submission); lock.lock(); + // Draining exhausted the task index. Cancel tickets before another worker + // claims an empty assignment while other CUs are still running. + cancel_pending_assignments(submission); if (--submission.active_workers == 0) submission.done_cv.notify_one(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/decoded_instruction_cache.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/decoded_instruction_cache.h new file mode 100644 index 000000000000..7b0eade759bd --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/decoded_instruction_cache.h @@ -0,0 +1,133 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" + +#include +#include +#include +#include +#include +#include + +namespace rocjitsu::amdgpu { + +/// @brief Reuse completed instructions on one CU's issue path. +/// +/// @details Fetchability and the byte I$ remain authoritative: lookups compare +/// all fetched words, including literals, so code maintenance and debugger +/// writes cannot reuse stale decodes. +/// Instructions that borrow the fetch buffer cannot be retained. Entries use +/// heap storage because a CU can run on different workers. Re-execution follows +/// the stateless Instruction::execute contract. +class DecodedInstructionCache { +public: + // Bounded host-speed choice; direct indexing requires a power of two. + static constexpr size_t kNumEntries = 256; + static_assert(kNumEntries != 0 && (kNumEntries & (kNumEntries - 1)) == 0); + // Match the four-word CU instruction-fetch window, including literals. + static constexpr size_t kNumWords = 4; + using Words = std::span; + + ~DecodedInstructionCache() { clear(); } + + /// @brief Decode from freshly fetched words, or take the matching cached object. + /// + /// @details The CU uses heap storage for every miss, including uncached + /// memory operations and debugger execution. The allocation guard also + /// bypasses an unrelated decoder pool installed by the caller. + /// @param enabled False bypasses reuse and table allocation while debugging. + DecodeResult decode(Decoder &decoder, uint64_t pc, uint32_t vmid, Words words, + const DecodeErrorEmitter &emit_error, bool enabled = true) { + if (enabled && !entries_) + entries_ = std::make_unique>(); + if (enabled && entries_) { + auto &entry = (*entries_)[index(pc)]; + if (entry.instruction && entry.pc == pc && entry.vmid == vmid && + std::equal(words.begin(), words.end(), entry.words.begin())) + return std::move(entry.instruction); + } + Instruction::ScopedHeapAllocation heap_allocation; + return decoder.decode(words.data(), emit_error); + } + + /// @brief Return only completed, stateless instructions. + /// + /// @details Construct immediately after decode() with the same key and policy. + /// Memory pipelines and async execution take ownership by releasing/moving + /// the caller's unique_ptr, so their in-flight instructions cannot become + /// cache entries. + class ScopedReturn { + public: + ScopedReturn(DecodedInstructionCache &cache, std::unique_ptr &instruction, + uint64_t pc, uint32_t vmid, Words words, bool enabled = true) + : cache_(cache), instruction_(instruction), pc_(pc), vmid_(vmid), words_(words), + enabled_(enabled), exceptions_(std::uncaught_exceptions()) {} + + ~ScopedReturn() { + if (enabled_ && instruction_ && !instruction_->is_memory_op() && !instruction_->data() && + !borrows_fetch_buffer(*instruction_, words_) && std::uncaught_exceptions() == exceptions_) + cache_.put(pc_, vmid_, words_, instruction_); + } + + ScopedReturn(const ScopedReturn &) = delete; + ScopedReturn &operator=(const ScopedReturn &) = delete; + + /// @brief Exclude a failed execution from reuse. + void discard() { enabled_ = false; } + + private: + DecodedInstructionCache &cache_; + std::unique_ptr &instruction_; + uint64_t pc_; + uint32_t vmid_; + Words words_; + bool enabled_; + int exceptions_; + }; + + /// @brief Drop entries before replacing the decoder/target, or destroying this CU. + void clear() { + Instruction::ScopedHeapAllocation heap_allocation; + entries_.reset(); + } + +private: + struct Entry { + uint64_t pc = 0; + uint32_t vmid = 0; + std::array words{}; + std::unique_ptr instruction; + }; + + static size_t index(uint64_t pc) { return (pc / sizeof(uint32_t)) & (kNumEntries - 1); } + + static bool borrows_fetch_buffer(const Instruction &instruction, Words words) { + const auto raw = reinterpret_cast(instruction.raw_encoding()); + const auto begin = reinterpret_cast(words.data()); + return raw >= begin && raw - begin < words.size_bytes(); + } + + void put(uint64_t pc, uint32_t vmid, Words words, std::unique_ptr &instruction) { + // A decode miss allocates the table before execution; returning an + // instruction during stack unwinding must not allocate. + if (!entries_) + return; + auto &entry = (*entries_)[index(pc)]; + if (entry.instruction) { + Instruction::ScopedHeapAllocation heap_allocation; + entry.instruction.reset(); + } + entry.pc = pc; + entry.vmid = vmid; + std::copy(words.begin(), words.end(), entry.words.begin()); + entry.instruction = std::move(instruction); + } + + std::unique_ptr> entries_; +}; + +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/device_cache_coherence.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/device_cache_coherence.h index fa1a0ab6681a..2296ec73a09b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/device_cache_coherence.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/device_cache_coherence.h @@ -88,6 +88,11 @@ class DeviceCacheCoherence { /// @brief Result of publishing dirty cache state for this boundary. [[nodiscard]] VmAccessOutcome outcome() const; + /// @brief Whether this boundary holds the specified coherence domain. + [[nodiscard]] bool belongs_to(const DeviceCacheCoherence *owner) const { + return owner_ != nullptr && owner_ == owner; + } + private: friend class DeviceCacheCoherence; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/dispatch_entry.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/dispatch_entry.h index e467b75947f7..99738e0b76a6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/dispatch_entry.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/dispatch_entry.h @@ -14,7 +14,10 @@ #include "rocjitsu/vm/amdgpu/aql/aql_packet_types.h" #include "rocjitsu/vm/amdgpu/consumer_cursor_journal.h" #include "rocjitsu/vm/amdgpu/gpu_handles.h" +#include "rocjitsu/vm/amdgpu/gpu_queue_registry.h" #include "rocjitsu/vm/amdgpu/interrupt_sink.h" +#include "rocjitsu/vm/amdgpu/pm4/pm4.h" +#include "rocjitsu/vm/amdgpu/pm4/pm4_packet_types.h" #include "rocjitsu/vm/amdgpu/queue_doorbell.h" #include "rocjitsu/vm/amdgpu/xcd_shard.h" @@ -57,8 +60,8 @@ class GpuVmAccess; (producer_pointer_address % alignof(uint64_t)) == 0; } -/// @brief Configuration supplied when registering an AQL queue with the CP. -struct AqlQueueConfig { +/// @brief Initial hardware state and root ring contract of one compute queue. +struct ComputeQueueConfig { AddressSpaceHandle address_space; InterruptSink interrupt_sink{}; uint32_t process_id = 0; @@ -82,6 +85,13 @@ struct AqlQueueConfig { /// creation path that models such a device; registering the queue replicates it /// onto the peer XCDs. bool xcd_fanout = false; + /// Root ring format stays fixed while an indirect stream changes the active decoder. + QueuePacketFormat packet_format = QueuePacketFormat::Aql; + std::optional initial_consumer_cursor = std::nullopt; + Pm4PacketCallbacks packet_callbacks{}; + /// DRM supplies command buffers directly rather than a user-mode ring. + bool submission_queue = false; + uint32_t scheduling_percentage = 100; }; struct WorkgroupCoord { @@ -408,6 +418,9 @@ struct DispatchEntry { bool wait_for_predecessors = false; /// Packet-type ordering: following packets cannot pass this packet. bool blocks_following = false; + /// An AMD vendor packet executes this PM4 buffer before publishing completion. + uint64_t aql_pm4_ib_address = 0; + uint32_t aql_pm4_ib_dwords = 0; /// Completion hooks and signal have already been delivered. bool completion_notified = false; bool execution_begun = false; @@ -669,16 +682,38 @@ inline constexpr uint32_t kPackedTidZShift = 20; return mask; } -/// @brief Configuration and runtime state for one CP-registered AQL queue. +/// @brief Shader dispatches currently executing within a PM4 command stream. +struct Pm4DispatchState { + std::deque entries; + void push_entry(DispatchEntry entry) { entries.push_back(std::move(entry)); } +}; + +/// @brief One compute queue owns its ring, CP execution state and all nested streams. /// /// @details Keeping the registration configuration and dispatch runtime in one /// record prevents queue mutation and teardown from desynchronizing parallel /// containers. Entries complete in submission order (in-order retirement per /// queue). -struct AqlQueueRecord : AqlQueueConfig { - AqlQueueRecord() : read_pointer_journal(read_ptr_va) {} - explicit AqlQueueRecord(AqlQueueConfig config) - : AqlQueueConfig(std::move(config)), read_pointer_journal(read_ptr_va) {} +struct ComputeQueueRecord : ComputeQueueConfig { + ComputeQueueRecord() : read_pointer_journal(read_ptr_va) {} + explicit ComputeQueueRecord(ComputeQueueConfig config) + : ComputeQueueConfig(std::move(config)), + read_pointer_journal(read_ptr_va, initial_consumer_cursor, + packet_format == QueuePacketFormat::Pm4 ? sizeof(uint32_t) + : sizeof(uint64_t)) {} + + /// Registers survive IB return; command frames retain their own fetch positions. + ComputeCommandState commands; + Pm4DispatchState dispatches; + /// Exact VM snapshot used for a root PM4 ring batch and its cursor writeback. + std::optional command_access; + // A transient VM access is retried only on a later CP pass. + bool command_retry_pending = false; + bool command_fault_pending = false; + [[nodiscard]] bool has_pending_commands() const { + return !entries.empty() || !dispatches.entries.empty() || !commands.submissions.empty() || + command_access.has_value() || read_pointer_journal.publication_pending(); + } enum class Status { Idle, Active, Blocked }; @@ -695,7 +730,7 @@ struct AqlQueueRecord : AqlQueueConfig { /// tears it down; fault notification is handled separately. bool faulted = false; bool debug_suspended = false; - bool runtime_suspended = false; + bool runtime_suspended = scheduling_percentage == 0; bool exception_suspended = false; [[nodiscard]] bool suspended() const { return debug_suspended || runtime_suspended || exception_suspended; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_queue_registry.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_queue_registry.h index bdf50bdf32c5..c53c5e6833a6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_queue_registry.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_queue_registry.h @@ -200,9 +200,9 @@ struct QueueRegistrationRequest { QueueDoorbellBinding doorbell; std::shared_ptr binding_factory; /// @brief Optional initial hardware read cursor. - /// @details MES supplies the cursor captured from the SDMA MQD; legacy/KFD - /// queues leave it unset so the binding loads the already-published cursor - /// from the consumer pointer address on first service. + /// @details MES supplies the cursor captured from the SDMA MQD; fresh KFD SDMA + /// and native PM4 queues supply zero. Bindings that leave it unset load the + /// already-published cursor from the consumer pointer address on first service. std::optional initial_consumer_cursor = std::nullopt; QueueAbi abi = QueueAbi::Generic; uint64_t queue_descriptor_address = 0; @@ -212,6 +212,7 @@ struct QueueRegistrationRequest { bool xcd_fanout = false; QueueType type = QueueType::Compute; QueuePacketFormat packet_format = QueuePacketFormat::Aql; + uint32_t scheduling_percentage = 100; }; /// @brief Shared queue registry used by legacy and PCI/VFIO frontends. diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_vm.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_vm.cpp index b3c315a0e0ec..946779b39e8c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_vm.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_vm.cpp @@ -10,8 +10,11 @@ #include #include #include +#include #include #include +#include +#include #include #include @@ -19,7 +22,42 @@ namespace rocjitsu::amdgpu { class GpuVmAccessState { public: - mutable std::shared_mutex mutex; + class FaultReporter { + public: + explicit FaultReporter(std::function callback) + : callback_(std::move(callback)) {} + + explicit operator bool() const { return static_cast(callback_); } + + void operator()(uint64_t address, VmAccessKind access) const { + // A const std::function can invoke a mutable target. Every generation + // shares this guard along with the target; a generation's shared access + // lease does not exclude other readers or pinned generations. Recursive + // acquisition allows same-thread callback reentry through independent + // access states; it does not make a state's revocation lease recursive. + std::lock_guard lock(mutex_); + callback_(address, access); + } + + private: + const std::function callback_; + mutable std::recursive_mutex mutex_; + }; + + GpuVmAccessState(std::shared_ptr translator = {}, + std::shared_ptr physical_memory = {}, + std::shared_ptr fault_reporter = {}) + : translator(std::move(translator)), physical_memory(std::move(physical_memory)), + fault_reporter(std::move(fault_reporter)) {} + + // One shared lifetime per snapshot; copying a snapshot must not copy the + // fault callback or separately contend on each backing object's reference count. + const std::shared_ptr translator; + const std::shared_ptr physical_memory; + // Generations share the callable too: pinning must not copy its target while + // a fault callback is running under another generation's shared lease. + const std::shared_ptr fault_reporter; + mutable util::DistributedSharedMutex mutex; // Written under mutex; atomic so copied policy hits can check retirement without a lease. std::atomic valid{true}; }; @@ -31,6 +69,19 @@ class GpuVmBindingState { namespace { +std::atomic next_vm_instance_id{1}; + +struct VmAccessBatchSnapshot { + const GpuVm *vm = nullptr; + uint64_t vm_instance = 0; + AddressSpaceHandle handle; + GpuVmAccess access; + bool routed = false; +}; + +thread_local uint32_t vm_access_batch_depth = 0; +thread_local std::deque vm_access_batch_snapshots; + constexpr uint64_t kGfx12PteValid = uint64_t{1} << 0; constexpr uint64_t kGfx12PteSystem = uint64_t{1} << 1; constexpr uint64_t kGfx12PteSnooped = uint64_t{1} << 2; @@ -99,46 +150,113 @@ Mtype gfx12_mtype(uint64_t entry) { } template -VmAccessOutcome -access_translated(const AddressSpaceTranslator &translator, PhysicalMemoryAccess &memory, - uint64_t address, Span bytes, std::size_t &completed_bytes, VmAccessKind access, - const std::function &fault_reporter = {}) { +VmTransferStep transfer_translated_span(const AddressSpaceTranslator &translator, + PhysicalMemoryAccess &memory, uint64_t address, Span bytes, + VmAccessKind access) { + const VmTranslationResult translated = + access == VmAccessKind::Execute ? translator.probe_translation(address, bytes.size(), access) + : translator.translate(address, bytes.size(), access); + if (!translated) + return {.outcome = translated.outcome, .report_translation_fault = true}; + if (translated.translation.contiguous_bytes == 0) + return {.outcome = VmAccessOutcome::Malformed}; + const auto chunk = static_cast( + std::min(bytes.size(), translated.translation.contiguous_bytes)); + const VmAccessOutcome outcome = [&] { + if constexpr (std::is_const_v) + return memory.write(translated.translation.domain, translated.translation.address, + bytes.first(chunk)); + else + return memory.read_for_access(translated.translation.domain, translated.translation.address, + bytes.first(chunk), access); + }(); + return {.completed_bytes = outcome == VmAccessOutcome::Complete ? chunk : 0, .outcome = outcome}; +} + +template +VmAccessOutcome access_translated(const AddressSpaceTranslator &translator, + PhysicalMemoryAccess &memory, uint64_t address, Span bytes, + std::size_t &completed_bytes, VmAccessKind access, + const GpuVmAccessState::FaultReporter *fault_reporter = nullptr) { if (completed_bytes > bytes.size()) return VmAccessOutcome::Malformed; while (completed_bytes < bytes.size()) { - const VmTranslationResult translated = - access == VmAccessKind::Execute - ? translator.probe_translation(address + completed_bytes, - bytes.size() - completed_bytes, access) - : translator.translate(address + completed_bytes, bytes.size() - completed_bytes, - access); - if (!translated) { - if (fault_reporter && translated.outcome != VmAccessOutcome::Unavailable) - fault_reporter(address + completed_bytes, access); - return translated.outcome; + const VmTransferStep step = [&] { + if constexpr (std::is_const_v) + return translator.write_step(memory, address + completed_bytes, + bytes.subspan(completed_bytes)); + else + return translator.read_step(memory, address + completed_bytes, + bytes.subspan(completed_bytes), access); + }(); + if (step.outcome != VmAccessOutcome::Complete) { + if (step.report_translation_fault && fault_reporter && *fault_reporter && + step.outcome != VmAccessOutcome::Unavailable) + (*fault_reporter)(address + completed_bytes, access); + return step.outcome; } - if (translated.translation.contiguous_bytes == 0) + if (step.completed_bytes == 0 || step.completed_bytes > bytes.size() - completed_bytes) return VmAccessOutcome::Malformed; - const std::size_t chunk = static_cast(std::min( - bytes.size() - completed_bytes, translated.translation.contiguous_bytes)); - const VmAccessOutcome outcome = [&]() { - if constexpr (std::is_const_v) { - return memory.write(translated.translation.domain, translated.translation.address, - bytes.subspan(completed_bytes, chunk)); - } else { - return memory.read_for_access(translated.translation.domain, translated.translation.address, - bytes.subspan(completed_bytes, chunk), access); - } - }(); - if (outcome != VmAccessOutcome::Complete) - return outcome; - completed_bytes += chunk; + completed_bytes += step.completed_bytes; } return VmAccessOutcome::Complete; } } // namespace +VmTransferStep AddressSpaceTranslator::read_step(PhysicalMemoryAccess &memory, uint64_t address, + std::span bytes, + VmAccessKind access) const { + return transfer_translated_span(*this, memory, address, bytes, access); +} + +VmTransferStep AddressSpaceTranslator::write_step(PhysicalMemoryAccess &memory, uint64_t address, + std::span bytes) const { + return transfer_translated_span(*this, memory, address, bytes, VmAccessKind::Write); +} + +GpuVmAccessBatchGuard::GpuVmAccessBatchGuard() { + if (vm_access_batch_depth++ == 0) { + vm_access_batch_snapshots.clear(); + } +} + +GpuVmAccessBatchGuard::~GpuVmAccessBatchGuard() { + assert(vm_access_batch_depth != 0); + if (--vm_access_batch_depth != 0) + return; + vm_access_batch_snapshots.clear(); +} + +bool GpuVmAccessBatchGuard::active() { return vm_access_batch_depth != 0; } + +const GpuVmAccess *GpuVmAccessBatchGuard::find_snapshot(const GpuVm *vm, + AddressSpaceHandle handle) { + if (vm_access_batch_depth == 0) + return nullptr; + const auto entry = std::ranges::find_if(vm_access_batch_snapshots, [&](const auto &candidate) { + return candidate.vm == vm && candidate.vm_instance == vm->instance_id_ && + candidate.handle == handle && candidate.access.is_current(); + }); + return entry != vm_access_batch_snapshots.end() ? &entry->access : nullptr; +} + +const GpuVmAccess *GpuVmAccessBatchGuard::find_snapshot_vmid(const GpuVm *vm, uint32_t vmid) { + if (vm_access_batch_depth == 0) + return nullptr; + const auto entry = std::ranges::find_if(vm_access_batch_snapshots, [&](const auto &candidate) { + return candidate.vm == vm && candidate.vm_instance == vm->instance_id_ && candidate.routed && + candidate.access.info().vmid == vmid && candidate.access.is_current(); + }); + return entry != vm_access_batch_snapshots.end() ? &entry->access : nullptr; +} + +void GpuVmAccessBatchGuard::retain_snapshot(const GpuVm *vm, AddressSpaceHandle handle, + const GpuVmAccess &access, bool routed) { + if (vm_access_batch_depth != 0) + vm_access_batch_snapshots.push_back({vm, vm->instance_id_, handle, access, routed}); +} + GpuVmBindingLease::GpuVmBindingLease(GpuVmBindingLease &&other) noexcept : state_(std::move(other.state_)), handle_(other.handle_), info_(other.info_) { other.handle_ = {}; @@ -170,7 +288,9 @@ void GpuVmBindingLease::release() noexcept { info_ = {}; } -GpuVm::GpuVm(Gfx12VmConfig gfx12_config) : gfx12_config_(gfx12_config) {} +GpuVm::GpuVm(Gfx12VmConfig gfx12_config) + : instance_id_(next_vm_instance_id.fetch_add(1, std::memory_order_relaxed)), + gfx12_config_(gfx12_config) {} VmAccessOutcome read_translated(const AddressSpaceTranslator &translator, PhysicalMemoryAccess &memory, uint64_t address, @@ -202,70 +322,129 @@ VmTranslationResult IdentityAddressSpaceTranslator::translate(uint64_t address, .permissions = {.readable = true, .writable = true, .executable = true}}}; } +bool GpuVmAccess::is_current() const { + return access_state_ != nullptr && access_state_->valid.load(std::memory_order_acquire); +} + VmTranslationResult GpuVmAccess::translate(uint64_t address, std::size_t size, VmAccessKind access) const { if (access_state_ == nullptr) return {.outcome = VmAccessOutcome::Unavailable, .translation = {}}; std::shared_lock state_lock(access_state_->mutex); - VmTranslationResult translated = !access_state_->valid || translator_ == nullptr + VmTranslationResult translated = !access_state_->valid || access_state_->translator == nullptr ? VmTranslationResult{ .outcome = VmAccessOutcome::Unavailable, .translation = {}, } - : translator_->translate(address, size, access); + : access_state_->translator->translate(address, size, access); report_terminal_fault(address, access, translated.outcome); return translated; } +bool GpuVmAccess::try_write_private_dwords(std::span stores, + Mtype instruction_mtype, Mtype expected_mtype) const { + if (stores.size() < 2 || stores.size() > VmRamDwordStore::kMaxBatch || !access_state_) + return false; + const uint64_t page = stores.front().address / IdentityAddressSpaceTranslator::kPageSize; + for (const auto &[address, source] : stores) + if (!source || (address & 3) || address / IdentityAddressSpaceTranslator::kPageSize != page) + return false; + std::shared_lock state_lock(access_state_->mutex); + return access_state_->valid && access_state_->translator && access_state_->physical_memory && + access_state_->translator->try_write_private_dwords( + *access_state_->physical_memory, stores, instruction_mtype, expected_mtype); +} + +bool GpuVmAccess::try_read_uncached_ram(uint64_t address, std::span bytes) const { + if (!access_state_) + return false; + std::shared_lock state_lock(access_state_->mutex); + return access_state_->valid && access_state_->translator && access_state_->physical_memory && + access_state_->translator->try_read_uncached_ram(*access_state_->physical_memory, address, + bytes); +} + bool GpuVmAccess::try_read_contiguous(uint64_t address, std::span bytes) const { if (!access_state_) return false; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || !translator_ || !physical_memory_) + if (!access_state_->valid || !access_state_->translator || !access_state_->physical_memory) return false; // Bypass the fault-reporting translate() wrapper for this optional span. - const auto translation = translator_->translate(address, bytes.size(), VmAccessKind::Read); + const auto translation = + access_state_->translator->translate(address, bytes.size(), VmAccessKind::Read); return translation && translation.translation.contiguous_bytes >= bytes.size() && - physical_memory_->try_read_contiguous(translation.translation.domain, - translation.translation.address, bytes); + access_state_->physical_memory->try_read_contiguous( + translation.translation.domain, translation.translation.address, bytes); } bool GpuVmAccess::try_write_contiguous(uint64_t address, std::span bytes) const { if (!access_state_) return false; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || !translator_ || !physical_memory_) + if (!access_state_->valid || !access_state_->translator || !access_state_->physical_memory) return false; // Bypass the fault-reporting translate() wrapper for this optional span. - const auto translation = translator_->translate(address, bytes.size(), VmAccessKind::Write); + const auto translation = + access_state_->translator->translate(address, bytes.size(), VmAccessKind::Write); return translation && translation.translation.contiguous_bytes >= bytes.size() && - physical_memory_->try_write_contiguous(translation.translation.domain, - translation.translation.address, bytes); + access_state_->physical_memory->try_write_contiguous( + translation.translation.domain, translation.translation.address, bytes); } std::optional GpuVmAccess::query_mtype(uint64_t address) const { if (access_state_ == nullptr) return std::nullopt; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr) return std::nullopt; - return translator_->query_mtype(address); + return access_state_->translator->query_mtype(address); } std::optional GpuVmAccess::query_mtype(uint64_t address, VmMtypeCache &cache) const { if (access_state_ == nullptr) return std::nullopt; - // A hit uses copied policy only. Retirement invalidates the retained state + // A hit uses copied policy only. Retirement invalidates the snapshot's state // before releasing any frontend storage; misses remain under its lease. - if (cache.access_state_ == access_state_ && - access_state_->valid.load(std::memory_order_acquire) && cache.snapshot_.unchanged(address)) - return cache.snapshot_.mtype; + // Compare generation ownership without acquiring another strong reference. + const bool same_generation = !cache.access_state_.owner_before(access_state_) && + !access_state_.owner_before(cache.access_state_); + auto &snapshot = cache.snapshots_[(address >> 12) % cache.snapshots_.size()]; + if (same_generation && access_state_->valid.load(std::memory_order_acquire) && + snapshot.unchanged(address)) + return snapshot.mtype; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr) + return std::nullopt; + if (!same_generation) { + cache.snapshots_ = {}; + cache.access_state_ = access_state_; + } + snapshot = access_state_->translator->snapshot_mtype(address); + return snapshot.mtype; +} + +bool GpuVmAccess::cached_private_uc_hint(uint64_t address, const VmMtypeCache &cache) const { + if (!access_state_) + return false; + const bool same_generation = !cache.access_state_.owner_before(access_state_) && + !access_state_.owner_before(cache.access_state_); + const auto &snapshot = cache.snapshots_[(address >> 12) % cache.snapshots_.size()]; + return same_generation && access_state_->valid.load(std::memory_order_acquire) && + snapshot.may_batch_private_uc && snapshot.unchanged(address); +} + +std::optional GpuVmAccess::cached_private_ram_mtype(uint64_t address, + const VmMtypeCache &cache) const { + if (!access_state_) return std::nullopt; - cache.snapshot_ = translator_->snapshot_mtype(address); - cache.access_state_ = access_state_; - return cache.snapshot_.mtype; + const bool same_generation = !cache.access_state_.owner_before(access_state_) && + !access_state_.owner_before(cache.access_state_); + const auto &snapshot = cache.snapshots_[(address >> 12) % cache.snapshots_.size()]; + if (same_generation && access_state_->valid.load(std::memory_order_acquire) && + snapshot.may_batch_private_ram && snapshot.unchanged(address)) + return snapshot.mtype; + return std::nullopt; } VmAccessOutcome GpuVmAccess::query_access(uint64_t address, std::size_t size, @@ -282,7 +461,7 @@ VmAccessOutcome GpuVmAccess::probe_impl(uint64_t address, std::size_t size, VmAc if (access_state_ == nullptr) return VmAccessOutcome::Unavailable; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr) return VmAccessOutcome::Unavailable; if (size == 0 || size - 1 > std::numeric_limits::max() - address) { if (report_fault) @@ -293,8 +472,8 @@ VmAccessOutcome GpuVmAccess::probe_impl(uint64_t address, std::size_t size, VmAc std::size_t completed_bytes = 0; while (completed_bytes < size) { const uint64_t current_address = address + completed_bytes; - const VmTranslationResult translated = - translator_->probe_translation(current_address, size - completed_bytes, access); + const VmTranslationResult translated = access_state_->translator->probe_translation( + current_address, size - completed_bytes, access); if (!translated) { if (report_fault) report_terminal_fault(current_address, access, translated.outcome); @@ -313,9 +492,9 @@ VmAccessOutcome GpuVmAccess::probe_impl(uint64_t address, std::size_t size, VmAc void GpuVmAccess::report_terminal_fault(uint64_t address, VmAccessKind access, VmAccessOutcome outcome) const { - if (fault_reporter_ && outcome != VmAccessOutcome::Complete && - outcome != VmAccessOutcome::Unavailable) { - fault_reporter_(address, access); + if (access_state_ && access_state_->fault_reporter && *access_state_->fault_reporter && + outcome != VmAccessOutcome::Complete && outcome != VmAccessOutcome::Unavailable) { + (*access_state_->fault_reporter)(address, access); } } @@ -332,10 +511,11 @@ VmAccessOutcome GpuVmAccess::read(uint64_t address, std::span bytes, if (access_state_ == nullptr) return VmAccessOutcome::Unavailable; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr || physical_memory_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr || + access_state_->physical_memory == nullptr) return VmAccessOutcome::Unavailable; - return access_translated(*translator_, *physical_memory_, address, bytes, completed_bytes, access, - fault_reporter_); + return access_translated(*access_state_->translator, *access_state_->physical_memory, address, + bytes, completed_bytes, access, access_state_->fault_reporter.get()); } VmAccessOutcome GpuVmAccess::write(uint64_t address, std::span bytes) const { @@ -348,10 +528,12 @@ VmAccessOutcome GpuVmAccess::write(uint64_t address, std::span if (access_state_ == nullptr) return VmAccessOutcome::Unavailable; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr || physical_memory_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr || + access_state_->physical_memory == nullptr) return VmAccessOutcome::Unavailable; - return access_translated(*translator_, *physical_memory_, address, bytes, completed_bytes, - VmAccessKind::Write, fault_reporter_); + return access_translated(*access_state_->translator, *access_state_->physical_memory, address, + bytes, completed_bytes, VmAccessKind::Write, + access_state_->fault_reporter.get()); } AtomicLoadResult GpuVmAccess::atomic_load(uint64_t address, uint32_t width) const { @@ -363,11 +545,12 @@ AtomicLoadResult GpuVmAccess::atomic_load(uint64_t address, uint32_t width) cons if (access_state_ == nullptr) return {.outcome = VmAccessOutcome::Unavailable, .value = 0}; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr || physical_memory_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr || + access_state_->physical_memory == nullptr) return {.outcome = VmAccessOutcome::Unavailable, .value = 0}; const VmTranslationResult translated = - translator_->translate(address, width, VmAccessKind::Atomic); + access_state_->translator->translate(address, width, VmAccessKind::Atomic); if (!translated) { report_terminal_fault(address, VmAccessKind::Atomic, translated.outcome); return {.outcome = translated.outcome, .value = 0}; @@ -376,8 +559,8 @@ AtomicLoadResult GpuVmAccess::atomic_load(uint64_t address, uint32_t width) cons report_terminal_fault(address, VmAccessKind::Atomic, VmAccessOutcome::Malformed); return {.outcome = VmAccessOutcome::Malformed, .value = 0}; } - return physical_memory_->atomic_load(translated.translation.domain, - translated.translation.address, width); + return access_state_->physical_memory->atomic_load(translated.translation.domain, + translated.translation.address, width); } VmAccessOutcome GpuVmAccess::atomic_store(uint64_t address, uint32_t width, uint64_t value) const { @@ -388,11 +571,12 @@ VmAccessOutcome GpuVmAccess::atomic_store(uint64_t address, uint32_t width, uint if (access_state_ == nullptr) return VmAccessOutcome::Unavailable; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr || physical_memory_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr || + access_state_->physical_memory == nullptr) return VmAccessOutcome::Unavailable; const VmTranslationResult translated = - translator_->translate(address, width, VmAccessKind::Atomic); + access_state_->translator->translate(address, width, VmAccessKind::Atomic); if (!translated) { report_terminal_fault(address, VmAccessKind::Atomic, translated.outcome); return translated.outcome; @@ -401,8 +585,8 @@ VmAccessOutcome GpuVmAccess::atomic_store(uint64_t address, uint32_t width, uint report_terminal_fault(address, VmAccessKind::Atomic, VmAccessOutcome::Malformed); return VmAccessOutcome::Malformed; } - return physical_memory_->atomic_store(translated.translation.domain, - translated.translation.address, width, value); + return access_state_->physical_memory->atomic_store(translated.translation.domain, + translated.translation.address, width, value); } AtomicCompareExchangeResult GpuVmAccess::compare_exchange(uint64_t address, uint32_t width, @@ -415,11 +599,12 @@ AtomicCompareExchangeResult GpuVmAccess::compare_exchange(uint64_t address, uint if (access_state_ == nullptr) return {.outcome = VmAccessOutcome::Unavailable}; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr || physical_memory_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr || + access_state_->physical_memory == nullptr) return {.outcome = VmAccessOutcome::Unavailable}; const VmTranslationResult translated = - translator_->translate(address, width, VmAccessKind::Atomic); + access_state_->translator->translate(address, width, VmAccessKind::Atomic); if (!translated) { report_terminal_fault(address, VmAccessKind::Atomic, translated.outcome); return {.outcome = translated.outcome}; @@ -428,7 +613,7 @@ AtomicCompareExchangeResult GpuVmAccess::compare_exchange(uint64_t address, uint report_terminal_fault(address, VmAccessKind::Atomic, VmAccessOutcome::Malformed); return {.outcome = VmAccessOutcome::Malformed}; } - return physical_memory_->compare_exchange( + return access_state_->physical_memory->compare_exchange( translated.translation.domain, translated.translation.address, width, expected, desired); } @@ -443,11 +628,12 @@ GpuVmAccess::atomic_modify(uint64_t address, uint32_t width, if (access_state_ == nullptr) return VmAccessOutcome::Unavailable; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr || physical_memory_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr || + access_state_->physical_memory == nullptr) return VmAccessOutcome::Unavailable; const VmTranslationResult translated = - translator_->translate(address, width, VmAccessKind::Atomic); + access_state_->translator->translate(address, width, VmAccessKind::Atomic); if (!translated) { report_terminal_fault(address, VmAccessKind::Atomic, translated.outcome); return translated.outcome; @@ -456,34 +642,38 @@ GpuVmAccess::atomic_modify(uint64_t address, uint32_t width, report_terminal_fault(address, VmAccessKind::Atomic, VmAccessOutcome::Malformed); return VmAccessOutcome::Malformed; } - return physical_memory_->atomic_modify(translated.translation.domain, - translated.translation.address, width, mutation); + return access_state_->physical_memory->atomic_modify( + translated.translation.domain, translated.translation.address, width, mutation); } std::byte *GpuVmAccess::resolve_host_pointer(uint64_t address, std::size_t size) const { if (access_state_ == nullptr) return nullptr; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr || physical_memory_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr || + access_state_->physical_memory == nullptr) return nullptr; - const VmTranslationResult translated = translator_->translate(address, size, VmAccessKind::Read); + const VmTranslationResult translated = + access_state_->translator->translate(address, size, VmAccessKind::Read); if (!translated || translated.translation.contiguous_bytes < size) return nullptr; - return physical_memory_->resolve_host_pointer(translated.translation.domain, - translated.translation.address, size); + return access_state_->physical_memory->resolve_host_pointer(translated.translation.domain, + translated.translation.address, size); } std::pair GpuVmAccess::host_range(uint64_t address) const { if (access_state_ == nullptr) return {0, 0}; std::shared_lock state_lock(access_state_->mutex); - if (!access_state_->valid || translator_ == nullptr || physical_memory_ == nullptr) + if (!access_state_->valid || access_state_->translator == nullptr || + access_state_->physical_memory == nullptr) return {0, 0}; - const VmTranslationResult translated = translator_->translate(address, 1, VmAccessKind::Read); + const VmTranslationResult translated = + access_state_->translator->translate(address, 1, VmAccessKind::Read); if (!translated) return {0, 0}; - return physical_memory_->host_range(translated.translation.domain, - translated.translation.address); + return access_state_->physical_memory->host_range(translated.translation.domain, + translated.translation.address); } Gfx12PageTableTranslator::Gfx12PageTableTranslator(std::shared_ptr memory, @@ -697,8 +887,8 @@ const GpuVm::Binding *GpuVm::find_locked(AddressSpaceHandle handle) const { return slot.generation == handle.generation && slot.binding ? &*slot.binding : nullptr; } -void GpuVm::advance_access_state_locked(Binding &binding) { - auto replacement = std::make_shared(); +void GpuVm::advance_access_state_locked(Binding &binding, + std::shared_ptr replacement) { if (binding.access_state != nullptr) { std::unique_lock state_lock(binding.access_state->mutex); binding.access_state->valid = false; @@ -730,17 +920,17 @@ GpuVm::register_address_space(uint32_t vmid, std::shared_ptr( + std::move(translator), std::move(physical_memory), + std::make_shared(std::move(fault_reporter))), + .pinned_access_states = {}, + .legacy_cache_compatible = legacy_cache_compatible, + .device_gart = false}); vmid_handles_.emplace(vmid, handle); return handle; } @@ -752,17 +942,17 @@ AddressSpaceHandle GpuVm::register_unrouted_address_space( if (translator == nullptr || physical_memory == nullptr) return {}; std::lock_guard lock(mutex_); - return allocate_locked({.vmid = vmid, - .translation_epoch = 1, - .queue_references = 0, - .binding_state = {}, - .translator = std::move(translator), - .physical_memory = std::move(physical_memory), - .access_state = {}, - .pinned_access_states = {}, - .fault_reporter = std::move(fault_reporter), - .legacy_cache_compatible = legacy_cache_compatible, - .device_gart = false}); + return allocate_locked( + {.vmid = vmid, + .translation_epoch = 1, + .queue_references = 0, + .binding_state = {}, + .access_state = std::make_shared( + std::move(translator), std::move(physical_memory), + std::make_shared(std::move(fault_reporter))), + .pinned_access_states = {}, + .legacy_cache_compatible = legacy_cache_compatible, + .device_gart = false}); } AddressSpaceHandle GpuVm::register_translated(uint32_t vmid, @@ -798,12 +988,12 @@ bool GpuVm::replace_translated(AddressSpaceHandle handle, return false; std::lock_guard lock(mutex_); Binding *binding = find_locked(handle); - if (binding == nullptr || binding->translator == nullptr || binding->legacy_cache_compatible || - binding->device_gart) + if (binding == nullptr || binding->access_state->translator == nullptr || + binding->legacy_cache_compatible || binding->device_gart) return false; - advance_access_state_locked(*binding); - binding->translator = std::move(translator); - binding->physical_memory = std::move(memory); + advance_access_state_locked( + *binding, std::make_shared(std::move(translator), std::move(memory), + binding->access_state->fault_reporter)); ++binding->translation_epoch; if (binding->translation_epoch == 0) ++binding->translation_epoch; @@ -828,11 +1018,8 @@ AddressSpaceHandle GpuVm::initialize_gart_address_space() { .translation_epoch = 1, .queue_references = 0, .binding_state = {}, - .translator = nullptr, - .physical_memory = nullptr, .access_state = {}, .pinned_access_states = {}, - .fault_reporter = {}, .legacy_cache_compatible = false, .device_gart = true}); return gart_address_space_; @@ -848,9 +1035,9 @@ bool GpuVm::publish_gart(const GartConfig &config, std::shared_ptrdevice_gart) return false; - advance_access_state_locked(*binding); - binding->translator = std::move(translator); - binding->physical_memory = std::move(memory); + advance_access_state_locked( + *binding, std::make_shared(std::move(translator), std::move(memory), + binding->access_state->fault_reporter)); ++binding->translation_epoch; if (binding->translation_epoch == 0) ++binding->translation_epoch; @@ -865,8 +1052,6 @@ bool GpuVm::clear_gart_binding() { return false; revoke_access_state_locked(*binding); binding->access_state = std::make_shared(); - binding->translator.reset(); - binding->physical_memory.reset(); ++binding->translation_epoch; if (binding->translation_epoch == 0) ++binding->translation_epoch; @@ -874,7 +1059,7 @@ bool GpuVm::clear_gart_binding() { } AddressSpaceHandle GpuVm::gart_address_space() const { - std::lock_guard lock(mutex_); + std::shared_lock lock(mutex_); return find_locked(gart_address_space_) != nullptr ? gart_address_space_ : AddressSpaceHandle{}; } @@ -883,8 +1068,11 @@ bool GpuVm::invalidate(AddressSpaceHandle handle) { Binding *binding = find_locked(handle); if (binding == nullptr) return false; + auto replacement = std::make_shared(binding->access_state->translator, + binding->access_state->physical_memory, + binding->access_state->fault_reporter); revoke_access_state_locked(*binding); - binding->access_state = std::make_shared(); + binding->access_state = std::move(replacement); ++binding->translation_epoch; if (binding->translation_epoch == 0) ++binding->translation_epoch; @@ -901,8 +1089,8 @@ std::optional GpuVm::retain_queue_address_space(AddressSpaceHa .translation_epoch = binding->translation_epoch, .queue_references = binding->queue_references, .legacy_cache_compatible = binding->legacy_cache_compatible, - .ready = binding->translator != nullptr && - binding->physical_memory != nullptr}; + .ready = binding->access_state->translator != nullptr && + binding->access_state->physical_memory != nullptr}; } bool GpuVm::retain_queue(AddressSpaceHandle handle) { @@ -929,8 +1117,8 @@ std::optional GpuVm::retain_binding(AddressSpaceHandle handle .translation_epoch = binding->translation_epoch, .queue_references = binding->queue_references, .legacy_cache_compatible = binding->legacy_cache_compatible, - .ready = - binding->translator != nullptr && binding->physical_memory != nullptr}; + .ready = binding->access_state->translator != nullptr && + binding->access_state->physical_memory != nullptr}; return GpuVmBindingLease(binding->binding_state, handle, info); } @@ -956,19 +1144,18 @@ bool GpuVm::unregister_address_space(AddressSpaceHandle handle) { } std::optional GpuVm::snapshot(AddressSpaceHandle handle) const { - std::lock_guard lock(mutex_); + std::shared_lock lock(mutex_); const Binding *binding = find_locked(handle); if (binding == nullptr) return std::nullopt; - return GpuVmAccess( - handle, - {.vmid = binding->vmid, - .translation_epoch = binding->translation_epoch, - .queue_references = binding->queue_references, - .legacy_cache_compatible = binding->legacy_cache_compatible, - .ready = binding->translator != nullptr && binding->physical_memory != nullptr}, - binding->translator, binding->physical_memory, binding->access_state, - binding->fault_reporter); + return GpuVmAccess(handle, + {.vmid = binding->vmid, + .translation_epoch = binding->translation_epoch, + .queue_references = binding->queue_references, + .legacy_cache_compatible = binding->legacy_cache_compatible, + .ready = binding->access_state->translator != nullptr && + binding->access_state->physical_memory != nullptr}, + binding->access_state); } std::optional GpuVm::snapshot_pinned(AddressSpaceHandle handle) { @@ -978,21 +1165,22 @@ std::optional GpuVm::snapshot_pinned(AddressSpaceHandle handle) { return std::nullopt; std::erase_if(binding->pinned_access_states, [](const std::weak_ptr &state) { return state.expired(); }); - auto access_state = std::make_shared(); + auto access_state = std::make_shared(binding->access_state->translator, + binding->access_state->physical_memory, + binding->access_state->fault_reporter); binding->pinned_access_states.emplace_back(access_state); - return GpuVmAccess( - handle, - {.vmid = binding->vmid, - .translation_epoch = binding->translation_epoch, - .queue_references = binding->queue_references, - .legacy_cache_compatible = binding->legacy_cache_compatible, - .ready = binding->translator != nullptr && binding->physical_memory != nullptr}, - binding->translator, binding->physical_memory, std::move(access_state), - binding->fault_reporter); + return GpuVmAccess(handle, + {.vmid = binding->vmid, + .translation_epoch = binding->translation_epoch, + .queue_references = binding->queue_references, + .legacy_cache_compatible = binding->legacy_cache_compatible, + .ready = binding->access_state->translator != nullptr && + binding->access_state->physical_memory != nullptr}, + std::move(access_state)); } std::optional GpuVm::snapshot_vmid(uint32_t vmid) const { - std::lock_guard lock(mutex_); + std::shared_lock lock(mutex_); const std::unordered_map::const_iterator found = vmid_handles_.find(vmid); if (found == vmid_handles_.end()) @@ -1000,15 +1188,39 @@ std::optional GpuVm::snapshot_vmid(uint32_t vmid) const { const Binding *binding = find_locked(found->second); if (binding == nullptr) return std::nullopt; - return GpuVmAccess( - found->second, - {.vmid = binding->vmid, - .translation_epoch = binding->translation_epoch, - .queue_references = binding->queue_references, - .legacy_cache_compatible = binding->legacy_cache_compatible, - .ready = binding->translator != nullptr && binding->physical_memory != nullptr}, - binding->translator, binding->physical_memory, binding->access_state, - binding->fault_reporter); + return GpuVmAccess(found->second, + {.vmid = binding->vmid, + .translation_epoch = binding->translation_epoch, + .queue_references = binding->queue_references, + .legacy_cache_compatible = binding->legacy_cache_compatible, + .ready = binding->access_state->translator != nullptr && + binding->access_state->physical_memory != nullptr}, + binding->access_state); +} + +const GpuVmAccess *GpuVm::borrow_snapshot(AddressSpaceHandle handle) const { + assert(GpuVmAccessBatchGuard::active()); + if (const GpuVmAccess *cached = GpuVmAccessBatchGuard::find_snapshot(this, handle)) + return cached; + if (const std::optional access = snapshot(handle)) { + GpuVmAccessBatchGuard::retain_snapshot(this, handle, *access, false); + // A concurrent revocation makes this access retryable, not an absent binding. + return &vm_access_batch_snapshots.back().access; + } + return nullptr; +} + +const GpuVmAccess *GpuVm::borrow_snapshot_vmid(uint32_t vmid) const { + assert(GpuVmAccessBatchGuard::active()); + if (const GpuVmAccess *cached = GpuVmAccessBatchGuard::find_snapshot_vmid(this, vmid)) + return cached; + if (const std::optional access = snapshot_vmid(vmid)) { + GpuVmAccessBatchGuard::retain_snapshot(this, access->cache_namespace().address_space, *access, + true); + // A concurrent revocation makes this access retryable, not an absent binding. + return &vm_access_batch_snapshots.back().access; + } + return nullptr; } VmTranslationResult GpuVm::translate(AddressSpaceHandle handle, uint64_t address, std::size_t size, @@ -1060,7 +1272,7 @@ AtomicCompareExchangeResult GpuVm::compare_exchange(AddressSpaceHandle handle, u } std::optional GpuVm::lookup(AddressSpaceHandle handle) const { - std::lock_guard lock(mutex_); + std::shared_lock lock(mutex_); const Binding *binding = find_locked(handle); if (binding == nullptr) return std::nullopt; @@ -1068,12 +1280,12 @@ std::optional GpuVm::lookup(AddressSpaceHandle handle) const { .translation_epoch = binding->translation_epoch, .queue_references = binding->queue_references, .legacy_cache_compatible = binding->legacy_cache_compatible, - .ready = binding->translator != nullptr && - binding->physical_memory != nullptr}; + .ready = binding->access_state->translator != nullptr && + binding->access_state->physical_memory != nullptr}; } std::optional GpuVm::find_vmid(uint32_t vmid) const { - std::lock_guard lock(mutex_); + std::shared_lock lock(mutex_); const std::unordered_map::const_iterator found = vmid_handles_.find(vmid); return found == vmid_handles_.end() ? std::nullopt @@ -1081,13 +1293,13 @@ std::optional GpuVm::find_vmid(uint32_t vmid) const { } std::size_t GpuVm::active_address_spaces() const { - std::lock_guard lock(mutex_); + std::shared_lock lock(mutex_); return static_cast( std::ranges::count_if(slots_, [](const Slot &slot) { return slot.binding.has_value(); })); } uint64_t GpuVm::reset_epoch() const { - std::lock_guard lock(mutex_); + std::shared_lock lock(mutex_); return reset_epoch_; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_vm.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_vm.h index 08533cef823e..6a1fb96e845a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_vm.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/gpu_vm.h @@ -8,15 +8,15 @@ #include "rocjitsu/vm/amdgpu/gpu_handles.h" #include "rocjitsu/vm/amdgpu/mtype.h" +#include "util/distributed_shared_mutex.h" +#include #include #include #include #include #include -#include #include -#include #include #include #include @@ -24,8 +24,35 @@ namespace rocjitsu::amdgpu { +class GpuVm; +class GpuVmAccess; class GpuVmAccessState; +/// @brief Cache immutable VM snapshots for one bounded execution quantum. +/// @details Nested guards share the outer cache. Borrowed objects remain alive +/// until the outer guard exits, but each operation independently checks and +/// leases its access state. Revoked entries are skipped on subsequent lookups. +class GpuVmAccessBatchGuard { +public: + GpuVmAccessBatchGuard(); + ~GpuVmAccessBatchGuard(); + + [[nodiscard]] static bool active(); + + GpuVmAccessBatchGuard(const GpuVmAccessBatchGuard &) = delete; + GpuVmAccessBatchGuard &operator=(const GpuVmAccessBatchGuard &) = delete; + GpuVmAccessBatchGuard(GpuVmAccessBatchGuard &&) = delete; + GpuVmAccessBatchGuard &operator=(GpuVmAccessBatchGuard &&) = delete; + +private: + friend class GpuVm; + + [[nodiscard]] static const GpuVmAccess *find_snapshot(const GpuVm *vm, AddressSpaceHandle handle); + [[nodiscard]] static const GpuVmAccess *find_snapshot_vmid(const GpuVm *vm, uint32_t vmid); + static void retain_snapshot(const GpuVm *vm, AddressSpaceHandle handle, const GpuVmAccess &access, + bool routed); +}; + /// @brief Operation whose permissions must be checked by an address-space walk. enum class VmAccessKind : uint8_t { Read, Write, Execute, Atomic }; @@ -88,6 +115,16 @@ class VmTranslationResult { explicit operator bool() const { return outcome == VmAccessOutcome::Complete; } }; +/// @brief Outcome of one complete physical request within a virtual transfer. +/// @details Failure advances no bytes and retains the physical transport's +/// no-effect guarantee. Only translation failures need the outer VM reporter; +/// a backing transport remains responsible for its own fault publication. +struct VmTransferStep { + std::size_t completed_bytes = 0; + VmAccessOutcome outcome = VmAccessOutcome::Faulted; + bool report_translation_fault = false; +}; + /// @brief Result of one strong compare/exchange against physical backing. /// /// @details The operation is indivisible with respect to every accessor of the @@ -154,6 +191,13 @@ class Gfx12VmConfig { } }; +/// @brief One ordered DWORD store from caller-owned, non-guest source storage. +struct VmRamDwordStore { + static constexpr std::size_t kMaxBatch = 32; + uint64_t address; + const uint8_t *source; +}; + /// @brief Stable physical-memory transport used after address translation. class PhysicalMemoryAccess { public: @@ -281,6 +325,10 @@ class PhysicalMemoryAccess { /// lease that excludes those mutations. No backing pointer is retained. struct VmMtypeSnapshot { std::optional mtype; + // A negative-only optimization hint. True still requires a live private-RAM + // proof before copying; it never grants access or retains backing storage. + bool may_batch_private_uc = false; + bool may_batch_private_ram = false; uint64_t begin = 0; uint64_t size = 0; std::shared_ptr> mutation_epoch{}; @@ -292,21 +340,54 @@ struct VmMtypeSnapshot { } }; -/// @brief Request-local policy cache, including the binding that owns it. +/// @brief Policy cache that does not retain the binding's backing. +/// @details Like an L1 cache, it requires externally serialized access. class VmMtypeCache { friend class GpuVmAccess; - std::shared_ptr access_state_; - VmMtypeSnapshot snapshot_; + std::weak_ptr access_state_; + // Interleaved lanes can revisit several pages in one request. + // Hashing by 4 KiB page avoids discarding their copied policy on every lane; + // each snapshot still validates its translator-provided range and epoch. + std::array snapshots_; }; /// @brief Address-space policy separated from the physical backing transport. class AddressSpaceTranslator { public: + /// @brief Optional ordered same-page DWORD stores. False has no guest effects. + /// @details Implementations must prove every request under the same admission held + /// through all copies, without allocation, callbacks or fault publication. + [[nodiscard]] virtual bool try_write_private_dwords(PhysicalMemoryAccess &, + std::span, Mtype, + Mtype) const { + return false; + } + + /// @brief Optional read from private, stable UC RAM. + /// @details Implementations must refuse unknown transports before touching guest + /// data and must not allocate, report faults, or invoke foreign callbacks. + /// False leaves bytes unchanged. + [[nodiscard]] virtual bool try_read_uncached_ram(PhysicalMemoryAccess &, uint64_t, + std::span) const { + return false; + } + virtual ~AddressSpaceTranslator() = default; [[nodiscard]] virtual VmTranslationResult translate(uint64_t address, std::size_t size, VmAccessKind access) const = 0; + /// @brief Execute the next ordinary translated physical request. + /// @details The default translates one span and invokes the supplied transport. + /// A combined implementation must preserve that span's policy, exact byte + /// effects and fault publication, and qualify the supplied transport's identity. + /// No translation pointer or admission may escape the call. + [[nodiscard]] virtual VmTransferStep read_step(PhysicalMemoryAccess &, uint64_t address, + std::span remaining, + VmAccessKind access) const; + [[nodiscard]] virtual VmTransferStep write_step(PhysicalMemoryAccess &, uint64_t address, + std::span remaining) const; + /// @brief Read only the page policy needed to resolve an effective MTYPE. /// @details The default uses an ordinary read translation. Compatibility /// translators override this so cache policy lookup does not query allocator @@ -463,13 +544,21 @@ class VmCacheNamespace { /// @brief Immutable, operation-scoped view of one GPU address-space binding. /// -/// @details The snapshot retains the translator and physical backing selected under the -/// GpuVm lock. A multi-page access therefore cannot observe half of an old -/// root and half of its replacement. Replacement or invalidation advances the -/// cache namespace used by subsequent snapshots; an already-started operation -/// is allowed to finish against the binding it captured. +/// @details The registry lock selects an immutable translator/backing generation. +/// Ordinary snapshots share that generation's retirement state; each pinned +/// snapshot has independent retirement state and survives root replacement. +/// Access methods hold a shared revocation lease, allowing an in-flight access +/// to finish without mixing roots. Invalidation and unregistration revoke both +/// kinds of snapshot. Functional instruction fetch reuses an ordinary snapshot +/// within a quantum, checking is_current() before reuse. class GpuVmAccess { public: + /// @brief Whether this snapshot's access state is still valid. + /// @details Replacement, invalidation, and unregistration retire ordinary snapshots. + /// Pinned snapshots remain valid across replacement until invalidation or unregistration. + /// This check does not prevent concurrent retirement; access methods still validate the state. + [[nodiscard]] bool is_current() const; + [[nodiscard]] AddressSpaceInfo info() const { return info_; } [[nodiscard]] VmCacheNamespace cache_namespace() const { return {.address_space = address_space_, .translation_epoch = info_.translation_epoch}; @@ -479,8 +568,11 @@ class GpuVmAccess { VmAccessKind access) const; /// @brief Query cache policy without exposing physical backing metadata. [[nodiscard]] std::optional query_mtype(uint64_t address) const; - /// @brief Reuse a copied policy while both its binding and mutation token are valid. + /// @brief Reuse a copied policy while its access state and mutation token are valid. [[nodiscard]] std::optional query_mtype(uint64_t address, VmMtypeCache &cache) const; + /// Read a copied optimization hint without refreshing policy or invoking a + /// translator. Uncertain/stale policy declines; true never grants access. + [[nodiscard]] bool cached_private_uc_hint(uint64_t address, const VmMtypeCache &cache) const; /// @brief Query whether a complete virtual range currently permits an access. /// @details This side-effect-free query is for provisioning and routing decisions /// where an absent mapping is expected and must not be delivered as a GPU fault. @@ -511,6 +603,15 @@ class GpuVmAccess { /// report a guest fault; the caller must issue its original smaller accesses. /// A successful copy is not atomic with respect to concurrent data accesses. [[nodiscard]] bool try_read_contiguous(uint64_t address, std::span bytes) const; + /// Optional fault-free read from private UC RAM, with no allocation or callbacks. + /// Refusal leaves the destination unchanged; issue original accesses afterward. + [[nodiscard]] bool try_read_uncached_ram(uint64_t address, std::span bytes) const; + /// No-effect refusal; sources must not overlap the admitted guest destination. + [[nodiscard]] bool try_write_private_dwords(std::span stores, + Mtype instruction_mtype, Mtype expected_mtype) const; + /// Copied policy only; never refreshes metadata or authorizes a transfer. + [[nodiscard]] std::optional cached_private_ram_mtype(uint64_t address, + const VmMtypeCache &cache) const; [[nodiscard]] bool try_write_contiguous(uint64_t address, std::span bytes) const; [[nodiscard]] AtomicLoadResult atomic_load(uint64_t address, uint32_t width) const; [[nodiscard]] VmAccessOutcome atomic_store(uint64_t address, uint32_t width, @@ -527,13 +628,8 @@ class GpuVmAccess { friend class GpuVm; GpuVmAccess(AddressSpaceHandle address_space, AddressSpaceInfo info, - std::shared_ptr translator, - std::shared_ptr physical_memory, - std::shared_ptr access_state, - std::function fault_reporter) - : address_space_(address_space), info_(info), translator_(std::move(translator)), - physical_memory_(std::move(physical_memory)), access_state_(std::move(access_state)), - fault_reporter_(std::move(fault_reporter)) {} + std::shared_ptr access_state) + : address_space_(address_space), info_(info), access_state_(std::move(access_state)) {} void report_terminal_fault(uint64_t address, VmAccessKind access, VmAccessOutcome outcome) const; [[nodiscard]] VmAccessOutcome probe_impl(uint64_t address, std::size_t size, VmAccessKind access, @@ -541,10 +637,7 @@ class GpuVmAccess { AddressSpaceHandle address_space_; AddressSpaceInfo info_; - std::shared_ptr translator_; - std::shared_ptr physical_memory_; std::shared_ptr access_state_; - std::function fault_reporter_; }; /// @brief Owns GPU address-space identities independently of their front end. @@ -562,6 +655,15 @@ class GpuVm { /// @details The VM retains the shared translator and backing in every access /// snapshot. @p legacy_cache_compatible describes cache-addressing behavior, /// not the frontend that supplied the binding. + /// The fault reporter is shared across snapshots and generations. Invocations + /// are serialized per registration. Same-thread callback reentry is allowed + /// only through independent access states, such as a fresh snapshot_pinned(). + /// Nested access through the reporting state is forbidden: its revocation + /// lease is not recursive, and ordinary snapshot()/snapshot_vmid() calls + /// share that state. A reporter must not wait for another thread to fault + /// through the same registration. Accesses can hold a revocation lease while + /// reporting, so a reporter must defer invalidating, replacing, or unregistering + /// that binding. [[nodiscard]] AddressSpaceHandle register_address_space(uint32_t vmid, std::shared_ptr translator, std::shared_ptr physical_memory, @@ -572,6 +674,7 @@ class GpuVm { /// @details Internal model queues can retain and snapshot this explicit /// handle while a frontend-owned address space with the same numeric VMID /// remains discoverable through find_vmid() and snapshot_vmid(). + /// The fault reporter follows the same contract as register_address_space(). [[nodiscard]] AddressSpaceHandle register_unrouted_address_space(uint32_t vmid, std::shared_ptr translator, std::shared_ptr physical_memory, @@ -623,7 +726,7 @@ class GpuVm { [[nodiscard]] std::optional retain_binding(AddressSpaceHandle handle); [[nodiscard]] bool unregister_address_space(AddressSpaceHandle handle); - /// @brief Capture one coherent translator/backing/epoch tuple for an operation. + /// @brief Capture current metadata and one coherent translator/backing/epoch tuple. [[nodiscard]] std::optional snapshot(AddressSpaceHandle handle) const; /// @brief Capture a transaction snapshot that survives root replacement. /// @details Packet execution and guest-visible publication may retry after @@ -635,6 +738,28 @@ class GpuVm { /// @details The lookup and snapshot occur under one lock, so unregister and /// VMID reuse cannot substitute a different generation between them. [[nodiscard]] std::optional snapshot_vmid(uint32_t vmid) const; + /// @brief Borrow a cached snapshot for the active functional quantum. + /// @details These forms avoid copying snapshot ownership on every emulated + /// instruction. The returned pointer remains valid until the outermost + /// GpuVmAccessBatchGuard on this thread exits. Calling without an active + /// batch is a programming error. Metadata reflects the cached capture; use + /// snapshot() or snapshot_vmid() when current metadata is required. + [[nodiscard]] const GpuVmAccess *borrow_snapshot(AddressSpaceHandle handle) const; + [[nodiscard]] const GpuVmAccess *borrow_snapshot_vmid(uint32_t vmid) const; + + /// @brief Run one operation against the current VMID snapshot. + /// @details Functional execution borrows the snapshot retained by the + /// active quantum. Other callers receive an owning operation snapshot, so + /// access-state invalidation is checked in both paths. The callback must not + /// retain snapshot pointers or references beyond the owning operation's + /// lifetime, or beyond the outer quantum for a borrowed snapshot. + template + decltype(auto) with_vmid_snapshot(uint32_t vmid, Operation &&operation) const { + if (GpuVmAccessBatchGuard::active()) + return std::invoke(std::forward(operation), borrow_snapshot_vmid(vmid)); + const std::optional access = snapshot_vmid(vmid); + return std::invoke(std::forward(operation), access ? &*access : nullptr); + } [[nodiscard]] VmTranslationResult translate(AddressSpaceHandle handle, uint64_t address, std::size_t size, VmAccessKind access) const; @@ -662,17 +787,16 @@ class GpuVm { [[nodiscard]] bool reset(); private: + friend class GpuVmAccessBatchGuard; + class Binding { public: uint32_t vmid = 0; uint64_t translation_epoch = 1; uint32_t queue_references = 0; std::shared_ptr binding_state; - std::shared_ptr translator; - std::shared_ptr physical_memory; std::shared_ptr access_state; std::vector> pinned_access_states; - std::function fault_reporter; bool legacy_cache_compatible = false; bool device_gart = false; }; @@ -686,15 +810,16 @@ class GpuVm { [[nodiscard]] AddressSpaceHandle allocate_locked(Binding binding); [[nodiscard]] Binding *find_locked(AddressSpaceHandle handle); [[nodiscard]] const Binding *find_locked(AddressSpaceHandle handle) const; - void advance_access_state_locked(Binding &binding); + void advance_access_state_locked(Binding &binding, std::shared_ptr replacement); void revoke_access_state_locked(Binding &binding); - mutable std::mutex mutex_; + mutable util::DistributedSharedMutex mutex_; std::vector slots_; std::vector free_slots_; std::unordered_map vmid_handles_; AddressSpaceHandle gart_address_space_; uint64_t reset_epoch_ = 1; + const uint64_t instance_id_; Gfx12VmConfig gfx12_config_; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/hwreg.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/hwreg.cpp index 92c602a3ea44..d5d556ddd9a3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/hwreg.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/hwreg.cpp @@ -34,6 +34,7 @@ enum class HwregState : uint8_t { WaveSchedMode, IbStsGfx1250, IbSts2Gfx1250, + WgpIdGfx1250, }; enum class HwregWritePolicy : uint8_t { @@ -351,7 +352,7 @@ constexpr HwregDescriptor GFX1250_HWREGS[] = { {19, "WAVE_TRAP_CTRL", HwregState::TrapCtrlGfx12, HwregWritePolicy::Privileged}, {20, "WAVE_SCRATCH_BASE_LO", HwregState::Unsupported, HwregWritePolicy::Privileged}, {21, "WAVE_SCRATCH_BASE_HI", HwregState::Unsupported, HwregWritePolicy::Privileged}, - {23, "WAVE_HW_ID1", HwregState::Unsupported, HwregWritePolicy::ReadOnly}, + {23, "WAVE_HW_ID1", HwregState::WgpIdGfx1250, HwregWritePolicy::ReadOnly}, {24, "WAVE_HW_ID2", HwregState::Unsupported, HwregWritePolicy::ReadOnly}, {26, "WAVE_SCHED_MODE", HwregState::WaveSchedMode, HwregWritePolicy::UserWritable}, {28, "IB_STS2", HwregState::IbSts2Gfx1250, HwregWritePolicy::ReadOnly}, @@ -470,6 +471,12 @@ HwregAccessResult read_raw_hwreg(Wavefront &wf, HwregState state, uint32_t &raw_ case HwregState::IbSts2Gfx1250: raw_value = gfx1250_ib_sts2_raw(wf); return HwregAccessResult::Success; + case HwregState::WgpIdGfx1250: + if (!wf.cu().cus_per_shader_array() || wf.cu().cus_per_shader_array() > 16) + return HwregAccessResult::Unsupported; + // Each gfx1250 ComputeUnit models one WGP, including its shared LDS. + raw_value = field_value(wf.cu().shader_array_cu_id(), 10, 4); + return HwregAccessResult::Success; case HwregState::Unsupported: return HwregAccessResult::Unsupported; } @@ -521,6 +528,7 @@ HwregAccessResult write_raw_hwreg(Wavefront &wf, HwregState state, uint32_t raw_ case HwregState::GprAllocCdna3_4: case HwregState::IbStsGfx1250: case HwregState::IbSts2Gfx1250: + case HwregState::WgpIdGfx1250: case HwregState::Unsupported: return HwregAccessResult::Unsupported; } @@ -628,6 +636,13 @@ HwregAccessResult read_hwreg_field(Wavefront &wf, uint16_t hwreg, uint32_t &valu return HwregAccessResult::Unsupported; } + // Only WGP_ID[13:10] has backing state; do not fabricate the other HW_ID1 fields. + if (desc->state == HwregState::WgpIdGfx1250 && + (decoded.offset < 10 || decoded.offset + decoded.size > 14)) { + value = 0; + return HwregAccessResult::Unsupported; + } + uint32_t raw_value = 0; HwregAccessResult result = read_raw_hwreg(wf, desc->state, raw_value); if (result != HwregAccessResult::Success) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/instruction_cache.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/instruction_cache.h index 78d2f06bca30..d36cfd8a40c7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/instruction_cache.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/instruction_cache.h @@ -138,8 +138,11 @@ class InstructionCache { /// @brief Discard every cached line (s_icache_inv). void invalidate_all() { ++epoch_; - for (Line &line : lines_) - line.valid = false; + if (has_cached_lines_) { + for (Line &line : lines_) + line.valid = false; + has_cached_lines_ = false; + } coherence_epoch_ = coherence_->current_instruction_epoch(); } @@ -185,6 +188,7 @@ class InstructionCache { line.cache_namespace = {}; line.translated = false; line.valid = true; + has_cached_lines_ = true; return line.data; } @@ -211,6 +215,7 @@ class InstructionCache { line.cache_namespace = cache_namespace; line.translated = true; line.valid = true; + has_cached_lines_ = true; data = line.data; return VmAccessOutcome::Complete; } @@ -219,6 +224,9 @@ class InstructionCache { uint64_t epoch_ = 0; std::shared_ptr coherence_; uint64_t coherence_epoch_ = 0; + // Only successful fills make a line valid. Repeated maintenance still + // advances the public epoch, but need not revisit already invalid lines. + bool has_cached_lines_ = false; }; } // namespace amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/instruction_compute_unit_view.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/instruction_compute_unit_view.h index d1f4319c738f..31d87d487d8d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/instruction_compute_unit_view.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/instruction_compute_unit_view.h @@ -48,7 +48,12 @@ class InstructionComputeUnitView { Lds &lds(); bool sram_ecc() const; bool setreg_vgpr_msb_fixup() const; + /// Whether register accesses have a diagnostic or plugin consumer. + bool observes_register_access() const; + bool debug_active() const; rj_code_arch_t arch() const; + uint32_t cus_per_shader_array() const; + uint32_t shader_array_cu_id() const; uint32_t wf_size() const; uint32_t sgprs_per_wf() const; uint32_t vgpr_allocation_block_size() const; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_scalar_cache.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_scalar_cache.cpp index daa147ca2343..97c67c3a19e9 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_scalar_cache.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_scalar_cache.cpp @@ -28,6 +28,7 @@ void L1ScalarCache::set_l2(L2Cache *l2) { void L1ScalarCache::set_gpu_vm(GpuVm *gpu_vm) { invalidate_all_lines(); gpu_vm_ = gpu_vm; + mtype_cache_ = {}; } VmAccessOutcome L1ScalarCache::ensure_line(uint64_t addr, uint32_t vmid, bool fetch_on_miss) { @@ -85,7 +86,7 @@ void L1ScalarCache::cache_partial_bytes(uint64_t addr, const uint8_t *src, uint3 VmAccessOutcome L1ScalarCache::store(uint64_t addr, uint32_t num_dwords, const uint32_t *src, uint32_t vmid) { synchronize_epoch(); - RequestMtypeResolver mtypes(gpu_vm_, vmid); + RequestMtypeResolver mtypes(gpu_vm_, vmid, mtype_cache_); for (uint32_t i = 0; i < num_dwords; ++i) { uint64_t ea = addr + i * 4; uint8_t buf[4]; @@ -171,9 +172,9 @@ void L1ScalarCache::flush_line(uint64_t addr, uint32_t vmid) { } VmAccessOutcome L1ScalarCache::load(uint64_t addr, uint32_t num_dwords, uint32_t *dst, - uint32_t vmid) { + uint32_t vmid, bool allow_private_batch) { synchronize_epoch(); - RequestMtypeResolver mtypes(gpu_vm_, vmid); + RequestMtypeResolver mtypes(gpu_vm_, vmid, mtype_cache_); for (uint32_t i = 0; i < num_dwords; ++i) { uint64_t ea = addr + i * 4; uint8_t buf[4]{}; @@ -188,6 +189,11 @@ VmAccessOutcome L1ScalarCache::load(uint64_t addr, uint32_t num_dwords, uint32_t if (mtype == Mtype::UC) { flush_line(chunk_addr, vmid); + if (allow_private_batch && i == 0 && copied == 0 && num_dwords >= 2 && num_dwords <= 16 && + (addr & 3) == 0 && num_dwords * 4 <= CacheStore::LINE_SIZE - line_offset && + mtypes.cached_private_uc_hint(addr) && + l2_->try_read_scalar_ram(addr, dst, num_dwords, vmid)) + return VmAccessOutcome::Complete; const VmAccessOutcome outcome = l2_->read(chunk_addr, buf + copied, chunk, Mtype::UC, vmid); if (outcome != VmAccessOutcome::Complete) return outcome; @@ -226,7 +232,7 @@ VmAccessOutcome L1ScalarCache::load(uint64_t addr, uint32_t num_dwords, uint32_t VmAccessOutcome L1ScalarCache::load_bytes(uint64_t addr, uint32_t num_bytes, uint8_t *dst, uint32_t vmid) { synchronize_epoch(); - RequestMtypeResolver mtypes(gpu_vm_, vmid); + RequestMtypeResolver mtypes(gpu_vm_, vmid, mtype_cache_); uint32_t copied = 0; while (copied < num_bytes) { uint64_t ea = addr + copied; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_scalar_cache.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_scalar_cache.h index aeb120fef096..6f4547e234d0 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_scalar_cache.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_scalar_cache.h @@ -3,7 +3,7 @@ #pragma once -#include "rocjitsu/vm/amdgpu/mtype.h" +#include "rocjitsu/vm/amdgpu/gpu_vm.h" #include "simdojo/components/cache.h" #include @@ -13,9 +13,7 @@ namespace rocjitsu { namespace amdgpu { class GpuMemory; -class GpuVm; class L2Cache; -enum class VmAccessOutcome : uint8_t; /// @brief L1 Scalar Cache (K$) controller for SMEM instructions. /// @@ -53,7 +51,9 @@ class L1ScalarCache { /// /// Fetches from K$ on hit, or fills from L2 on miss. Handles requests /// that span multiple cache lines. - VmAccessOutcome load(uint64_t addr, uint32_t num_dwords, uint32_t *dst, uint32_t vmid = 0); + /// Private RAM batching must be enabled only for unobserved functional requests. + VmAccessOutcome load(uint64_t addr, uint32_t num_dwords, uint32_t *dst, uint32_t vmid = 0, + bool allow_private_batch = false); /// @brief Scalar load: read num_bytes contiguous bytes from addr. VmAccessOutcome load_bytes(uint64_t addr, uint32_t num_bytes, uint8_t *dst, uint32_t vmid = 0); @@ -81,6 +81,7 @@ class L1ScalarCache { CacheStore cache_; L2Cache *l2_; GpuVm *gpu_vm_ = nullptr; + VmMtypeCache mtype_cache_; uint64_t coherence_epoch_ = 0; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.cpp index 076230c24af8..12107e309916 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.cpp @@ -3,12 +3,14 @@ #include "rocjitsu/vm/amdgpu/l1_vector_cache.h" +#include "rocjitsu/base/rj_compiler.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" #include "rocjitsu/vm/amdgpu/l2_cache.h" #include "rocjitsu/vm/amdgpu/request_mtype_resolver.h" #include "util/log.h" #include +#include #include #include #include @@ -76,6 +78,7 @@ void L1VectorCache::set_l2(L2Cache *l2) { void L1VectorCache::set_gpu_vm(GpuVm *gpu_vm) { invalidate_all_lines(); gpu_vm_ = gpu_vm; + mtype_cache_ = {}; } VmAccessOutcome L1VectorCache::ensure_line(uint64_t addr, uint32_t vmid, bool fetch_on_miss) { @@ -272,7 +275,7 @@ VmAccessOutcome L1VectorCache::load(const uint64_t *addrs, uint64_t lane_mask, u std::span element_lane_masks, uint32_t swizzle_unit) { synchronize_epoch(); - RequestMtypeResolver mtypes(gpu_vm_, vmid, mtype); + RequestMtypeResolver mtypes(gpu_vm_, vmid, mtype_cache_, mtype); uint32_t stride = num_elems * elem_size; // Scratch and buffer swizzling: consecutive units of a lane sit // addr_stride bytes apart. Addresses are materialized per element, so this honours @@ -345,14 +348,112 @@ VmAccessOutcome L1VectorCache::load(const uint64_t *addrs, uint64_t lane_mask, u return outcome; } +// Keep bounded descriptor storage and swizzled-loop live ranges out of plain stores. +RJ_NOINLINE VmAccessOutcome L1VectorCache::store_swizzled( + const uint64_t *addrs, uint64_t lane_mask, uint32_t elem_size, uint32_t num_elems, + const uint8_t *src, Mtype mtype, bool non_temporal, uint32_t vmid, uint32_t addr_stride, + uint32_t addr_base_offset, std::span element_lane_masks, uint32_t swizzle_unit, + bool allow_private_batch, RequestMtypeResolver &mtypes) { + const uint32_t stride = num_elems * elem_size; + assert((swizzle_unit == 4 || swizzle_unit == 16) && addr_base_offset < swizzle_unit); + const uint32_t astride = addr_stride; + const bool batch_dwords = + allow_private_batch && elem_size == sizeof(uint32_t) && (addr_base_offset & 3) == 0; + uint64_t refused_page = UINT64_MAX; + for (uint32_t elem = 0; elem < num_elems; ++elem) { + uint64_t mask = element_lane_masks.empty() ? lane_mask : (element_lane_masks[elem] & lane_mask); + store_l2_writes_ += std::popcount(mask); + uint64_t attempted_line = UINT64_MAX; + while (mask) { + const uint32_t lane = std::countr_zero(mask); + mask &= mask - 1; + const uint64_t base = addrs[lane]; + const uint32_t first_byte_in_unit = + static_cast((base - addr_base_offset) % swizzle_unit); + uint32_t copied = elem * elem_size; + const uint32_t elem_end = copied + elem_size; + while (copied < elem_end) { + const uint32_t logical_byte = first_byte_in_unit + copied; + const uint32_t byte_in_unit = logical_byte % swizzle_unit; + const uint32_t chunk = std::min(elem_end - copied, swizzle_unit - byte_in_unit); + const uint64_t ea = + base - first_byte_in_unit + logical_byte / swizzle_unit * astride + byte_in_unit; + const VmAccessOutcome outcome = + write_bytes(ea, src + lane * stride + copied, chunk, non_temporal, vmid, mtypes); + if (outcome != VmAccessOutcome::Complete) + return outcome; + copied += chunk; + // Keep the first request ordinary. It owns all existing allocation, + // dirty-prefix and fault behavior and warms the copied policy hint. + const uint64_t line = ea >> LINE_SIZE_BITS; + if (!batch_dwords || chunk != sizeof(uint32_t) || (ea & 3) || !mask || + attempted_line == line || refused_page == (ea >> 12)) + continue; + attempted_line = line; + const auto policy = mtypes.cached_private_ram_mtype(ea); + if (!policy) { + // False is always safe to retain for this instruction; never refresh + // a policy or call an unknown translator just to attempt batching. + refused_page = ea >> 12; + continue; + } + const auto *resident = cache_.peek(ea, vmid); + if (*policy == Mtype::UC || non_temporal) { + if (resident) + continue; + } else if (!resident || resident->dirty) { + continue; + } + std::array stores; + size_t count = 0; + uint64_t remaining = mask; + while (remaining && count < stores.size()) { + const uint32_t next_lane = std::countr_zero(remaining); + const uint64_t next_base = addrs[next_lane]; + const uint32_t first_byte = + static_cast((next_base - addr_base_offset) % swizzle_unit); + const uint32_t logical = first_byte + elem * elem_size; + const uint64_t address = + next_base - first_byte + logical / swizzle_unit * astride + logical % swizzle_unit; + if ((next_base & 3) || (address >> LINE_SIZE_BITS) != line) + break; + stores[count++] = {address, src + next_lane * stride + elem * elem_size}; + remaining &= remaining - 1; + } + const std::span group(stores.data(), count); + if (count < 2) + continue; + if (!l2_->try_write_private_dwords(group, mtype, *policy, vmid)) { + // A refusal grants no access and changes no guest state. Retaining it + // only selects ordinary requests for later elements on this page. + refused_page = ea >> 12; + continue; + } + if (*policy != Mtype::UC && !non_temporal) { + for (const auto &store : group) { + cache_.write_line(store.address, store.source, CacheStore::line_offset(store.address), + sizeof(uint32_t), vmid); + simdojo::CacheTag *tag = nullptr; + cache_.lookup(store.address, &tag, vmid); + tag->coherence = *policy == Mtype::CC ? simdojo::CoherenceState::SHARED + : simdojo::CoherenceState::EXCLUSIVE; + } + } + mask = remaining; + } + } + } + return VmAccessOutcome::Complete; +} + VmAccessOutcome L1VectorCache::store(const uint64_t *addrs, uint64_t lane_mask, uint32_t elem_size, uint32_t num_elems, const uint8_t *src, Mtype mtype, bool non_temporal, uint32_t wf_size, uint32_t vmid, uint32_t addr_stride, uint32_t addr_base_offset, std::span element_lane_masks, - uint32_t swizzle_unit) { + uint32_t swizzle_unit, bool allow_private_batch) { synchronize_epoch(); - RequestMtypeResolver mtypes(gpu_vm_, vmid, mtype); + RequestMtypeResolver mtypes(gpu_vm_, vmid, mtype_cache_, mtype); uint32_t stride = num_elems * elem_size; const uint32_t active_lanes = std::popcount(lane_mask); ++store_count_; @@ -364,37 +465,10 @@ VmAccessOutcome L1VectorCache::store(const uint64_t *addrs, uint64_t lane_mask, // rather than strided over. With no element masks this walks exactly the // lanes and bytes the uniform path would. addr_base_offset identifies the // low bits added after swizzling so they do not move the logical swizzle-unit boundary. - if (addr_stride != 0) { - assert((swizzle_unit == 4 || swizzle_unit == 16) && addr_base_offset < swizzle_unit); - const uint32_t astride = addr_stride; - for (uint32_t elem = 0; elem < num_elems; ++elem) { - uint64_t mask = - element_lane_masks.empty() ? lane_mask : (element_lane_masks[elem] & lane_mask); - store_l2_writes_ += std::popcount(mask); - while (mask) { - const uint32_t lane = std::countr_zero(mask); - mask &= mask - 1; - const uint64_t base = addrs[lane]; - const uint32_t first_byte_in_unit = - static_cast((base - addr_base_offset) % swizzle_unit); - uint32_t copied = elem * elem_size; - const uint32_t elem_end = copied + elem_size; - while (copied < elem_end) { - const uint32_t logical_byte = first_byte_in_unit + copied; - const uint32_t byte_in_unit = logical_byte % swizzle_unit; - const uint32_t chunk = std::min(elem_end - copied, swizzle_unit - byte_in_unit); - const uint64_t ea = - base - first_byte_in_unit + logical_byte / swizzle_unit * astride + byte_in_unit; - const VmAccessOutcome outcome = - write_bytes(ea, src + lane * stride + copied, chunk, non_temporal, vmid, mtypes); - if (outcome != VmAccessOutcome::Complete) - return outcome; - copied += chunk; - } - } - } - return VmAccessOutcome::Complete; - } + if (addr_stride != 0) + return store_swizzled(addrs, lane_mask, elem_size, num_elems, src, mtype, non_temporal, vmid, + addr_stride, addr_base_offset, element_lane_masks, swizzle_unit, + allow_private_batch, mtypes); if (!all_elements_use_lane_mask(element_lane_masks, lane_mask, num_elems)) { uint64_t full_lane_mask = fully_valid_lane_mask(element_lane_masks, lane_mask); VmAccessOutcome outcome = VmAccessOutcome::Complete; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.h index fb227373fdc7..f275060df7a6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.h @@ -3,7 +3,7 @@ #pragma once -#include "rocjitsu/vm/amdgpu/mtype.h" +#include "rocjitsu/vm/amdgpu/gpu_vm.h" #include "simdojo/components/cache.h" #include @@ -13,10 +13,8 @@ namespace rocjitsu { namespace amdgpu { class GpuMemory; -class GpuVm; class L2Cache; class RequestMtypeResolver; -enum class VmAccessOutcome : uint8_t; /// @brief L1 Vector Cache (V$) controller for FLAT/MUBUF/MTBUF instructions. /// @@ -66,12 +64,14 @@ class L1VectorCache { /// @param element_lane_masks Empty when every element uses @p lane_mask; /// otherwise contains exactly @p num_elems masks. In the latter form, /// @p lane_mask is the union of lanes valid for at least one element. + /// Private batching is opt-in for unobserved functional requests whose source + /// data remain in instruction-owned storage until the operation completes. VmAccessOutcome store(const uint64_t *addrs, uint64_t lane_mask, uint32_t elem_size, uint32_t num_elems, const uint8_t *src, Mtype mtype, bool non_temporal, uint32_t wf_size, uint32_t vmid = 0, uint32_t addr_stride = 0, uint32_t addr_base_offset = 0, std::span element_lane_masks = {}, - uint32_t swizzle_unit = 4); + uint32_t swizzle_unit = 4, bool allow_private_batch = false); void invalidate(uint64_t addr, uint32_t vmid = 0); void invalidate_all(); @@ -88,6 +88,13 @@ class L1VectorCache { bool request_l1_bypass, uint32_t vmid, RequestMtypeResolver &mtypes); VmAccessOutcome write_bytes(uint64_t addr, const uint8_t *src, uint32_t size, bool non_temporal, uint32_t vmid, RequestMtypeResolver &mtypes); + VmAccessOutcome store_swizzled(const uint64_t *addrs, uint64_t lane_mask, uint32_t elem_size, + uint32_t num_elems, const uint8_t *src, Mtype mtype, + bool non_temporal, uint32_t vmid, uint32_t addr_stride, + uint32_t addr_base_offset, + std::span element_lane_masks, + uint32_t swizzle_unit, bool allow_private_batch, + RequestMtypeResolver &mtypes); void cache_partial_bytes(uint64_t addr, const uint8_t *src, uint32_t size, uint32_t vmid); // Store allocation leaves untouched bytes absent instead of reading backing. VmAccessOutcome ensure_line(uint64_t addr, uint32_t vmid, bool fetch_on_miss = true); @@ -95,6 +102,7 @@ class L1VectorCache { CacheStore cache_; L2Cache *l2_; GpuVm *gpu_vm_ = nullptr; + VmMtypeCache mtype_cache_; uint64_t coherence_epoch_ = 0; uint64_t store_count_ = 0; uint64_t store_active_count_ = 0; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l2_cache.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l2_cache.cpp index 11674c285900..43eeb01b1093 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l2_cache.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l2_cache.cpp @@ -191,7 +191,7 @@ VmAccessOutcome L2Cache::publish_dirty_bytes_to_legacy_backing(uint64_t line_add clear_dirty_bytes(line_addr, start, static_cast(completed_bytes), vmid); if (outcome != VmAccessOutcome::Complete) return outcome; - backing_write_transactions_.fetch_add(1, std::memory_order_relaxed); + diagnostics(line_addr + start).backing_writes.fetch_add(1, std::memory_order_relaxed); } } return VmAccessOutcome::Complete; @@ -220,17 +220,18 @@ VmAccessOutcome L2Cache::access_outcome(simdojo::MessageStatus status) { bool L2Cache::can_fetch_range(uint64_t addr, uint32_t size, uint32_t vmid) const { if (vmid == 0 || gpu_vm_ == nullptr) return true; - const std::optional vm_access = gpu_vm_->snapshot_vmid(vmid); - return vm_access && - vm_access->query_access(addr, size, VmAccessKind::Read) == VmAccessOutcome::Complete; + return gpu_vm_->with_vmid_snapshot(vmid, [&](const GpuVmAccess *vm_access) { + return vm_access && + vm_access->query_access(addr, size, VmAccessKind::Read) == VmAccessOutcome::Complete; + }); } VmAccessOutcome L2Cache::send_backing(uint64_t addr, uint8_t *data, uint32_t size, simdojo::MessageOp op, uint32_t vmid) { if (op == simdojo::MessageOp::WRITE) - backing_write_transactions_.fetch_add(1, std::memory_order_relaxed); + diagnostics(addr).backing_writes.fetch_add(1, std::memory_order_relaxed); else - backing_read_transactions_.fetch_add(1, std::memory_order_relaxed); + diagnostics(addr).backing_reads.fetch_add(1, std::memory_order_relaxed); if (backing_memory_) { if (vmid == 0 && op == simdojo::MessageOp::WRITE) { thread_local uint64_t wb_count = 0; @@ -246,14 +247,13 @@ VmAccessOutcome L2Cache::send_backing(uint64_t addr, uint8_t *data, uint32_t siz } if (gpu_vm_ == nullptr) return VmAccessOutcome::Unavailable; - std::optional vm_access = gpu_vm_->snapshot_vmid(vmid); - if (!vm_access) - return VmAccessOutcome::Faulted; - const VmAccessOutcome outcome = - op == simdojo::MessageOp::WRITE - ? vm_access->write(addr, std::as_bytes(std::span(data, size))) - : vm_access->read(addr, std::as_writable_bytes(std::span(data, size))); - return outcome; + return gpu_vm_->with_vmid_snapshot(vmid, [&](const GpuVmAccess *vm_access) { + if (!vm_access) + return VmAccessOutcome::Faulted; + return op == simdojo::MessageOp::WRITE + ? vm_access->write(addr, std::as_bytes(std::span(data, size))) + : vm_access->read(addr, std::as_writable_bytes(std::span(data, size))); + }); } assert(req_port_ != nullptr && "L2Cache: req_port_ not set"); if (req_port_->link() == nullptr || @@ -275,8 +275,8 @@ VmAccessOutcome L2Cache::send_backing(uint64_t addr, uint8_t *data, uint32_t siz VmAccessOutcome L2Cache::send_atomic_backing(uint64_t addr, uint32_t size, const simdojo::MemoryAtomicMutation &mutation, uint32_t vmid) { - backing_read_transactions_.fetch_add(1, std::memory_order_relaxed); - backing_write_transactions_.fetch_add(1, std::memory_order_relaxed); + diagnostics(addr).backing_reads.fetch_add(1, std::memory_order_relaxed); + diagnostics(addr).backing_writes.fetch_add(1, std::memory_order_relaxed); assert(req_port_ != nullptr && "L2Cache: req_port_ not set"); if (req_port_->link() == nullptr || req_port_->link()->exec_mode() != simdojo::ExecMode::FUNCTIONAL) @@ -396,6 +396,30 @@ VmAccessOutcome L2Cache::cache_partial_bytes(uint64_t addr, const uint8_t *src, return VmAccessOutcome::Complete; } +bool L2Cache::try_read_scalar_ram(uint64_t addr, uint32_t *dst, uint32_t num_dwords, + uint32_t vmid) { + if (!backing_memory_ || !gpu_vm_ || vmid == 0 || num_dwords < 2 || num_dwords > 16 || + (addr & 3) || num_dwords * 4 > 64 - (addr & 63)) + return false; + // Discover the snapshot before cache admission: a cold quantum snapshot may + // allocate its retained entry. The optional copy itself cannot allocate. + return gpu_vm_->with_vmid_snapshot(vmid, [&](const GpuVmAccess *access) { + if (!access) + return false; + auto maintenance_lock = acquire_cache_access(); + std::lock_guard set_lock(set_mutex(addr)); + // Resident lines retain the existing flush/fault path. A miss neither + // updates replacement state nor publishes or invalidates cached bytes. + if (cache_.peek(addr, vmid)) + return false; + const bool copied = + access->try_read_uncached_ram(addr, std::as_writable_bytes(std::span(dst, num_dwords))); + if (copied) + diagnostics(addr).backing_reads.fetch_add(1, std::memory_order_relaxed); + return copied; + }); +} + VmAccessOutcome L2Cache::read(uint64_t addr, uint8_t *dst, uint32_t size, Mtype mtype, uint32_t vmid) { auto maintenance_lock = acquire_cache_access(); @@ -464,6 +488,43 @@ VmAccessOutcome L2Cache::read(uint64_t addr, uint8_t *dst, uint32_t size, Mtype return VmAccessOutcome::Complete; } +bool L2Cache::try_write_private_dwords(std::span stores, + Mtype instruction_mtype, Mtype mtype, uint32_t vmid) { + if (!backing_memory_ || !gpu_vm_ || !vmid || stores.size() < 2 || + stores.size() > VmRamDwordStore::kMaxBatch) + return false; + const uint64_t address = stores.front().address; + for (const auto &store : stores) + if ((store.address & 3) || + CacheStore::line_address(store.address) != CacheStore::line_address(address)) + return false; + // A cold snapshot can allocate retained storage; prepare it before cache admission. + return gpu_vm_->with_vmid_snapshot(vmid, [&](const GpuVmAccess *access) { + if (!access) + return false; + auto maintenance_lock = acquire_cache_access(); + std::lock_guard set_lock(set_mutex(address)); + const auto *resident = cache_.peek(address, vmid); + if (mtype == Mtype::UC ? resident != nullptr : !resident || resident->dirty) + return false; + if (!access->try_write_private_dwords(stores, instruction_mtype, mtype)) + return false; + diagnostics(address).backing_writes.fetch_add(stores.size(), std::memory_order_relaxed); + if (mtype != Mtype::UC) { + for (const auto &store : stores) { + simdojo::CacheTag *tag = nullptr; + cache_.lookup(store.address, &tag, vmid); + cache_.write_line(store.address, store.source, CacheStore::line_offset(store.address), + sizeof(uint32_t), vmid); + tag->coherence = mtype == Mtype::CC ? simdojo::CoherenceState::SHARED + : simdojo::CoherenceState::EXCLUSIVE; + } + diagnostics(address).writes.fetch_add(stores.size(), std::memory_order_relaxed); + } + return true; + }); +} + VmAccessOutcome L2Cache::write(uint64_t addr, const uint8_t *src, uint32_t size, Mtype mtype, uint32_t vmid) { auto maintenance_lock = acquire_cache_access(); @@ -518,7 +579,7 @@ VmAccessOutcome L2Cache::write(uint64_t addr, const uint8_t *src, uint32_t size, : (mtype == Mtype::CC ? simdojo::CoherenceState::SHARED : simdojo::CoherenceState::EXCLUSIVE); - write_count_.fetch_add(1, std::memory_order_relaxed); + diagnostics(ea).writes.fetch_add(1, std::memory_order_relaxed); copied += chunk; } return VmAccessOutcome::Complete; @@ -632,8 +693,7 @@ VmAccessOutcome L2Cache::flush_dirty_locked() { }); if (dirty_count != 0) util::Logger::vm("L2 flush: ", dirty_count, " dirty lines [0x", std::hex, min_addr, "-0x", - max_addr, "]", std::dec, - " total_writes=", write_count_.load(std::memory_order_relaxed)); + max_addr, "]", std::dec, " total_writes=", write_count()); return outcome; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l2_cache.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l2_cache.h index 919f772f9d60..8e63a2b83d26 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l2_cache.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l2_cache.h @@ -117,11 +117,13 @@ class L2Cache : public simdojo::Component { void set_coherence_domain(std::shared_ptr coherence); const std::shared_ptr &coherence_domain() const { return coherence_; } + /// Diagnostic totals are exact at quiescence. Concurrent queries sample + /// independent relaxed shards, not one instantaneous global snapshot. uint64_t backing_read_transactions() const { - return backing_read_transactions_.load(std::memory_order_relaxed); + return diagnostic_total(&DiagnosticCounters::backing_reads); } uint64_t backing_write_transactions() const { - return backing_write_transactions_.load(std::memory_order_relaxed); + return diagnostic_total(&DiagnosticCounters::backing_writes); } /// @brief Return whether a range may be read speculatively for a cache fill. @@ -141,6 +143,12 @@ class L2Cache : public simdojo::Component { VmAccessOutcome read(uint64_t addr, uint8_t *dst, uint32_t size, Mtype mtype = Mtype::RW, uint32_t vmid = 0); + /// Batch an unobserved scalar UC request only when this line is absent and + /// the direct VM backing proves private, fault-free RAM. False has no guest + /// effects; the caller must retain its original per-dword fallback. + [[nodiscard]] bool try_read_scalar_ram(uint64_t addr, uint32_t *dst, uint32_t num_dwords, + uint32_t vmid); + /// @brief Write data to L2 (and possibly through to HBM). /// /// Used by L1 for write-through (CC) and write-back evictions. @@ -151,7 +159,15 @@ class L2Cache : public simdojo::Component { VmAccessOutcome write(uint64_t addr, const uint8_t *src, uint32_t size, Mtype mtype = Mtype::RW, uint32_t vmid = 0); - uint64_t write_count() const { return write_count_.load(std::memory_order_relaxed); } + /// Optional same-line stores with a live private-RAM proof. Refusal changes + /// neither memory nor cache replacement state. Logical counters stay per store. + [[nodiscard]] bool try_write_private_dwords(std::span stores, + Mtype instruction_mtype, Mtype effective_mtype, + uint32_t vmid); + + /// Successful cached write chunks; UC bypass writes are not included. + /// Like backing transaction totals, concurrent queries sample each shard. + uint64_t write_count() const { return diagnostic_total(&DiagnosticCounters::writes); } /// @brief Fetch an entire cache line into the given buffer. /// @@ -192,13 +208,36 @@ class L2Cache : public simdojo::Component { template [[nodiscard]] VmAccessOutcome atomic_rmw(uint64_t addr, uint32_t size, F &&fn, uint32_t vmid = 0) { + const auto vm_access = snapshot_atomic_access(vmid); DeviceCacheCoherence::AtomicBoundary boundary = coherence_->acquire_atomic_boundary(); + return atomic_rmw(boundary, addr, size, std::forward(fn), vmid, vm_access); + } + + /// @brief Select local VM backing before acquiring the device coherence boundary. + /// @details Take a fresh snapshot for each attempt, including retries. VMID-zero + /// physical accesses and transport-backed atomics do not need a local snapshot. + [[nodiscard]] std::optional snapshot_atomic_access(uint32_t vmid) const { + if (!backing_memory_ || vmid == 0 || !gpu_vm_) + return std::nullopt; + return gpu_vm_->snapshot_vmid(vmid); + } + + /// @brief Perform one backing atomic within an existing device boundary. + /// @details A vector instruction can prepare the cache hierarchy once for all + /// its lanes. Each backing RMW remains atomic with respect to host accesses. + /// @param vm_access Result of snapshot_atomic_access(vmid) for this attempt. + template + [[nodiscard]] VmAccessOutcome atomic_rmw(const DeviceCacheCoherence::AtomicBoundary &boundary, + uint64_t addr, uint32_t size, F &&fn, uint32_t vmid, + const std::optional &vm_access) { if (boundary.outcome() != VmAccessOutcome::Complete) return boundary.outcome(); + if (!boundary.belongs_to(coherence_.get())) + return VmAccessOutcome::Malformed; if (backing_memory_) { - backing_read_transactions_.fetch_add(1, std::memory_order_relaxed); - backing_write_transactions_.fetch_add(1, std::memory_order_relaxed); + diagnostics(addr).backing_reads.fetch_add(1, std::memory_order_relaxed); + diagnostics(addr).backing_writes.fetch_add(1, std::memory_order_relaxed); if (vmid == 0) { const bool modified = backing_memory_->atomic_modify(addr, size, [&](uint8_t *target) { fn(target, 0); }); @@ -206,7 +245,6 @@ class L2Cache : public simdojo::Component { } if (gpu_vm_ == nullptr) return VmAccessOutcome::Unavailable; - std::optional vm_access = gpu_vm_->snapshot_vmid(vmid); if (!vm_access) return VmAccessOutcome::Faulted; return vm_access->atomic_modify(addr, size, [&](std::span target) { @@ -371,11 +409,28 @@ class L2Cache : public simdojo::Component { std::map, DirtyMask> dirty_bytes_; std::atomic has_dirty_lines_{false}; std::vector cpl_ports_; - std::atomic write_count_ = 0; ///< Debug: total L2 writes (for trace). - // Relaxed atomics: independent cache operations can update these counters - // concurrently; the values are diagnostic only. - std::atomic backing_read_transactions_{0}; - std::atomic backing_write_transactions_{0}; + struct alignas(64) DiagnosticCounters { + std::atomic writes{0}; + std::atomic backing_reads{0}; + std::atomic backing_writes{0}; + }; + static constexpr uint32_t kDiagnosticShards = 64; + + DiagnosticCounters &diagnostics(uint64_t address) { + const uint32_t set = CacheStore::set_index(address); + // Fold the high set bits so page-aligned accesses use distinct shards. + return diagnostics_[(set ^ (set >> 6)) & (kDiagnosticShards - 1)]; + } + + uint64_t diagnostic_total(std::atomic DiagnosticCounters::*counter) const { + uint64_t total = 0; + for (const auto &shard : diagnostics_) + total += (shard.*counter).load(std::memory_order_relaxed); + return total; + } + + // Accounting has no cache/coherence role and never shares a dirty-state line. + std::array diagnostics_{}; }; } // namespace amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/legacy_address_space.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/legacy_address_space.h index d373fcee3ff4..ac8281c3c12f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/legacy_address_space.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/legacy_address_space.h @@ -9,6 +9,7 @@ #include "rocjitsu/kmd/linux/host_access_guard.h" #include "rocjitsu/kmd/linux/host_mapping_lock.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" +#include "rocjitsu/vm/amdgpu/gpu_vm.h" #include "rocjitsu/vm/amdgpu/legacy_page_table.h" #include "simdojo/components/sparse_memory.h" #include "simdojo/sim/component.h" @@ -31,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -150,6 +152,8 @@ class LegacyAddressSpaceRegistration { const uint64_t *page_table_generation = nullptr; std::shared_ptr request_mutex; std::shared_ptr> mutation_epoch{}; + /// Requires request_mutex so all users serialize policy-cache invalidations. + std::shared_ptr page_table_cache_state{}; pid_t client_pid = 0; int client_mem_fd = -1; bool passthrough = false; @@ -174,6 +178,13 @@ enum class CopyOutcome : uint8_t { Faulted, ///< An endpoint does not exist; retrying will never help. }; +/// @brief Progress and policy outcome for one ordinary compatibility transfer. +struct LegacyTransferStep { + size_t completed_bytes = 0; + CopyOutcome outcome = CopyOutcome::Faulted; + bool policy_fault = false; +}; + /// @brief AMDGPU address space with VMID-based per-process page table resolution. /// /// @details Mirrors the GFXHUB's VMID register file. Each process registers its @@ -247,9 +258,14 @@ class LegacyAddressSpace { /// Omitting it disables the per-thread fast path for this page table. /// @param request_mutex Optional lease that stabilizes batched page-table /// lookups. Omitting it disables cross-chunk MTYPE reuse. + /// @param page_table_cache_state Copied-policy admissions; requires request_mutex. + /// @throws std::invalid_argument if a policy cache has no shared request mutex. void register_process(uint32_t pid, LegacyPageTable *pt, util::DistributedSharedMutex *mu, const uint64_t *generation = nullptr, - std::shared_ptr request_mutex = {}) { + std::shared_ptr request_mutex = {}, + std::shared_ptr page_table_cache_state = {}) { + if (page_table_cache_state && !request_mutex) + throw std::invalid_argument("cached page-table policies require a shared request mutex"); util::Logger::cp("VMID_REG pid=", pid, " mem=0x", std::hex, reinterpret_cast(this), std::dec, " pt_size=", pt->size()); update_vmid_registration(pid, [&](auto) { @@ -260,6 +276,7 @@ class LegacyAddressSpace { .client_mem_fd = {}, .generation = generation, .request_mutex = std::move(request_mutex), + .page_table_cache_state = std::move(page_table_cache_state), .passthrough = false, .fault_reporter = nullptr, }; @@ -271,9 +288,22 @@ class LegacyAddressSpace { /// @details Page-table mutations take the exclusive side of this lease before /// the ordinary page-table lock. Callers must release the lease before a /// backing-memory access, whose allocator metadata query may reenter KFD. - PageTableRequestGuard acquire_page_table_request(uint32_t vmid) const { + PageTableRequestGuard acquire_page_table_request( + uint32_t vmid, const std::shared_ptr &expected = {}) const { if (vmid == 0) return {}; + if (expected) { + // The caller retains this owner beyond its outer operation lock. Refuse + // replacement rather than acquiring/destroying an unknown mutex owner. + PageTableRequestGuard guard(expected); + std::shared_lock lock(vmid_mutex_); + auto it = vmid_table_.find(vmid); + if (it == vmid_table_.end() || it->second.request_mutex != expected) + return {}; + guard.bind(it->second.page_table, it->second.mutex, it->second.generation, + vmid_registry_generation_); + return guard; + } while (true) { std::shared_ptr request_mutex; { @@ -563,34 +593,50 @@ class LegacyAddressSpace { return pte != vmid_entry->second.page_table->end() ? pte->second.mtype : Mtype::RW; } - /// @brief Bound one compatibility translation to the current mapped extent. + struct TranslationPolicy { + std::size_t contiguous_bytes; + Mtype mtype = Mtype::RW; + }; + + /// @brief Read one compatibility translation's extent bound and page policy. /// @details A sub-page KFD mapping must be exposed as a separate translation /// span so GpuVm can report the exact prefix completed before a later gap or /// inaccessible extent faults. Unknown and passthrough mappings retain the /// ordinary page boundary; the backing access performs their live check. - std::size_t translation_contiguous_bytes(uint64_t addr, uint32_t vmid = 0) const { + TranslationPolicy translation_policy(uint64_t addr, uint32_t vmid = 0) const { const std::size_t page_offset = addr & PAGE_MASK; const std::size_t page_bytes = PAGE_SIZE - page_offset; - std::size_t contiguous_bytes = page_bytes; - (void)with_page_mapping(addr, vmid, [&](const LegacyPageTableEntry *pte, IdentityPage) { - if (pte != nullptr) { - const LegacyHostExtent *extent = host_extent_at(*pte, page_offset); - if (extent != nullptr) { - contiguous_bytes = std::min(page_bytes, extent->gpu_page_offset + - extent->host_backed_bytes - page_offset); - } - } - return true; - }); - return contiguous_bytes; + TranslationPolicy policy{.contiguous_bytes = page_bytes}; + const bool resolved = + with_page_mapping(addr, vmid, [&](const LegacyPageTableEntry *pte, IdentityPage) { + if (pte != nullptr) + policy = mapped_translation_policy(addr, *pte); + return true; + }); + // A PTE without a host extent still contributes page policy. Ordinary + // mapped accesses obtain both fields under the same page-table lease. + if (!resolved) + policy.mtype = pte_mtype(addr, vmid); + return policy; } /// @brief Look up PTE MTYPE through an already validated request lease. - Mtype pte_mtype(uint64_t addr, const PageTableRequestGuard &guard) const { + Mtype pte_mtype(uint64_t addr, const PageTableRequestGuard &guard, + bool *may_batch_private_uc = nullptr, + bool *may_batch_private_ram = nullptr) const { assert(guard.owns_lock()); assert(guard.page_table_ != nullptr && guard.page_table_mutex_ != nullptr); std::shared_lock page_table_lock(*guard.page_table_mutex_); auto pte = guard.page_table_->find(addr >> PAGE_SHIFT); + if (may_batch_private_uc) + *may_batch_private_uc = + pte != guard.page_table_->end() && pte->second.mtype == Mtype::UC && + pte->second.host_extents.size() == 1 && + pte->second.host_extents.front().owner == LegacyHostExtentOwner::DriverSealedRam; + if (may_batch_private_ram) + *may_batch_private_ram = + pte != guard.page_table_->end() && pte->second.host_extents.size() == 1 && + pte->second.host_extents.front().owner == LegacyHostExtentOwner::DriverSealedRam; return pte != guard.page_table_->end() ? pte->second.mtype : Mtype::RW; } @@ -637,6 +683,91 @@ class LegacyAddressSpace { return copied; }); } + /// Ordered private stores with one live admission and no effects on refusal. + /// The binding must retain expected beyond its enclosing VM-state lock. + bool + try_write_private_dwords(std::span stores, uint32_t vmid, + Mtype instruction_mtype, Mtype expected_mtype, + const std::shared_ptr &expected) const { + if (!expected || stores.size() < 2 || stores.size() > VmRamDwordStore::kMaxBatch) + return false; + const uint64_t page = stores.front().address >> PAGE_SHIFT; + for (const auto &[address, source] : stores) + if (!source || (address & 3) || (address >> PAGE_SHIFT) != page) + return false; + auto request = acquire_page_table_request(vmid, expected); + if (!request.owns_lock()) + return false; + std::shared_lock page_table_lock(*request.page_table_mutex_); + const auto it = request.page_table_->find(stores.front().address >> PAGE_SHIFT); + if (it == request.page_table_->end() || it->second.host_extents.size() != 1 || + effective_mtype(instruction_mtype, it->second.mtype) != expected_mtype) + return false; + const auto &extent = it->second.host_extents.front(); + if (extent.owner != LegacyHostExtentOwner::DriverSealedRam || !extent.host_ptr) + return false; + const uintptr_t host = reinterpret_cast(extent.host_ptr); + if (extent.host_backed_bytes > UINTPTR_MAX - host) + return false; + uintptr_t first = UINTPTR_MAX, last = 0; + for (const auto &[address, source] : stores) { + const size_t page_offset = address & PAGE_MASK; + if (page_offset < extent.gpu_page_offset) + return false; + const size_t offset = page_offset - extent.gpu_page_offset; + if (offset > extent.host_backed_bytes || sizeof(uint32_t) > extent.host_backed_bytes - offset) + return false; + first = std::min(first, host + offset); + last = std::max(last, host + offset + sizeof(uint32_t)); + } + // Keep subsequent cache copies equivalent too: a store must not change a + // source word that a later descriptor or cache update still needs to read. + for (const auto &store : stores) { + const uintptr_t source = reinterpret_cast(store.source); + if (source > UINTPTR_MAX - sizeof(uint32_t) || + (source < last && first < source + sizeof(uint32_t))) + return false; + } + const auto mapping = rocjitsu::host_mapping_lock().lock_shared(); + for (const auto &[address, source] : stores) + std::memcpy(extent.host_ptr + (address & PAGE_MASK) - extent.gpu_page_offset, source, + sizeof(uint32_t)); + return true; + } + + /// @brief Copy private UC RAM under one live admission without allocation. + /// Exactly one extent is required: adjacent or overlapping extents can change + /// original dword fault boundaries even when their host bytes are contiguous. + /// The binding must retain expected beyond its enclosing VM-state lock. + bool try_read_uncached_ram(uint64_t addr, std::span dst, uint32_t vmid, + const std::shared_ptr &expected) const { + if (!expected || (addr & 3) || dst.empty() || (dst.size() & 3) || + dst.size() > PAGE_SIZE - (addr & PAGE_MASK)) + return false; + auto request = acquire_page_table_request(vmid, expected); + if (!request.owns_lock()) + return false; + std::shared_lock page_table_lock(*request.page_table_mutex_); + auto it = request.page_table_->find(addr >> PAGE_SHIFT); + if (it == request.page_table_->end() || it->second.mtype != Mtype::UC || + it->second.host_extents.size() != 1) + return false; + const size_t offset = addr & PAGE_MASK; + const auto *extent = host_extent_at(it->second, offset); + if (!extent || !extent->host_ptr || extent->owner != LegacyHostExtentOwner::DriverSealedRam || + dst.size() > extent->host_backed_bytes - (offset - extent->gpu_page_offset)) + return false; + const auto *source = extent->host_ptr + offset - extent->gpu_page_offset; + const uintptr_t source_begin = reinterpret_cast(source); + const uintptr_t destination_begin = reinterpret_cast(dst.data()); + if (source_begin > UINTPTR_MAX - dst.size() || destination_begin > UINTPTR_MAX - dst.size() || + (source_begin < destination_begin + dst.size() && + destination_begin < source_begin + dst.size())) + return false; + const auto mapping = rocjitsu::host_mapping_lock().lock_shared(); + std::memcpy(dst.data(), source, dst.size()); + return true; + } uint32_t fetch32(uint64_t addr, uint32_t vmid = 0) const { return read32(addr, vmid); } @@ -792,26 +923,9 @@ class LegacyAddressSpace { auto out = std::span(staged).subspan(offset, chunk); if (copy_from_mapped(ea, out.data(), chunk, vmid)) return true; - if (faults.observed()) { - outcome = CopyOutcome::Faulted; - return false; - } - if (vmid > 0 && has_client_backing(vmid)) { - if (read_client_memory(ea, out.data(), chunk, vmid)) - return true; - std::ranges::fill(out, uint8_t{0}); - note_rejected_identity_access(ea, vmid); - outcome = CopyOutcome::Faulted; - return false; - } - if (has_page_mapping(ea, vmid)) { - note_clipped_mapped_access("read", ea, chunk, vmid); - note_rejected_identity_access(ea, vmid); - outcome = CopyOutcome::Faulted; - return false; - } - outcome = CopyOutcome::Unavailable; - return false; + outcome = finish_strict_copy(ea, out.data(), chunk, vmid, + /*into_memory=*/false, faults); + return outcome == CopyOutcome::Complete; }); if (!completed) return outcome; @@ -839,29 +953,26 @@ class LegacyAddressSpace { auto in = src.subspan(offset, chunk); if (copy_to_mapped(ea, in.data(), chunk, vmid)) return true; - if (faults.observed()) { - outcome = CopyOutcome::Faulted; - return false; - } - if (vmid > 0 && has_client_backing(vmid)) { - if (write_client_memory(ea, in.data(), chunk, vmid)) - return true; - note_rejected_identity_access(ea, vmid, MemoryFaultCause::Indeterminate); - outcome = CopyOutcome::Faulted; - return false; - } - if (has_page_mapping(ea, vmid)) { - note_clipped_mapped_access("write", ea, chunk, vmid); - note_rejected_identity_access(ea, vmid); - outcome = CopyOutcome::Faulted; - return false; - } - outcome = CopyOutcome::Unavailable; - return false; + outcome = finish_strict_copy(ea, const_cast(in.data()), chunk, vmid, + /*into_memory=*/true, faults); + return outcome == CopyOutcome::Complete; }); return completed ? CopyOutcome::Complete : outcome; } + /// @brief Translate policy and read the next page span under one admission. + [[nodiscard]] LegacyTransferStep read_step(uint64_t addr, std::span remaining, + uint32_t vmid = 0) const { + return transfer_step(addr, remaining.data(), remaining.size(), vmid); + } + + /// @brief Translate policy and write the next page span under one admission. + [[nodiscard]] LegacyTransferStep write_step(uint64_t addr, std::span remaining, + uint32_t vmid = 0) const { + return transfer_step(addr, const_cast(remaining.data()), remaining.size(), + vmid); + } + /// @brief Perform an atomic read-modify-write on resolved backing storage. /// @details Storage classification and the page-table shared lock remain /// stable through the callback. Mapped aliases rendezvous on a process-wide @@ -1458,7 +1569,8 @@ class LegacyAddressSpace { if (registration.page_table == nullptr || registration.page_table_mutex == nullptr) return false; register_process(vmid, registration.page_table, registration.page_table_mutex, - registration.page_table_generation, std::move(registration.request_mutex)); + registration.page_table_generation, std::move(registration.request_mutex), + std::move(registration.page_table_cache_state)); set_process_client_pid(vmid, registration.client_pid); set_process_mem_fd(vmid, registration.client_mem_fd); set_process_passthrough(vmid, registration.passthrough); @@ -2135,7 +2247,8 @@ class LegacyAddressSpace { [[nodiscard]] static bool extent_is_writable(const LegacyHostExtent &extent, const uint8_t *target, size_t size, MemoryFaultCause &cause) { - if (extent.owner == LegacyHostExtentOwner::Driver) + if (extent.owner == LegacyHostExtentOwner::Driver || + extent.owner == LegacyHostExtentOwner::DriverSealedRam) return true; const auto writability = host_range_writability(target, size); if (writability == PageWritability::Writable) @@ -2369,6 +2482,7 @@ class LegacyAddressSpace { util::UniqueHandle client_mem_fd; const uint64_t *generation = nullptr; std::shared_ptr request_mutex; + std::shared_ptr page_table_cache_state; bool passthrough = false; MemoryFaultReporter *fault_reporter = nullptr; }; @@ -2399,6 +2513,8 @@ class LegacyAddressSpace { if (current_request_mutex != request_mutex) continue; + if (it != vmid_table_.end() && it->second.page_table_cache_state) + it->second.page_table_cache_state->invalidate_and_wait(); if (update(it)) ++vmid_registry_generation_; return; @@ -2418,6 +2534,15 @@ class LegacyAddressSpace { LegacyPageTable *page_table = nullptr; util::DistributedSharedMutex *mutex = nullptr; const uint64_t *generation_ptr = nullptr; + std::shared_ptr page_table_cache_state; + uint64_t page_table_cache_generation = 0; + }; + + class PteCacheSet { + public: + static constexpr size_t kEntryCount = 64; + static_assert((kEntryCount & (kEntryCount - 1)) == 0); + std::array entries; }; #if defined(RJ_GPU_MEMORY_WITH_ASAN) @@ -2553,7 +2678,8 @@ class LegacyAddressSpace { /// @brief Walk a VMID page table with a generation-keyed thread-local cache. /// @details A mapped-PTE callback runs while both VMID registration and the - /// selected page table are shared-locked. Miss callbacks and ASan allocator + /// selected page table are shared-locked or a copied-PTE admission is held. + /// Each admission ends when its callback returns. Miss callbacks and ASan allocator /// queries run without either lock. After an unlocked query, the VMID binding /// and exact PTE contents are revalidated before the bounded copy is published. /// Addressability checks in the mapped-span helpers remain the final guard @@ -2562,6 +2688,23 @@ class LegacyAddressSpace { auto cached_walk(uint64_t addr, uint32_t vmid, PteCache &cache, F &&fn) const -> std::invoke_result_t { const uint64_t page_key = addr >> PAGE_SHIFT; + + // Cached PTEs are immutable copies. A two-check admission against the + // retained process state lets a hit avoid both page-table reader locks; + // mutation advances the generation before draining admitted readers. The + // admission covers only this callback, never the surrounding quantum. + if (cache.memory == this && cache.memory_instance == instance_id_ && cache.vmid == vmid && + cache.page_key == page_key && cache.found && cache.page_table_cache_state) { + auto *state = cache.page_table_cache_state.get(); + if (state->try_acquire(cache.page_table_cache_generation)) { + struct ReaderGuard { + LegacyPageTableCacheState *state; + ~ReaderGuard() { state->release(); } + } reader_guard{state}; + return fn(&cache.pte); + } + } + #if defined(RJ_GPU_MEMORY_WITH_ASAN) size_t metadata_retries = 0; #endif @@ -2575,6 +2718,8 @@ class LegacyAddressSpace { LegacyPageTable *page_table = cache.page_table; util::DistributedSharedMutex *page_table_mutex = cache.mutex; const uint64_t *generation_ptr = cache.generation_ptr; + std::shared_ptr page_table_cache_state = + cache.page_table_cache_state; if (!cached_table) { auto vmid_entry = vmid_table_.find(vmid); if (vmid_entry == vmid_table_.end()) { @@ -2585,6 +2730,7 @@ class LegacyAddressSpace { page_table = vmid_entry->second.page_table; page_table_mutex = vmid_entry->second.mutex; generation_ptr = vmid_entry->second.generation; + page_table_cache_state = vmid_entry->second.page_table_cache_state; } std::shared_lock page_table_lock(*page_table_mutex); @@ -2602,11 +2748,18 @@ class LegacyAddressSpace { .page_table = page_table, .mutex = page_table_mutex, .generation_ptr = generation_ptr, + .page_table_cache_state = page_table_cache_state, + .page_table_cache_generation = + page_table_cache_state ? page_table_cache_state->generation() : 0, }; }; const bool cached_page = allow_cache_hit && cached_table && generation_ptr && cache.generation == generation && cache.page_key == page_key; if (cached_page) { + // A no-op mutation revokes admissions without changing the PTE. Renew + // the copied entry's admission generation while both locks are held. + cache.page_table_cache_generation = + page_table_cache_state ? page_table_cache_state->generation() : 0; if (cache.found) return fn(&cache.pte); page_table_lock.unlock(); @@ -2698,6 +2851,12 @@ class LegacyAddressSpace { } if (generation_ptr) generation = *generation_ptr; + // Allocator bounds can change without a PTE mutation when a pool reuses + // the same mapped page. Keep shortened bounds local to this access. + if (candidate != raw_pte) { + cache = {}; + return fn(&candidate); + } #endif publish_cache(true, std::move(candidate)); return fn(&cache.pte); @@ -2728,7 +2887,11 @@ class LegacyAddressSpace { return fn(nullptr, IdentityPage(page)); } - static thread_local PteCache cache; + static thread_local PteCacheSet caches; + const uint64_t page = addr >> PAGE_SHIFT; + const uint64_t cache_key = + page ^ (page >> 6) ^ (page >> 12) ^ (static_cast(vmid) * 0x9e3779b97f4a7c15ULL); + PteCache &cache = caches.entries[cache_key & (PteCacheSet::kEntryCount - 1)]; return cached_walk(addr, vmid, cache, [&](const LegacyPageTableEntry *pte) { if (pte) { if (pte->host_extents.empty()) @@ -2829,6 +2992,79 @@ class LegacyAddressSpace { }); } + static TranslationPolicy mapped_translation_policy(uint64_t addr, + const LegacyPageTableEntry &pte) { + const size_t offset = addr & PAGE_MASK; + TranslationPolicy policy{.contiguous_bytes = PAGE_SIZE - offset, .mtype = pte.mtype}; + if (const LegacyHostExtent *extent = host_extent_at(pte, offset)) + policy.contiguous_bytes = std::min( + policy.contiguous_bytes, extent->gpu_page_offset + extent->host_backed_bytes - offset); + return policy; + } + + CopyOutcome finish_strict_copy(uint64_t addr, void *bytes, size_t size, uint32_t vmid, + bool into_memory, const FaultScope &faults) const { + if (faults.observed()) + return CopyOutcome::Faulted; + if (vmid > 0 && has_client_backing(vmid)) { + if (into_memory ? write_client_memory(addr, bytes, size, vmid) + : read_client_memory(addr, bytes, size, vmid)) + return CopyOutcome::Complete; + if (!into_memory) + std::memset(bytes, 0, size); + note_rejected_identity_access( + addr, vmid, into_memory ? MemoryFaultCause::Indeterminate : MemoryFaultCause::NotPresent); + return CopyOutcome::Faulted; + } + if (has_page_mapping(addr, vmid)) { + note_clipped_mapped_access(into_memory ? "write" : "read", addr, size, vmid); + note_rejected_identity_access(addr, vmid); + return CopyOutcome::Faulted; + } + return CopyOutcome::Unavailable; + } + + template + LegacyTransferStep transfer_step(uint64_t addr, void *bytes, size_t remaining, + uint32_t vmid) const { + const FaultDispatch fault_dispatch(*this); + const FaultScope faults; + if (!range_within_address_space(addr, remaining)) { + note_rejected_identity_access(addr, vmid); + return {.outcome = CopyOutcome::Faulted}; + } + if (remaining == 0) + return {.outcome = CopyOutcome::Complete}; + const size_t chunk = std::min(remaining, PAGE_SIZE - (addr & PAGE_MASK)); + // Allocate staging before any mapping admission. A failed physical read + // leaves its destination unchanged, including a guarded host-access fault. + std::vector staged; + if constexpr (!IntoMemory) + staged.resize(chunk); + void *copy_bytes = IntoMemory ? bytes : staged.data(); + bool policy_fault = false; + const bool copied = + with_page_mapping(addr, vmid, [&](const LegacyPageTableEntry *pte, IdentityPage page) { + // A sub-page extent is a hard translation boundary even when another + // adjacent extent could satisfy the strict transport's coverage test. + if (pte && mapped_translation_policy(addr, *pte).contiguous_bytes < chunk) { + policy_fault = true; + return false; + } + return copy_resolved_span(addr, copy_bytes, chunk, vmid, IntoMemory, pte, page); + }); + if (policy_fault) + return {.outcome = CopyOutcome::Faulted, .policy_fault = true}; + const CopyOutcome outcome = + copied ? CopyOutcome::Complete + : finish_strict_copy(addr, copy_bytes, chunk, vmid, IntoMemory, faults); + if (outcome != CopyOutcome::Complete) + return {.outcome = outcome}; + if constexpr (!IntoMemory) + std::memcpy(bytes, staged.data(), chunk); + return {.completed_bytes = chunk, .outcome = CopyOutcome::Complete}; + } + /// @brief Copy a page-bounded span in or out without exposing a bare pointer. /// /// @details A page-table span is memcpy'd from its extent. An identity span is @@ -2840,109 +3076,115 @@ class LegacyAddressSpace { if (size == 0 || (addr & PAGE_MASK) + size > PAGE_SIZE) return false; return with_page_mapping(addr, vmid, [&](const LegacyPageTableEntry *pte, IdentityPage page) { - const size_t page_offset = addr & PAGE_MASK; - if (pte) { - // Collect and validate every extent before copying. A strict write may - // span adjacent sub-page extents, but it must never publish the extents - // before discovering a gap or a read-only destination. - struct Span { - size_t value_offset = 0; - uint8_t *host_ptr = nullptr; - size_t size = 0; - const LegacyHostExtent *extent = nullptr; - }; - std::vector spans; - const auto mapping_lease = rocjitsu::host_mapping_lock().lock_shared(); - const size_t mapped_bytes = - for_each_mapped_span(*pte, page_offset, size, - [&](size_t value_offset, uint8_t *host_ptr, size_t span_size, - const LegacyHostExtent &extent) { - spans.push_back({value_offset, host_ptr, span_size, &extent}); - }); - std::ranges::sort(spans, {}, &Span::value_offset); - size_t covered = 0; - for (const Span &span : spans) { - if (span.value_offset != covered) - return false; - covered += span.size; - } - if (mapped_bytes != size || covered != size) + return copy_resolved_span(addr, bytes, size, vmid, into_memory, pte, page); + }); + } + + // The caller owns the mapped-PTE admission. No pointer or guard escapes. + bool copy_resolved_span(uint64_t addr, void *bytes, size_t size, uint32_t vmid, bool into_memory, + const LegacyPageTableEntry *pte, IdentityPage page) const { + const size_t page_offset = addr & PAGE_MASK; + if (pte) { + // Collect and validate every extent before copying. A strict write may + // span adjacent sub-page extents, but it must never publish the extents + // before discovering a gap or a read-only destination. + struct Span { + size_t value_offset = 0; + uint8_t *host_ptr = nullptr; + size_t size = 0; + const LegacyHostExtent *extent = nullptr; + }; + std::vector spans; + const auto mapping_lease = rocjitsu::host_mapping_lock().lock_shared(); + const size_t mapped_bytes = + for_each_mapped_span(*pte, page_offset, size, + [&](size_t value_offset, uint8_t *host_ptr, size_t span_size, + const LegacyHostExtent &extent) { + spans.push_back({value_offset, host_ptr, span_size, &extent}); + }); + std::ranges::sort(spans, {}, &Span::value_offset); + size_t covered = 0; + for (const Span &span : spans) { + if (span.value_offset != covered) + return false; + covered += span.size; + } + if (mapped_bytes != size || covered != size) + return false; + // A write wholly contained in one host page cannot partially cross a + // protection boundary: an unwritable page faults on its first store. + // Let the guarded copy take that exceptional fault instead of scanning + // /proc/self/maps before every ordinary GPU store. Multi-span and + // cross-page strict writes still validate every destination first, + // because a later fault there could otherwise leave an earlier span + // modified even though this operation reports failure. + bool write_can_fault_atomically = false; + if (into_memory && spans.size() == 1) { + const Span &span = spans.front(); + const uintptr_t first_page = + reinterpret_cast(span.host_ptr) & ~static_cast(PAGE_MASK); + const uintptr_t last_page = reinterpret_cast(span.host_ptr + span.size - 1) & + ~static_cast(PAGE_MASK); + write_can_fault_atomically = first_page == last_page; + if (write_can_fault_atomically && + addressable_prefix(span.host_ptr, span.size) != span.size) { + note_rejected_identity_access(addr, vmid, MemoryFaultCause::NotPresent); return false; - // A write wholly contained in one host page cannot partially cross a - // protection boundary: an unwritable page faults on its first store. - // Let the guarded copy take that exceptional fault instead of scanning - // /proc/self/maps before every ordinary GPU store. Multi-span and - // cross-page strict writes still validate every destination first, - // because a later fault there could otherwise leave an earlier span - // modified even though this operation reports failure. - bool write_can_fault_atomically = false; - if (into_memory && spans.size() == 1) { - const Span &span = spans.front(); - const uintptr_t first_page = - reinterpret_cast(span.host_ptr) & ~static_cast(PAGE_MASK); - const uintptr_t last_page = reinterpret_cast(span.host_ptr + span.size - 1) & - ~static_cast(PAGE_MASK); - write_can_fault_atomically = first_page == last_page; - if (write_can_fault_atomically && - addressable_prefix(span.host_ptr, span.size) != span.size) { - note_rejected_identity_access(addr, vmid, MemoryFaultCause::NotPresent); - return false; - } } - if (into_memory && !write_can_fault_atomically) { - for (const Span &span : spans) { - MemoryFaultCause cause = MemoryFaultCause::NotPresent; - if (!extent_is_writable(*span.extent, span.host_ptr, span.size, cause)) { - note_rejected_identity_access(addr + span.value_offset, vmid, cause); - return false; - } + } + if (into_memory && !write_can_fault_atomically) { + for (const Span &span : spans) { + MemoryFaultCause cause = MemoryFaultCause::NotPresent; + if (!extent_is_writable(*span.extent, span.host_ptr, span.size, cause)) { + note_rejected_identity_access(addr + span.value_offset, vmid, cause); + return false; } } - if (!rocjitsu::with_host_access_guard([&] { - for (const Span &span : spans) { + } + if (!rocjitsu::with_host_access_guard([&] { + for (const Span &span : spans) { #if defined(RJ_GPU_MEMORY_WITH_TSAN) - constexpr uintptr_t kHostPageMask = PAGE_SIZE - 1; - const uintptr_t begin = reinterpret_cast(span.host_ptr) & - ~static_cast(kHostPageMask); - const uintptr_t end = (reinterpret_cast(span.host_ptr) + span.size - 1) & + constexpr uintptr_t kHostPageMask = PAGE_SIZE - 1; + const uintptr_t begin = reinterpret_cast(span.host_ptr) & ~static_cast(kHostPageMask); - for (uintptr_t host_page = begin;; host_page += PAGE_SIZE) { - __tsan_acquire(reinterpret_cast(host_page)); - if (host_page == end) - break; - } + const uintptr_t end = (reinterpret_cast(span.host_ptr) + span.size - 1) & + ~static_cast(kHostPageMask); + for (uintptr_t host_page = begin;; host_page += PAGE_SIZE) { + __tsan_acquire(reinterpret_cast(host_page)); + if (host_page == end) + break; + } #endif - if (into_memory) { - std::memcpy(span.host_ptr, - static_cast(bytes) + span.value_offset, span.size); - } else { - std::memcpy(static_cast(bytes) + span.value_offset, span.host_ptr, - span.size); - } + if (into_memory) { + std::memcpy(span.host_ptr, static_cast(bytes) + span.value_offset, + span.size); + } else { + std::memcpy(static_cast(bytes) + span.value_offset, span.host_ptr, + span.size); + } #if defined(RJ_GPU_MEMORY_WITH_TSAN) - for (uintptr_t host_page = begin;; host_page += PAGE_SIZE) { - __tsan_release(reinterpret_cast(host_page)); - if (host_page == end) - break; - } -#endif + for (uintptr_t host_page = begin;; host_page += PAGE_SIZE) { + __tsan_release(reinterpret_cast(host_page)); + if (host_page == end) + break; } - })) { - note_rejected_identity_access(addr, vmid, guarded_fault_cause()); - return false; - } - return true; - } - if (addr >= kUserSpaceLimit || size > kUserSpaceLimit - addr) +#endif + } + })) { + note_rejected_identity_access(addr, vmid, guarded_fault_cause()); return false; - MemoryFaultCause write_cause = MemoryFaultCause::NotPresent; - const bool moved = into_memory ? page.write_strict(page_offset, bytes, size, write_cause) - : page.read(page_offset, bytes, size); - if (!moved) - note_rejected_identity_access(addr, vmid, - into_memory ? write_cause : MemoryFaultCause::NotPresent); - return moved; - }); + } + return true; + } + if (addr >= kUserSpaceLimit || size > kUserSpaceLimit - addr) + return false; + MemoryFaultCause write_cause = MemoryFaultCause::NotPresent; + const bool moved = into_memory ? page.write_strict(page_offset, bytes, size, write_cause) + : page.read(page_offset, bytes, size); + if (!moved) + note_rejected_identity_access(addr, vmid, + into_memory ? write_cause : MemoryFaultCause::NotPresent); + return moved; } /// @brief Read a mapped span into @p dst without ever exposing a bare pointer. @@ -3056,7 +3298,7 @@ class LegacyAddressSpace { return true; } - bool write_client_memory(uint64_t addr, const void *src, size_t len, uint32_t vmid) { + bool write_client_memory(uint64_t addr, const void *src, size_t len, uint32_t vmid) const { // See read_client_memory(): the authorized fd is the only path that works // for a debuggee the daemon did not ptrace-attach to itself. if (util::UniqueHandle mem_fd = duplicate_client_mem_fd(vmid); mem_fd.get() >= 0) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/legacy_page_table.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/legacy_page_table.h index 6acde9a567d4..40cc0b3b9683 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/legacy_page_table.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/legacy_page_table.h @@ -8,9 +8,13 @@ #include "rocjitsu/vm/amdgpu/mtype.h" +#include +#include #include #include +#include #include +#include #include #include @@ -22,6 +26,8 @@ inline constexpr uint64_t kLegacyPageSize = uint64_t{1} << kLegacyPageShift; /// @brief Who owns host storage named by a compatibility mapping. enum class LegacyHostExtentOwner : uint8_t { Driver, + /// Private read-write driver mapping backed by a file sealed against shrinking. + DriverSealedRam, Application, }; @@ -56,4 +62,73 @@ class LegacyPageTableEntry { using LegacyPageTable = std::pmr::unordered_map; +/// @brief Lifetime-safe admission state for lock-free copied legacy PTEs. +/// +/// @details A reader joins only while the generation it cached is current. +/// Mutations advance the generation first, then wait for already-admitted +/// readers before changing page-table storage. The shared state may safely +/// outlive the process and address-space objects retained by thread-local +/// caches. Admissions must be released by the acquiring thread. +class LegacyPageTableCacheState { +public: + [[nodiscard]] uint64_t generation() const { return generation_.load(std::memory_order_seq_cst); } + + [[nodiscard]] bool try_acquire(uint64_t expected_generation) const { + if (generation_.load(std::memory_order_seq_cst) != expected_generation) + return false; + reader_count().fetch_add(1, std::memory_order_seq_cst); + if (generation_.load(std::memory_order_seq_cst) == expected_generation) + return true; + release(); + return false; + } + + void release() const { + auto &active = reader_count(); + if (active.fetch_sub(1, std::memory_order_seq_cst) == 1 && + draining_.load(std::memory_order_seq_cst)) + active.notify_all(); + } + + /// @brief Exclude new cached readers and drain the previous generation. + /// @pre The caller owns the mutation-side serialization for the page table + /// or address-space registration it is about to change. + void invalidate_and_wait() { + // Publish the wakeup requirement before invalidating admissions. A last + // reader either sees this flag and wakes us, or leaves before the scan + // observes its count. Ordinary releases never touch the library's shared + // atomic-wait bookkeeping. + draining_.store(true, std::memory_order_seq_cst); + generation_.fetch_add(1, std::memory_order_seq_cst); + for (auto &shard : readers_) { + uint64_t active = shard.count.load(std::memory_order_seq_cst); + while (active != 0) { + shard.count.wait(active, std::memory_order_seq_cst); + active = shard.count.load(std::memory_order_seq_cst); + } + } + draining_.store(false, std::memory_order_seq_cst); + } + +private: + // Reads dominate mutations. Keep the generation off the counters' cache + // lines, and let unrelated simulation threads admit independently. Hash + // collisions preserve exclusion, including nested reads on one thread. + static constexpr std::size_t kReaderShards = 128; + struct alignas(64) ReaderShard { + std::atomic count{0}; + }; + alignas(64) std::atomic generation_{1}; + std::atomic draining_{false}; + mutable std::array readers_; + + std::atomic &reader_count() const { + // Thread identity gives inline calls in different shared libraries the + // same slot, without retaining thread-owned state in the cache object. + static thread_local const std::size_t shard = + std::hash{}(std::this_thread::get_id()) % kReaderShards; + return readers_[shard].count; + } +}; + } // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/matrix_coexecution.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/matrix_coexecution.h index 897bbb81ae96..35990c8c3726 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/matrix_coexecution.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/matrix_coexecution.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -135,6 +137,12 @@ class Helper { std::thread thread_; }; +// Map the issuer hash into the pool before selecting a bit in its current word. +// Pools above 64 helpers keep first-word priority. +constexpr unsigned preferred_helper_start(uint32_t issuer_hash, std::size_t capacity) { + return static_cast((uint64_t{issuer_hash} * capacity) >> 32) & 63; +} + // Each claim covers publication through joining, so another issuer cannot // reuse a helper's payload while its previous owner still reads completion. // There is no pending-job queue and no wait for capacity. Two bitmap words @@ -203,19 +211,31 @@ class SharedPool { Slot *try_submit(Instruction &instruction, void *wave) { if (!helpers_enabled()) return nullptr; + // A stable preference may reuse a still-awake helper without pinning or a + // per-job cursor. Single-issuer counters support lower sleep/wakeup costs, + // but neither this preference nor more complex allocation policies gave + // consistent throughput gains; multi-issuer wake delays can increase. + // Keep the simple policy: useful issuer overlap and spare CPU capacity + // determine the payoff. Bound claims and fall back inline on a miss. + // // One or two loads detect an exhausted pool. Reservation is bounded even // under contention: after two failed CAS attempts per word, run inline. // A conservative miss is allowed; waiting for worker capacity is not. + // Scale the hash to the pool size so small pools do not send out-of-range + // starts to the lowest helper. + // Pools above 64 helpers retain first-word priority; this preference does + // not distribute issuers uniformly across both words. + static thread_local const uint32_t issuer_hash = + static_cast(std::hash{}(std::this_thread::get_id())); + const unsigned start = preferred_helper_start(issuer_hash, slots_.size()); for (unsigned word = 0; word != (slots_.size() > 64 ? 2u : 1u); ++word) { auto free = free_[word].load(std::memory_order_relaxed); for (unsigned attempt = 0; free && attempt != 2; ++attempt) { - thread_local unsigned next = std::hash{}(std::this_thread::get_id()) & 63; - const unsigned bit = (std::countr_zero(std::rotr(free, int(next))) + next) & 63; + const unsigned bit = (std::countr_zero(std::rotr(free, int(start))) + start) & 63; if (!free_[word].compare_exchange_strong(free, free & ~(uint64_t{1} << bit), std::memory_order_acquire, std::memory_order_relaxed)) continue; - next = (bit + 1) & 63; auto *slot = slots_[word * 64 + bit].get(); slot->helper.submit(instruction, wave); return slot; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h index b0468a65c480..f483dcdf0cfe 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h @@ -170,6 +170,9 @@ class VectorMemState : public DynamicInstState { bool is_load = true; Mtype mtype = Mtype::RW; WaitCounterType wait_counter_type = WaitCounterType::VMCNT; + /// FLAT counter participation selected from the resolved requesting lanes. + /// Unset before routing; decoded metadata continues to describe all domains. + std::optional routed_issue_info; bool non_temporal = false; // Keep this outside Mtype: cluster loads force only the request-side vector // L1 lookup to miss, while mtype must still preserve the instruction/PTE @@ -226,6 +229,9 @@ class VectorMemState : public DynamicInstState { /// Older MIN/MAX compare flushed inputs but return the original selected bits. /// They also propagate signaling NaNs instead of treating them as missing numbers. bool atomic_legacy_minmax = true; + /// L2 ADD on qualified RDNA4 targets selects the incoming NaN first. + /// Indexed LDS has an independent source-first policy. + bool atomic_source_nan_first = false; bool lds_dst = false; ///< Buffer load with LDS bit: write to LDS, not VGPRs. /// Reference LDS address for LDS-destination loads. For ordinary LDS-dst /// paths this may include the lane-0 destination offset. For cluster diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp index 2cb0be6df63c..dc6b84a9f38e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp @@ -47,7 +47,14 @@ MemoryPipeline::~MemoryPipeline() { MemoryPipeline::WaitCounterTokens MemoryPipeline::issue_counters(const Instruction &inst) const { WaitCounterTokens counters; - if (const auto *issue = inst.amdgpu_memory_issue_info()) { + const DynamicInstState *state = inst.data(); + const auto *issue = inst.amdgpu_memory_issue_info(); + if (state && (state->tag() == GLOBAL_MEM || state->tag() == LOCAL_MEM)) { + const auto &routed = inst.data_as()->routed_issue_info; + if (routed) + issue = &*routed; + } + if (issue) { for (const auto obligation : issue->counter_obligations()) { for (uint8_t token = 0; token < obligation.counter_increment(); ++token) counters.types[counters.size++] = obligation.wait_counter_type(); @@ -56,7 +63,6 @@ MemoryPipeline::WaitCounterTokens MemoryPipeline::issue_counters(const Instructi } WaitCounterType counter = counter_type_; - const DynamicInstState *state = inst.data(); if (state != nullptr) { switch (state->tag()) { case SCALAR_MEM: @@ -462,8 +468,11 @@ VmAccessOutcome ScalarMemPipeline::initiate_access(Instruction &inst, Wavefront return outcome; } } else { + auto &cu = wf.raw_cu(); + const bool allow_private_batch = + GpuVmAccessBatchGuard::active() && !cu.debug_active() && cu.plugin_group().empty(); const VmAccessOutcome outcome = - l1_->load(d.addr, d.num_dwords, d.response_data, wf.process_id()); + l1_->load(d.addr, d.num_dwords, d.response_data, wf.process_id(), allow_private_batch); if (outcome != VmAccessOutcome::Complete) return outcome; } @@ -504,66 +513,10 @@ ScalarMemPipeline::complete_access(Instruction &inst, Wavefront &wf, namespace { -/// @brief Apply an integer atomic RMW operation (32-bit or 64-bit). -template T apply_int_atomic(AtomicOp op, T old_val, T src_val, T cmp_val = 0) { - using S = std::make_signed_t; - switch (op) { - case AtomicOp::SWAP: - return src_val; - case AtomicOp::CONDXCHG32: { - static_assert(sizeof(T) == 4 || sizeof(T) == 8); - T result = old_val; - for (uint32_t shift = 0; shift < sizeof(T) * 8; shift += 32) { - const T half_mask = T{0xffffffffu} << shift; - const T store_mask = T{0x7fffffffu} << shift; - if ((src_val >> shift) & 0x80000000u) - result = (result & ~half_mask) | (src_val & store_mask); - } - return result; - } - case AtomicOp::CMPSWAP: - return (old_val == cmp_val) ? src_val : old_val; - case AtomicOp::MSKOR: - return (old_val & ~src_val) | cmp_val; - case AtomicOp::ADD: - return old_val + src_val; - case AtomicOp::SUB: - return old_val - src_val; - case AtomicOp::SUB_CLAMP: - return old_val >= src_val ? old_val - src_val : T{0}; - case AtomicOp::COND_SUB: - return old_val >= src_val ? old_val - src_val : old_val; - case AtomicOp::WRAP: - return old_val >= src_val ? old_val - src_val : old_val + cmp_val; - case AtomicOp::RSUB: - return src_val - old_val; - case AtomicOp::SMIN: - return static_cast(std::min(static_cast(old_val), static_cast(src_val))); - case AtomicOp::UMIN: - return std::min(old_val, src_val); - case AtomicOp::SMAX: - return static_cast(std::max(static_cast(old_val), static_cast(src_val))); - case AtomicOp::UMAX: - return std::max(old_val, src_val); - case AtomicOp::AND: - return old_val & src_val; - case AtomicOp::OR: - return old_val | src_val; - case AtomicOp::XOR: - return old_val ^ src_val; - case AtomicOp::INC: - return (old_val >= src_val) ? T{0} : old_val + 1; - case AtomicOp::DEC: - return (old_val == 0 || old_val > src_val) ? src_val : old_val - 1; - default: - return old_val; - } -} - // Translate the deferred request into the shared ISA floating-point contract. template Bits apply_fp_atomic(const VectorMemState &state, Bits old_bits, Bits source_bits, - Bits compare_bits, uint32_t denorm_mode) { + Bits compare_bits, uint32_t denorm_mode, bool local = false) { fp_mode::ScalarAtomicOp operation; switch (state.atomic_op) { case AtomicOp::FADD: @@ -582,7 +535,7 @@ Bits apply_fp_atomic(const VectorMemState &state, Bits old_bits, Bits source_bit return old_bits; } return fp_mode::atomic_scalar(operation, old_bits, source_bits, compare_bits, denorm_mode, - state.atomic_legacy_minmax); + state.atomic_legacy_minmax, local || state.atomic_source_nan_first); } bool is_packed_add(AtomicOp op) { @@ -777,8 +730,12 @@ VmAccessOutcome execute_translated_atomic_rmw(VectorMemState &d) { std::memcpy(&compare, &d.store_data[lane * source_stride + sizeof(source)], sizeof(compare)); compare_value = compare; - new_value = is_fp ? apply_fp_atomic(d, old32, source, compare, d.atomic_denorm_mode) - : apply_int_atomic(d.atomic_op, old32, source, compare); + new_value = + is_packed_add(d.atomic_op) + ? fp_mode::atomic_add_packed_16(old32, source, d.atomic_op == AtomicOp::PK_ADD_BF16, + 3, d.atomic_source_nan_first) + : is_fp ? apply_fp_atomic(d, old32, source, compare, d.atomic_denorm_mode) + : apply_int_atomic(d.atomic_op, old32, source, compare); } else if (width == sizeof(uint64_t)) { uint64_t source = 0; uint64_t compare = 0; @@ -832,15 +789,23 @@ VmAccessOutcome execute_atomic_rmw(VectorMemState &d, L2Cache *l2, uint32_t vmid const bool is_fp = (d.atomic_op == AtomicOp::FADD || d.atomic_op == AtomicOp::FMIN || d.atomic_op == AtomicOp::FMAX || d.atomic_op == AtomicOp::FCMPSWAP); - for (uint32_t lane = 0; lane < d.wf_size; ++lane) { + if (d.lane_mask == 0) + return VmAccessOutcome::Complete; + // A retry keeps completed lanes but must not reuse a retired VM generation. + const auto vm_access = l2->snapshot_atomic_access(vmid); + DeviceCacheCoherence::AtomicBoundary boundary = l2->coherence_domain()->acquire_atomic_boundary(); + if (boundary.outcome() != VmAccessOutcome::Complete) + return boundary.outcome(); + + for (uint32_t lane = d.translated.atomic_lane; lane < d.wf_size; ++lane) { if (!(d.lane_mask & (1ULL << lane))) continue; uint64_t ea = d.per_lane_addr[lane]; - // Perform the atomic RMW under L2's atomic lock. + // Reuse the vector's cache-coherence boundary; each backing update remains atomic. const VmAccessOutcome outcome = l2->atomic_rmw( - ea, esz, + boundary, ea, esz, [&](uint8_t *line_data, uint32_t offset) { if (esz == 4) { uint32_t old_val; @@ -852,7 +817,7 @@ VmAccessOutcome execute_atomic_rmw(VectorMemState &d, L2Cache *l2, uint32_t vmid std::memcpy(&src_val, &d.store_data[lane * src_stride], 4); new_val = fp_mode::atomic_add_packed_16(old_val, src_val, d.atomic_op == AtomicOp::PK_ADD_BF16, - /*denorm_mode=*/3); + /*denorm_mode=*/3, d.atomic_source_nan_first); } else if (is_fp) { uint32_t source_bits = 0, compare_bits = 0; std::memcpy(&source_bits, &d.store_data[lane * src_stride], 4); @@ -894,9 +859,12 @@ VmAccessOutcome execute_atomic_rmw(VectorMemState &d, L2Cache *l2, uint32_t vmid std::memcpy(&d.response_data[lane * 8], &old_val, 8); } }, - vmid); - if (outcome != VmAccessOutcome::Complete) + vmid, vm_access); + if (outcome != VmAccessOutcome::Complete) { + d.translated.atomic_lane = lane; return outcome; + } + d.translated.atomic_lane = lane + 1; } return VmAccessOutcome::Complete; } @@ -954,14 +922,16 @@ void execute_lds_atomic_rmw(VectorMemState &d, Lds *lds, if (is_packed_add(d.atomic_op)) { uint32_t src_val; std::memcpy(&src_val, &store_data[lane * src_stride], 4); - new_val = fp_mode::atomic_add_packed_16( - old_val, src_val, d.atomic_op == AtomicOp::PK_ADD_BF16, d.packed_denorm_mode); + new_val = + fp_mode::atomic_add_packed_16(old_val, src_val, d.atomic_op == AtomicOp::PK_ADD_BF16, + d.packed_denorm_mode, /*source_first=*/true); } else if (is_fp) { uint32_t source_bits = 0, compare_bits = 0; std::memcpy(&source_bits, &store_data[lane * src_stride], 4); if (uses_two_sources) std::memcpy(&compare_bits, &store_data[lane * src_stride + 4], 4); - new_val = apply_fp_atomic(d, old_val, source_bits, compare_bits, d.atomic_lds_denorm_mode); + new_val = apply_fp_atomic(d, old_val, source_bits, compare_bits, d.atomic_lds_denorm_mode, + /*local=*/true); } else { uint32_t src_val = 0, cmp_val = 0; std::memcpy(&src_val, &store_data[lane * src_stride], 4); @@ -985,7 +955,8 @@ void execute_lds_atomic_rmw(VectorMemState &d, Lds *lds, std::memcpy(&source_bits, &store_data[lane * src_stride], 8); if (uses_two_sources) std::memcpy(&compare_bits, &store_data[lane * src_stride + 8], 8); - new_val = apply_fp_atomic(d, old_val, source_bits, compare_bits, d.atomic_lds_denorm_mode); + new_val = apply_fp_atomic(d, old_val, source_bits, compare_bits, d.atomic_lds_denorm_mode, + /*local=*/true); } else { uint64_t src_val = 0, cmp_val = 0; std::memcpy(&src_val, &store_data[lane * src_stride], 8); @@ -1075,7 +1046,9 @@ VmAccessOutcome GlobalMemPipeline::initiate_access(Instruction &inst, Wavefront const VmAccessOutcome outcome = l1_->store(d.per_lane_addr.data(), swizzled_lanes, d.elem_size, d.num_elems, d.store_data.data(), d.mtype, d.non_temporal, d.wf_size, wf.process_id(), - stride, base_offset, d.element_lane_masks.view(), d.scratch_swizzle_unit); + stride, base_offset, d.element_lane_masks.view(), d.scratch_swizzle_unit, + GpuVmAccessBatchGuard::active() && !wf.raw_cu().debug_active() && + wf.raw_cu().plugin_group().empty()); if (outcome != VmAccessOutcome::Complete) return outcome; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_wait_scoreboard.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_wait_scoreboard.cpp index 5c826bd03512..c54cab3c3b2e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_wait_scoreboard.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_wait_scoreboard.cpp @@ -358,6 +358,12 @@ void MemoryWaitScoreboard::before(const Instruction &inst, rj_code_arch_t arch) const auto events = WaitcheckTarget::classify_events(inst, arch); if (events.succeeded()) for (const auto &event : events.value()) { + // FLAT may not use either memory domain. Its resolved requests apply + // the corresponding capacity constraint after address calculation. + if (inst.mnemonic().starts_with("flat_") && + (event.counter == WaitCounterKind::Load || event.counter == WaitCounterKind::Store || + event.counter == WaitCounterKind::Ds)) + continue; if (event.counter == WaitCounterKind::VmVsrc || event.counter == WaitCounterKind::VaVdst || event.counter == WaitCounterKind::Depctr) continue; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mes_engine.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mes_engine.h index 115d305f1227..a049d9d65ec3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mes_engine.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mes_engine.h @@ -10,7 +10,7 @@ #include "rocjitsu/vm/amdgpu/gpu_queue_registry.h" #include "rocjitsu/vm/amdgpu/gpu_vm.h" #include "rocjitsu/vm/amdgpu/interrupt_sink.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h" +#include "rocjitsu/vm/amdgpu/pm4/pm4_packet_types.h" #include #include diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4.h similarity index 75% rename from emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4.h rename to emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4.h index dfe3ac87ca53..460d1c4fa377 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4.h @@ -4,7 +4,7 @@ #pragma once /// @file -/// @brief Compute PM4 packets, register offsets and submission lifetime state. +/// @brief PM4 packets, shader registers and submission lifetime state. #include #include @@ -32,11 +32,14 @@ inline constexpr uint32_t kPm4ComputePgmRsrc2 = 0x213; inline constexpr uint32_t kPm4ComputeTmpringSize = 0x218; inline constexpr uint32_t kPm4ComputeUserData0 = 0x240; -/// @brief Type-3 packet opcodes accepted by the compute command processor. +/// @brief Type-3 packet opcodes accepted by the command processor. enum class Pm4Opcode : uint32_t { Nop = 0x10, SetBase = 0x11, ClearState = 0x12, + SetPredication = 0x20, + CondExec = 0x22, + PredExec = 0x23, ContextControl = 0x28, PfpSyncMe = 0x42, SetContextReg = 0x69, @@ -44,17 +47,29 @@ enum class Pm4Opcode : uint32_t { SetContextRegPairsPacked = 0xb9, DispatchDirect = 0x15, DispatchIndirect = 0x16, + DispatchDirectInterleaved = 0xa7, + DispatchIndirectInterleaved = 0xa8, + AtomicMem = 0x1e, WriteData = 0x37, WaitRegMem = 0x3c, IndirectBuffer = 0x3f, CopyData = 0x40, EventWrite = 0x46, + StreamoutStatsQuery = 0xc3, ReleaseMem = 0x49, DmaData = 0x50, + ContextRegRmw = 0x51, AcquireMem = 0x58, + PrimeUtcl2 = 0x5d, + LoadUconfigReg = 0x5e, + LoadShReg = 0x5f, + LoadContextReg = 0x61, LoadShRegIndex = 0x63, + LoadContextRegIndex = 0x9f, SetShReg = 0x76, + SetShRegIndex = 0x9b, SetUconfigReg = 0x79, + SetUconfigRegIndex = 0x7a, SetShRegPairs = 0xba, SetUconfigRegPairs = 0xbe, }; @@ -98,6 +113,9 @@ struct Pm4IndirectBuffer { uint64_t address = 0; uint32_t dwords = 0; uint32_t depth = 0; + /// Nonzero only for a root ring frame; nested IBs always use linear addresses. + uint64_t ring_base = 0; + uint32_t ring_bytes = 0; }; /// @brief Shared launch/wave failure status; wake the CP to cancel the owning queue. @@ -111,11 +129,12 @@ struct Pm4FailureState { } }; -/// @brief One DRM command submission and its asynchronous fence callbacks. +/// @brief One PM4 stream and its return or frontend completion callback. /// @details The CP evaluates dependencies without blocking its engine thread and /// publishes completion only after all commands retire. struct Pm4Submission { bool graphics_engine = false; + bool allow_dispatch = true; ///< AQL vendor IBs disable this to reject shader launches. std::deque buffers; uint32_t indirect_expansions = 0; static constexpr uint32_t kMaxIndirectDepth = 64; @@ -125,10 +144,17 @@ struct Pm4Submission { std::function complete; }; -/// @brief Compute register file and ordered DRM submissions for one CP queue. -struct Pm4QueueState { +/// @brief Shader and graphics register files and ordered command streams for one CP queue. +struct ComputeCommandState { + using ContextRegisters = std::array; + uint64_t indirect_base = 0; + bool predicate_pass = true; std::array sh_registers{}; + ContextRegisters context_registers{}; + std::array uconfig_registers{}; + /// One saved context bank; nested CLEAR_STATE pushes are unsupported. + std::unique_ptr saved_context_registers; std::deque submissions; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_clear_state.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_clear_state.h new file mode 100644 index 000000000000..ed1235313ec0 --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_clear_state.h @@ -0,0 +1,57 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include + +namespace rocjitsu::amdgpu { + +/// @brief Restore the GFX11 context-shadow ranges to their CLEAR_STATE values. +/// @details Register data follows InitializeContextRegistersGfx11 in PAL c5e8000, +/// src/core/hw/gfxip/gfx9/g_gfx9ShadowedRegistersInit.h. Registers outside those +/// ranges retain their values; shader and user-config registers are unaffected. +inline void reset_gfx11_context_registers(std::array ®isters) { + static constexpr uint32_t kRanges[][2] = { + {0x000, 0x00000022}, {0x07a, 0x0000005e}, {0x0f4, 0x00000006}, {0x0fc, 0x00000003}, + {0x103, 0x00000084}, {0x191, 0x0000002c}, {0x1c2, 0x00000004}, {0x1d4, 0x00000014}, + {0x1f5, 0x00000004}, {0x1ff, 0x00000014}, {0x280, 0x00000004}, {0x286, 0x00000002}, + {0x292, 0x00000003}, {0x2a1, 0x00000001}, {0x2a3, 0x00000001}, {0x2a6, 0x00000001}, + {0x2ab, 0x00000007}, {0x2ca, 0x00000005}, {0x2d3, 0x00000004}, {0x2db, 0x0000000a}, + {0x2f5, 0x00000021}, {0x318, 0x00000001}, {0x31b, 0x00000004}, {0x325, 0x00000003}, + {0x32a, 0x00000004}, {0x334, 0x00000003}, {0x339, 0x00000004}, {0x343, 0x00000003}, + {0x348, 0x00000004}, {0x352, 0x00000003}, {0x357, 0x00000004}, {0x361, 0x00000003}, + {0x366, 0x00000004}, {0x370, 0x00000003}, {0x375, 0x00000004}, {0x37f, 0x00000003}, + {0x384, 0x00000004}, {0x38e, 0x0000000a}, {0x3a8, 0x00000018}, + }; + static constexpr uint32_t kNonzeroDefaults[][2] = { + {0x00d, 0x40004000}, {0x01f, 0x00150055}, {0x081, 0x80000000}, {0x082, 0x40004000}, + {0x083, 0x0000ffff}, {0x085, 0x40004000}, {0x087, 0x40004000}, {0x089, 0x40004000}, + {0x08b, 0x40004000}, {0x08c, 0xaa99aaaa}, {0x08e, 0xffffffff}, {0x08f, 0xffffffff}, + {0x090, 0x80000000}, {0x091, 0x40004000}, {0x094, 0x80000000}, {0x095, 0x40004000}, + {0x096, 0x80000000}, {0x097, 0x40004000}, {0x098, 0x80000000}, {0x099, 0x40004000}, + {0x09a, 0x80000000}, {0x09b, 0x40004000}, {0x09c, 0x80000000}, {0x09d, 0x40004000}, + {0x09e, 0x80000000}, {0x09f, 0x40004000}, {0x0a0, 0x80000000}, {0x0a1, 0x40004000}, + {0x0a2, 0x80000000}, {0x0a3, 0x40004000}, {0x0a4, 0x80000000}, {0x0a5, 0x40004000}, + {0x0a6, 0x80000000}, {0x0a7, 0x40004000}, {0x0a8, 0x80000000}, {0x0a9, 0x40004000}, + {0x0aa, 0x80000000}, {0x0ab, 0x40004000}, {0x0ac, 0x80000000}, {0x0ad, 0x40004000}, + {0x0ae, 0x80000000}, {0x0af, 0x40004000}, {0x0b0, 0x80000000}, {0x0b1, 0x40004000}, + {0x0b2, 0x80000000}, {0x0b3, 0x40004000}, {0x0b5, 0x3f800000}, {0x0b7, 0x3f800000}, + {0x0b9, 0x3f800000}, {0x0bb, 0x3f800000}, {0x0bd, 0x3f800000}, {0x0bf, 0x3f800000}, + {0x0c1, 0x3f800000}, {0x0c3, 0x3f800000}, {0x0c5, 0x3f800000}, {0x0c7, 0x3f800000}, + {0x0c9, 0x3f800000}, {0x0cb, 0x3f800000}, {0x0cd, 0x3f800000}, {0x0cf, 0x3f800000}, + {0x0d1, 0x3f800000}, {0x0d3, 0x3f800000}, {0x104, 0x00550055}, {0x10c, 0x01000000}, + {0x10d, 0x01000000}, {0x1b6, 0x00000002}, {0x204, 0x00090000}, {0x205, 0x00000004}, + {0x2f7, 0x00001000}, {0x2f9, 0x00000005}, {0x2fa, 0x3f800000}, {0x2fb, 0x3f800000}, + {0x2fc, 0x3f800000}, {0x2fd, 0x3f800000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff}, + {0x311, 0x00000003}, {0x313, 0x00100000}, + }; + for (const auto &[first, count] : kRanges) + std::fill_n(registers.begin() + first, count, 0u); + for (const auto &[reg, value] : kNonzeroDefaults) + registers[reg] = value; +} + +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.cpp index 085c066aa263..7a64b5a770d9 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.cpp @@ -3,81 +3,770 @@ #include "rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h" -namespace rocjitsu::amdgpu { -namespace { - -constexpr uint32_t kPacketType3 = 3; -constexpr uint32_t kPacket3SetUconfigRegister = 0x79; -constexpr uint32_t kPacket3OneDwordNop = 0xffff1000; -constexpr uint32_t kPacket3SetUconfigRegisterOffsetMask = 0xffff; -constexpr uint64_t kPacket3SetUconfigRegisterStart = 0xc000; - -} // namespace +#include "rocjitsu/vm/amdgpu/atomic_op.h" +#include "rocjitsu/vm/amdgpu/circular_ring_reader.h" +#include "rocjitsu/vm/amdgpu/dispatch_entry.h" +#include "rocjitsu/vm/amdgpu/gpu_vm.h" +#include "rocjitsu/vm/amdgpu/hsa_clock.h" +#include "rocjitsu/vm/amdgpu/pm4/pm4_clear_state.h" +#include "util/log.h" -Pm4PacketProcessResult Pm4PacketProcessor::process(Request request) const { - Pm4PacketProcessResult result; - if (request.available_dwords.empty()) { - result.packet.status = PacketProcessStatus::NeedInput; - result.packet.required_bytes = sizeof(uint32_t); - return result; - } - - result.packet_header = request.available_dwords.front(); - const uint32_t type = result.packet_header >> 30; - const uint32_t opcode = (result.packet_header >> 8) & 0xff; +#include +#include +#include +#include +#include +#include +#include +#include +#include - // The supported compute-queue stream uses the driver's special one-dword NOP - // encoding, whose count field is padding rather than a packet length. - if (result.packet_header == kPacket3OneDwordNop) { - result.packet = {.status = PacketProcessStatus::Complete, - .retirement = PacketRetirement::Retire, - .retirement_bytes = sizeof(uint32_t)}; - return result; - } +namespace rocjitsu::amdgpu { - const uint32_t packet_dwords = ((result.packet_header >> 16) & 0x3fff) + 2; - if (type != kPacketType3 || opcode != kPacket3SetUconfigRegister || packet_dwords != 3) { - result.packet.status = PacketProcessStatus::Unsupported; - result.diagnostic = Pm4PacketDiagnostic::UnsupportedPacket; - return result; - } +std::optional normalize_pm4_producer_cursor(uint64_t producer_cursor, + uint64_t reference_cursor, + uint64_t ring_dwords) noexcept { + if (ring_dwords == 0) + return std::nullopt; + if (producer_cursor >= reference_cursor && producer_cursor - reference_cursor <= ring_dwords) + return producer_cursor; + if (producer_cursor >= ring_dwords) + return std::nullopt; - result.packet.retirement_bytes = packet_dwords * sizeof(uint32_t); - if (request.available_dwords.size() < packet_dwords) { - result.packet.status = PacketProcessStatus::NeedInput; - result.packet.required_bytes = result.packet.retirement_bytes; - return result; + const uint64_t epoch = reference_cursor - reference_cursor % ring_dwords; + if (producer_cursor > std::numeric_limits::max() - epoch) + return std::nullopt; + uint64_t normalized = epoch + producer_cursor; + if (normalized < reference_cursor) { + if (normalized > std::numeric_limits::max() - ring_dwords) + return std::nullopt; + normalized += ring_dwords; } + if (normalized - reference_cursor > ring_dwords) + return std::nullopt; + return normalized; +} - const uint32_t encoded_register_offset = request.available_dwords[1]; - if ((encoded_register_offset & ~kPacket3SetUconfigRegisterOffsetMask) != 0) { - result.packet.status = PacketProcessStatus::Malformed; - result.diagnostic = Pm4PacketDiagnostic::InvalidRegisterOffset; - return result; +void process_pm4_packets(ComputeQueueRecord &queue, GpuVm *gpu_vm, + const Pm4ExecutionContext &context) { + if (queue.faulted || queue.command_retry_pending) + return; + auto &state = queue.commands; + if (state.submissions.empty()) + return; + if (state.submissions.front().failure->failed.load(std::memory_order_acquire)) { + context.fault_queue(); + return; } - result.register_dword = kPacket3SetUconfigRegisterStart + - (encoded_register_offset & kPacket3SetUconfigRegisterOffsetMask); - if (!callbacks_.write_uconfig_register) { - result.packet.status = PacketProcessStatus::Unsupported; - result.diagnostic = Pm4PacketDiagnostic::RegisterWriteRejected; - return result; + if (!queue.dispatches.entries.empty()) + return; + try { + if (!queue.command_access) + queue.command_access = gpu_vm->snapshot_pinned(queue.address_space); + const auto &access = queue.command_access; + // Bound one event's packet work, including IB chains. + const uint32_t packet_budget = queue.submission_queue ? 4096 : 256; + for (uint32_t budget = 0; budget < packet_budget && !state.submissions.empty(); ++budget) { + if (!access) + throw std::runtime_error("PM4 queue has no GPU address space"); + // A snapshot captured before GART publication can never become ready. + if (!access->info().ready) { + queue.command_access.reset(); + context.retry(); + return; + } + // A committed root packet must publish before another packet can execute. + const auto published = queue.read_pointer_journal.publish(); + if (published == VmAccessOutcome::Unavailable) { + context.retry(); + return; + } + if (published != VmAccessOutcome::Complete) { + queue.publication_faulted = true; + util::Logger::warn("PM4 read pointer publication failed"); + context.fault_queue(); + return; + } + auto &submission = state.submissions.front(); + if (submission.ready && !submission.ready()) { + context.retry(); + return; + } + if (submission.buffers.empty()) { + context.flush_caches(); + if (submission.complete) + submission.complete(true); + // Completion can release an earlier queue in this same service pass. + context.wake(); + state.submissions.pop_front(); + queue.command_access.reset(); + if (!state.submissions.empty()) + queue.command_access = gpu_vm->snapshot_pinned(queue.address_space); + continue; + } + auto &ib = submission.buffers.front(); + if (!ib.dwords) { + submission.buffers.pop_front(); + continue; + } + const auto read_commands = [&](uint64_t offset, std::span bytes) { + return ib.ring_bytes + ? CircularRingReader(ib.ring_base, ib.ring_bytes).read(*access, offset, bytes) + : access->read(offset, bytes); + }; + uint32_t header = 0; + auto outcome = + read_commands(ib.address, {reinterpret_cast(&header), sizeof(header)}); + if (outcome == VmAccessOutcome::Unavailable) { + context.retry(); + return; + } + if (outcome != VmAccessOutcome::Complete) + throw std::runtime_error("unmapped PM4 command buffer"); + const uint32_t type = header >> 30; + const uint32_t count = type == 2 || header == 0xffff1000 ? 1 : ((header >> 16) & 0x3fff) + 2; + if (count > ib.dwords || (type != 2 && type != 3)) + throw std::runtime_error("invalid PM4 packet size or type"); + std::vector words(count - 1); + if (!words.empty()) { + outcome = read_commands(ib.address + 4, + {reinterpret_cast(words.data()), words.size() * 4}); + if (outcome == VmAccessOutcome::Unavailable) { + context.retry(); + return; + } + if (outcome != VmAccessOutcome::Complete) + throw std::runtime_error("unmapped PM4 packet payload"); + } + ib.address += count * 4; + ib.dwords -= count; + // Save root retirement before an IB command replaces the front frame. + const uint32_t root_ring_bytes = ib.ring_bytes; + uint64_t root_cursor = ib.address / 4; + const auto skip_commands = [&](uint32_t dwords) { + ib.address += uint64_t{dwords} * 4; + ib.dwords -= dwords; + root_cursor = ib.address / 4; + }; + const auto require = [&](size_t size) { + if (words.size() != size) + throw std::runtime_error(std::format("PM4 opcode {:#x} expects {} payload words, got {}", + (header >> 8) & 0xff, size, words.size())); + }; + const auto address = [&](size_t index) { + return uint64_t{words[index]} | (uint64_t{words[index + 1]} << 32); + }; + const uint32_t opcode = + type == 2 || header == 0xffff1000 ? uint32_t(Pm4Opcode::Nop) : (header >> 8) & 0xff; + util::Logger::vm([&](auto &os) { + os << "PM4 packet " << std::hex << opcode << " payload:"; + for (uint32_t word : words) + os << ' ' << word; + }); + if ((header & 1) && !state.predicate_pass) + continue; + switch (static_cast(opcode)) { + case Pm4Opcode::Nop: + break; + case Pm4Opcode::PredExec: { + require(1); + if ((context.arch != ROCJITSU_CODE_ARCH_CDNA3 && + context.arch != ROCJITSU_CODE_ARCH_CDNA4) || + (words[0] & 0x00ffc000) || context.xcc_id >= 8) + throw std::runtime_error("unsupported PRED_EXEC control or target"); + const uint32_t skip = words[0] & 0x3fff; + if (skip > ib.dwords) + throw std::runtime_error("PRED_EXEC exceeds its indirect buffer"); + if (!(words[0] & (1u << (24 + context.xcc_id)))) + skip_commands(skip); + break; + } + case Pm4Opcode::ContextControl: + case Pm4Opcode::PfpSyncMe: + if (!submission.graphics_engine) + throw std::runtime_error("graphics state packet on compute engine"); + // The single CP retires preceding work before these packets. + if (opcode == uint32_t(Pm4Opcode::ContextControl)) + require(2); + else + require(1); + break; + case Pm4Opcode::ClearState: { + require(1); + const uint32_t command = words[0]; + const bool reset = command == 0 || command == 3; + const bool push = command == 1 || command == 3; + // GFX12 has push/pop only. Reset values are qualified for GFX11. + if (!submission.graphics_engine || command > 3 || + (reset && (context.arch == ROCJITSU_CODE_ARCH_INVALID || + (context.arch != ROCJITSU_CODE_ARCH_RDNA3 && + context.arch != ROCJITSU_CODE_ARCH_RDNA3_5)))) + throw std::runtime_error("unsupported CLEAR_STATE mode or engine"); + if (push) { + if (state.saved_context_registers) + throw std::runtime_error("nested CLEAR_STATE push is unsupported"); + state.saved_context_registers = + std::make_unique(state.context_registers); + } + if (command == 2) { + if (!state.saved_context_registers) + throw std::runtime_error("CLEAR_STATE pop without a saved context"); + state.context_registers = *state.saved_context_registers; + state.saved_context_registers.reset(); + } else if (reset) { + reset_gfx11_context_registers(state.context_registers); + } + break; + } + case Pm4Opcode::CondExec: { + require(4); + if ((words[0] & 3) || words[2] || (words[3] & ~0x3fffu) || words[3] > ib.dwords) + throw std::runtime_error("invalid COND_EXEC address, control, or extent"); + context.flush_caches(); + uint32_t value = 0; + if (access->read(address(0), {reinterpret_cast(&value), sizeof(value)}) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 COND_EXEC read failed"); + if (!value) + skip_commands(words[3]); + break; + } + case Pm4Opcode::SetPredication: { + require(3); + const uint32_t operation = (words[0] >> 16) & 7; + // Boolean predicates use the common GFX9+ packet layout. + // Query accumulation and its CONTINUE/HINT controls are not modeled. + if (!submission.graphics_engine || (words[0] & ~0x70100u) || + (operation != 0 && operation != 3 && operation != 4)) + throw std::runtime_error("unsupported SET_PREDICATION control"); + state.predicate_pass = true; + if (operation) { + const size_t bytes = operation == 4 ? 4 : 8; + if (address(1) % bytes) + throw std::runtime_error("unaligned SET_PREDICATION address"); + context.flush_caches(); + uint64_t value = 0; + if (access->read(address(1), {reinterpret_cast(&value), bytes}) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 SET_PREDICATION read failed"); + state.predicate_pass = (value != 0) == bool(words[0] & (1u << 8)); + } + break; + } + case Pm4Opcode::SetBase: + require(3); + if (!submission.graphics_engine || words[0] != 1) + throw std::runtime_error("unsupported SET_BASE index"); + state.indirect_base = address(1); + break; + case Pm4Opcode::WriteData: { + if (words.size() < 4) + throw std::runtime_error("invalid WRITE_DATA payload"); + const uint32_t destination = (words[0] >> 8) & 15; + if ((destination != 1 && destination != 2 && destination != 5) || (words[0] & (1u << 16))) + throw std::runtime_error("unsupported WRITE_DATA destination"); + context.flush_caches(); + if (access->write(address(1), {reinterpret_cast(words.data() + 3), + (words.size() - 3) * 4}) != VmAccessOutcome::Complete) + throw std::runtime_error("PM4 WRITE_DATA failed"); + break; + } + case Pm4Opcode::AtomicMem: { + require(8); + // GFX9+ ME/MEC share the integer TC operations. Cache policy does not + // change their values. Loop-until-compare commands require CP retry + // state and are rejected rather than executed as a single pass. + if ((words[0] & ~0x0600007fu) || (words[7] & ~0x1fffu)) + throw std::runtime_error("unsupported ATOMIC_MEM control"); + const uint32_t operation = words[0] & 0x1f; + const uint32_t width = (words[0] & 0x20) ? 8 : 4; + if ((operation != 7 && operation != 8 && (operation < 15 || operation > 25)) || + address(1) % width) + throw std::runtime_error("unsupported ATOMIC_MEM operation or alignment"); + context.flush_caches(); + // TC opcodes are shared by the ME and MEC. Reuse shader-memory atomic + // arithmetic after decoding the packet's operation and operand order. + static constexpr AtomicOp tc_operations[] = {AtomicOp::ADD, AtomicOp::SUB, AtomicOp::SMIN, + AtomicOp::UMIN, AtomicOp::SMAX, AtomicOp::UMAX, + AtomicOp::AND, AtomicOp::OR, AtomicOp::XOR, + AtomicOp::INC, AtomicOp::DEC}; + const AtomicOp atomic = operation == 7 ? AtomicOp::SWAP + : operation == 8 ? AtomicOp::CMPSWAP + : tc_operations[operation - 15]; + const auto mutate = [&]() { + return access->atomic_modify(address(1), sizeof(T), [&](std::span bytes) { + T value; + std::memcpy(&value, bytes.data(), sizeof(value)); + value = apply_int_atomic(atomic, value, static_cast(address(3)), + static_cast(address(5))); + std::memcpy(bytes.data(), &value, sizeof(value)); + }); + }; + const auto outcome = width == 8 ? mutate.template operator()() + : mutate.template operator()(); + if (outcome != VmAccessOutcome::Complete) + throw std::runtime_error("PM4 ATOMIC_MEM access failed"); + break; + } + case Pm4Opcode::ReleaseMem: { + require(7); + context.flush_caches(); + const uint32_t selection = words[1] >> 29; + uint64_t value = address(4); + if (selection == 3) + value = hsa_system_timestamp(); + else if (selection != 0 && selection != 1 && selection != 2) + throw std::runtime_error("unsupported RELEASE_MEM data source"); + const size_t bytes = selection == 0 ? 0 : selection == 1 ? 4 : 8; + if (bytes && access->write(address(2), {reinterpret_cast(&value), + bytes}) != VmAccessOutcome::Complete) + throw std::runtime_error("PM4 RELEASE_MEM failed"); + break; + } + case Pm4Opcode::CopyData: { + require(5); + const uint32_t source = words[0] & 15, destination = (words[0] >> 8) & 15; + const size_t bytes = (words[0] & (1u << 16)) ? 8 : 4; + if (destination != 1 && destination != 2 && destination != 5) + throw std::runtime_error("unsupported COPY_DATA destination"); + context.flush_caches(); + uint64_t value = 0; + if (source == 0) { + if (!submission.graphics_engine || words[2] || words[1] > 0x3ffff) + throw std::runtime_error("unsupported COPY_DATA register source"); + for (uint32_t i = 0; i < bytes / 4; ++i) { + const uint32_t reg = words[1] + i; + uint32_t data; + if (reg >= 0x2c00 && reg < 0x3000) + data = state.sh_registers[reg - 0x2c00]; + else if (reg >= 0xa000 && reg < 0xc000) + data = state.context_registers[reg - 0xa000]; + else if (reg >= 0xc000 && reg < 0x10000) + data = state.uconfig_registers[reg - 0xc000]; + else + throw std::runtime_error("COPY_DATA register outside modeled apertures"); + value |= uint64_t{data} << (32 * i); + } + } else if (source == 5) + value = address(1); + else if (source == 9) + value = hsa_system_timestamp(); + else if (source == 1 || source == 2) { + if (access->read(address(1), {reinterpret_cast(&value), bytes}) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 COPY_DATA read failed"); + } else { + throw std::runtime_error("unsupported COPY_DATA source"); + } + if (access->write(address(3), {reinterpret_cast(&value), bytes}) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 COPY_DATA write failed"); + break; + } + case Pm4Opcode::WaitRegMem: { + require(6); + context.flush_caches(); + uint32_t value = 0; + const uint32_t space = (words[0] >> 4) & 3; + const uint32_t operation = (words[0] >> 6) & 3; + if (space == 0 && operation == 1 && + (context.arch == ROCJITSU_CODE_ARCH_RDNA3 || + context.arch == ROCJITSU_CODE_ARCH_RDNA3_5 || + context.arch == ROCJITSU_CODE_ARCH_CDNA3 || context.arch == ROCJITSU_CODE_ARCH_CDNA4 || + context.arch == ROCJITSU_CODE_ARCH_RDNA4) && + words[1] == 0xe26 && words[2] == 0xe27) { + // NBIO 4.3/7.9/7.11 and NBIF 6.3 share this HDP_FLUSH_REQ/DONE pair. + // WR_WAIT_WR_REG requests host-data + // visibility and waits for the same engine bits in the done register. + // Host mappings are coherent; the synchronous cache flush above + // completes the modeled request before its acknowledgement. + value = words[3]; + } else if (space == 1 && (operation == 0 || operation == 3)) { + const auto loaded = + access->read(address(1), {reinterpret_cast(&value), sizeof(value)}); + if (loaded == VmAccessOutcome::Unavailable) { + ib.address -= count * 4; + ib.dwords += count; + context.retry(); + return; + } + if (loaded != VmAccessOutcome::Complete) + throw std::runtime_error("PM4 WAIT_REG_MEM read failed"); + } else { + throw std::runtime_error("unsupported WAIT_REG_MEM register space or operation"); + } + value &= words[4]; + uint32_t reference = words[3] & words[4]; + bool ready; + switch (words[0] & 7) { + case 0: + ready = true; + break; + case 1: + ready = value < reference; + break; + case 2: + ready = value <= reference; + break; + case 3: + ready = value == reference; + break; + case 4: + ready = value != reference; + break; + case 5: + ready = value >= reference; + break; + case 6: + ready = value > reference; + break; + default: + throw std::runtime_error("invalid WAIT_REG_MEM comparison"); + } + if (!ready) { + ib.address -= count * 4; + ib.dwords += count; + context.retry(); + return; + } + break; + } + case Pm4Opcode::LoadUconfigReg: + case Pm4Opcode::LoadShReg: + case Pm4Opcode::LoadContextReg: { + if (words.size() < 4 || words.size() % 2 || (words[0] & 3) || (words[1] & 0xffff0000u)) + throw std::runtime_error("invalid shadow register load payload"); + if (opcode != uint32_t(Pm4Opcode::LoadShReg) && !submission.graphics_engine) + throw std::runtime_error("graphics register load on compute engine"); + const std::span registers = opcode == uint32_t(Pm4Opcode::LoadUconfigReg) + ? std::span(state.uconfig_registers) + : opcode == uint32_t(Pm4Opcode::LoadShReg) + ? std::span(state.sh_registers) + : std::span(state.context_registers); + for (size_t i = 2; i < words.size(); i += 2) { + const uint32_t first = words[i], count = words[i + 1]; + if (!count || (count & ~0x3fffu) || first >= registers.size() || + count > registers.size() - first) + throw std::runtime_error("invalid shadow register load range"); + } + context.flush_caches(); + for (size_t i = 2; i < words.size(); i += 2) { + const uint32_t first = words[i], count = words[i + 1]; + // Shadow storage is indexed by the register offset, not packed by + // the order of the ranges in this packet. + if (access->read(address(0) + uint64_t{first} * 4, + {reinterpret_cast(registers.data() + first), count * 4}) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 shadow register load read failed"); + } + break; + } + case Pm4Opcode::LoadShRegIndex: + case Pm4Opcode::LoadContextRegIndex: { + require(4); + if (opcode == uint32_t(Pm4Opcode::LoadContextRegIndex) && !submission.graphics_engine) + throw std::runtime_error("graphics register load on compute engine"); + const uint32_t first = words[2], count = words[3]; + const std::span registers = opcode == uint32_t(Pm4Opcode::LoadShRegIndex) + ? std::span(state.sh_registers) + : std::span(state.context_registers); + // Direct-address mode, contiguous values (no register/value pairs). + if ((words[0] & 3) || !count || first >= registers.size() || + count > registers.size() - first) + throw std::runtime_error("unsupported register load range or mode"); + context.flush_caches(); + if (access->read(address(0), {reinterpret_cast(registers.data() + first), + count * 4}) != VmAccessOutcome::Complete) + throw std::runtime_error("PM4 register load read failed"); + break; + } + case Pm4Opcode::SetShReg: + case Pm4Opcode::SetShRegIndex: { + if (words.size() < 2) + throw std::runtime_error("invalid shader register payload"); + const uint32_t index = words[0] >> 28; + const uint32_t first = words[0] & 0xffff; + const bool interleave = submission.graphics_engine && index == 2 && first == 0x22f && + words.size() == 2 && context.arch != ROCJITSU_CODE_ARCH_INVALID && + context.arch == ROCJITSU_CODE_ARCH_RDNA4; + // Index 3 applies the KMD CU mask to RSRC3/4. CU affinity does not + // change functional shader results; retain the programmed resources. + // GFX12 index 2 updates the dispatch-interleave shadow. Scheduling + // interleave likewise does not change register values or shader results. + if ((words[0] & 0x0fff0000) || + (index && + (opcode != uint32_t(Pm4Opcode::SetShRegIndex) || (index != 3 && !interleave))) || + first >= state.sh_registers.size() || + words.size() - 1 > state.sh_registers.size() - first) + throw std::runtime_error("invalid shader register range or index"); + std::copy(words.begin() + 1, words.end(), state.sh_registers.begin() + first); + break; + } + case Pm4Opcode::ContextRegRmw: + require(3); + if (!submission.graphics_engine || words[0] >= state.context_registers.size()) + throw std::runtime_error("invalid CONTEXT_REG_RMW register or engine"); + state.context_registers[words[0]] = + (state.context_registers[words[0]] & ~words[1]) | (words[2] & words[1]); + break; + case Pm4Opcode::PrimeUtcl2: + require(4); + // PAL also sets the PFP engine selector on compute queues, where the + // hardware ignores it. Retain the same accepted control bits there. + if ((words[0] & ~0x4000000fu) || (words[1] & 0xfffu) || (words[3] & ~0x3fffu) || + address(1) > UINT64_MAX - (uint64_t{words[3]} << 12)) + throw std::runtime_error("invalid PRIME_UTCL2 control or range"); + // Translation prefetch has no data result. Address translation already + // completes synchronously when an instruction accesses memory. + break; + case Pm4Opcode::SetShRegPairs: // SET_SH_REG_PAIRS + if (words.size() % 2) + throw std::runtime_error("invalid SET_SH_REG_PAIRS payload"); + for (size_t i = 0; i < words.size(); i += 2) { + if (words[i] >= state.sh_registers.size()) + throw std::runtime_error("invalid SET_SH_REG_PAIRS register"); + state.sh_registers[words[i]] = words[i + 1]; + } + break; + case Pm4Opcode::SetContextReg: + case Pm4Opcode::SetContextRegPairs: + case Pm4Opcode::SetContextRegPairsPacked: + case Pm4Opcode::SetUconfigReg: + case Pm4Opcode::SetUconfigRegIndex: + case Pm4Opcode::SetUconfigRegPairs: { + if (opcode == uint32_t(Pm4Opcode::SetUconfigReg)) { + if (!queue.packet_callbacks.write_uconfig_register && !queue.submission_queue) + throw std::runtime_error("SET_UCONFIG_REG requires a register write callback"); + if (queue.packet_callbacks.write_uconfig_register) { + if (words.size() != 2 || (words[0] & ~0xffffu)) { + util::Logger::warn("invalid SET_UCONFIG_REG payload"); + context.fault_queue(); + return; + } + const auto result = + queue.packet_callbacks.write_uconfig_register(0xc000 + words[0], words[1]); + if (result == Pm4RegisterWriteStatus::Blocked) { + ib.address -= count * 4; + ib.dwords += count; + context.retry(); + return; + } + if (result != Pm4RegisterWriteStatus::Complete) { + util::Logger::warn("SET_UCONFIG_REG write rejected"); + context.fault_queue(); + return; + } + } + } + if (opcode == uint32_t(Pm4Opcode::SetUconfigRegPairs) && !queue.submission_queue) + throw std::runtime_error("SET_UCONFIG_REG_PAIRS is unsupported on a native ring"); + const bool context = opcode == uint32_t(Pm4Opcode::SetContextReg) || + opcode == uint32_t(Pm4Opcode::SetContextRegPairs) || + opcode == uint32_t(Pm4Opcode::SetContextRegPairsPacked); + if (context && !submission.graphics_engine) + throw std::runtime_error("graphics state packet on compute engine"); + std::span registers = context ? std::span(state.context_registers) + : std::span(state.uconfig_registers); + const auto write = [&](uint32_t reg, uint32_t value) { + if (reg >= registers.size()) + throw std::runtime_error("graphics register outside aperture"); + registers[reg] = value; + }; + if (opcode == uint32_t(Pm4Opcode::SetContextRegPairsPacked)) { + if (words.empty() || words[0] == 0 || (words[0] & 1) || + uint64_t{words[0]} / 2 * 3 + 1 != words.size()) + throw std::runtime_error("invalid packed graphics register payload"); + for (size_t i = 1; i < words.size(); i += 3) { + write(words[i] & 0xffff, words[i + 1]); + write(words[i] >> 16, words[i + 2]); + } + } else if (opcode == uint32_t(Pm4Opcode::SetContextRegPairs) || + opcode == uint32_t(Pm4Opcode::SetUconfigRegPairs)) { + if (words.empty() || words.size() % 2) + throw std::runtime_error("invalid graphics register pairs"); + for (size_t i = 0; i < words.size(); i += 2) + write(words[i], words[i + 1]); + } else { + if (words.size() < 2) + throw std::runtime_error("invalid graphics register payload"); + const uint32_t index = words[0] >> 28; + const uint32_t first = words[0] & 0xffff; + if ((words[0] & 0x0fff0000) || (index && (context || index > 4))) + throw std::runtime_error( + std::format("unsupported graphics register index {:#x}", words[0])); + for (size_t i = 1; i < words.size(); ++i) + write(first + i - 1, words[i]); + } + break; + } + case Pm4Opcode::AcquireMem: // ACQUIRE_MEM: earlier dispatches and DMA are already retired. + // GFX9 carries CP_COHER_CNTL in ordinal 2 and ends with POLL_INTERVAL. + // Later generations append a GCR control word. + require( + (context.arch == ROCJITSU_CODE_ARCH_CDNA1 || context.arch == ROCJITSU_CODE_ARCH_CDNA2 || + context.arch == ROCJITSU_CODE_ARCH_CDNA3 || context.arch == ROCJITSU_CODE_ARCH_CDNA4) + ? 6 + : 7); + context.flush_caches(); + break; + case Pm4Opcode::EventWrite: { + const uint32_t event = words[0] & 0x3f; + const uint32_t event_index = (words[0] >> 8) & 15; + if (event == 15 && event_index >= 8 && event_index <= 11) { + require(3); + const bool supports_streamout_query = context.arch != ROCJITSU_CODE_ARCH_INVALID && + (context.arch == ROCJITSU_CODE_ARCH_RDNA3 || + context.arch == ROCJITSU_CODE_ARCH_RDNA3_5); + if (!submission.graphics_engine || !supports_streamout_query || + words[0] != (15 | (event_index << 8)) || (words[1] & 7)) + throw std::runtime_error("unsupported PM4 streamout query event"); + context.flush_caches(); + // Compute execution has no streamout counter producer. Return valid + // zero samples until graphics supplies the counters. + constexpr uint64_t kSampleValid = uint64_t{1} << 63; + constexpr std::array values{kSampleValid, kSampleValid}; + if (access->write(address(1), std::as_bytes(std::span{values})) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 streamout query write failed"); + break; + } + if (submission.graphics_engine && event == 56) { + // PIXEL_PIPE_STAT_CONTROL configures graphics counters, not a memory write. + require(3); + break; + } + require(1); + // Counter START/STOP events also configure compute counters. SQ_NON_EVENT + // drains graphics pipeline messages without writing a sampled result. + if (event != 7 && event != 23 && event != 24 && event != 25 && event != 26 && + !(submission.graphics_engine && + (event == 15 || event == 16 || event == 36 || event == 38 || event == 44 || + event == 46 || event == 49))) + throw std::runtime_error(std::format("unsupported PM4 EVENT_WRITE event {}", event)); + context.flush_caches(); + break; + } + case Pm4Opcode::StreamoutStatsQuery: { + require(5); + if (!submission.graphics_engine || context.arch == ROCJITSU_CODE_ARCH_INVALID || + context.arch != ROCJITSU_CODE_ARCH_RDNA4 || (words[0] & 7) || (words[3] & 7) || + words[2] > 3 || address(0) > UINT64_MAX - 79) + throw std::runtime_error("unsupported PM4 streamout statistics query"); + // GFX12 shaders maintain four needed/written counter pairs in ordinary memory, + // following four dwords of streamout buffer offsets. + context.flush_caches(); + std::array values{}; + if (access->read(address(0) + 16 + 16 * words[2], + std::as_writable_bytes(std::span{values})) != VmAccessOutcome::Complete) + throw std::runtime_error("PM4 streamout query read failed"); + for (auto &value : values) + value |= uint64_t{1} << 63; + if (access->write(address(3), std::as_bytes(std::span{values})) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 streamout query write failed"); + break; + } + case Pm4Opcode::DmaData: { + require(6); + const uint32_t src_select = (words[0] >> 29) & 3; + const uint32_t dst_select = (words[0] >> 20) & 3; + const uint32_t bytes = words[5] & 0x3ffffff; + if (!bytes || dst_select == 2) // DMA drain or prefetch only. + break; + if (dst_select != 0 && dst_select != 3) + throw std::runtime_error("unsupported PM4 DMA destination"); + context.flush_caches(); + if (src_select == 2) { + std::vector data(bytes); + for (uint32_t i = 0; i < bytes; ++i) + data[i] = words[1] >> ((i % 4) * 8); + if (access->write(address(3), std::as_bytes(std::span(data))) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 DMA fill failed"); + } else if (src_select == 0 || src_select == 3) { + std::vector data(bytes); + if (access->read(address(1), data) != VmAccessOutcome::Complete || + access->write(address(3), data) != VmAccessOutcome::Complete) + throw std::runtime_error("PM4 DMA copy failed"); + } else { + throw std::runtime_error("unsupported PM4 DMA source"); + } + break; + } + case Pm4Opcode::IndirectBuffer: { + require(3); + const bool chained = words[2] & (1u << 20); + const uint32_t depth = ib.depth + (chained ? 0 : 1); + if (++submission.indirect_expansions > Pm4Submission::kMaxIndirectExpansions || + depth >= Pm4Submission::kMaxIndirectDepth) { + context.fault_queue(); + return; + } + // CHAIN jumps at the current IB level; a non-chained IB returns here. + // Other root IBs supplied by the same CS remain queued behind this one. + if (words[2] & (1u << 20)) + submission.buffers.pop_front(); + submission.buffers.push_front({address(0), words[2] & 0xfffff, depth}); + break; + } + case Pm4Opcode::DispatchDirectInterleaved: + if (!submission.graphics_engine || context.arch == ROCJITSU_CODE_ARCH_INVALID || + context.arch != ROCJITSU_CODE_ARCH_RDNA4) + throw std::runtime_error("interleaved dispatch requires a GFX12 graphics queue"); + [[fallthrough]]; + case Pm4Opcode::DispatchDirect: + require(4); + context.dispatch({words[0], words[1], words[2], words[3]}); + break; + case Pm4Opcode::DispatchIndirectInterleaved: + if (!submission.graphics_engine || context.arch == ROCJITSU_CODE_ARCH_INVALID || + context.arch != ROCJITSU_CODE_ARCH_RDNA4) + throw std::runtime_error("interleaved dispatch requires a GFX12 graphics queue"); + [[fallthrough]]; + case Pm4Opcode::DispatchIndirect: { + require(submission.graphics_engine ? 2 : 3); + context.flush_caches(); + const uint64_t arguments = + submission.graphics_engine ? state.indirect_base + words[0] : address(0); + std::array dimensions{0, 0, 0, words.back()}; + if (access->read(arguments, {reinterpret_cast(dimensions.data()), 12}) != + VmAccessOutcome::Complete) + throw std::runtime_error("PM4 DISPATCH_INDIRECT read failed"); + context.dispatch(dimensions); + break; + } + default: + throw std::runtime_error(std::format("unsupported PM4 opcode {:#x}", opcode)); + } + if (root_ring_bytes) { + // An IB enters a child frame; only the root packet advances the root ring. + // The journal retains this stream's snapshot until writeback is durable. + uint64_t retired = root_cursor; + queue.read_pointer_journal.retire(retired, retired % (root_ring_bytes / 4), *access); + const auto published = queue.read_pointer_journal.publish(); + if (published == VmAccessOutcome::Unavailable) { + context.retry(); + return; + } + if (published != VmAccessOutcome::Complete) { + queue.publication_faulted = true; + util::Logger::warn("PM4 read pointer publication failed"); + context.fault_queue(); + return; + } + } + if (!queue.dispatches.entries.empty()) + return; + } + if (!state.submissions.empty()) { + // Yielding the packet budget is runnable work, not an external wait. + context.wake(); + context.retry(); + } + } catch (const std::exception &error) { + util::Logger::warn("PM4 queue failed: ", error.what()); + context.fault_queue(); } - - const Pm4RegisterWriteStatus write_status = - callbacks_.write_uconfig_register(result.register_dword, request.available_dwords[2]); - if (write_status != Pm4RegisterWriteStatus::Complete) { - result.packet.status = - write_status == Pm4RegisterWriteStatus::Blocked ? PacketProcessStatus::Blocked - : write_status == Pm4RegisterWriteStatus::Faulted ? PacketProcessStatus::Faulted - : PacketProcessStatus::Unsupported; - if (write_status == Pm4RegisterWriteStatus::Rejected) - result.diagnostic = Pm4PacketDiagnostic::RegisterWriteRejected; - return result; - } - - result.packet.status = PacketProcessStatus::Complete; - result.packet.retirement = PacketRetirement::Retire; - return result; } } // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h index c8d0cc31bca9..dade0d47e1c0 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h @@ -1,70 +1,83 @@ // Copyright (c) 2026 Advanced Micro Devices, Inc. // SPDX-License-Identifier: MIT -/// @file pm4/pm4_packet_processor.h -/// @brief Processing for the supported PM4 compute-queue packet subset. - #pragma once -#include "rocjitsu/vm/amdgpu/packet_processor.h" +#include "rocjitsu/code/rj_code.h" +#include #include #include -#include -#include +#include namespace rocjitsu::amdgpu { -enum class Pm4PacketDiagnostic : uint8_t { - None, - UnsupportedPacket, - InvalidRegisterOffset, - RegisterWriteRejected, -}; - -enum class Pm4RegisterWriteStatus : uint8_t { - Complete, - Blocked, - Faulted, - Rejected, -}; - -class Pm4PacketCallbacks { -public: - /// Apply one SET_UCONFIG_REG write through the owning register model. - /// Non-complete outcomes must mean that no register effect occurred. - std::function - write_uconfig_register; -}; - -struct Pm4PacketProcessRequest { - std::span available_dwords; -}; - -struct Pm4PacketProcessResult { - [[nodiscard]] const PacketProcessResult &packet_result() const noexcept { return packet; } - - PacketProcessResult packet; - Pm4PacketDiagnostic diagnostic = Pm4PacketDiagnostic::None; - uint32_t packet_header = 0; - uint64_t register_dword = 0; -}; - -/// @brief Processes one PM4 packet without owning its ring or queue lifetime. -class Pm4PacketProcessor { -public: - using Request = Pm4PacketProcessRequest; - using Result = Pm4PacketProcessResult; - - explicit Pm4PacketProcessor(Pm4PacketCallbacks callbacks = {}) - : callbacks_(std::move(callbacks)) {} - - [[nodiscard]] Result process(Request request) const; - -private: - Pm4PacketCallbacks callbacks_; +struct ComputeQueueRecord; +class GpuVm; + +/// @brief CP services used by packet execution. +/// @details Queue state remains owned by CP. All callbacks must be populated and +/// run synchronously during process_pm4_packets(); the processor does not retain +/// the context or callbacks. +struct Pm4ExecutionContext { + /// @brief Target architecture for architecture-specific packet behavior. + /// @details INVALID represents a CP without a compute unit. + rj_code_arch_t arch = ROCJITSU_CODE_ARCH_INVALID; + /// @brief XCD index used by PRED_EXEC in CDNA vendor command buffers. + uint32_t xcc_id = 0; + /// @brief Flush GPU caches before packet memory effects and stream completion. + std::function flush_caches; + /// @brief Admit a dispatch through CP using X, Y, Z, and the initiator word. + /// @details The initiator determines whether dimensions count threads or groups. + /// CP appends admitted work to queue.dispatches.entries; processing pauses + /// until CP retires those entries. + std::function &)> dispatch; + /// @brief Preserve the owner's retry scheduling at the original execution points. + /// @details Set queue.command_retry_pending and arrange a later CP scheduling + /// turn. CP clears the flag before resuming processing. + std::function retry; + /// @brief Wake CP promptly after stream completion or yielding runnable work. + /// @details Reset retry backoff and arrange a scheduling turn without marking + /// this queue blocked; packet processing may continue in the current turn. + std::function wake; + /// @brief Cancel through CP at the original execution points, including exception scope. + /// @details Set queue.faulted, cancel pending work, and notify failed submissions. + /// The processor sets queue.publication_faulted before calling this callback + /// on terminal cursor-publication failures; CP preserves that failure state. + std::function fault_queue; }; -static_assert(PacketProcessor); +/// @brief Process a bounded turn of a native ring or DRM indirect-buffer stream. +/// @details Fetch, opcode effects, IB traversal, and cursor publication live here. +/// The caller owns serialization, scheduling, dispatch admission, and cancellation. +/// For native rings, CP captures queue.command_access, initializes the consumer +/// cursor, creates root submissions from doorbells, and retries pending cursor +/// publication even while execution is suspended. This processor traverses those +/// submissions and nested IBs, captures access for submitted streams as needed, +/// and commits/publishes packet retirement through queue.read_pointer_journal. +/// queue.commands and queue.command_access retain progress across turns; the +/// retry and dispatch callbacks must update the shared fields documented above. +/// An unready VM snapshot is released before retry so a later GART publication +/// can be observed. Ready snapshots stay pinned through execution and publication. +/// A VM must be attached before commands can execute; idle queues do not access it. +/// Returns when idle, waiting for dispatch retirement, blocked, faulted, or at the +/// packet budget. Retry and fault handling are reported through context callbacks. +/// Standard exceptions during execution are translated to fault_queue(); exceptions +/// from cancellation before execution or from the catch handler propagate. +/// @param queue CP-owned queue whose command state and cursor are updated in place. +/// @param gpu_vm Borrowed VM used to acquire pinned access to the queue address space. +/// @param context Borrowed architecture and synchronous CP services for this turn. +void process_pm4_packets(ComputeQueueRecord &queue, GpuVm *gpu_vm, + const Pm4ExecutionContext &context); + +/// @brief Convert a native ring-relative producer cursor into its monotonic epoch. +/// @param producer_cursor Ring-relative cursor or an already monotonic cursor. +/// @param reference_cursor Monotonic consumer cursor defining the current ring epoch. +/// @param ring_dwords Ring capacity in dwords. +/// @return A monotonic cursor at or after the reference, at most one ring ahead; +/// nullopt for zero capacity, arithmetic overflow, or a cursor outside that window. +[[nodiscard]] std::optional normalize_pm4_producer_cursor(uint64_t producer_cursor, + uint64_t reference_cursor, + uint64_t ring_dwords) noexcept; } // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_types.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_types.h new file mode 100644 index 000000000000..d65ab6b0bee1 --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_types.h @@ -0,0 +1,33 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include + +namespace rocjitsu::amdgpu { + +/// @brief Outcome of a write through the owning user-config register model. +enum class Pm4RegisterWriteStatus : uint8_t { + Complete, ///< The register effect occurred; packet processing may continue. + Blocked, ///< No effect occurred; retry the same packet later. + Faulted, ///< No effect occurred; cancel the queue after a register-access failure. + Rejected, ///< No effect occurred; cancel the queue for an unsupported write. +}; + +/// @brief Optional register-model services copied into each CP-owned PM4 queue. +/// @details Captured objects must outlive every queue that holds these callbacks. +class Pm4PacketCallbacks { +public: + /// @brief Apply one SET_UCONFIG_REG write through the owning register model. + /// @details Non-complete outcomes must mean that no register effect occurred. + /// register_dword is an absolute register dword index (0xc000 plus the packet + /// offset); value is the 32-bit register value. A native ring requires this + /// callback for SET_UCONFIG_REG. A submission queue may omit it and use its + /// local user-config register state. + std::function + write_uconfig_register; +}; + +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.cpp deleted file mode 100644 index c383b94454e6..000000000000 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.cpp +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -#include "rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.h" - -#include "rocjitsu/vm/amdgpu/command_processor.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h" - -#include -#include - -namespace rocjitsu::amdgpu { -namespace { - -class Pm4QueueBinding final : public QueueBinding { -public: - Pm4QueueBinding(CommandProcessor &command_processor, uint64_t registration_id) - : command_processor_(command_processor), registration_id_(registration_id) {} - - ~Pm4QueueBinding() override { - if (registration_id_ != 0) - (void)command_processor_.unregister_pm4_queue_registration(registration_id_); - } - - QueuePrepareCloseStatus prepare_close() noexcept override { - if (registration_id_ == 0) - return QueuePrepareCloseStatus::Ready; - const QueuePrepareCloseStatus status = - command_processor_.prepare_unregister_pm4_queue_registration(registration_id_); - if (status != QueuePrepareCloseStatus::Ready) - return status; - registration_id_ = 0; - return QueuePrepareCloseStatus::Ready; - } - - QueueReconfigureStatus reconfigure(const QueueReconfigureRequest &request) override { - return registration_id_ != 0 - ? command_processor_.update_pm4_queue_registration(registration_id_, request) - : QueueReconfigureStatus::Stale; - } - - QueueSubmissionStatus submit_producer(uint64_t producer_cursor) override { - return registration_id_ != 0 - ? command_processor_.notify_pm4_queue_doorbell(registration_id_, producer_cursor) - : QueueSubmissionStatus::Faulted; - } - -private: - CommandProcessor &command_processor_; - uint64_t registration_id_ = 0; -}; - -class Pm4QueueBindingFactory final : public QueueBindingFactory { -public: - Pm4QueueBindingFactory(CommandProcessor &command_processor, Pm4PacketCallbacks packet_callbacks) - : command_processor_(command_processor), packet_callbacks_(std::move(packet_callbacks)) {} - - QueueBindingCreateResult create_binding(const QueueRegistrationRequest &request) override { - if (request.type != QueueType::Compute || request.packet_format != QueuePacketFormat::Pm4) - return {}; - // PM4 currently advances only through explicit frontend notifications. - // Accepting either polling mode would discard its publication source and - // leave a queue that appears live but cannot discover new packets. - if (request.doorbell.mode != QueueDoorbellMode::Explicit) - return {}; - const uint64_t registration_id = command_processor_.register_pm4_queue({ - .address_space = request.identity.address_space, - .ring_base = request.ring.base_address, - .ring_size_bytes = request.ring.size_bytes, - .consumer_pointer_address = request.ring.consumer_pointer_address, - .initial_consumer_cursor = request.initial_consumer_cursor, - .packet_callbacks = packet_callbacks_, - }); - if (registration_id == 0) - return {}; - try { - return {.status = QueueBindingCreateStatus::Bound, - .binding = std::make_unique(command_processor_, registration_id)}; - } catch (...) { - (void)command_processor_.unregister_pm4_queue_registration(registration_id); - throw; - } - } - -private: - CommandProcessor &command_processor_; - Pm4PacketCallbacks packet_callbacks_; -}; - -} // namespace - -std::shared_ptr -make_pm4_queue_binding_factory(CommandProcessor &command_processor, - Pm4PacketCallbacks packet_callbacks) { - return std::make_shared(command_processor, std::move(packet_callbacks)); -} - -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.h deleted file mode 100644 index 8c549a0b1972..000000000000 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -/// @file pm4/pm4_queue_binding_factory.h -/// @brief Registry binding for synchronous PM4 compute queues. - -#pragma once - -#include "rocjitsu/vm/amdgpu/gpu_queue_registry.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h" - -#include - -namespace rocjitsu::amdgpu { - -class CommandProcessor; - -/// @brief Create a reusable binding factory for PM4 queues assigned to one CP. -/// @details The factory and bindings borrow the command processor. Its owner must -/// outlive the factory and every queue binding created from it. Packet callbacks -/// are copied into each CP-owned PM4 queue and must obey their own captured -/// lifetime contracts. -[[nodiscard]] std::shared_ptr -make_pm4_queue_binding_factory(CommandProcessor &command_processor, - Pm4PacketCallbacks packet_callbacks); - -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.cpp deleted file mode 100644 index 8773ebd4a105..000000000000 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.cpp +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -#include "rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h" - -#include "rocjitsu/vm/amdgpu/gpu_vm.h" - -#include -#include -#include - -namespace rocjitsu::amdgpu { - -Pm4QueueController::QueueRecord::QueueRecord(GpuVm &gpu_vm, Pm4QueueConfig config, - GpuVmBindingLease retained_address_space) - : address_space_lease(std::move(retained_address_space)), - packet_processor(std::move(config.packet_callbacks)), - ring_consumer(packet_processor, gpu_vm, config.address_space, config.ring_base, - config.ring_size_bytes, config.consumer_pointer_address, - config.initial_consumer_cursor), - producer_cursor(config.initial_consumer_cursor.value_or(0)), - ring_dwords(config.ring_size_bytes / sizeof(uint32_t)) {} - -Pm4QueueController::RegistrationId Pm4QueueController::attach(Pm4QueueConfig config) { - if (!config.address_space || config.ring_base == 0 || config.consumer_pointer_address == 0 || - config.ring_size_bytes == 0 || (config.ring_base % sizeof(uint32_t)) != 0 || - (config.consumer_pointer_address % sizeof(uint64_t)) != 0 || - (config.ring_size_bytes % sizeof(uint32_t)) != 0) { - return 0; - } - - std::optional address_space_lease = - gpu_vm_.retain_binding(config.address_space); - if (!address_space_lease) - return 0; - auto queue = - std::make_shared(gpu_vm_, std::move(config), std::move(*address_space_lease)); - std::lock_guard lock(mutex_); - RegistrationId registration_id = next_registration_id_++; - if (next_registration_id_ == 0) - next_registration_id_ = 1; - while (registration_id == 0 || queues_.contains(registration_id)) { - registration_id = next_registration_id_++; - if (next_registration_id_ == 0) - next_registration_id_ = 1; - } - queues_.emplace(registration_id, std::move(queue)); - return registration_id; -} - -QueuePrepareCloseStatus -Pm4QueueController::prepare_detach(RegistrationId registration_id) noexcept { - return detach(registration_id, false); -} - -bool Pm4QueueController::detach(RegistrationId registration_id) noexcept { - return detach(registration_id, true) == QueuePrepareCloseStatus::Ready; -} - -QueuePrepareCloseStatus Pm4QueueController::detach(RegistrationId registration_id, - bool force) noexcept { - std::shared_ptr queue; - std::unique_lock lock(mutex_); - const auto found = queues_.find(registration_id); - if (found == queues_.end() || found->second->closing) - return QueuePrepareCloseStatus::Faulted; - queue = found->second; - queue->closing = true; - queue->ready = false; - condition_.wait(lock, [&queue]() { return !queue->service_active; }); - if (!force) { - const QueuePrepareCloseStatus status = queue->ring_consumer.prepare_close(); - if (status != QueuePrepareCloseStatus::Ready) { - queue->closing = false; - queue->ready = !queue->terminal; - condition_.notify_all(); - return status; - } - } else { - queue->ring_consumer.reset(); - } - queues_.erase(registration_id); - return QueuePrepareCloseStatus::Ready; -} - -QueueReconfigureStatus Pm4QueueController::update(RegistrationId registration_id, - const QueueReconfigureRequest &request) { - std::unique_lock lock(mutex_); - const auto found = queues_.find(registration_id); - if (found == queues_.end()) - return QueueReconfigureStatus::Stale; - const std::shared_ptr queue_record = found->second; - if (queue_record->closing) - return QueueReconfigureStatus::Stale; - ++queue_record->management_operations; - condition_.wait( - lock, [&queue_record]() { return queue_record->closing || !queue_record->service_active; }); - if (queue_record->closing) { - --queue_record->management_operations; - condition_.notify_all(); - return QueueReconfigureStatus::Stale; - } - QueueRecord &queue = *queue_record; - if (queue.ring_consumer.in_flight()) { - --queue.management_operations; - condition_.notify_all(); - return QueueReconfigureStatus::Busy; - } - QueueReconfigureStatus status = QueueReconfigureStatus::Applied; - if (request.ring_base_address == 0 || request.ring_size_bytes == 0 || - request.scheduling_percentage == 0) { - queue.enabled = false; - queue.ready = false; - queue.terminal = false; - queue.ring_consumer.reconfigure(request.ring_base_address, request.ring_size_bytes); - queue.producer_cursor = 0; - queue.ring_dwords = 0; - status = QueueReconfigureStatus::Disabled; - } else if ((request.ring_base_address % sizeof(uint32_t)) != 0 || - (request.ring_size_bytes % sizeof(uint32_t)) != 0) { - status = QueueReconfigureStatus::Invalid; - } else { - queue.enabled = true; - queue.ready = false; - queue.terminal = false; - queue.ring_consumer.reconfigure(request.ring_base_address, request.ring_size_bytes); - queue.producer_cursor = 0; - queue.ring_dwords = request.ring_size_bytes / sizeof(uint32_t); - } - --queue.management_operations; - condition_.notify_all(); - return status; -} - -QueueSubmissionStatus Pm4QueueController::notify(RegistrationId registration_id, - uint64_t producer_cursor) { - std::lock_guard lock(mutex_); - const auto found = queues_.find(registration_id); - if (found == queues_.end() || found->second->closing || !found->second->enabled || - found->second->terminal) - return QueueSubmissionStatus::Faulted; - const std::optional normalized = normalize_pm4_producer_cursor( - producer_cursor, found->second->producer_cursor, found->second->ring_dwords); - if (!normalized) { - found->second->terminal = true; - found->second->ready = false; - return QueueSubmissionStatus::Faulted; - } - if (*normalized != found->second->producer_cursor) { - found->second->producer_cursor = *normalized; - found->second->ready = true; - } - return QueueSubmissionStatus::Accepted; -} - -QueueSubmissionStatus Pm4QueueController::service_queue(QueueRecord &queue, - uint64_t producer_cursor) { - const Pm4RingResult result = queue.ring_consumer.consume(producer_cursor, kPacketsPerTurn); - if (result.status == Pm4RingStatus::Blocked) - return QueueSubmissionStatus::Retry; - if (result.status != Pm4RingStatus::Complete) - return QueueSubmissionStatus::Faulted; - return QueueSubmissionStatus::Accepted; -} - -bool Pm4QueueController::service() { - std::vector registrations; - { - std::lock_guard lock(mutex_); - registrations.reserve(queues_.size()); - for (const auto &[registration_id, queue] : queues_) { - (void)queue; - registrations.push_back(registration_id); - } - } - - bool retry_pending = false; - for (const RegistrationId registration_id : registrations) { - std::shared_ptr queue; - uint64_t producer_cursor = 0; - { - std::lock_guard lock(mutex_); - const auto found = queues_.find(registration_id); - if (found == queues_.end()) - continue; - queue = found->second; - if (!queue->ready || queue->terminal || !queue->enabled || queue->closing || - queue->service_active || queue->management_operations != 0) { - continue; - } - queue->ready = false; - queue->service_active = true; - producer_cursor = queue->producer_cursor; - } - - QueueSubmissionStatus status = QueueSubmissionStatus::Faulted; - try { - status = service_queue(*queue, producer_cursor); - } catch (...) { - status = QueueSubmissionStatus::Faulted; - } - - { - std::lock_guard lock(mutex_); - queue->service_active = false; - if (!queue->closing) { - if (status == QueueSubmissionStatus::Retry) { - queue->ready = true; - retry_pending = true; - } else if (status == QueueSubmissionStatus::Faulted) { - queue->terminal = true; - queue->ready = false; - } else { - queue->ready = queue->ring_consumer.consumer_cursor() != queue->producer_cursor; - retry_pending = retry_pending || queue->ready; - } - } - condition_.notify_all(); - } - } - return retry_pending; -} - -void Pm4QueueController::close_all() noexcept { - std::unordered_map> queues; - { - std::unique_lock lock(mutex_); - for (auto &[registration_id, queue] : queues_) { - (void)registration_id; - queue->closing = true; - queue->ready = false; - } - condition_.wait(lock, [this]() { - return std::ranges::none_of(queues_, - [](const auto &entry) { return entry.second->service_active; }); - }); - for (auto &[registration_id, queue] : queues_) { - (void)registration_id; - queue->ring_consumer.reset(); - } - queues.swap(queues_); - } -} - -std::size_t Pm4QueueController::active_queues() const { - std::lock_guard lock(mutex_); - return queues_.size(); -} - -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h deleted file mode 100644 index 4307426555e5..000000000000 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -/// @file pm4/pm4_queue_controller.h -/// @brief Command-processor-owned PM4 queue state and service. - -#pragma once - -#include "rocjitsu/vm/amdgpu/gpu_queue_registry.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace rocjitsu::amdgpu { - -class GpuVm; - -/// @brief PM4 queue configuration admitted by one CommandProcessor. -struct Pm4QueueConfig { - AddressSpaceHandle address_space; - uint64_t ring_base = 0; - uint32_t ring_size_bytes = 0; - uint64_t consumer_pointer_address = 0; - std::optional initial_consumer_cursor; - Pm4PacketCallbacks packet_callbacks; -}; - -/// @brief Per-CP owner of PM4 queue state below GpuQueueRegistry. -/// @details Submission only records producer progress. The owning CP invokes -/// service() on its event context, keeping PM4 ring traversal and packet effects -/// out of MES and PCI/VFIO callbacks. A service lease keeps the queue alive while -/// VM accesses and packet callbacks run without the controller mutex; detach and -/// reconfiguration synchronously wait for that lease to finish. -class Pm4QueueController { -public: - using RegistrationId = uint64_t; - - explicit Pm4QueueController(GpuVm &gpu_vm) : gpu_vm_(gpu_vm) {} - - [[nodiscard]] RegistrationId attach(Pm4QueueConfig config); - /// @brief Gracefully remove a queue without discarding committed progress. - [[nodiscard]] QueuePrepareCloseStatus prepare_detach(RegistrationId registration_id) noexcept; - /// @brief Unconditionally cancel and remove a queue during rollback or reset. - [[nodiscard]] bool detach(RegistrationId registration_id) noexcept; - [[nodiscard]] QueueReconfigureStatus update(RegistrationId registration_id, - const QueueReconfigureRequest &request); - [[nodiscard]] QueueSubmissionStatus notify(RegistrationId registration_id, - uint64_t producer_cursor); - - /// @brief Service every runnable PM4 queue once. - /// @returns true when blocked or newly arrived work needs another service pass. - [[nodiscard]] bool service(); - - /// @brief Force-cancel every queue and release all retained VM bindings. - void close_all() noexcept; - - [[nodiscard]] std::size_t active_queues() const; - -private: - static constexpr std::size_t kPacketsPerTurn = 256; - - class QueueRecord { - public: - QueueRecord(GpuVm &gpu_vm, Pm4QueueConfig config, GpuVmBindingLease address_space_lease); - - GpuVmBindingLease address_space_lease; - Pm4PacketProcessor packet_processor; - Pm4RingConsumer ring_consumer; - uint64_t producer_cursor = 0; - uint64_t ring_dwords = 0; - uint32_t management_operations = 0; - bool enabled = true; - bool ready = false; - bool service_active = false; - bool closing = false; - bool terminal = false; - }; - - [[nodiscard]] QueueSubmissionStatus service_queue(QueueRecord &queue, uint64_t producer_cursor); - [[nodiscard]] QueuePrepareCloseStatus detach(RegistrationId registration_id, bool force) noexcept; - - mutable std::mutex mutex_; - std::condition_variable condition_; - GpuVm &gpu_vm_; - std::unordered_map> queues_; - RegistrationId next_registration_id_ = 1; -}; - -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.cpp deleted file mode 100644 index 5144648ad9c9..000000000000 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.cpp +++ /dev/null @@ -1,333 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -#include "rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace rocjitsu::amdgpu { -namespace { - -Pm4RingStatus map(VmAccessOutcome outcome) { - switch (outcome) { - case VmAccessOutcome::Complete: - return Pm4RingStatus::Complete; - case VmAccessOutcome::Unavailable: - return Pm4RingStatus::Blocked; - case VmAccessOutcome::Faulted: - return Pm4RingStatus::Faulted; - case VmAccessOutcome::Malformed: - return Pm4RingStatus::Malformed; - } - return Pm4RingStatus::Malformed; -} - -Pm4RingStatus map(const Pm4PacketProcessResult &result) { - if (result.diagnostic == Pm4PacketDiagnostic::UnsupportedPacket) - return Pm4RingStatus::UnsupportedPacket; - if (result.diagnostic == Pm4PacketDiagnostic::RegisterWriteRejected) - return Pm4RingStatus::RegisterWriteRejected; - switch (result.packet.status) { - case PacketProcessStatus::Complete: - return Pm4RingStatus::Complete; - case PacketProcessStatus::NeedInput: - case PacketProcessStatus::Blocked: - return Pm4RingStatus::Blocked; - case PacketProcessStatus::Faulted: - return Pm4RingStatus::Faulted; - case PacketProcessStatus::Malformed: - return Pm4RingStatus::Malformed; - case PacketProcessStatus::Unsupported: - return Pm4RingStatus::UnsupportedPacket; - } - return Pm4RingStatus::Malformed; -} - -} // namespace - -std::optional normalize_pm4_producer_cursor(uint64_t producer_cursor, - uint64_t reference_cursor, - uint64_t ring_dwords) noexcept { - if (ring_dwords == 0) - return std::nullopt; - if (producer_cursor >= reference_cursor && producer_cursor - reference_cursor <= ring_dwords) - return producer_cursor; - if (producer_cursor >= ring_dwords) - return std::nullopt; - - const uint64_t epoch = reference_cursor - reference_cursor % ring_dwords; - if (producer_cursor > std::numeric_limits::max() - epoch) - return std::nullopt; - uint64_t normalized = epoch + producer_cursor; - if (normalized < reference_cursor) { - if (normalized > std::numeric_limits::max() - ring_dwords) - return std::nullopt; - normalized += ring_dwords; - } - if (normalized - reference_cursor > ring_dwords) - return std::nullopt; - return normalized; -} - -Pm4RingConsumer::Pm4RingConsumer(Pm4PacketProcessor &packet_processor, GpuVm &gpu_vm, - AddressSpaceHandle address_space, uint64_t ring_base, - uint64_t ring_bytes, uint64_t consumer_pointer_address, - std::optional initial_consumer_cursor) - : packet_processor_(&packet_processor), gpu_vm_(&gpu_vm), address_space_(address_space), - ring_base_(ring_base), ring_bytes_(ring_bytes), - cursor_journal_(consumer_pointer_address, initial_consumer_cursor, sizeof(uint64_t)) {} - -bool Pm4RingConsumer::in_flight() const noexcept { - return access_.has_value() || transaction_producer_cursor_.has_value() || - cursor_journal_.publication_pending(); -} - -bool Pm4RingConsumer::publication_pending() const noexcept { - return cursor_journal_.publication_pending(); -} - -uint64_t Pm4RingConsumer::consumer_cursor() const noexcept { return cursor_journal_.cursor(); } - -QueuePrepareCloseStatus Pm4RingConsumer::prepare_close() noexcept { - if (cursor_journal_.publication_pending()) - return publication_faulted_ ? QueuePrepareCloseStatus::Faulted : QueuePrepareCloseStatus::Busy; - reset(); - return QueuePrepareCloseStatus::Ready; -} - -void Pm4RingConsumer::reconfigure(uint64_t ring_base, uint64_t ring_bytes) noexcept { - reset(); - ring_base_ = ring_base; - ring_bytes_ = ring_bytes; -} - -void Pm4RingConsumer::release_transaction() noexcept { - access_.reset(); - transaction_producer_cursor_.reset(); - producer_cursor_normalized_ = false; - terminal_after_publication_.reset(); -} - -void Pm4RingConsumer::reset() noexcept { - release_transaction(); - cursor_journal_.reset(); - publication_faulted_ = false; -} - -Pm4RingResult Pm4RingConsumer::consume(uint64_t producer_cursor, std::size_t packet_budget) { - Pm4RingResult result{.read_pointer = cursor_journal_.cursor()}; - if (packet_budget == 0) { - result.status = Pm4RingStatus::Complete; - return result; - } - if (ring_base_ == 0 || ring_bytes_ == 0 || (ring_bytes_ % sizeof(uint32_t)) != 0) { - return result; - } - if (!access_) { - access_ = gpu_vm_->snapshot_pinned(address_space_); - if (!access_) { - result.status = Pm4RingStatus::Faulted; - return result; - } - if (!access_->info().ready) { - result.status = Pm4RingStatus::Blocked; - release_transaction(); - return result; - } - transaction_producer_cursor_ = producer_cursor; - } - if (!transaction_producer_cursor_) { - result.status = Pm4RingStatus::Faulted; - release_transaction(); - return result; - } - - const VmAccessOutcome initialized = cursor_journal_.initialize(*access_); - if (initialized == VmAccessOutcome::Unavailable) { - result.status = Pm4RingStatus::Blocked; - return result; - } - if (initialized != VmAccessOutcome::Complete) { - result.status = map(initialized); - release_transaction(); - return result; - } - result.read_pointer = cursor_journal_.cursor(); - - const uint64_t ring_dwords = ring_bytes_ / sizeof(uint32_t); - if (!producer_cursor_normalized_) { - const std::optional normalized = normalize_pm4_producer_cursor( - *transaction_producer_cursor_, cursor_journal_.cursor(), ring_dwords); - if (!normalized) { - result.status = Pm4RingStatus::Malformed; - release_transaction(); - return result; - } - transaction_producer_cursor_ = *normalized; - producer_cursor_normalized_ = true; - } - const uint64_t transaction_producer_cursor = *transaction_producer_cursor_; - if (transaction_producer_cursor < cursor_journal_.cursor() || - transaction_producer_cursor - cursor_journal_.cursor() > ring_dwords) { - result.status = Pm4RingStatus::Malformed; - release_transaction(); - return result; - } - - if (cursor_journal_.publication_pending()) { - const VmAccessOutcome published = cursor_journal_.publish(); - result.read_pointer = cursor_journal_.cursor(); - if (published == VmAccessOutcome::Unavailable) { - result.status = Pm4RingStatus::Blocked; - return result; - } - if (published != VmAccessOutcome::Complete) { - publication_faulted_ = published != VmAccessOutcome::Complete; - result.status = map(published); - if (!cursor_journal_.publication_pending()) - release_transaction(); - return result; - } - publication_faulted_ = false; - if (terminal_after_publication_) { - result = *terminal_after_publication_; - result.read_pointer = cursor_journal_.cursor(); - release_transaction(); - return result; - } - } - - if (cursor_journal_.cursor() == transaction_producer_cursor) { - result.status = Pm4RingStatus::Complete; - result.read_pointer = cursor_journal_.cursor(); - release_transaction(); - return result; - } - - result = consume_packets(*access_, cursor_journal_.cursor(), transaction_producer_cursor, - packet_budget); - if (result.read_pointer != cursor_journal_.cursor()) { - cursor_journal_.retire(result.read_pointer, result.read_pointer % ring_dwords, *access_); - if (result.status != Pm4RingStatus::Complete && result.status != Pm4RingStatus::Blocked) { - terminal_after_publication_ = result; - } - const VmAccessOutcome published = cursor_journal_.publish(); - if (published == VmAccessOutcome::Unavailable) { - result.status = Pm4RingStatus::Blocked; - return result; - } - if (published != VmAccessOutcome::Complete || terminal_after_publication_) { - publication_faulted_ = published != VmAccessOutcome::Complete; - if (published == VmAccessOutcome::Complete) { - publication_faulted_ = false; - result = *terminal_after_publication_; - result.read_pointer = cursor_journal_.cursor(); - } else { - result.status = map(published); - } - if (!cursor_journal_.publication_pending()) - release_transaction(); - return result; - } - publication_faulted_ = false; - } - - result.read_pointer = cursor_journal_.cursor(); - if (result.status == Pm4RingStatus::Blocked) - return result; - if (result.status != Pm4RingStatus::Complete || - cursor_journal_.cursor() != transaction_producer_cursor) { - release_transaction(); - return result; - } - release_transaction(); - return result; -} - -Pm4RingResult Pm4RingConsumer::consume_packets(const GpuVmAccess &access, uint64_t read_pointer, - uint64_t write_pointer, - std::size_t packet_budget) const { - Pm4RingResult ring_result{.read_pointer = read_pointer}; - const uint64_t ring_dwords = ring_bytes_ / sizeof(uint32_t); - if (ring_base_ == 0 || ring_dwords == 0 || - ring_dwords > std::numeric_limits::max() / sizeof(uint32_t) || - ring_base_ > std::numeric_limits::max() - ring_dwords * sizeof(uint32_t) || - write_pointer < read_pointer || write_pointer - read_pointer > ring_dwords) { - return ring_result; - } - - const CircularRingReader ring_reader(ring_base_, ring_dwords * sizeof(uint32_t)); - std::size_t retired_packets = 0; - while (ring_result.read_pointer != write_pointer) { - std::array raw_packet{}; - const uint64_t byte_cursor = (ring_result.read_pointer % ring_dwords) * sizeof(uint32_t); - const VmAccessOutcome header_outcome = - ring_reader.read(access, byte_cursor, std::span(raw_packet).first(sizeof(uint32_t))); - if (header_outcome != VmAccessOutcome::Complete) { - ring_result.status = map(header_outcome); - return ring_result; - } - - std::array packet = std::bit_cast>(raw_packet); - Pm4PacketProcessResult packet_result = - packet_processor_->process({.available_dwords = std::span(packet).first(1)}); - std::size_t supplied_bytes = sizeof(uint32_t); - ring_result.packet_header = packet_result.packet_header; - ring_result.register_dword = packet_result.register_dword; - if (!valid_packet_process_result(packet_result.packet_result(), sizeof(uint32_t), - sizeof(uint32_t))) { - ring_result.status = Pm4RingStatus::Malformed; - return ring_result; - } - - if (packet_result.packet_result().status == PacketProcessStatus::NeedInput) { - if (packet_result.packet_result().required_bytes > raw_packet.size() || - packet_result.packet_result().required_bytes / sizeof(uint32_t) > - write_pointer - ring_result.read_pointer) { - ring_result.status = Pm4RingStatus::Malformed; - return ring_result; - } - const VmAccessOutcome packet_outcome = ring_reader.read( - access, byte_cursor, - std::span(raw_packet).first(packet_result.packet_result().required_bytes)); - if (packet_outcome != VmAccessOutcome::Complete) { - ring_result.status = map(packet_outcome); - return ring_result; - } - packet = std::bit_cast>(raw_packet); - supplied_bytes = packet_result.packet_result().required_bytes; - packet_result = packet_processor_->process( - {.available_dwords = std::span(packet).first(supplied_bytes / sizeof(uint32_t))}); - ring_result.packet_header = packet_result.packet_header; - ring_result.register_dword = packet_result.register_dword; - if (!valid_packet_process_result(packet_result.packet_result(), supplied_bytes, - sizeof(uint32_t))) { - ring_result.status = Pm4RingStatus::Malformed; - return ring_result; - } - } - - ring_result.status = map(packet_result); - const PacketProcessResult &common_result = packet_result.packet_result(); - if (common_result.retirement != PacketRetirement::Retire) - return ring_result; - if (common_result.retirement_bytes > supplied_bytes) { - ring_result.status = Pm4RingStatus::Malformed; - return ring_result; - } - ring_result.read_pointer += common_result.retirement_bytes / sizeof(uint32_t); - if (++retired_packets >= packet_budget) - break; - } - - ring_result.status = Pm4RingStatus::Complete; - return ring_result; -} - -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.h deleted file mode 100644 index ae08fb0ba792..000000000000 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.h +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -/// @file pm4/pm4_ring_consumer.h -/// @brief Ring traversal for the supported PM4 compute-queue subset. - -#pragma once - -#include "rocjitsu/vm/amdgpu/circular_ring_reader.h" -#include "rocjitsu/vm/amdgpu/consumer_cursor_journal.h" -#include "rocjitsu/vm/amdgpu/gpu_queue_registry.h" -#include "rocjitsu/vm/amdgpu/gpu_vm.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h" - -#include -#include -#include -#include - -namespace rocjitsu::amdgpu { - -enum class Pm4RingStatus : uint8_t { - Complete, - Blocked, - Faulted, - Malformed, - UnsupportedPacket, - RegisterWriteRejected, -}; - -struct Pm4RingResult { - Pm4RingStatus status = Pm4RingStatus::Malformed; - uint64_t read_pointer = 0; - uint32_t packet_header = 0; - uint64_t register_dword = 0; -}; - -/// @brief Convert a native ring-relative producer cursor into its monotonic epoch. -[[nodiscard]] std::optional normalize_pm4_producer_cursor(uint64_t producer_cursor, - uint64_t reference_cursor, - uint64_t ring_dwords) noexcept; - -/// @brief Owns one PM4 ring transaction, including cursor retirement and publication. -class Pm4RingConsumer { -public: - Pm4RingConsumer(Pm4PacketProcessor &packet_processor, GpuVm &gpu_vm, - AddressSpaceHandle address_space, uint64_t ring_base, uint64_t ring_bytes, - uint64_t consumer_pointer_address, - std::optional initial_consumer_cursor); - - Pm4RingConsumer(const Pm4RingConsumer &) = delete; - Pm4RingConsumer &operator=(const Pm4RingConsumer &) = delete; - Pm4RingConsumer(Pm4RingConsumer &&) = delete; - Pm4RingConsumer &operator=(Pm4RingConsumer &&) = delete; - - /// @brief Consume through one producer boundary using one immutable VM snapshot. - [[nodiscard]] Pm4RingResult - consume(uint64_t producer_cursor, - std::size_t packet_budget = std::numeric_limits::max()); - - /// @brief Whether retry-critical transaction or publication state is retained. - [[nodiscard]] bool in_flight() const noexcept; - [[nodiscard]] bool publication_pending() const noexcept; - [[nodiscard]] uint64_t consumer_cursor() const noexcept; - - /// @brief Prepare for graceful removal without losing committed progress. - [[nodiscard]] QueuePrepareCloseStatus prepare_close() noexcept; - - /// @brief Replace ring geometry after the owner has established quiescence. - void reconfigure(uint64_t ring_base, uint64_t ring_bytes) noexcept; - - /// @brief Force-cancel retained transaction state and reset the ring cursor. - void reset() noexcept; - -private: - [[nodiscard]] Pm4RingResult consume_packets(const GpuVmAccess &access, uint64_t read_pointer, - uint64_t write_pointer, - std::size_t packet_budget) const; - void release_transaction() noexcept; - - Pm4PacketProcessor *packet_processor_ = nullptr; - GpuVm *gpu_vm_ = nullptr; - AddressSpaceHandle address_space_; - uint64_t ring_base_ = 0; - uint64_t ring_bytes_ = 0; - ConsumerCursorJournal cursor_journal_; - std::optional access_; - std::optional transaction_producer_cursor_; - bool producer_cursor_normalized_ = false; - std::optional terminal_after_publication_; - bool publication_faulted_ = false; -}; - -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/request_mtype_resolver.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/request_mtype_resolver.h index 5e54c10fe809..1bacfbdeca32 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/request_mtype_resolver.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/request_mtype_resolver.h @@ -13,20 +13,36 @@ namespace rocjitsu { namespace amdgpu { /// Resolves effective MTYPE through one lifetime-safe VM binding snapshot. +/// The caller's cache retains copied policy across requests, but no VM backing. class RequestMtypeResolver { public: - RequestMtypeResolver(GpuVm *gpu_vm, uint32_t vmid) - : access_(vmid != 0 && gpu_vm != nullptr ? gpu_vm->snapshot_vmid(vmid) : std::nullopt), - fallback_(Mtype::RW), combine_(false) {} + RequestMtypeResolver(GpuVm *gpu_vm, uint32_t vmid, VmMtypeCache &cache, + std::optional instruction_mtype = std::nullopt) + : mtype_cache_(cache), fallback_(instruction_mtype.value_or(Mtype::RW)), + combine_(instruction_mtype.has_value()) { + bind_access(gpu_vm, vmid); + } - RequestMtypeResolver(GpuVm *gpu_vm, uint32_t vmid, Mtype instruction_mtype) - : access_(vmid != 0 && gpu_vm != nullptr ? gpu_vm->snapshot_vmid(vmid) : std::nullopt), - fallback_(instruction_mtype), combine_(true) {} + RequestMtypeResolver(const RequestMtypeResolver &) = delete; + RequestMtypeResolver &operator=(const RequestMtypeResolver &) = delete; + RequestMtypeResolver(RequestMtypeResolver &&) = delete; + RequestMtypeResolver &operator=(RequestMtypeResolver &&) = delete; Mtype fallback() const { return fallback_; } - Mtype at(uint64_t addr) { + bool cached_private_uc_hint(uint64_t addr) const { + return access_ && access_->cached_private_uc_hint(addr, mtype_cache_); + } + + std::optional cached_private_ram_mtype(uint64_t addr) const { if (!access_) + return std::nullopt; + const auto policy = access_->cached_private_ram_mtype(addr, mtype_cache_); + return policy && combine_ ? std::optional{effective_mtype(fallback_, *policy)} : policy; + } + + Mtype at(uint64_t addr) { + if (access_ == nullptr) return fallback_; const std::optional mtype = access_->query_mtype(addr, mtype_cache_); if (!mtype) @@ -35,8 +51,20 @@ class RequestMtypeResolver { } private: - std::optional access_; - VmMtypeCache mtype_cache_; + void bind_access(GpuVm *gpu_vm, uint32_t vmid) { + if (vmid == 0 || gpu_vm == nullptr) + return; + if (GpuVmAccessBatchGuard::active()) { + access_ = gpu_vm->borrow_snapshot_vmid(vmid); + return; + } + owned_access_ = gpu_vm->snapshot_vmid(vmid); + access_ = owned_access_ ? &*owned_access_ : nullptr; + } + + std::optional owned_access_; + const GpuVmAccess *access_ = nullptr; + VmMtypeCache &mtype_cache_; Mtype fallback_; bool combine_; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.cpp index cd169b3ae4b5..4cd98efeca1e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.cpp @@ -294,7 +294,10 @@ class SdmaPacketProcessor::Impl { return frame.words[frame.at + offset]; } - bool gfx11_plus() const { return dialect_ != SdmaPacketDialect::Legacy; } + bool gfx11_plus() const { + // Extended copy counts do not opt legacy dialects into gfx11 packet layouts. + return dialect_ == SdmaPacketDialect::Gfx11Plus || dialect_ == SdmaPacketDialect::Gfx1250; + } PacketExtent packet_extent(std::span words) const { const auto need = [&](std::size_t dwords) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.h index 722e17f40ca1..8cd726dff240 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.h @@ -18,8 +18,12 @@ namespace rocjitsu::amdgpu { +/// @brief Selects the SDMA packet layout and ordinary linear-copy count width. +/// @details LegacyExtendedCount retains the legacy layout but extends ordinary +/// linear-copy counts from 22 to 30 bits. enum class SdmaPacketDialect { Legacy, + LegacyExtendedCount, Gfx11Plus, Gfx1250, }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/shader_engine.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/shader_engine.h index 25cf00fb2a47..52aa190c952c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/shader_engine.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/shader_engine.h @@ -32,6 +32,7 @@ class ShaderEngine : public simdojo::CompositeComponent { struct Config { uint32_t num_compute_units; ///< Number of CUs in this SE. ComputeUnitCore::Config compute_unit; ///< Configuration applied to each CU. + uint32_t cus_per_shader_array = 0; ///< Shader-array width; zero means unknown. }; /// @brief Construct a shader engine from configuration. diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/spi.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/spi.h index 19b945953cc7..93f556b65e63 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/spi.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/spi.h @@ -152,14 +152,6 @@ class ShaderProcessorInput { return false; } - /// @brief Append this SPI's active CUs to a command-processor work batch. - void append_active_cus(std::vector &active) const { - for (auto *cu : cus_) { - if (cu->has_active_wfs()) - active.push_back(cu); - } - } - /// @brief Check if any WGs are queued or any CU is active. bool has_pending() const { for (auto &q : pipe_queues_) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/wavefront.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/wavefront.cpp index 6a8fd14a0533..8519818affb3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/wavefront.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/wavefront.cpp @@ -7,7 +7,7 @@ #include "rocjitsu/vm/amdgpu/compute_unit.h" #include "rocjitsu/vm/amdgpu/gpu_vm.h" #include "rocjitsu/vm/amdgpu/hsa_clock.h" -#include "rocjitsu/vm/amdgpu/pm4.h" +#include "rocjitsu/vm/amdgpu/pm4/pm4.h" namespace rocjitsu { namespace amdgpu { @@ -21,6 +21,20 @@ Wavefront::Wavefront(ComputeUnitCore &cu, uint32_t wf_id, uint32_t default_wf_si memory_wait_checks_enabled_(cu.config().memory_wait_diagnostics != MemoryWaitDiagnostics::Off) {} +void Wavefront::update_activity_counts(bool was_active, bool was_runnable) { + if (!activity_tracked_) + return; + const bool active = !is_halted(); + const bool runnable = active && !debug_paused(); + const uint64_t delta = (uint64_t(active) - uint64_t(was_active)) + + ((uint64_t(runnable) - uint64_t(was_runnable)) << 32); + const uint64_t previous = cu_.wave_activity_.fetch_add(delta, std::memory_order_release); + assert((active || !was_active || static_cast(previous) != 0) && + "active wave count underflow"); + assert((runnable || !was_runnable || (previous >> 32) != 0) && "runnable wave count underflow"); + (void)previous; +} + Lds &Wavefront::lds() { return lds_ ? *lds_ : cu_.lds(); } const Lds &Wavefront::lds() const { return lds_ ? *lds_ : cu_.lds(); } @@ -117,7 +131,7 @@ void Wavefront::halt(CpCompletionNotice notice) { void Wavefront::release_wait_counter(WaitCounterType type) { wait_counters_.decrement(type); if (state_ == WfState::WAITCNT && wait_satisfied()) - state_ = WfState::RUNNING; + set_state(WfState::RUNNING); if (state_ == WfState::ENDING && wait_counters_.empty()) halt(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/wavefront.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/wavefront.h index e8a67194d660..f769c07d5c9c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/wavefront.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/wavefront.h @@ -476,6 +476,7 @@ class Wavefront : public ThreadContext { static constexpr uint32_t DX10_CLAMP_BIT = 1u << 8; static constexpr uint32_t IEEE_BIT = 1u << 9; static constexpr uint32_t GPR_IDX_EN_BIT = 1u << 27; + static constexpr uint32_t VSKIP_BIT = 1u << 28; static constexpr uint32_t FP16_OVFL_BIT = 1u << 23; /// STATUS.HALT. Bit 13 on every modelled architecture -- see StatusReg::HALT @@ -580,7 +581,7 @@ class Wavefront : public ThreadContext { void set_wait_all() { wait_target_ = {0, 0, 0, 0, 0, 0, 0, 0}; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } void set_wait_target(uint8_t vmcnt, uint8_t lgkmcnt, uint8_t expcnt) { @@ -588,56 +589,56 @@ class Wavefront : public ThreadContext { wait_target_.lgkmcnt = lgkmcnt; wait_target_.expcnt = expcnt; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set the VSCNT target (GFX10 S_WAITCNT_VSCNT). void set_wait_target_vscnt(uint8_t threshold) { wait_target_.vscnt = threshold; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set the LOADCNT target (GFX11+ S_WAITCNT_VMCNT / S_WAIT_LOADCNT). void set_wait_target_loadcnt(uint8_t threshold) { wait_target_.vmcnt = threshold; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set the STORECNT target (GFX11+ S_WAITCNT_VSCNT / S_WAIT_STORECNT). void set_wait_target_storecnt(uint8_t threshold) { wait_target_.vscnt = threshold; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set the DSCNT target (GFX11+ S_WAITCNT_LGKMCNT / S_WAIT_DSCNT). void set_wait_target_dscnt(uint8_t threshold) { wait_target_.dscnt = threshold; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set the KMCNT target (GFX11+ S_WAIT_KMCNT). void set_wait_target_kmcnt(uint8_t threshold) { wait_target_.kmcnt = threshold; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set the TENSORCNT target (GFX12.5 S_WAIT_TENSORCNT). void set_wait_target_tensorcnt(uint8_t threshold) { wait_target_.tensorcnt = threshold; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set the ASYNCCNT target (GFX12.5 S_WAIT_ASYNCCNT). void set_wait_target_asynccnt(uint8_t threshold) { wait_target_.asynccnt = threshold; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set combined STORECNT + DSCNT targets (GFX12 S_WAIT_STORECNT_DSCNT). @@ -645,7 +646,7 @@ class Wavefront : public ThreadContext { wait_target_.vscnt = storecnt; wait_target_.dscnt = dscnt; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set combined LOADCNT + DSCNT targets (GFX12 S_WAIT_LOADCNT_DSCNT). @@ -653,7 +654,7 @@ class Wavefront : public ThreadContext { wait_target_.vmcnt = loadcnt; wait_target_.dscnt = dscnt; if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } /// @brief Set a single split-wait counter threshold by name. @@ -679,11 +680,11 @@ class Wavefront : public ThreadContext { else if (name == "wait_expcnt") { wait_target_.expcnt = static_cast(threshold & 0x07); if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } else if (name == "wait_samplecnt" || name == "wait_bvhcnt") { wait_target_.vmcnt = t; // map to vmcnt if (!wait_satisfied()) - state_ = WfState::WAITCNT; + set_state(WfState::WAITCNT); } else if (name == "wait_loadcnt_dscnt") { set_wait_target_loadcnt_dscnt(static_cast((threshold >> 8) & 0x3F), static_cast(threshold & 0x3F)); @@ -735,7 +736,12 @@ class Wavefront : public ThreadContext { /// @brief Set the execution state. /// @param s New execution state. - void set_state(WfState s) { state_ = s; } + void set_state(WfState s) { + const bool was_active = !is_halted(); + state_ = s; + if (was_active != !is_halted()) + update_activity_counts(was_active, was_active && !debug_paused()); + } /// @brief Check whether this wavefront slot is halted. /// @retval true Slot is halted and available for dispatch. @@ -850,19 +856,34 @@ class Wavefront : public ThreadContext { /// @details A debug-halted wave keeps its slot and all register state; the /// scheduler skips it so the CU can go quiescent without retiring the wave. bool debug_halted() const { return debug_halted_; } - void set_debug_halted(bool v) { debug_halted_ = v; } + void set_debug_halted(bool v) { + const bool was_runnable = !is_halted() && !debug_paused(); + debug_halted_ = v; + if (was_runnable != (!is_halted() && !debug_paused())) + update_activity_counts(!is_halted(), was_runnable); + } /// @brief Whether KFD has temporarily suspended this wave's queue. /// @details Queue suspension freezes execution for a stable CWSR snapshot, /// but unlike debug_halted it does not imply an architectural stop reason. bool debug_suspended() const { return debug_suspended_; } - void set_debug_suspended(bool v) { debug_suspended_ = v; } + void set_debug_suspended(bool v) { + const bool was_runnable = !is_halted() && !debug_paused(); + debug_suspended_ = v; + if (was_runnable != (!is_halted() && !debug_paused())) + update_activity_counts(!is_halted(), was_runnable); + } /// @brief Runtime-suspended: the queue's queue_percentage went to zero. /// @details A separate reason from the debugger's, because the two overlap. /// Sharing one bit let a runtime resume clear a debugger pause, and a /// debugger or CWSR resume clear an active runtime pause. bool runtime_suspended() const { return runtime_suspended_; } - void set_runtime_suspended(bool v) { runtime_suspended_ = v; } + void set_runtime_suspended(bool v) { + const bool was_runnable = !is_halted() && !debug_paused(); + runtime_suspended_ = v; + if (was_runnable != (!is_halted() && !debug_paused())) + update_activity_counts(!is_halted(), was_runnable); + } /// @brief Whether a *debugger* currently holds this wave stopped. /// @details Deliberately excludes runtime_suspended_: KFD uses this to decide @@ -914,7 +935,7 @@ class Wavefront : public ThreadContext { /// the saved PC points just after the trap, matching the ROCr trap handler. void debug_trap(uint32_t trap_id) { trap_id_ = trap_id; - debug_halted_ = true; + set_debug_halted(true); single_step_ = false; } @@ -950,7 +971,7 @@ class Wavefront : public ThreadContext { set_mode_raw(saved.mode_raw); gfx12_trap_ctrl_raw_ = saved.gfx12_trap_ctrl_raw; trap_id_ = saved.trap_id; - debug_halted_ = saved.debug_halted; + set_debug_halted(saved.debug_halted); single_step_ = saved.single_step; fatal_exception_pending_ = saved.fatal_exception_pending; fatal_exception_cwsr_valid_ = saved.fatal_exception_cwsr_valid; @@ -981,7 +1002,7 @@ class Wavefront : public ThreadContext { if (wait_counters_.empty()) halt(); else - state_ = WfState::ENDING; + set_state(WfState::ENDING); } /// @brief Log instruction count at end for trace/debug. @@ -1046,7 +1067,7 @@ class Wavefront : public ThreadContext { wait_counters_ = {}; wait_target_ = {}; ready_cycle_ = 0; - state_ = WfState::HALTED; + set_state(WfState::HALTED); for (auto &t : ttmp_) t = 0; trapsts_ = 0; @@ -1081,6 +1102,9 @@ class Wavefront : public ThreadContext { Wavefront(ComputeUnitCore &cu, uint32_t wf_id, uint32_t default_wf_size, uint32_t max_wf_size, uint32_t max_sgprs, uint32_t max_vgprs, bool mode_has_gpr_idx_en); + void update_activity_counts(bool was_active, bool was_runnable); + bool activity_tracked_ = false; + ComputeUnitCore &cu_; ///< Parent CU (permanent, set at construction). InstructionComputeUnitView cu_view_; uint32_t wf_id_ = 0; ///< Slot index within the CU (permanent). diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/xcd.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/xcd.cpp index 499f17def432..cc32a0127d83 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/xcd.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/xcd.cpp @@ -44,7 +44,7 @@ Xcd::Xcd(std::string name, const Config &config, rj_code_arch_t arch, GpuMemory // This also sets up on_idle callbacks from CUs to CP::check_all_idle(). for (uint32_t c = 0; c < se->num_compute_units(); ++c) { auto *cu = se->compute_unit(c); - cu->set_shader_engine_location(i, c); + cu->set_shader_engine_location(i, c, config.shader_engine.cus_per_shader_array); cp_->add_compute_unit(cu); } // Register the SE's SPI with the CP so ace_dispatch_all() can use the diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/execution_plugin.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/execution_plugin.h index 799a75c9bf26..e175d76b2a63 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/execution_plugin.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/execution_plugin.h @@ -78,10 +78,11 @@ class ExecutionPlugin { /// return false after its wavefront-dispatch callback has decided that the /// wave is outside its observation scope. The group samples the decision once /// after every plugin's wavefront-dispatch callback and reuses it until halt. - /// Reentrant hooks during dispatch/halt and hooks without a wavefront use a - /// live query, so this method must remain lock-free and thread-safe. The - /// decision must be stable between dispatch completion and the matching halt. - /// Lifecycle and dispatch/workgroup/wavefront callbacks are unaffected. + /// Reentrant hooks during dispatch/halt, hooks without a wavefront, and hooks + /// on a resident wave after live plugin-group replacement use a live query, + /// so this method must remain lock-free and thread-safe. The decision must be + /// stable between dispatch completion and the matching halt. Lifecycle and + /// dispatch/workgroup/wavefront callbacks are unaffected. virtual bool observes_hot_hooks_for_wavefront(const amdgpu::Wavefront * /*wf*/) const { return true; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/README.md b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/README.md index 33136c73b431..5a025f4420c1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/README.md +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/README.md @@ -61,6 +61,55 @@ For a larger trace, the budget can be raised explicitly: The 1 GiB value is only an example, not a recommended default. A larger budget trades host memory for trace coverage, and some known traces exceed even 1 GiB. +To profile one kernel in a multi-dispatch application, set `dispatch_name` to +the exact normalized display name shown in quotes in RocJITsu's VM dispatch +log: + +```json +"perfsim": { + "library_path": "/absolute/path/to/libgpucsim_ffm_plugin.so", + "dispatch_name": "_topk_topp_kernel" +} +``` + +For example, the mangled ELF symbol `_Z11racy_kernelPKfPf` has the normalized +display name `racy_kernel`: demangling removes its argument list, a leading +`void `, and whitespace. A VM dispatch log line contains both forms: + +```text +dispatch #1 d=0 "racy_kernel" symbol="_Z11racy_kernelPKfPf" ... +``` + +Copy the quoted display name, not the `symbol="..."` field. Copying a mangled +symbol can therefore miss, and overloads that normalize to the same display +name are selected together. + +Nonmatching dispatches still execute normally in RocJITsu, including their +functional memory effects. The adapter only suppresses their observer event +staging and replay into Perfsim. Matching is exact; it is neither a prefix nor +a regular-expression match. If `dispatch_name` is absent, every supported +dispatch is forwarded exactly as before. If no dispatch matches the configured +name, the adapter reports that name once when the plugin shuts down. + +For diagnostic runs, `max_observed_workgroups` can additionally cap the number +of distinct workgroups whose events are staged for each selected dispatch: + +```json +"perfsim": { + "library_path": "/absolute/path/to/libgpucsim_ffm_plugin.so", + "dispatch_name": "_topk_topp_kernel", + "max_observed_workgroups": 1 +} +``` + +The cap admits the first distinct RocJITsu workgroup IDs encountered in +wave-dispatch order; after an ID is admitted, all of that workgroup's observed +waves remain eligible. The cap does not skip functional execution. It only +limits observer events, and is disabled when omitted. A capped trace is +incomplete and must not be treated as an exact full-grid result unless the +backend explicitly reconstructs the full population from dispatch geometry and +the workload satisfies that backend's scaling assumptions. + Configure Perfsim through its own environment, then launch the workload: ```bash @@ -68,6 +117,15 @@ Configure Perfsim through its own environment, then launch the workload: --config /absolute/path/to/gfx1250-config.json -- ./application ``` +`GPUCSIM_INTERNAL_DETAILED_REPORT=1` is a diagnostic option implemented by +recent `libgpucsim_ffm_plugin.so` builds, not by the RocJITsu adapter. It asks +the backend to emit its internal detailed JSON schema, including fields such as +`dispatch_id`. Leave it unset (or set it to `0`) for the stable public summary +schema. The detailed schema is intended for backend qualification and tests, +may change with the backend, and should not be treated as a customer-facing +report contract. The real-backend parity test enables it itself because that +test compares dispatches individually. + ## Backend ABI The adapter contains a private, non-installed declaration of only the latest @@ -100,7 +158,10 @@ a helper module on first use. ### Use Rocjitsu Use a Python environment containing ROCm Torch, Triton, and NumPy to run the -GPT-OSS `_rms_norm_kernel/D1` corpus case. Run it directly with RocJITsu: +GPT-OSS `_rms_norm_kernel/D1` corpus case. A tested stack used Python 3.12.14, +ROCm PyTorch `2.10.0+rocm7.13.0a20260511` (HIP 7.13.0), Triton `3.6.0`, and +NumPy `2.5.2`. These are example versions, not minimum plugin requirements. +Run it directly with RocJITsu: ```bash export GPT_OSS=/absolute/path/to/gpt-oss-kernel-harness diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin.cpp index 96226327a802..384031b7e9d8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin.cpp @@ -118,6 +118,8 @@ class RetainedSharedLibrary { struct AdapterConfig { std::string library_path; size_t max_staged_bytes = kDefaultMaxStagedBytes; + std::optional dispatch_name; + std::optional max_observed_workgroups; }; AdapterConfig parse_config(const char *config_json) { @@ -141,6 +143,14 @@ AdapterConfig parse_config(const char *config_json) { if (!std::filesystem::path(result.library_path).is_absolute()) throw std::invalid_argument("Perfsim plugin 'library_path' must be absolute"); + const auto dispatch_name = config["dispatch_name"]; + if (!dispatch_name.IsNull()) { + if (!dispatch_name.IsString() || dispatch_name.AsString().size() == 0) { + throw std::invalid_argument("Perfsim plugin 'dispatch_name' must be a non-empty string"); + } + result.dispatch_name.emplace(dispatch_name.AsString().c_str(), dispatch_name.AsString().size()); + } + const auto max_staged_bytes = config["max_staged_bytes"]; if (!max_staged_bytes.IsNull()) { if (!max_staged_bytes.IsIntOrUint() || @@ -154,6 +164,22 @@ AdapterConfig parse_config(const char *config_json) { throw std::invalid_argument("Perfsim plugin 'max_staged_bytes' is too large"); result.max_staged_bytes = static_cast(value); } + + const auto max_observed_workgroups = config["max_observed_workgroups"]; + if (!max_observed_workgroups.IsNull()) { + if (!max_observed_workgroups.IsIntOrUint() || + (max_observed_workgroups.IsInt() && max_observed_workgroups.AsInt64() <= 0)) { + throw std::invalid_argument( + "Perfsim plugin 'max_observed_workgroups' must be a positive integer"); + } + const uint64_t value = max_observed_workgroups.AsUInt64(); + if (value == 0) + throw std::invalid_argument( + "Perfsim plugin 'max_observed_workgroups' must be a positive integer"); + if (value > std::numeric_limits::max()) + throw std::invalid_argument("Perfsim plugin 'max_observed_workgroups' is too large"); + result.max_observed_workgroups = static_cast(value); + } return result; } @@ -333,7 +359,21 @@ size_t staged_dynamic_size(const OrderedEvent &event) { return 0; } +struct DispatchState; + +struct StaticInstructionInfo { + uint64_t pc = 0; + std::array encoding{}; + FfmInstructionCounters counters{}; + FfmWaitInfo wait{FFM_WAIT_TYPE_NONE, FFM_WAIT_NAME_NONE}; + uint8_t encoding_dwords = 0; + bool scratch_incompatible = false; + bool terminates = false; + bool valid = false; +}; + struct PerfsimWavefrontState final : WavefrontState { + DispatchState *dispatch_state = nullptr; FfmWaveInfo wave_info{}; uint32_t compute_unit_id = 0; uint32_t physical_wavefront_id = 0; @@ -352,16 +392,75 @@ struct PerfsimWavefrontState final : WavefrontState { bool last_instruction_terminates = false; }; +/// Cache immutable instruction classification per callback thread. +/// +/// A wave-local cache repeats the same mnemonic work once for every resident +/// wave even though all waves execute the same code object. GPUCompilerSim v13 +/// invokes a wave from one callback thread at a time, so a thread-local cache +/// shares those immutable results across waves without adding a lock to this +/// hot path. The complete encoding remains part of the key: patched/self- +/// modifying code at an existing PC is classified again rather than inheriting +/// stale counters or wait semantics. +class StaticInstructionCache { +public: + const StaticInstructionInfo &classify(uint64_t pc, const Instruction &inst, + const std::array &encoding, + uint32_t encoding_dwords) { + StaticInstructionInfo &cached = entries_[index(pc, encoding, encoding_dwords)]; + if (cached.valid && cached.pc == pc && cached.encoding_dwords == encoding_dwords && + cached.encoding == encoding) + return cached; + + cached.pc = pc; + cached.encoding = encoding; + cached.counters = instruction_counters(inst); + cached.wait = wait_info(inst); + cached.encoding_dwords = static_cast(encoding_dwords); + cached.scratch_incompatible = inst.mnemonic().starts_with("scratch_"); + cached.terminates = (inst.flags() & PROGRAM_TERMINATOR) != 0; + cached.valid = true; + return cached; + } + +private: + static constexpr size_t kEntries = 4096; + static_assert(std::has_single_bit(kEntries)); + + static size_t index(uint64_t pc, const std::array &encoding, + uint32_t encoding_dwords) { + // Classification belongs to the instruction at a PC; the full encoding is + // validated above, so it need not be re-hashed on every hit. Fold code + // offsets one cache span apart to avoid mapping every 16 KiB page to the + // same slots while keeping the hit path to a few integer operations. + (void)encoding; + (void)encoding_dwords; + const uint64_t word_address = pc >> 2; + return static_cast(word_address ^ (word_address >> 12)) & (kEntries - 1); + } + + std::array entries_{}; +}; + +const StaticInstructionInfo &classify_instruction(uint64_t pc, const Instruction &inst, + const std::array &encoding, + uint32_t encoding_dwords) { + static thread_local StaticInstructionCache cache; + return cache.classify(pc, inst, encoding, encoding_dwords); +} + struct DispatchState { FfmDispatchMetadata metadata{}; std::string dispatch_name; bool metadata_seen = false; bool begun = false; bool ended = false; - bool supported = true; + bool selected = true; + std::atomic supported{true}; + std::mutex rejection_mutex; bool diagnostic_emitted = false; std::string rejection_reason; size_t live_waves = 0; + std::unordered_set observed_workgroups; }; std::optional validate_dispatch(const KernelDispatchInfo &info) { @@ -540,7 +639,7 @@ struct PerfsimPlugin::Impl { std::vector incomplete_dispatches; for (const auto &[dispatch_id, state] : dispatches) - if (state.begun && !state.ended) + if (state.selected && state.begun && !state.ended) incomplete_dispatches.push_back(dispatch_id); for (uint32_t dispatch_id : incomplete_dispatches) reject(dispatch_id, "dispatch was incomplete at plugin shutdown"); @@ -548,31 +647,77 @@ struct PerfsimPlugin::Impl { drain_epoch(/*shutdown=*/true); physical_waves.clear(); + if (initialized && config.dispatch_name && !dispatch_name_matched) { + write_sink(std::format("[rocjitsu:perfsim] configured dispatch_name '{}' matched no " + "dispatches\n", + *config.dispatch_name)); + } + if (initialized) invoke_foreign_abi(api.on_shutdown); deactivate_host_log(); shutdown_called = true; } + static bool supported(const DispatchState &state) { + return state.supported.load(std::memory_order_acquire); + } + + static bool mark_rejected(DispatchState &state, std::string reason) { + std::lock_guard lock(state.rejection_mutex); + if (!state.supported.load(std::memory_order_relaxed)) + return false; + state.rejection_reason = std::move(reason); + state.supported.store(false, std::memory_order_release); + return true; + } + void reject(uint32_t dispatch_id, std::string reason) { auto &state = dispatches[dispatch_id]; - if (!state.supported) + if (!mark_rejected(state, std::move(reason))) return; - state.supported = false; - state.rejection_reason = std::move(reason); replay_blockers.erase(dispatch_id); purge_events(dispatch_id); drain_epoch(/*shutdown=*/false); } + void reject_wave(PerfsimWavefrontState *wave, uint32_t dispatch_id, std::string reason) { + (void)wave; + reject(dispatch_id, std::move(reason)); + } + DispatchState *active_dispatch(uint32_t dispatch_id) { auto iter = dispatches.find(dispatch_id); - if (iter == dispatches.end() || !iter->second.supported || !iter->second.begun || - iter->second.ended) + if (iter == dispatches.end() || !iter->second.selected || !supported(iter->second) || + !iter->second.begun || iter->second.ended) return nullptr; return &iter->second; } + bool intentionally_unselected(uint32_t dispatch_id) const { + const auto iter = dispatches.find(dispatch_id); + return iter != dispatches.end() && iter->second.metadata_seen && !iter->second.selected; + } + + bool observe_workgroup(DispatchState &dispatch, uint32_t workgroup_id) const { + if (!config.max_observed_workgroups) + return true; + if (dispatch.observed_workgroups.contains(workgroup_id)) + return true; + if (dispatch.observed_workgroups.size() >= *config.max_observed_workgroups) + return false; + dispatch.observed_workgroups.insert(workgroup_id); + return true; + } + + bool intentionally_unobserved(uint32_t dispatch_id, uint32_t workgroup_id) const { + if (!config.max_observed_workgroups) + return false; + const auto iter = dispatches.find(dispatch_id); + return iter != dispatches.end() && iter->second.metadata_seen && iter->second.selected && + !iter->second.observed_workgroups.contains(workgroup_id); + } + PerfsimWavefrontState *find_wave(uint32_t compute_unit_id, uint32_t wavefront_id) { const auto iter = physical_waves.find(physical_wave_key(compute_unit_id, wavefront_id)); return iter == physical_waves.end() ? nullptr : iter->second; @@ -616,7 +761,7 @@ struct PerfsimPlugin::Impl { bool can_stage(uint32_t dispatch_id, size_t bytes) { const auto iter = dispatches.find(dispatch_id); - if (iter == dispatches.end() || !iter->second.supported) + if (iter == dispatches.end() || !iter->second.selected || !supported(iter->second)) return false; if (bytes > config.max_staged_bytes || staged_bytes > config.max_staged_bytes - bytes) { reject(dispatch_id, @@ -734,21 +879,25 @@ struct PerfsimPlugin::Impl { staged_bytes -= staged.dynamic_bytes; OrderedEvent &event = staged.event; const auto iter = dispatches.find(event.dispatch_id); - if (iter != dispatches.end() && iter->second.supported && iter->second.ended) + if (iter != dispatches.end() && iter->second.selected && supported(iter->second) && + iter->second.ended) replay(event); } } assert(staged_bytes == 0); for (auto &[dispatch_id, state] : dispatches) { - if (!state.supported && !state.diagnostic_emitted) { + if (!supported(state) && !state.diagnostic_emitted) { write_sink(std::format("[rocjitsu:perfsim] skipped dispatch {}: {}\n", dispatch_id, state.rejection_reason)); state.diagnostic_emitted = true; } } for (auto iter = dispatches.begin(); iter != dispatches.end();) { - if (shutdown || iter->second.ended) + // Subscribed wave state caches a pointer to its dispatch entry. A + // malformed execution-end callback can arrive while waves are still + // live, so retain that entry until every such wave has been retired. + if (shutdown || (iter->second.ended && iter->second.live_waves == 0)) iter = dispatches.erase(iter); else ++iter; @@ -785,7 +934,8 @@ struct PerfsimPlugin::Impl { if ((lane_mask & (uint64_t{1} << lane)) == 0) continue; if (addresses[lane] < wave.lds_base) { - reject(access.dispatch_id, "LDS address precedes the workgroup allocation base"); + reject_wave(&wave, access.dispatch_id, + "LDS address precedes the workgroup allocation base"); return false; } local_addresses[lane] = addresses[lane] - wave.lds_base; @@ -794,7 +944,14 @@ struct PerfsimPlugin::Impl { return true; } - void memory_access(const amdgpu::MemoryAccessObservation &access) { + void memory_access(const amdgpu::MemoryAccessObservation &access, + PerfsimWavefrontState *wave_hint = nullptr) { + if (!wave_hint) { + if (intentionally_unselected(access.dispatch_id)) + return; + if (intentionally_unobserved(access.dispatch_id, access.workgroup_id)) + return; + } // Current FFM does not emit ON_MEMORY_ACCESS for scalar memory. if (access.route == amdgpu::MemoryRoute::SCALAR) return; @@ -804,30 +961,32 @@ struct PerfsimPlugin::Impl { if (ffm_omits_memory_access(access.mnemonic)) return; - DispatchState *dispatch = active_dispatch(access.dispatch_id); - if (!dispatch) { - reject(access.dispatch_id, "memory callback arrived outside an active dispatch"); + PerfsimWavefrontState *wave = + wave_hint ? wave_hint : find_wave(access.compute_unit_id, access.wavefront_id); + DispatchState *dispatch = wave ? wave->dispatch_state : active_dispatch(access.dispatch_id); + if (!dispatch || !supported(*dispatch)) { + if (!dispatch) + reject(access.dispatch_id, "memory callback arrived outside an active dispatch"); return; } - - PerfsimWavefrontState *wave = find_wave(access.compute_unit_id, access.wavefront_id); if (!wave || wave->wave_info.workgroup_info.cluster_info.dispatch_info.dispatch_id != access.dispatch_id || wave->rocjitsu_workgroup_id != access.workgroup_id || wave->queue_id != access.queue_id || wave->process_id != access.process_id || !wave->has_current_instruction || wave->current_pc != access.pc) { - reject(access.dispatch_id, "memory callback could not be matched to its issuing wave"); + reject_wave(wave, access.dispatch_id, + "memory callback could not be matched to its issuing wave"); return; } - if (access.decoded_space == amdgpu::DecodedMemorySpace::SCRATCH) { - reject(access.dispatch_id, "dedicated SCRATCH instruction addresses are not FFM-compatible"); + reject_wave(wave, access.dispatch_id, + "dedicated SCRATCH instruction addresses are not FFM-compatible"); return; } if (access.route == amdgpu::MemoryRoute::UNKNOWN || access.decoded_space == amdgpu::DecodedMemorySpace::UNKNOWN) { - reject(access.dispatch_id, "memory route or decoded address space is unknown"); + reject_wave(wave, access.dispatch_id, "memory route or decoded address space is unknown"); return; } if (access.wavefront_size != 32 || access.addresses.size() != access.wavefront_size || @@ -839,7 +998,8 @@ struct PerfsimPlugin::Impl { access.element_lane_masks.size() != access.elements_per_lane) || access.element_size_bytes == 0 || access.elements_per_lane == 0 || access.bytes_per_lane() > std::numeric_limits::max()) { - reject(access.dispatch_id, "memory observation has malformed dimensions or addresses"); + reject_wave(wave, access.dispatch_id, + "memory observation has malformed dimensions or addresses"); return; } @@ -856,7 +1016,7 @@ struct PerfsimPlugin::Impl { (access.flat_local_lane_mask & access.scratch_lane_mask) != 0 || (access.flat_dds_lane_mask & access.scratch_lane_mask) != 0 || (access.flat_dds_lane_mask & access.flat_local_lane_mask) != 0) { - reject(access.dispatch_id, "memory observation has inconsistent lane masks"); + reject_wave(wave, access.dispatch_id, "memory observation has inconsistent lane masks"); return; } // FFM has no per-element validity field. Its native gfx1250 VBUFFER @@ -867,7 +1027,8 @@ struct PerfsimPlugin::Impl { // preserve the direct-FFM callback contract. for (uint64_t element_mask : access.element_lane_masks) { if ((element_mask & ~access.valid_lane_mask) != 0) { - reject(access.dispatch_id, "memory observation has an invalid per-element lane mask"); + reject_wave(wave, access.dispatch_id, + "memory observation has an invalid per-element lane mask"); return; } } @@ -881,7 +1042,7 @@ struct PerfsimPlugin::Impl { if (access.route != amdgpu::MemoryRoute::LOCAL || access.normalized_to_local || access.scratch_lane_mask != 0 || access.flat_local_lane_mask != 0 || access.flat_dds_lane_mask != 0 || !access.pre_routing_addresses.empty()) { - reject(access.dispatch_id, "explicit LDS observation has inconsistent routing"); + reject_wave(wave, access.dispatch_id, "explicit LDS observation has inconsistent routing"); return; } { @@ -889,8 +1050,8 @@ struct PerfsimPlugin::Impl { if (access.mnemonic == "ds_load_tr16_b128") { callback_requests = access.architectural_exec_lane_mask; if ((callback_requests & ~requests) != 0) { - reject(access.dispatch_id, - "DS TR16 architectural EXEC is not a subset of requesting lanes"); + reject_wave(wave, access.dispatch_id, + "DS TR16 architectural EXEC is not a subset of requesting lanes"); return; } } @@ -907,7 +1068,8 @@ struct PerfsimPlugin::Impl { access.scratch_lane_mask != 0 || access.flat_local_lane_mask != 0 || access.flat_dds_lane_mask != 0 || !access.pre_routing_addresses.empty() || !access.secondary_addresses.empty()) { - reject(access.dispatch_id, "explicit global observation has inconsistent routing"); + reject_wave(wave, access.dispatch_id, + "explicit global observation has inconsistent routing"); return; } record_memory(*wave, access, requests, FFM_RESOURCE_GLOBAL, access.addresses); @@ -916,7 +1078,7 @@ struct PerfsimPlugin::Impl { case amdgpu::DecodedMemorySpace::FLAT: if (access.route != amdgpu::MemoryRoute::LOCAL && access.route != amdgpu::MemoryRoute::GLOBAL) { - reject(access.dispatch_id, "FLAT observation has an unsupported route"); + reject_wave(wave, access.dispatch_id, "FLAT observation has an unsupported route"); return; } { @@ -928,17 +1090,19 @@ struct PerfsimPlugin::Impl { if (access.route == amdgpu::MemoryRoute::LOCAL) { if (!access.normalized_to_local || !first_lane_is_shared || access.pre_routing_addresses.size() != access.wavefront_size) { - reject(access.dispatch_id, - "FLAT local route lacks consistent per-lane aperture metadata"); + reject_wave(wave, access.dispatch_id, + "FLAT local route lacks consistent per-lane aperture metadata"); return; } } else if (access.normalized_to_local || first_lane_is_shared || !access.pre_routing_addresses.empty()) { - reject(access.dispatch_id, "FLAT global route has inconsistent aperture metadata"); + reject_wave(wave, access.dispatch_id, + "FLAT global route has inconsistent aperture metadata"); return; } if (!access.secondary_addresses.empty()) { - reject(access.dispatch_id, "FLAT observation unexpectedly has secondary addresses"); + reject_wave(wave, access.dispatch_id, + "FLAT observation unexpectedly has secondary addresses"); return; } @@ -946,14 +1110,15 @@ struct PerfsimPlugin::Impl { access.pre_routing_addresses.empty() ? access.addresses : access.pre_routing_addresses; const uint64_t dds_requests = requests & access.flat_dds_lane_mask; if (dds_requests != 0 && (!access.is_load || access.atomic_op != amdgpu::AtomicOp::NONE)) { - reject(access.dispatch_id, "FLAT DDS store or atomic is not FFM-compatible"); + reject_wave(wave, access.dispatch_id, "FLAT DDS store or atomic is not FFM-compatible"); return; } const uint64_t local_requests = requests & flat_shared_lane_mask; const uint64_t scratch_requests = requests & access.scratch_lane_mask; const uint64_t global_requests = requests & ~(local_requests | scratch_requests); if (access.atomic_op != amdgpu::AtomicOp::NONE && scratch_requests != 0) { - reject(access.dispatch_id, "FLAT atomic resolving to scratch is not FFM-compatible"); + reject_wave(wave, access.dispatch_id, + "FLAT atomic resolving to scratch is not FFM-compatible"); return; } @@ -970,11 +1135,16 @@ struct PerfsimPlugin::Impl { add_record(global_requests, FFM_RESOURCE_GLOBAL); add_record(scratch_requests, FFM_RESOURCE_SCRATCH); add_record(local_requests, FFM_RESOURCE_LDS); - std::ranges::sort(records.begin(), records.begin() + record_count, - [](const ResourceRecord &lhs, const ResourceRecord &rhs) { - return std::countr_zero(lhs.lane_mask) < - std::countr_zero(rhs.lane_mask); - }); + const auto swap_if_out_of_order = [&](size_t lhs, size_t rhs) { + if (std::countr_zero(records[lhs].lane_mask) > std::countr_zero(records[rhs].lane_mask)) + std::swap(records[lhs], records[rhs]); + }; + if (record_count >= 2) + swap_if_out_of_order(0, 1); + if (record_count == 3) { + swap_if_out_of_order(1, 2); + swap_if_out_of_order(0, 1); + } for (size_t i = 0; i < record_count; ++i) record_memory(*wave, access, records[i].lane_mask, records[i].resource, callback_addresses); @@ -986,27 +1156,36 @@ struct PerfsimPlugin::Impl { case amdgpu::DecodedMemorySpace::UNKNOWN: break; } - reject(access.dispatch_id, "unsupported memory observation"); + reject_wave(wave, access.dispatch_id, "unsupported memory observation"); } - void tensor_dma_memory_access(const amdgpu::TensorDmaMemoryAccessObservation &access) { - DispatchState *dispatch = active_dispatch(access.dispatch_id); - if (!dispatch) { - reject(access.dispatch_id, "tensor-DMA callback arrived outside an active dispatch"); + void tensor_dma_memory_access(const amdgpu::TensorDmaMemoryAccessObservation &access, + PerfsimWavefrontState *wave_hint = nullptr) { + if (!wave_hint) { + if (intentionally_unselected(access.dispatch_id)) + return; + if (intentionally_unobserved(access.dispatch_id, access.workgroup_id)) + return; + } + PerfsimWavefrontState *wave = + wave_hint ? wave_hint : find_wave(access.compute_unit_id, access.wavefront_id); + DispatchState *dispatch = wave ? wave->dispatch_state : active_dispatch(access.dispatch_id); + if (!dispatch || !supported(*dispatch)) { + if (!dispatch) + reject(access.dispatch_id, "tensor-DMA callback arrived outside an active dispatch"); return; } - PerfsimWavefrontState *wave = find_wave(access.compute_unit_id, access.wavefront_id); if (!wave || wave->wave_info.workgroup_info.cluster_info.dispatch_info.dispatch_id != access.dispatch_id || wave->rocjitsu_workgroup_id != access.workgroup_id || wave->queue_id != access.queue_id || wave->process_id != access.process_id || !wave->has_current_instruction || wave->current_pc != access.pc) { - reject(access.dispatch_id, "tensor-DMA callback could not be matched to its issuing wave"); + reject_wave(wave, access.dispatch_id, + "tensor-DMA callback could not be matched to its issuing wave"); return; } - const bool is_load = access.mnemonic == "tensor_load_to_lds"; const bool is_store = access.mnemonic == "tensor_store_from_lds"; if ((!is_load && !is_store) || access.is_load != is_load || access.addresses.empty() || @@ -1015,12 +1194,11 @@ struct PerfsimPlugin::Impl { access.element_size_bytes != (uint32_t{1} << access.data_size) || access.tile_dim0 == 0 || access.tensor_dim0_stride < 0 || access.tensor_dim1_stride < 0 || access.addresses.size() > std::numeric_limits::max()) { - reject(access.dispatch_id, "tensor-DMA observation is malformed"); + reject_wave(wave, access.dispatch_id, "tensor-DMA observation is malformed"); return; } if (access.addresses.size() > std::numeric_limits::max() / sizeof(uint64_t)) { - reject(access.dispatch_id, - std::format("staging budget of {} bytes exceeded", config.max_staged_bytes)); + reject_wave(wave, access.dispatch_id, "tensor-DMA address payload is too large"); return; } if (!can_stage_event(access.dispatch_id, access.addresses.size() * sizeof(uint64_t))) { @@ -1034,7 +1212,7 @@ struct PerfsimPlugin::Impl { event.addresses = std::make_unique_for_overwrite(event.num_addresses); if (!access.addresses.copy_to( std::span(event.addresses.get(), event.num_addresses))) { - reject(access.dispatch_id, "tensor-DMA observation is malformed"); + reject_wave(wave, access.dispatch_id, "tensor-DMA observation is malformed"); return; } event.data_size_bytes = access.element_size_bytes; @@ -1056,6 +1234,7 @@ struct PerfsimPlugin::Impl { uint32_t negotiated_api_version = 0; bool initialized = false; bool shutdown_called = false; + bool dispatch_name_matched = false; std::unordered_map dispatches; // Only supported in-flight dispatches block ordered replay. Rejected // dispatch state remains until its real end callback, but it retains no @@ -1075,6 +1254,12 @@ void PerfsimPlugin::onInit() { impl_->init(); } void PerfsimPlugin::onShutdown() { impl_->shutdown(); } +bool PerfsimPlugin::requires_serial_hot_hooks() const { return true; } + +bool PerfsimPlugin::observes_hot_hooks_for_wavefront(const amdgpu::Wavefront *wf) const { + return wf != nullptr && wf->has_plugin_state(slot_index()); +} + void PerfsimPlugin::onAmdgpuDispatchPacketProcessed(const KernelDispatchInfo &info) { auto [iter, inserted] = impl_->dispatches.try_emplace(info.dispatch_id); DispatchState &state = iter->second; @@ -1086,6 +1271,11 @@ void PerfsimPlugin::onAmdgpuDispatchPacketProcessed(const KernelDispatchInfo &in state.dispatch_name = info.kernelNameOrUnknown(); state.metadata.dispatch_name = state.dispatch_name.c_str(); state.metadata_seen = true; + state.selected = + !impl_->config.dispatch_name || state.dispatch_name == *impl_->config.dispatch_name; + if (!state.selected) + return; + impl_->dispatch_name_matched = true; if (const auto reason = validate_dispatch(info)) impl_->reject(info.dispatch_id, *reason); } @@ -1100,7 +1290,7 @@ void PerfsimPlugin::onAmdgpuDispatchExecutionBegin(uint32_t dispatch_id) { } state.begun = true; state.ended = false; - if (!state.supported) + if (!state.selected || !Impl::supported(state)) return; impl_->replay_blockers.insert(dispatch_id); impl_->record_event({dispatch_id, BeginEvent{state.metadata}}); @@ -1117,16 +1307,53 @@ void PerfsimPlugin::onAmdgpuDispatchExecutionEnd(uint32_t dispatch_id) { impl_->reject(dispatch_id, "dispatch ended more than once"); return; } - if (state.live_waves != 0) + if (state.selected && state.live_waves != 0) impl_->reject(dispatch_id, "dispatch ended with live wavefronts"); state.ended = true; impl_->replay_blockers.erase(dispatch_id); - if (state.supported) + if (state.selected && Impl::supported(state)) impl_->record_event({dispatch_id, EndEvent{state.metadata}}); impl_->drain_epoch(/*shutdown=*/false); } void PerfsimPlugin::onAmdgpuWavefrontDispatched(amdgpu::Wavefront &wf) { + const uint32_t compute_unit_id = static_cast(wf.cu().id()); + const uint64_t physical_key = physical_wave_key(compute_unit_id, wf.wf_id()); + + // Wavefront slots are reused and plugin state deliberately survives the + // generic Wavefront::reset(). A terminal dispatch fault resets the wave + // without a halt callback, so retire that incarnation while its state still + // owns the raw pointer recorded in physical_waves. + if (wf.has_plugin_state(slot_index())) { + auto *previous = static_cast(wf.plugin_state(slot_index())); + const uint32_t previous_dispatch = static_cast( + previous->wave_info.workgroup_info.cluster_info.dispatch_info.dispatch_id); + const auto existing = impl_->physical_waves.find(physical_key); + if (existing != impl_->physical_waves.end() && existing->second != previous) { + impl_->physical_waves.erase(existing); + wf.clear_plugin_state(slot_index()); + impl_->reject(wf.dispatch_id(), "physical wavefront slot retained mismatched state"); + return; + } + if (existing != impl_->physical_waves.end()) + impl_->physical_waves.erase(existing); + if (const auto dispatch = impl_->dispatches.find(previous_dispatch); + dispatch != impl_->dispatches.end()) { + if (dispatch->second.live_waves != 0) + --dispatch->second.live_waves; + impl_->reject(previous_dispatch, "wavefront slot was reset without a halt callback"); + } + wf.clear_plugin_state(slot_index()); + } else if (const auto existing = impl_->physical_waves.find(physical_key); + existing != impl_->physical_waves.end()) { + // Never dereference a map entry after its owning Wavefront state has gone. + impl_->physical_waves.erase(existing); + impl_->reject(wf.dispatch_id(), "physical wavefront slot retained orphaned state"); + return; + } + + if (impl_->intentionally_unselected(wf.dispatch_id())) + return; DispatchState *dispatch = impl_->active_dispatch(wf.dispatch_id()); if (!dispatch) { impl_->reject(wf.dispatch_id(), "wavefront dispatched outside an active dispatch"); @@ -1137,20 +1364,11 @@ void PerfsimPlugin::onAmdgpuWavefrontDispatched(amdgpu::Wavefront &wf) { return; } - const uint32_t compute_unit_id = static_cast(wf.cu().id()); - const uint64_t physical_key = physical_wave_key(compute_unit_id, wf.wf_id()); - if (const auto existing = impl_->physical_waves.find(physical_key); - existing != impl_->physical_waves.end()) { - const uint32_t old_dispatch = static_cast( - existing->second->wave_info.workgroup_info.cluster_info.dispatch_info.dispatch_id); - impl_->reject(old_dispatch, "physical wavefront slot was reused before halt"); - impl_->reject(wf.dispatch_id(), "physical wavefront slot was reused before halt"); - impl_->physical_waves.erase(existing); - return; - } - auto state = std::make_unique(); + state->dispatch_state = dispatch; state->wave_info = make_wave_info(wf.dispatch_id(), wf.wg_coord(), wf.wave_in_group()); + if (!impl_->observe_workgroup(*dispatch, wf.wg_id())) + return; state->compute_unit_id = compute_unit_id; state->physical_wavefront_id = wf.wf_id(); state->rocjitsu_workgroup_id = wf.wg_id(); @@ -1165,6 +1383,10 @@ void PerfsimPlugin::onAmdgpuWavefrontDispatched(amdgpu::Wavefront &wf) { } void PerfsimPlugin::onAmdgpuWavefrontHalted(amdgpu::Wavefront &wf) { + if (impl_->intentionally_unselected(wf.dispatch_id())) + return; + if (impl_->intentionally_unobserved(wf.dispatch_id(), wf.wg_id())) + return; const uint32_t compute_unit_id = static_cast(wf.cu().id()); const uint64_t physical_key = physical_wave_key(compute_unit_id, wf.wf_id()); const auto wave_iter = impl_->physical_waves.find(physical_key); @@ -1176,6 +1398,7 @@ void PerfsimPlugin::onAmdgpuWavefrontHalted(amdgpu::Wavefront &wf) { PerfsimWavefrontState &wave = *wave_iter->second; const uint32_t dispatch_id = static_cast(wave.wave_info.workgroup_info.cluster_info.dispatch_info.dispatch_id); + bool retired_ended_dispatch = false; auto dispatch_iter = impl_->dispatches.find(dispatch_id); if (dispatch_iter != impl_->dispatches.end()) { DispatchState &dispatch = dispatch_iter->second; @@ -1185,8 +1408,12 @@ void PerfsimPlugin::onAmdgpuWavefrontHalted(amdgpu::Wavefront &wf) { impl_->reject(dispatch_id, "wavefront halted without a terminating instruction"); if (dispatch.live_waves != 0) --dispatch.live_waves; + retired_ended_dispatch = dispatch.ended && dispatch.live_waves == 0; } impl_->physical_waves.erase(wave_iter); + wf.clear_plugin_state(slot_index()); + if (retired_ended_dispatch) + impl_->drain_epoch(/*shutdown=*/false); } void PerfsimPlugin::onAmdgpuBeforeExecuteInstruction(uint64_t pc, const Instruction &inst, @@ -1202,33 +1429,39 @@ void PerfsimPlugin::onAmdgpuBeforeExecuteInstruction(uint64_t pc, const Instruct void PerfsimPlugin::record_instruction(uint64_t pc, const Instruction &inst, amdgpu::Wavefront &wf, std::span fetch_window) { - const auto dispatch_iter = impl_->dispatches.find(wf.dispatch_id()); - if (dispatch_iter != impl_->dispatches.end() && !dispatch_iter->second.supported) + PerfsimWavefrontState *wave = + wf.has_plugin_state(slot_index()) + ? static_cast(wf.plugin_state(slot_index())) + : nullptr; + if (!wave) { + const auto dispatch_iter = impl_->dispatches.find(wf.dispatch_id()); + if (dispatch_iter != impl_->dispatches.end() && + (!dispatch_iter->second.selected || !Impl::supported(dispatch_iter->second))) + return; + if (impl_->intentionally_unobserved(wf.dispatch_id(), wf.wg_id())) + return; + const uint32_t compute_unit_id = static_cast(wf.cu().id()); + wave = impl_->find_wave(compute_unit_id, wf.wf_id()); + } + if (wave && wave->dispatch_state && !Impl::supported(*wave->dispatch_state)) return; - - const uint32_t compute_unit_id = static_cast(wf.cu().id()); - PerfsimWavefrontState *wave = impl_->find_wave(compute_unit_id, wf.wf_id()); if (!wave || wave->wave_info.workgroup_info.cluster_info.dispatch_info.dispatch_id != wf.dispatch_id() || wave->rocjitsu_workgroup_id != wf.wg_id()) { - impl_->reject(wf.dispatch_id(), "instruction callback could not be matched to its wavefront"); + impl_->reject_wave(wave, wf.dispatch_id(), + "instruction callback could not be matched to its wavefront"); return; } - const int encoding_bytes = inst.size(); if (encoding_bytes <= 0 || encoding_bytes % 4 != 0 || encoding_bytes > 16 || !inst.raw_encoding()) { - impl_->reject(wf.dispatch_id(), - std::format("instruction encoding is not representable by FFM v{}", - impl_->negotiated_api_version)); - return; - } - if (inst.mnemonic().starts_with("scratch_")) { - impl_->reject(wf.dispatch_id(), "dedicated SCRATCH instructions are not FFM-compatible"); + impl_->reject_wave(wave, wf.dispatch_id(), + std::format("instruction encoding is not representable by FFM v{}", + impl_->negotiated_api_version)); return; } if (!fetch_window.empty() && fetch_window.size() != 4) { - impl_->reject(wf.dispatch_id(), "instruction fetch window is not four dwords"); + impl_->reject_wave(wave, wf.dispatch_id(), "instruction fetch window is not four dwords"); return; } @@ -1238,12 +1471,19 @@ void PerfsimPlugin::record_instruction(uint64_t pc, const Instruction &inst, amd const uint32_t encoding_dwords = static_cast(encoding_bytes / 4); std::array instruction_encoding{}; std::ranges::copy_n(inst.raw_encoding(), encoding_dwords, instruction_encoding.begin()); + const StaticInstructionInfo &static_info = + classify_instruction(pc, inst, instruction_encoding, encoding_dwords); + if (static_info.scratch_incompatible) { + impl_->reject_wave(wave, wf.dispatch_id(), + "dedicated SCRATCH instructions are not FFM-compatible"); + return; + } if (fetch_window.empty()) event.raw_isa = instruction_encoding; else std::ranges::copy(fetch_window, event.raw_isa.begin()); - event.counters = instruction_counters(inst); - event.wait = wait_info(inst); + event.counters = static_info.counters; + event.wait = static_info.wait; // FFM decrements its per-wave ordinal when a wait yields and reissues at the // same PC. RocJITsu's hook has no source ordinal, but a consecutive callback @@ -1265,7 +1505,7 @@ void PerfsimPlugin::record_instruction(uint64_t pc, const Instruction &inst, amd wave->current_wait = event.wait; wave->has_current_instruction = true; wave->saw_instruction = true; - wave->last_instruction_terminates = (inst.flags() & PROGRAM_TERMINATOR) != 0; + wave->last_instruction_terminates = static_info.terminates; impl_->record_event({wf.dispatch_id(), std::move(event)}); } @@ -1273,9 +1513,25 @@ void PerfsimPlugin::onAmdgpuMemoryAccessRouted(const amdgpu::MemoryAccessObserva impl_->memory_access(access); } +void PerfsimPlugin::onAmdgpuMemoryAccessRouted(const amdgpu::MemoryAccessObservation &access, + const amdgpu::Wavefront &wf) { + auto *state = wf.has_plugin_state(slot_index()) + ? static_cast(wf.plugin_state(slot_index())) + : nullptr; + impl_->memory_access(access, state); +} + void PerfsimPlugin::onAmdgpuTensorDmaMemoryAccess( const amdgpu::TensorDmaMemoryAccessObservation &access) { impl_->tensor_dma_memory_access(access); } +void PerfsimPlugin::onAmdgpuTensorDmaMemoryAccess( + const amdgpu::TensorDmaMemoryAccessObservation &access, const amdgpu::Wavefront &wf) { + auto *state = wf.has_plugin_state(slot_index()) + ? static_cast(wf.plugin_state(slot_index())) + : nullptr; + impl_->tensor_dma_memory_access(access, state); +} + } // namespace rocjitsu::plugins::perfsim diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin.h index 5b03f8ad490f..357c14dc530c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin.h @@ -15,15 +15,23 @@ namespace rocjitsu::plugins::perfsim { class PerfsimPlugin final : public ExecutionPlugin { public: /// @param config_json Resolved plugin configuration containing the required - /// string field `library_path` and optional positive integer - /// `max_staged_bytes`. + /// non-empty string `library_path`; optional positive integers + /// `max_staged_bytes` and `max_observed_workgroups`; and the optional + /// non-empty string `dispatch_name`. explicit PerfsimPlugin(const char *config_json); ~PerfsimPlugin() override; PerfsimPlugin(const PerfsimPlugin &) = delete; PerfsimPlugin &operator=(const PerfsimPlugin &) = delete; - bool requires_serial_hot_hooks() const override { return true; } + bool requires_serial_hot_hooks() const override; + bool observes_hot_hooks_for_wavefront(const amdgpu::Wavefront *wf) const override; + bool observes_after_execute_instruction() const override { return false; } + bool observes_async_instruction_issued() const override { return false; } + bool observes_memory_instruction_routing() const override { return false; } + bool observes_vgpr_reads() const override { return false; } + bool observes_vgpr_writes() const override { return false; } + bool observes_scalar_register_writes() const override { return false; } bool observes_memory_routing() const override { return true; } bool observes_tensor_dma_memory_access() const override { return true; } bool observes_sgpr_reads() const override { return false; } @@ -40,8 +48,12 @@ class PerfsimPlugin final : public ExecutionPlugin { void onAmdgpuBeforeExecuteInstruction(uint64_t pc, const Instruction &inst, amdgpu::Wavefront &wf, std::span fetch_window) override; void onAmdgpuMemoryAccessRouted(const amdgpu::MemoryAccessObservation &access) override; + void onAmdgpuMemoryAccessRouted(const amdgpu::MemoryAccessObservation &access, + const amdgpu::Wavefront &wf) override; void onAmdgpuTensorDmaMemoryAccess(const amdgpu::TensorDmaMemoryAccessObservation &access) override; + void onAmdgpuTensorDmaMemoryAccess(const amdgpu::TensorDmaMemoryAccessObservation &access, + const amdgpu::Wavefront &wf) override; private: void record_instruction(uint64_t pc, const Instruction &inst, amdgpu::Wavefront &wf, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin_export.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin_export.cpp index fae35ababc0c..ff56ca70c7bc 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin_export.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/perfsim/plugin_export.cpp @@ -12,4 +12,4 @@ ROCJITSU_DEFINE_PLUGIN( rocjitsu::plugins::perfsim::PerfsimPlugin, "perfsim", - R"({"library_path":{"type":"string","description":"Absolute path to the separately supplied Perfsim backend shared object, such as libgpucsim_ffm_plugin.so"},"max_staged_bytes":{"type":"number","description":"Maximum charged bytes for staged event records","default":268435456}})") + R"({"library_path":{"type":"string","description":"Absolute path to the separately supplied Perfsim backend shared object, such as libgpucsim_ffm_plugin.so"},"max_staged_bytes":{"type":"number","description":"Maximum charged bytes for staged event records","default":268435456},"dispatch_name":{"type":"string","description":"Optional exact RocJITsu normalized display name shown in quotes in the VM dispatch log; nonmatching dispatches still execute functionally but are not staged or replayed to Perfsim","optional":true},"max_observed_workgroups":{"type":"number","description":"Optional diagnostic cap admitting the first distinct RocJITsu workgroup IDs encountered in wave-dispatch order for each selected dispatch","optional":true}})") diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/plugin_config_resolver.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/plugin_config_resolver.cpp index 6b259eb66d00..d64344a41479 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/plugin_config_resolver.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/plugin_config_resolver.cpp @@ -142,6 +142,9 @@ bool resolve_config(const std::string &plugin_name, const char *schema_json, return false; } emit(arg, def); + } else if (const auto optional = spec.AsMap()["optional"]; + optional.IsBool() && optional.AsBool()) { + continue; } else { util::Logger::warn("plugin '", plugin_name, "': missing required config arg '", arg, "'"); return false; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/event_registry.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/event_registry.h index 599dd50b6b82..012cc89fd557 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/event_registry.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/event_registry.h @@ -118,6 +118,15 @@ class EventRegistry { } return std::nullopt; } + uint8_t pendingCounterIncrement(EventId id, amdgpu::WaitCounterType waitType) const { + const auto &event = entries_[index(id)]; + for (uint8_t i = 0; i < event.numCounterObligations; ++i) { + if ((event.satisfiedCounterMask & (uint8_t{1} << i)) == 0 && + amdgpu::wait_counter_covers(waitType, event.counterObligations[i].wait_counter_type())) + return event.counterObligations[i].counter_increment(); + } + return 0; + } bool satisfyWaitCounter(EventId id, amdgpu::WaitCounterType waitType) { auto &event = entries_[index(id)]; assert(event.status == EventStatus::ACTIVE); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/race_detector.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/race_detector.cpp index 9602d5997bd5..e23bf060d99a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/race_detector.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/race_detector.cpp @@ -9,11 +9,12 @@ namespace rocjitsu::plugins::race_detector { RaceDetector::RaceDetector(int nWaves, int vgprCount, int sgprCount, Dim3d workgroupId, - std::function raceHandler) + std::function raceHandler, + CounterCapacities counterCapacities) : workgroupId(workgroupId), raceHandler(std::move(raceHandler)) { waveRaceStates.reserve(nWaves); for (int i = 0; i < nWaves; ++i) { - waveRaceStates.emplace_back(vgprCount, sgprCount, WaveId{i}, this); + waveRaceStates.emplace_back(vgprCount, sgprCount, WaveId{i}, this, counterCapacities); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/race_detector.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/race_detector.h index 04cf4e28586a..efae73770f32 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/race_detector.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/race_detector.h @@ -21,8 +21,9 @@ namespace rocjitsu::plugins::race_detector { /// /// Event lifecycle: /// 1. allocateEventId() — registers a new event (ACTIVE). -/// 2. markEventWaveComplete() — transitions to WAVE_COMPLETE after all of -/// the event's wait-counter obligations have completed. +/// 2. markEventWaveComplete() — transitions to WAVE_COMPLETE after a wait or +/// ordered counter-capacity backpressure satisfies all of the event's +/// wait-counter obligations. /// 3. retireEvent() — removes from live lists, decrements byte counts, /// marks RETIRED (for LDS events, called at s_barrier via /// flushBarrierPendingEvents). @@ -37,7 +38,7 @@ class RaceDetector { public: RaceDetector(int nWaves, int vgprCount, int sgprCount, Dim3d workgroupId, - std::function raceHandler); + std::function raceHandler, CounterCapacities); /// Allocate an event using the counter and ordering implied by its type. EventId allocateEventId(WaveId wave, uint64_t pc, MemoryEventType type, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/wave_race_state.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/wave_race_state.cpp index 0f314558c56c..c5557ac425af 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/wave_race_state.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/wave_race_state.cpp @@ -2,11 +2,13 @@ // SPDX-License-Identifier: MIT #include "rocjitsu/vm/plugins/race_detector/core/wave_race_state.h" +#include "rocjitsu/vm/amdgpu/wait_counters.h" #include "rocjitsu/vm/plugins/race_detector/core/interval_set.h" #include "rocjitsu/vm/plugins/race_detector/core/race_detector.h" #include #include #include +#include namespace rocjitsu::plugins::race_detector { namespace { @@ -18,8 +20,9 @@ struct ProfileScope { }; } // namespace -WaveRaceState::WaveRaceState(int vgprCount, int sgprCount, WaveId waveId, RaceDetector *detector) - : waveId(waveId), detector(detector) { +WaveRaceState::WaveRaceState(int vgprCount, int sgprCount, WaveId waveId, RaceDetector *detector, + CounterCapacities counterCapacities) + : counterCapacities(counterCapacities), waveId(waveId), detector(detector) { vgprMemoryEvents.resize(vgprCount); sgprMemoryEvents.resize(sgprCount); sgprEventCount.resize(sgprCount, 0); @@ -138,6 +141,63 @@ void WaveRaceState::registerEventWithIntervals( waveMemoryEvents.push_back(eventId); } +void WaveRaceState::prepareForCounterIncrement(amdgpu::WaitCounterType type, uint8_t increment) { + // The runtime plugin calls this only for pre-GFX12 capacity models, where + // these internal split names alias the combined architectural counters. + if (amdgpu::wait_counter_covers(amdgpu::WaitCounterType::VMCNT, type)) + type = amdgpu::WaitCounterType::VMCNT; + else if (amdgpu::wait_counter_covers(amdgpu::WaitCounterType::LGKMCNT, type)) + type = amdgpu::WaitCounterType::LGKMCNT; + + int capacity; + switch (type) { + case amdgpu::WaitCounterType::VMCNT: + capacity = counterCapacities.vmcnt; + break; + case amdgpu::WaitCounterType::LGKMCNT: + capacity = counterCapacities.lgkmcnt; + break; + case amdgpu::WaitCounterType::EXPCNT: + case amdgpu::WaitCounterType::VSCNT: + case amdgpu::WaitCounterType::LOADCNT: + case amdgpu::WaitCounterType::STORECNT: + case amdgpu::WaitCounterType::DSCNT: + case amdgpu::WaitCounterType::KMCNT: + case amdgpu::WaitCounterType::TENSORCNT: + case amdgpu::WaitCounterType::ASYNCCNT: + return; + default: + assert(false && "invalid wait counter type"); + return; + } + if (capacity <= 0) + throw std::logic_error("counter capacity is not configured"); + assert(increment > 0 && increment <= capacity && + "counter increment is outside the configured capacity"); + if (increment == 0 || increment > capacity) + return; + + int pending_tokens = 0; + for (const EventId event_id : waveMemoryEvents) + pending_tokens += detector->events().pendingCounterIncrement(event_id, type); + if (pending_tokens + increment <= capacity) + return; + + // An instruction cannot issue if adding its counter token would overflow the + // finite counter. Once participation is known, admission establishes the + // same upper bound as an implicit partial wait. + applyCounterConstraint(type, capacity - increment, /*includeUnordered=*/false); +} + +void WaveRaceState::prepareForMemoryIssue(const amdgpu::MemoryIssueInfo &info) { + for (const auto obligation : info.counter_obligations()) { + const auto counter = obligation.wait_counter_type(); + if (amdgpu::wait_counter_covers(amdgpu::WaitCounterType::VMCNT, counter) || + amdgpu::wait_counter_covers(amdgpu::WaitCounterType::LGKMCNT, counter)) + prepareForCounterIncrement(counter, obligation.counter_increment()); + } +} + void WaveRaceState::registerLdsEvent(uint64_t pc, MemoryEventType type, std::vector registers, uint64_t execMask, int waveSize, std::span laneBaseAddresses, @@ -266,7 +326,7 @@ void WaveRaceState::satisfyOldestCounterObligations(int count, amdgpu::WaitCount const EventId eventId = waveMemoryEvents[read]; bool eventComplete = false; if (matches(eventId) && satisfied < count) { - ++satisfied; + satisfied += detector->events().pendingCounterIncrement(eventId, waitCounter); eventComplete = detector->satisfyEventWaitCounter(eventId, waitCounter); if (eventComplete) { retireEventRegisters(eventId); @@ -289,10 +349,16 @@ void WaveRaceState::applyCounterConstraint(amdgpu::WaitCounterType type, int max auto targetsCounter = [&](EventId eventId) { return detector->events().hasPendingWaitCounter(eventId, type); }; + auto pending_tokens = [&](auto matches) { + int total = 0; + for (const EventId event_id : waveMemoryEvents) { + if (matches(event_id)) + total += detector->events().pendingCounterIncrement(event_id, type); + } + return total; + }; if (includeUnordered) { - satisfyOldestCounterObligations( - static_cast(std::ranges::count_if(waveMemoryEvents, targetsCounter)), type, - targetsCounter); + satisfyOldestCounterObligations(pending_tokens(targetsCounter), type, targetsCounter); return; } @@ -305,13 +371,40 @@ void WaveRaceState::applyCounterConstraint(amdgpu::WaitCounterType type, int max auto belongsToOrder = [&](EventId eventId) { return detector->events().pendingCompletionClass(eventId, type) == order; }; - const int pending = static_cast(std::ranges::count_if(waveMemoryEvents, belongsToOrder)); + const int pending = pending_tokens(belongsToOrder); satisfyOldestCounterObligations(pending - maximumRemaining, type, belongsToOrder); } } +int WaveRaceState::doNotWaitValue(amdgpu::WaitCounterType type) const { + switch (type) { + case amdgpu::WaitCounterType::VMCNT: + return counterCapacities.vmcnt; + case amdgpu::WaitCounterType::LGKMCNT: + return counterCapacities.lgkmcnt; + case amdgpu::WaitCounterType::EXPCNT: + return amdgpu::WaitCounters::EXPCNT_MAX; + case amdgpu::WaitCounterType::VSCNT: + return amdgpu::WaitCounters::VSCNT_MAX; + case amdgpu::WaitCounterType::LOADCNT: + return amdgpu::WaitCounters::VMCNT_MAX; + case amdgpu::WaitCounterType::STORECNT: + return amdgpu::WaitCounters::VSCNT_MAX; + case amdgpu::WaitCounterType::DSCNT: + return amdgpu::WaitCounters::DSCNT_MAX; + case amdgpu::WaitCounterType::KMCNT: + return amdgpu::WaitCounters::KMCNT_MAX; + case amdgpu::WaitCounterType::TENSORCNT: + return amdgpu::WaitCounters::TENSORCNT_MAX; + case amdgpu::WaitCounterType::ASYNCCNT: + return amdgpu::WaitCounters::ASYNCCNT_MAX; + } + assert(false && "invalid wait counter type"); + return -1; +} + void WaveRaceState::applyWaitCounter(amdgpu::WaitCounterType type, int threshold) { - if (threshold < 0) + if (threshold < 0 || threshold == doNotWaitValue(type)) return; if (threshold == 0) { diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/wave_race_state.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/wave_race_state.h index 2744240689e4..e9aac5aa3f04 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/wave_race_state.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/core/wave_race_state.h @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT #pragma once +#include "rocjitsu/isa/arch/amdgpu/generated/shared/isa_properties.h" #include "rocjitsu/isa/register_set.h" #include "rocjitsu/vm/plugins/race_detector/core/common_register.h" #include "rocjitsu/vm/plugins/race_detector/core/profiler_interface.h" @@ -17,13 +18,37 @@ namespace rocjitsu::plugins::race_detector { class IntervalSet; class RaceDetector; +struct CounterCapacities { + int vmcnt; + int lgkmcnt; +}; + +/// Adapt the generated architecture properties to the race core's compact +/// counter model. A zero capacity means that the architecture has no combined +/// counter of that type. +[[nodiscard]] inline constexpr CounterCapacities counterCapacitiesForArch(rj_code_arch_t arch) { + const auto properties = isa_properties(arch); + return { + .vmcnt = properties.vmcnt_capacity, + .lgkmcnt = properties.lgkmcnt_capacity, + }; +} + /// Per-wave race detection state. Owns VGPR event lists, wave-level event /// queues, and provides event registration, waitcnt resolution, and VGPR /// race checking. Holds a pointer to the workgroup level RaceDetector for /// event allocation and lifecycle transitions. class WaveRaceState { public: - WaveRaceState(int vgprCount, int sgprCount, WaveId, RaceDetector *); + WaveRaceState(int vgprCount, int sgprCount, WaveId, RaceDetector *, CounterCapacities); + + /// Apply the issue backpressure required before an instruction adds one + /// token to a finite hardware counter. Call before operand checks when the + /// counter is known statically, or after routing for address-dependent FLAT. + void prepareForCounterIncrement(amdgpu::WaitCounterType, uint8_t increment = 1); + + /// Apply backpressure to every counter incremented by a resolved issue. + void prepareForMemoryIssue(const amdgpu::MemoryIssueInfo &); /// Register an in-flight memory event that does not involve LDS. /// \param pc The PC of the instruction that produced the event. @@ -171,6 +196,7 @@ class WaveRaceState { void applyCounterConstraint(amdgpu::WaitCounterType type, int maximumRemaining, bool includeUnordered); void applyWaitCounter(amdgpu::WaitCounterType type, int threshold); + int doNotWaitValue(amdgpu::WaitCounterType type) const; void regEventCountInc(MemoryEventType type, int reg) { regEventCount[static_cast(type)][reg]++; @@ -191,6 +217,8 @@ class WaveRaceState { std::vector waveMemoryEvents; std::vector barrierPendingEvents; + CounterCapacities counterCapacities; + WaveId waveId; RaceDetector *detector; ProfilerInterface *profiler_ = &NullProfiler::instance(); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/plugin.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/plugin.cpp index 6d1654767031..d91948f05719 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/plugin.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/plugin.cpp @@ -14,10 +14,12 @@ #include "rocjitsu/vm/plugins/race_detector/core/common_register.h" #include "rocjitsu/vm/plugins/race_detector/core/wave_race_state.h" +#include #include #include #include #include +#include #include #include #include @@ -54,12 +56,18 @@ uint8_t vector_memory_byte_mask(const amdgpu::VectorMemState &state, const amdgp return ExecutionPlugin::kFullByteMask; } -MemoryOrderClass memory_order_for(const Instruction &inst) { - const auto *info = inst.amdgpu_memory_issue_info(); - assert(info && "memory instruction reached the race detector without completion metadata"); - if (!info || info->empty()) - return MemoryOrderClass::UNORDERED; - return memoryOrderForObligations(info->counter_obligations()); +bool supports_counter_capacity(rj_code_arch_t arch) { + switch (arch) { + case ROCJITSU_CODE_ARCH_CDNA1: + case ROCJITSU_CODE_ARCH_CDNA2: + case ROCJITSU_CODE_ARCH_CDNA3: + case ROCJITSU_CODE_ARCH_CDNA4: + case ROCJITSU_CODE_ARCH_RDNA3: + case ROCJITSU_CODE_ARCH_RDNA3_5: + return true; + default: + return false; + } } std::optional> @@ -84,6 +92,28 @@ validated_load_destinations(const amdgpu::VectorMemState &state, const amdgpu::W return registers; } +std::optional +routed_flat_issue_info(const amdgpu::MemoryAccessObservation &access, + const amdgpu::MemoryIssueInfo &decoded) { + if (access.decoded_space != amdgpu::DecodedMemorySpace::FLAT || + (access.route != amdgpu::MemoryRoute::GLOBAL && access.route != amdgpu::MemoryRoute::LOCAL)) + return std::nullopt; + + if (access.request_lane_mask == 0) + return decoded.for_flat_memory_domains(false, false); + + const uint64_t shared_lanes = + (access.flat_local_lane_mask | access.flat_dds_lane_mask) & access.request_lane_mask; + // Mixed-space execution still routes by the first requesting lane (#11456). + // Narrow dependencies only when every requesting lane uses the same pipeline. + if (shared_lanes != 0 && shared_lanes != access.request_lane_mask) + return decoded; + if (access.route != (shared_lanes ? amdgpu::MemoryRoute::LOCAL : amdgpu::MemoryRoute::GLOBAL)) + return std::nullopt; + + return decoded.for_flat_memory_domains(shared_lanes == 0, shared_lanes != 0); +} + } // namespace // Declared in plugin.h (used by formatTrace tests in execution_plugin_test.cpp). @@ -290,7 +320,8 @@ void RaceDetectorPlugin::onAmdgpuWorkgroupDispatched(uint32_t dispatch_id, uint3 std::lock_guard lock(dispatch_mutex_); detectors_[key] = std::make_unique( static_cast(num_waves), static_cast(physical_vgpr_count), - static_cast(physical_sgpr_count), Dim3d(static_cast(wg_id)), std::move(handler)); + static_cast(physical_sgpr_count), Dim3d(static_cast(wg_id)), std::move(handler), + counterCapacitiesForArch(wavefronts.front()->cu().arch())); auto &det = *detectors_[key]; auto &dc = dispatch_disasm_[dispatch_id]; @@ -304,14 +335,24 @@ void RaceDetectorPlugin::onAmdgpuWorkgroupDispatched(uint32_t dispatch_id, uint3 } } -void RaceDetectorPlugin::onAmdgpuMemoryAccessRouted( - const amdgpu::MemoryAccessObservation & /*access*/, const Instruction &inst, - amdgpu::Wavefront &wf) { +void RaceDetectorPlugin::onAmdgpuMemoryAccessRouted(const amdgpu::MemoryAccessObservation &access, + const Instruction &inst, + amdgpu::Wavefront &wf) { auto *s = get_state(wf); assert(s && s->race_state); auto *rs = s->race_state; auto *detector = rs->getDetector(); auto waveId = rs->getWaveId(); + const auto *decoded = inst.amdgpu_memory_issue_info(); + assert(decoded && "memory instruction reached the race detector without issue metadata"); + if (!decoded) + return; + const auto routed = routed_flat_issue_info(access, *decoded); + const auto &issue = routed ? *routed : *decoded; + const auto obligations = issue.counter_obligations(); + const MemoryOrderClass memoryOrder = memoryOrderForObligations(obligations); + if (routed && access.request_lane_mask != 0 && supports_counter_capacity(wf.cu().arch())) + rs->prepareForMemoryIssue(issue); if (inst.data()->tag() == amdgpu::LOCAL_MEM) { auto &d = *inst.data_as(); @@ -319,9 +360,6 @@ void RaceDetectorPlugin::onAmdgpuMemoryAccessRouted( if (execMask == 0) return; auto type = d.is_load ? MemoryEventType::LDS_TO_VGPR : MemoryEventType::VGPR_TO_LDS; - const MemoryOrderClass memoryOrder = memory_order_for(inst); - const auto *issue = inst.amdgpu_memory_issue_info(); - assert(issue && "memory instruction reached the race detector without issue metadata"); const uint32_t perLaneBytes = d.num_elems * d.elem_size; std::vector registers; @@ -356,6 +394,8 @@ void RaceDetectorPlugin::onAmdgpuMemoryAccessRouted( for (uint32_t reg : registers) rs->checkVgprWrite(static_cast(reg), execMask, byte_mask, memoryOrder); } + if (obligations.empty()) + return; if (d.ds2_active) { uint32_t secondLaneAddrs[64]; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) @@ -363,11 +403,11 @@ void RaceDetectorPlugin::onAmdgpuMemoryAccessRouted( rs->registerLdsEvent(wf.pc, type, std::move(registers), execMask, wf.wf_size(), std::span(laneAddrs, wf.wf_size()), std::span(secondLaneAddrs, wf.wf_size()), perLaneBytes, - byte_mask, issue->counter_obligations(), memoryOrder); + byte_mask, obligations, memoryOrder); } else { rs->registerLdsEvent(wf.pc, type, std::move(registers), execMask, wf.wf_size(), std::span(laneAddrs, wf.wf_size()), perLaneBytes, - byte_mask, issue->counter_obligations(), memoryOrder); + byte_mask, obligations, memoryOrder); } } @@ -375,10 +415,9 @@ void RaceDetectorPlugin::onAmdgpuMemoryAccessRouted( auto &d = *inst.data_as(); if (d.exec_mask == 0) return; - const auto *issue = inst.amdgpu_memory_issue_info(); - assert(issue && "memory instruction reached the race detector without issue metadata"); if (d.lds_dst) { - const MemoryOrderClass memoryOrder = memory_order_for(inst); + if (obligations.empty()) + return; uint32_t perLaneBytes = d.num_elems * d.elem_size; if (d.cluster_multicast && d.cluster_mcast_mask != 0) { uint32_t selfMask = amdgpu::cluster_multicast_rank_mask(wf.cluster_rank()); @@ -398,9 +437,8 @@ void RaceDetectorPlugin::onAmdgpuMemoryAccessRouted( } rs->registerLdsEvent(wf.pc, MemoryEventType::GLOBAL_TO_LDS, {}, validLaneMask, wf.wf_size(), std::span(ldsAddrs, wf.wf_size()), perLaneBytes, 0xF, - issue->counter_obligations(), memoryOrder); + obligations, memoryOrder); } else if (d.is_load) { - const MemoryOrderClass memoryOrder = memory_order_for(inst); auto destinations = validated_load_destinations(d, wf); if (!destinations) return; @@ -410,29 +448,28 @@ void RaceDetectorPlugin::onAmdgpuMemoryAccessRouted( for (uint32_t i = 0; i < registers.size(); ++i) rs->checkVgprWrite(static_cast(registers[i]), d.exec_mask, vector_memory_byte_mask(d, wf, i), memoryOrder); + if (obligations.empty()) + return; rs->registerEvent(wf.pc, MemoryEventType::GLOBAL_TO_VGPR, std::move(registers), d.exec_mask, - byte_mask, issue->counter_obligations(), memoryOrder, last_byte_mask); - } else if (!d.is_load) { - rs->registerEvent(wf.pc, MemoryEventType::VGPR_TO_GLOBAL, {}, d.exec_mask, 0xF, - issue->counter_obligations(), memory_order_for(inst)); + byte_mask, obligations, memoryOrder, last_byte_mask); + } else if (!obligations.empty()) { + rs->registerEvent(wf.pc, MemoryEventType::VGPR_TO_GLOBAL, {}, d.exec_mask, 0xF, obligations, + memoryOrder); } } if (inst.data()->tag() == amdgpu::SCALAR_MEM) { auto &d = *inst.data_as(); - const MemoryOrderClass memoryOrder = memory_order_for(inst); - const auto *issue = inst.amdgpu_memory_issue_info(); - assert(issue && "memory instruction reached the race detector without issue metadata"); if (d.is_load) { if (const auto reg = d.dst_register.register_ref()) { - rs->registerScalarLoad(wf.pc, *reg, wf.exec(), issue->counter_obligations(), memoryOrder); + rs->registerScalarLoad(wf.pc, *reg, wf.exec(), obligations, memoryOrder); } else { - rs->registerEvent(wf.pc, MemoryEventType::GLOBAL_TO_SGPR, {}, wf.exec(), 0xF, - issue->counter_obligations(), memoryOrder); + rs->registerEvent(wf.pc, MemoryEventType::GLOBAL_TO_SGPR, {}, wf.exec(), 0xF, obligations, + memoryOrder); } } else { - rs->registerEvent(wf.pc, MemoryEventType::SCALAR_TO_GLOBAL, {}, wf.exec(), 0xF, - issue->counter_obligations(), memoryOrder); + rs->registerEvent(wf.pc, MemoryEventType::SCALAR_TO_GLOBAL, {}, wf.exec(), 0xF, obligations, + memoryOrder); } } } @@ -469,6 +506,21 @@ void RaceDetectorPlugin::onAmdgpuBeforeExecuteInstruction(uint64_t pc, const Ins amdgpu::Wavefront &wf) { auto *s = get_state(wf); assert(s && s->race_state); + if (supports_counter_capacity(wf.cu().arch())) { + // FLAT's address-dependent admission is applied after routing. Before its + // operands are read, neither memory domain is guaranteed to be used. + if (inst.is_memory_op() && !inst.mnemonic().starts_with("flat_")) { + const auto *info = inst.amdgpu_memory_issue_info(); + assert(info && "memory instruction reached the race detector without issue metadata"); + if (info && (!info->exec_masked || wf.exec() != 0)) + s->race_state->prepareForMemoryIssue(*info); + } + + // Messages share LGKMCNT on the capacity-modeled targets but are not memory + // instructions, so they intentionally remain outside memory issue metadata. + if (std::string_view(inst.mnemonic()).starts_with("s_sendmsg")) + s->race_state->prepareForCounterIncrement(amdgpu::WaitCounterType::LGKMCNT); + } s->trace.push(pc); s->disasm->record(pc, inst); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/plugin.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/plugin.h index bcf0b2ade232..ba9f7ec58792 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/plugin.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/plugins/race_detector/plugin.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/soc.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/soc.cpp index b07fa1b32a1a..cf0bade5a95f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/soc.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/soc.cpp @@ -3,8 +3,7 @@ #include "rocjitsu/vm/soc.h" -#include "rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.h" +#include "rocjitsu/vm/amdgpu/compute_queue_binding_factory.h" #include "simdojo/sim/simulation.h" #include "simdojo/sim/topology.h" @@ -25,6 +24,10 @@ amdgpu::SdmaPacketDialect sdma_dialect(rj_code_arch_t arch) { arch == ROCJITSU_CODE_ARCH_RDNA4) { return amdgpu::SdmaPacketDialect::Gfx11Plus; } + if (arch == ROCJITSU_CODE_ARCH_CDNA2 || arch == ROCJITSU_CODE_ARCH_CDNA3 || + arch == ROCJITSU_CODE_ARCH_CDNA4 || arch == ROCJITSU_CODE_ARCH_RDNA2) { + return amdgpu::SdmaPacketDialect::LegacyExtendedCount; + } return amdgpu::SdmaPacketDialect::Legacy; } @@ -232,8 +235,8 @@ std::optional SoC::make_aql(uint32_t /*queue_or // must therefore stay on its selected owner: fanning it out would execute on // XCDs the guest did not size resources such as scratch for. The simulated // KFD path enables fan-out separately when it publishes every XCD. - return amdgpu::ComputeQueueBindingPlan{.factory = amdgpu::make_aql_queue_binding_factory(*owner), - .xcd_fanout = false}; + return amdgpu::ComputeQueueBindingPlan{ + .factory = amdgpu::make_compute_queue_binding_factory(*owner), .xcd_fanout = false}; } std::optional SoC::make_pm4(uint32_t /*queue_ordinal*/, @@ -242,7 +245,7 @@ std::optional SoC::make_pm4(uint32_t /*queue_or if (owner == nullptr) return std::nullopt; return amdgpu::ComputeQueueBindingPlan{ - .factory = amdgpu::make_pm4_queue_binding_factory(*owner, std::move(callbacks)), + .factory = amdgpu::make_compute_queue_binding_factory(*owner, std::move(callbacks)), .xcd_fanout = false}; } @@ -258,11 +261,16 @@ void SoC::set_dispatch_threads(uint32_t threads) { } void SoC::apply_dispatch_threads() { - size_t max_cp_cus = 1; - for_each_cp( - [&max_cp_cus](auto *cp) { max_cp_cus = std::max(max_cp_cus, cp->compute_units().size()); }); + // Each XCD caller uses at most CUs-1 workers. Concurrent callers share the + // SoC pool, so its capacity is not limited to the largest individual XCD. + // Fewer engine partitions can leave some of this maximum capacity unused. + size_t capacity = 1; + for_each_cp([&capacity](auto *cp) { + if (!cp->compute_units().empty()) + capacity += cp->compute_units().size() - 1; + }); uint32_t effective_threads = - static_cast(std::min(requested_dispatch_threads_, max_cp_cus)); + static_cast(std::min(requested_dispatch_threads_, capacity)); if (exec_mode_ != simdojo::ExecMode::FUNCTIONAL) effective_threads = 1; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/soc.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/soc.h index b72fca25da5e..d8e8d5131ac2 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/soc.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/soc.h @@ -245,11 +245,12 @@ class SoC : public simdojo::CompositeComponent, private amdgpu::ComputeQueueBind /// /// @details This controls host acceleration rather than modeled GPU /// resources or timing. The count includes the command-processor thread that - /// calls the pool. The effective width is clamped to the largest CU count of - /// any command processor in the SoC. One pool retains width-1 workers shared - /// across the SoC. CPs can submit and execute concurrently; each submission - /// uses its caller and at most width-1 workers, and joins only its own work. - /// With E engine threads, the total CU execution capacity is E + width - 1. + /// calls the pool. The effective width is clamped to one plus the sum of + /// CUs-1 across nonempty XCDs. One pool retains width-1 workers shared across + /// the SoC. CPs can submit and execute concurrently; each submission uses its + /// caller and at most min(width, active CUs)-1 workers, and joins only its own + /// work. With E engine threads, the total engine/CU thread count is + /// E + width - 1; fewer concurrent XCDs can leave some workers idle. void set_dispatch_threads(uint32_t threads); /// @returns The effective functional dispatch width after mode and CU-capacity clamps. uint32_t dispatch_threads() const { return dispatch_threads_; } diff --git a/emulation/rocjitsu/lib/simdojo/include/simdojo/components/cache.h b/emulation/rocjitsu/lib/simdojo/include/simdojo/components/cache.h index e0f670e6a1fe..d76c82ff5b6a 100644 --- a/emulation/rocjitsu/lib/simdojo/include/simdojo/components/cache.h +++ b/emulation/rocjitsu/lib/simdojo/include/simdojo/components/cache.h @@ -165,6 +165,18 @@ class Cache { return false; } + /// Inspect residency without changing replacement order or cache contents. + const CacheTag *peek(uint64_t addr, uint32_t vmid = 0) const { + const uint32_t set = set_index(addr); + const uint64_t tag = tag_bits(addr); + for (uint32_t way = 0; way < Associativity; ++way) { + const auto &entry = tag_at(set, way); + if (entry.valid && entry.tag == tag && entry.vmid == vmid) + return &entry; + } + return nullptr; + } + /// @brief Allocate a cache line for an address, evicting the LRU victim if needed. /// /// @param addr The memory address to allocate for. @@ -263,17 +275,26 @@ class Cache { /// @brief Invalidate all cache lines. void invalidate_all() { - for (uint32_t s = 0; s < NumSets; ++s) { - if (!touched_sets_[s]) + for (uint32_t first = 0; first < NumSets; first += sizeof(uint64_t)) { + const uint32_t end = std::min(first + sizeof(uint64_t), NumSets); + uint64_t touched = 0; + // Full-cache maintenance already excludes concurrent set accesses. + // Scan empty marker groups together without changing fill-side writes. + std::memcpy(&touched, touched_sets_.data() + first, end - first); + if (touched == 0) continue; - for (uint32_t w = 0; w < Associativity; ++w) { - auto &t = tag_at(s, w); - t.coherence_epoch = 0; - t.valid = false; - t.dirty = false; - t.coherence = CoherenceState::INVALID; + for (uint32_t s = first; s < end; ++s) { + if (!touched_sets_[s]) + continue; + for (uint32_t w = 0; w < Associativity; ++w) { + auto &t = tag_at(s, w); + t.coherence_epoch = 0; + t.valid = false; + t.dirty = false; + t.coherence = CoherenceState::INVALID; + } + touched_sets_[s] = 0; } - touched_sets_[s] = 0; } } diff --git a/emulation/rocjitsu/lib/simdojo/include/simdojo/components/register_file.h b/emulation/rocjitsu/lib/simdojo/include/simdojo/components/register_file.h index ae8137e073fa..bcbd8e6c0811 100644 --- a/emulation/rocjitsu/lib/simdojo/include/simdojo/components/register_file.h +++ b/emulation/rocjitsu/lib/simdojo/include/simdojo/components/register_file.h @@ -129,7 +129,13 @@ template class SoftwareLazyRegisterStora const size_t clear_begin = std::max(static_cast(base), chunk_base); const size_t clear_end = std::min(range_end, valid_chunk_end); if (clear_begin == chunk_base && clear_end == valid_chunk_end) { - chunk.reset(); + // Retain up to 64 KiB of retired chunks per register file for reuse. + // They are logically absent and are zeroed when materialized again; + // excess chunks release their allocations immediately. + if (spare_count_ < spares_.size()) + spares_[spare_count_++] = std::move(chunk); + else + chunk.reset(); } else { std::ranges::fill( chunk->registers.begin() + static_cast(clear_begin - chunk_base), @@ -145,7 +151,7 @@ template class SoftwareLazyRegisterStora [[nodiscard]] static constexpr uint32_t registers_per_chunk() noexcept { return REGS_PER_CHUNK; } /// @brief Count chunks with materialized register storage. - /// @returns Number of currently materialized chunks. + /// @returns Number of currently materialized chunks, excluding retained spares. [[nodiscard]] size_t materialized_chunk_count() const noexcept { return static_cast( std::ranges::count_if(chunks_, [](const auto &chunk) { return chunk != nullptr; })); @@ -155,7 +161,7 @@ template class SoftwareLazyRegisterStora assert(idx < total_regs_); auto &chunk = chunks_[idx / REGS_PER_CHUNK]; if (!chunk) - chunk = std::make_unique(); + chunk = acquire_chunk(); return chunk->registers[idx % REGS_PER_CHUNK]; } @@ -176,7 +182,7 @@ template class SoftwareLazyRegisterStora auto &chunk = chunks_[chunk_idx]; if (!chunk) - chunk = std::make_unique(); + chunk = acquire_chunk(); for (; local_idx < chunk_end; ++local_idx) function(chunk->registers[local_idx]); @@ -228,7 +234,7 @@ template class SoftwareLazyRegisterStora [&](size_t chunk_idx, size_t chunk_offset, size_t run_size, size_t source_offset) { auto &chunk = chunks_[chunk_idx]; if (!chunk) - chunk = std::make_unique(); + chunk = acquire_chunk(); auto *destination = reinterpret_cast(chunk->registers.data()); std::memcpy(destination + chunk_offset, source.data() + source_offset, run_size); }); @@ -245,7 +251,7 @@ template class SoftwareLazyRegisterStora return; auto &chunk = chunks_[chunk_idx]; if (!chunk) - chunk = std::make_unique(); + chunk = acquire_chunk(); auto *destination = reinterpret_cast(chunk->registers.data()); std::memcpy(destination + chunk_offset, run.data(), run.size()); }); @@ -261,6 +267,18 @@ template class SoftwareLazyRegisterStora std::array registers{}; }; + std::unique_ptr acquire_chunk() { + if (spare_count_ == 0) + return std::make_unique(); + auto chunk = std::move(spares_[--spare_count_]); + std::ranges::fill(chunk->registers, RegType{}); + return chunk; + } + + // At most 64 KiB per register file, independent of the logical capacity. + std::array, (64 * 1024) / sizeof(Chunk)> spares_{}; + size_t spare_count_ = 0; + template static void visit_byte_runs(uint32_t base, size_t byte_count, Function &&function) { constexpr size_t CHUNK_BYTES = REGS_PER_CHUNK * sizeof(RegType); @@ -327,6 +345,7 @@ class RegisterFile : public Component { data_.init(total_regs); uint32_t num_blocks = (regs_per_block > 0) ? (total_regs / regs_per_block) : 0; free_blocks_.assign(num_blocks, true); + free_block_count_ = num_blocks; needs_reset_.assign(num_blocks, false); } @@ -335,12 +354,13 @@ class RegisterFile : public Component { /// @returns Base register index, or -1 if no free block. /// @post On success, every register in the returned allocation block is zero. int32_t allocate(uint32_t count) { - if (count == 0 || regs_per_block_ == 0) + if (count == 0 || regs_per_block_ == 0 || free_block_count_ == 0) return -1; assert(count <= regs_per_block_ && "requested register count exceeds block size"); for (size_t i = 0; i < free_blocks_.size(); ++i) { if (free_blocks_[i]) { free_blocks_[i] = false; + --free_block_count_; uint32_t base = static_cast(i * regs_per_block_); if (needs_reset_[i]) { data_.reset(base, regs_per_block_); @@ -364,14 +384,15 @@ class RegisterFile : public Component { return; assert(!free_blocks_[block] && "double-free of register block"); free_blocks_[block] = true; + ++free_block_count_; if constexpr (Storage == RegisterFileStorage::SOFTWARE_LAZY) { - // Immediately restore the retired block's zero state and release wholly - // covered chunks. Layouts whose blocks share chunks receive one final - // whole-file reset so their boundary chunks can also be released. + // Immediately restore the retired block's logical zero state. Wholly + // covered chunks become absent; up to 64 KiB is retained for reuse. + // Layouts whose blocks share chunks receive one final whole-file reset + // so their boundary chunks can also be reclaimed. const bool independently_reclaimable = data_.can_reclaim_independently(regs_per_block_); const bool all_blocks_free = - !independently_reclaimable && - std::ranges::all_of(free_blocks_, [](bool is_free) { return is_free; }); + !independently_reclaimable && free_block_count_ == free_blocks_.size(); if (all_blocks_free) { data_.reset(0, total_regs_); } else { @@ -480,16 +501,10 @@ class RegisterFile : public Component { /// @brief Count the number of free allocation blocks. /// @returns Number of blocks available for allocation. - uint32_t free_block_count() const { - uint32_t count = 0; - for (bool b : free_blocks_) - if (b) - ++count; - return count; - } + uint32_t free_block_count() const { return free_block_count_; } /// @brief Count chunks with materialized backing storage. - /// @returns Number of currently materialized chunks. + /// @returns Number of currently materialized chunks, excluding retained spares. [[nodiscard]] size_t materialized_chunk_count() const requires requires(const detail::RegisterStorage &storage) { storage.materialized_chunk_count(); @@ -522,6 +537,7 @@ class RegisterFile : public Component { uint32_t total_regs_ = 0; ///< Total registers. uint32_t regs_per_block_ = 0; ///< Registers per block. std::vector free_blocks_; ///< One bit per block (true = free). + uint32_t free_block_count_ = 0; std::vector needs_reset_; ///< Blocks dirtied by prior allocation. }; diff --git a/emulation/rocjitsu/lib/util/include/util/amdgpu_exp.h b/emulation/rocjitsu/lib/util/include/util/amdgpu_exp.h new file mode 100644 index 000000000000..deae30c96490 --- /dev/null +++ b/emulation/rocjitsu/lib/util/include/util/amdgpu_exp.h @@ -0,0 +1,91 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +/// @file Integer EXP reduction and staged approximation from RDNA3/4 captures. + +#include +#include + +namespace util::detail::exp { +// Constant and linear coefficients use Q28, quadratic Q35, and cubic Q24. +// The 32 intervals cover the fractional part of x. Evaluation uses only +// integer operations, preserving the caller's complete floating-point state. +struct Coefficient { + uint32_t constant, linear, quadratic, cubic; +}; +inline constexpr Coefficient coefficients[32] = { + {268435456u, 5814541u, 8060268u, 29u}, {274313427u, 5941862u, 8236765u, 29u}, + {280320109u, 6071972u, 8417127u, 30u}, {286458319u, 6204931u, 8601438u, 31u}, + {292730941u, 6340801u, 8789785u, 31u}, {299140913u, 6479647u, 8982256u, 32u}, + {305691246u, 6621533u, 9178942u, 33u}, {312385013u, 6766525u, 9379934u, 33u}, + {319225354u, 6914693u, 9585328u, 34u}, {326215479u, 7066105u, 9795219u, 35u}, + {333358667u, 7220833u, 10009707u, 36u}, {340658273u, 7378948u, 10228891u, 36u}, + {348117717u, 7540526u, 10452874u, 37u}, {355740503u, 7705642u, 10681762u, 38u}, + {363530205u, 7874374u, 10915663u, 39u}, {371490480u, 8046800u, 11154685u, 40u}, + {379625062u, 8223002u, 11398940u, 41u}, {387937769u, 8403062u, 11648545u, 42u}, + {396432501u, 8587066u, 11903614u, 42u}, {405113242u, 8775098u, 12164270u, 43u}, + {413984066u, 8967247u, 12430633u, 44u}, {423049137u, 9163605u, 12702828u, 45u}, + {432312707u, 9364261u, 12980984u, 46u}, {441779122u, 9569312u, 13265230u, 47u}, + {451452826u, 9778853u, 13555701u, 48u}, {461338356u, 9992982u, 13852532u, 49u}, + {471440350u, 10211799u, 14155863u, 50u}, {481763549u, 10435409u, 14465836u, 52u}, + {492312797u, 10663914u, 14782597u, 53u}, {503093043u, 10897424u, 15106293u, 54u}, + {514109347u, 11136046u, 15437078u, 55u}, {525366876u, 11379894u, 15775106u, 56u}}; +inline uint64_t round_even(uint64_t value, unsigned shift) { + const uint64_t whole = value >> shift, rest = value & ((uint64_t{1} << shift) - 1), + midpoint = uint64_t{1} << (shift - 1); + return whole + (rest > midpoint || (rest == midpoint && (whole & 1))); +} +inline uint32_t evaluate(uint32_t input, bool quiet_snan = true) { + const uint32_t mag = input & 0x7fffffffu; + if (mag >= 0x7f800000u) { + if (mag > 0x7f800000u) + return input | (quiet_snan ? 0x00400000u : 0u); + return input >> 31 ? 0u : 0x7f800000u; + } + // The small-input interval returns one, including negative inputs. + if (mag < 0x33800000u) + return 0x3f800000u; + const bool negative = input >> 31; + if (!negative && mag >= 0x43000000u) + return 0x7f800000u; + if (negative && mag > 0x42fc0000u) + return 0; + // Retain 29 fractional bits. Negative arguments use a ones-complement + // reduction, so exact negative integers approach the previous interval. + const int exponent = static_cast(mag >> 23) - 127; + const uint64_t mantissa = (mag & 0x7fffffu) | 0x800000u; + const int shift = exponent + 6; + const uint64_t absolute = shift >= 0 ? mantissa << shift : mantissa >> -shift; + const int64_t phase = + negative ? -static_cast(absolute) - 1 : static_cast(absolute); + const uint32_t fraction = static_cast(phase) & 0xffffffu; + const auto c = coefficients[(static_cast(phase) >> 24) & 31]; + // Round the linear product to Q29 (capped at Q28 for large products), + // then discard its low guard bit before accumulation. + const uint64_t lp = uint64_t{c.linear} * fraction; + const uint64_t linear = lp >= (uint64_t{1} << 47) ? round_even(lp, 24) : round_even(lp, 23) >> 1; + // The quadratic stage retains 18 coordinate bits and a Q24 square. + // Round the inner coefficient to Q35 before multiplying by that square. + const uint32_t coordinate = fraction >> 6; + const uint64_t square = (uint64_t{coordinate} * coordinate) >> 12; + const uint64_t inner = + round_even((uint64_t{c.quadratic} << 7) + uint64_t{c.cubic} * coordinate, 7); + const uint64_t product = inner * square; + // Round the product to Q36, capped at 24 significant bits. + const uint64_t quad = + product >= (uint64_t{1} << 47) ? round_even(product, 24) * 2 : round_even(product, 23); + // Combine the terms in Q36 and round once to the FP32 significand. + const uint64_t sum = ((uint64_t{c.constant} + linear) << 8) + quad; + return static_cast(((phase >> 29) + 126) * 0x800000 + + static_cast(round_even(sum, 13))); +} +} // namespace util::detail::exp + +namespace util { +/// @brief Base-2 exponential matching the captured RDNA3/4 instruction mapping. +inline float amdgpu_exp_f32(float value, bool quiet_snan = true) { + return std::bit_cast(detail::exp::evaluate(std::bit_cast(value), quiet_snan)); +} +} // namespace util diff --git a/emulation/rocjitsu/lib/util/include/util/amdgpu_log.h b/emulation/rocjitsu/lib/util/include/util/amdgpu_log.h new file mode 100644 index 000000000000..24acb59c5890 --- /dev/null +++ b/emulation/rocjitsu/lib/util/include/util/amdgpu_log.h @@ -0,0 +1,164 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +/// @file Integer LOG reduction and staged approximation from RDNA3/4 captures. + +#include +#include + +namespace util::detail::log { +// Coefficients and intermediate rounding reproduce the captured instruction, +// including cancellation near one. No host floating arithmetic is required. +struct Coefficient { + uint32_t constant, linear, quadratic, cubic; +}; +// Q30, Q28, Q34, Q37. Entry32 refines the upper half of mantissa interval1. +inline constexpr Coefficient coefficients[] = { + {15u, 12102179u, 12095174u, 1925858u}, {47667825u, 11735468u, 11378345u, 1799053u}, + {93912517u, 11390291u, 10714741u, 1610016u}, {138816594u, 11064855u, 10111507u, 1477533u}, + {182455591u, 10757500u, 9557803u, 1359325u}, {224898847u, 10466758u, 9048378u, 1253542u}, + {266210154u, 10191316u, 8578611u, 1158417u}, {306448312u, 9930003u, 8144494u, 1072666u}, + {345667679u, 9681753u, 7742500u, 995087u}, {383918558u, 9445614u, 7369567u, 924970u}, + {421247645u, 9220718u, 7022931u, 861214u}, {457698312u, 9006284u, 6700175u, 803096u}, + {493310964u, 8801596u, 6399165u, 750066u}, {528123258u, 8606005u, 6118007u, 701755u}, + {562170399u, 8418918u, 5854955u, 657310u}, {595485263u, 8239793u, 5608532u, 616715u}, + {628098719u, 8068131u, 5377341u, 579376u}, {660039693u, 7903475u, 5160150u, 544955u}, + {691335339u, 7745406u, 4955875u, 513361u}, {722011229u, 7593536u, 4763477u, 484038u}, + {752091435u, 7447506u, 4582059u, 456830u}, {781598650u, 7306987u, 4410807u, 431594u}, + {810554297u, 7171673u, 4248984u, 408222u}, {838978616u, 7041279u, 4095937u, 386761u}, + {866890752u, 6915542u, 3950988u, 366593u}, {894308853u, 6794217u, 3813578u, 347636u}, + {921250083u, 6677075u, 3683220u, 330396u}, {947730761u, 6563905u, 3559471u, 313859u}, + {973766367u, 6454506u, 3441820u, 298469u}, {999371610u, 6348694u, 3329908u, 284397u}, + {1024560488u, 6246298u, 3223482u, 270890u}, {1049346329u, 6147160u, 3121894u, 258238u}, + {47667912u, 11735350u, 11363743u, 1719989u}, +}; + +inline uint64_t round_even(uint64_t x, unsigned shift) { + const uint64_t whole = x >> shift, rest = x & ((uint64_t{1} << shift) - 1), + half = uint64_t{1} << (shift - 1); + return whole + (rest > half || (rest == half && (whole & 1))); +} + +// Below one, k measures the distance in units of 2^-24. Normalize the +// linear and squared products separately in four-bit groups. The inner +// coefficient uses the ones-complement distance; its result is rounded to +// Q24. The second coefficient row starts at distance 1/128 from one. +// Keep Q50 guard bits for the smallest inputs, then round magnitude upward. +inline uint32_t near_one_negative(uint32_t bits) { + const uint32_t k = 0x3f800000u - bits; + const bool upper = k >= 0x20000u; + const uint32_t c1 = upper ? 12102221u : 12102203u; + const uint32_t c2 = upper ? 12097323u : 12101979u; + const uint32_t c3 = upper ? 2088428u : 2035248u; + const unsigned width = std::bit_width(k); + const unsigned normalization = width < 18 ? 4 * ((18 - width) / 4) : 0; + const unsigned linear_shift = normalization < 14 ? 14 - normalization : 0; + const unsigned square_shift = normalization < 12 ? 12 - normalization : 0; + const uint64_t linear = ((uint64_t{2} * c1 * k) >> linear_shift) << linear_shift; + const uint64_t square = ((uint64_t{k} * k) >> square_shift) << square_shift; + const uint64_t inner = round_even((uint64_t{c2} << 22) + uint64_t{c3} * (k - 1), 22); + const unsigned product_shift = 38 - normalization; + const uint64_t quadratic = + (inner * square + ((uint64_t{1} << product_shift) - 1)) >> product_shift; + const uint64_t fixed = + (linear << 2) + (quadratic << (16 - normalization)) - (upper ? (uint64_t{100} << 16) : 0); + unsigned leading = std::bit_width(fixed) - 1; + const unsigned shift = leading - 23; + uint64_t mantissa = (fixed + ((uint64_t{1} << shift) - 1)) >> shift; + if (mantissa == (uint64_t{1} << 24)) { + mantissa >>= 1; + ++leading; + } + return 0x80000000u | ((leading - 50 + 127) << 23) | (static_cast(mantissa) & 0x7fffffu); +} +// Above one, i measures the distance in units of 2^-23. Constants use Q35; +// the zero-origin row gains precision through four-bit normalization. The +// rounded quadratic product retains at most 24 significant bits. Packing +// advances the truncated positive significand, including exact grid hits. +inline uint32_t near_one_positive(uint32_t bits) { + const uint32_t i = bits - 0x3f800000u; + const unsigned row = i < 0x10000u ? 0 : i < 0x20000u ? 1 : 2; + constexpr uint32_t constants[] = {0u, 192u, 2856u}; + constexpr uint32_t linears[] = {12102203u, 12102186u, 12102072u}; + constexpr uint32_t quadratics[] = {12102094u, 12097614u, 12084310u}; + constexpr uint32_t cubics[] = {1999052u, 1948887u, 1883467u}; + const unsigned width = std::bit_width(i); + const unsigned normalization = row ? 0 : 4 * ((18 - width) / 4); + const uint64_t normalized = uint64_t{i} << normalization; + const uint64_t linear = ((uint64_t{linears[row]} * normalized) >> 13) << 2; + const uint64_t square = (normalized * i) >> 12; + const uint64_t inner = + round_even((uint64_t{quadratics[row]} << 22) - uint64_t{cubics[row]} * (2 * i + 1), 22); + const uint64_t product = inner * square; + const uint64_t quadratic = + product >= (uint64_t{1} << 47) ? round_even(product, 24) * 2 : round_even(product, 23); + const uint64_t fixed = constants[row] + linear - quadratic; + unsigned leading = std::bit_width(fixed) - 1; + const unsigned shift = leading - 23; + uint64_t mantissa = (fixed >> shift) + 1; + if (mantissa == (uint64_t{1} << 24)) { + mantissa >>= 1; + ++leading; + } + return ((leading - 35 - normalization + 127) << 23) | + (static_cast(mantissa) & 0x7fffffu); +} +// The ordinary path retains the extracted exponent until the final signed +// accumulation. Its quadratic product is rounded to Q35, capped at 24 bits; +// the final conversion rounds downward, including negative results. +inline uint32_t ordinary(uint32_t bits) { + const uint32_t fraction = bits & 0x3ffffu; + unsigned index = (bits >> 18) & 31; + if (index == 1 && fraction >= 0x20000u) + index = 32; + const auto c = coefficients[index]; + const uint64_t linear = (uint64_t{c.linear} * fraction) >> 16; + const uint64_t inner = + round_even((uint64_t{c.quadratic} << 22) - uint64_t{c.cubic} * (2 * fraction + 1), 22); + const uint64_t square = (uint64_t{fraction} * fraction) >> 12, product = inner * square; + const uint64_t quadratic = + product >= (uint64_t{1} << 47) ? round_even(product, 24) * 2 : round_even(product, 23); + const int exponent = static_cast(bits >> 23) - 127; + const int64_t fixed = int64_t{exponent} * (int64_t{1} << 35) + + static_cast(((uint64_t{c.constant} + linear) << 5) - quadratic); + const bool negative = fixed < 0; + const uint64_t magnitude = + negative ? static_cast(-fixed) : static_cast(fixed); + unsigned leading = 63 - std::countl_zero(magnitude), shift = leading - 23; + uint64_t mantissa = (magnitude + (negative ? ((uint64_t{1} << shift) - 1) : 0)) >> shift; + if (mantissa == (1u << 24)) { + mantissa >>= 1; + ++leading; + } + return (negative ? 0x80000000u : 0) | ((leading - 35 + 127) << 23) | + (static_cast(mantissa) & 0x7fffffu); +} +inline uint32_t evaluate(uint32_t bits, bool quiet_snan = true) { + const uint32_t magnitude = bits & 0x7fffffffu; + if (magnitude > 0x7f800000u) + return bits | (quiet_snan ? 0x00400000u : 0u); + // LOG flushes either sign of subnormal input independently of MODE. + if (magnitude < 0x00800000u) + return 0xff800000u; + if (bits & 0x80000000u) + return 0xffc00000u; + if (magnitude == 0x7f800000u) + return 0x7f800000u; + if (bits == 0x3f800000u) + return 0; + if (bits >= 0x3f7c0000u && bits < 0x3f800000u) + return near_one_negative(bits); + if (bits > 0x3f800000u && bits < 0x3f840000u) + return near_one_positive(bits); + return ordinary(bits); +} +} // namespace util::detail::log + +namespace util { +/// @brief Base-2 logarithm matching the captured RDNA3/4 instruction mapping. +inline float amdgpu_log_f32(float value, bool quiet_snan = true) { + return std::bit_cast(detail::log::evaluate(std::bit_cast(value), quiet_snan)); +} +} // namespace util diff --git a/emulation/rocjitsu/lib/util/include/util/amdgpu_rcp.h b/emulation/rocjitsu/lib/util/include/util/amdgpu_rcp.h new file mode 100644 index 000000000000..824f535a68ae --- /dev/null +++ b/emulation/rocjitsu/lib/util/include/util/amdgpu_rcp.h @@ -0,0 +1,370 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +/// @file +/// @brief Shared FP32 reciprocal mapping for AMDGPU instruction execution. + +#include "util/big_int.h" + +#include +#include + +namespace util::detail { + +// A fixed-point cubic and sparse residuals reproduce every normalized FP32 +// reciprocal captured on physical gfx1100 and gfx1201. The residuals encode +// one-ULP differences from the polynomial, not an assumed internal hardware +// implementation. The complete mapping remains within the ISA's 1-ULP bound. +// Coefficients use quarter units of the normalized result's FP32 ULP. +inline uint32_t amdgpu_rcp_polynomial(uint32_t mantissa) { + struct Coefficient { + uint32_t constant; + uint32_t linear; + uint16_t quadratic; + uint16_t cubic; + }; + static constexpr Coefficient coefficients[] = { + {67108864u, 2097138u, 65462u, 1925u}, {65075262u, 1971966u, 59693u, 1705u}, + {63161283u, 1857674u, 54582u, 1516u}, {61356675u, 1753039u, 50039u, 1352u}, + {59652323u, 1657001u, 45986u, 1210u}, {58040098u, 1568644u, 42359u, 1086u}, + {56512727u, 1487171u, 39104u, 977u}, {55063683u, 1411884u, 36174u, 882u}, + {53687091u, 1342172u, 33529u, 798u}, {52377649u, 1277499u, 31136u, 724u}, + {51130563u, 1217390u, 28966u, 658u}, {49941480u, 1161426u, 26992u, 599u}, + {48806446u, 1109234u, 25194u, 547u}, {47721858u, 1060483u, 23552u, 501u}, + {46684427u, 1014876u, 22050u, 459u}, {45691141u, 972149u, 20673u, 421u}, + {44739242u, 932065u, 19408u, 388u}, {43826196u, 894410u, 18244u, 357u}, + {42949672u, 858991u, 17171u, 330u}, {42107522u, 825636u, 16181u, 305u}, + {41297762u, 794186u, 15266u, 282u}, {40518559u, 764500u, 14418u, 262u}, + {39768215u, 736447u, 13632u, 243u}, {39045157u, 709911u, 12902u, 226u}, + {38347922u, 684783u, 12223u, 210u}, {37675151u, 660966u, 11591u, 196u}, + {37025580u, 638371u, 11002u, 183u}, {36398027u, 616915u, 10452u, 171u}, + {35791394u, 596522u, 9938u, 160u}, {35204649u, 577124u, 9458u, 150u}, + {34636833u, 558658u, 9007u, 140u}, {34087042u, 541063u, 8585u, 131u}, + }; + const Coefficient coefficient = coefficients[mantissa >> 18]; + const uint32_t fraction = mantissa & 0x3ffffu; + const uint64_t linear_product = uint64_t{coefficient.linear} * fraction; + const uint32_t shift = linear_product >= (uint64_t{1} << 38) ? 15 : 14; + const uint64_t quotient = linear_product >> shift; + const uint64_t remainder = linear_product & ((uint64_t{1} << shift) - 1); + const uint64_t midpoint = uint64_t{1} << (shift - 1); + const uint64_t rounded = + quotient + (remainder > midpoint || (remainder == midpoint && (quotient & 1))); + const uint64_t linear = rounded << (shift - 14); + const uint64_t inner = + (uint64_t{coefficient.quadratic} << 18) - uint64_t{coefficient.cubic} * fraction; + const uint32_t square_input = fraction & ~1u; + const uint64_t quadratic = + static_cast((uint128_t{inner} * square_input * square_input) >> 51); + const uint32_t value = (uint64_t{coefficient.constant} * 16 - linear + quadratic * 2 + 32) >> 6; + return 0x3e800000u + value; +} + +// Bucket by the top eight mantissa bits. Each entry stores the remaining +// fifteen bits and a high bit selecting +1 rather than -1 ULP. +inline constexpr uint16_t kAmdgpuRcpCorrectionOffsets[] = { + 0x0000, 0x000c, 0x0026, 0x0038, 0x004d, 0x005e, 0x0077, 0x0090, 0x00a8, 0x00b8, 0x00c5, 0x00da, + 0x00e9, 0x00fb, 0x010a, 0x011b, 0x0135, 0x0149, 0x0158, 0x0166, 0x0173, 0x0184, 0x0194, 0x01ac, + 0x01cb, 0x01dd, 0x01eb, 0x01f7, 0x0207, 0x021c, 0x022b, 0x0241, 0x0255, 0x025f, 0x026f, 0x0276, + 0x0285, 0x028f, 0x02a5, 0x02bb, 0x02ca, 0x02da, 0x02e2, 0x02f6, 0x0302, 0x030f, 0x031d, 0x0337, + 0x0349, 0x0350, 0x035f, 0x0368, 0x0371, 0x0382, 0x0393, 0x039f, 0x03ae, 0x03b8, 0x03c3, 0x03ca, + 0x03d4, 0x03dc, 0x03e8, 0x03ef, 0x0401, 0x0411, 0x0416, 0x0421, 0x042f, 0x0436, 0x0443, 0x044e, + 0x0462, 0x046e, 0x047e, 0x0486, 0x0492, 0x049e, 0x04a6, 0x04b5, 0x04be, 0x04c5, 0x04c9, 0x04d3, + 0x04df, 0x04e4, 0x04ed, 0x04fd, 0x0510, 0x0518, 0x0523, 0x052e, 0x053b, 0x0542, 0x054f, 0x0557, + 0x055f, 0x056b, 0x0572, 0x0575, 0x0585, 0x0592, 0x05a2, 0x05ac, 0x05b9, 0x05bf, 0x05c7, 0x05cd, + 0x05d6, 0x05dc, 0x05e5, 0x05f2, 0x05fc, 0x0609, 0x0610, 0x061c, 0x0629, 0x0630, 0x0638, 0x063e, + 0x064a, 0x0656, 0x065e, 0x0666, 0x066d, 0x0673, 0x067c, 0x0685, 0x0690, 0x06a0, 0x06ad, 0x06b5, + 0x06c1, 0x06cc, 0x06dc, 0x06e1, 0x06eb, 0x06f3, 0x0700, 0x070c, 0x0710, 0x0714, 0x071e, 0x0729, + 0x0733, 0x0739, 0x0741, 0x0747, 0x074e, 0x0759, 0x0760, 0x076b, 0x0774, 0x077f, 0x0786, 0x078d, + 0x0792, 0x079a, 0x07a0, 0x07a6, 0x07ae, 0x07bc, 0x07c4, 0x07c6, 0x07ce, 0x07d6, 0x07dd, 0x07e9, + 0x07f1, 0x07f8, 0x0803, 0x080a, 0x0814, 0x081e, 0x0827, 0x082f, 0x0834, 0x083b, 0x0841, 0x0849, + 0x0852, 0x0858, 0x085d, 0x0867, 0x0873, 0x087b, 0x0881, 0x088c, 0x0890, 0x0898, 0x08a4, 0x08ac, + 0x08b2, 0x08bc, 0x08bf, 0x08cc, 0x08d2, 0x08d6, 0x08df, 0x08e6, 0x08f2, 0x08f9, 0x0902, 0x090a, + 0x0913, 0x091e, 0x092b, 0x0935, 0x0945, 0x094f, 0x0956, 0x095c, 0x0962, 0x096d, 0x0975, 0x097f, + 0x0985, 0x0993, 0x099e, 0x09a7, 0x09b2, 0x09b5, 0x09bd, 0x09c1, 0x09cb, 0x09d0, 0x09dd, 0x09e8, + 0x09f1, 0x09f5, 0x09fe, 0x0a08, 0x0a16, 0x0a1d, 0x0a27, 0x0a31, 0x0a3c, 0x0a3f, 0x0a42, 0x0a47, + 0x0a4c, 0x0a55, 0x0a59, 0x0a5f, 0x0a68, 0x0a71, 0x0a75, 0x0a7b, 0x0a7f, 0x0a89, 0x0a91, 0x0a99, + 0x0aa0, 0x0aac, 0x0ab4, 0x0ac0, 0x0ac9, +}; +inline constexpr uint16_t kAmdgpuRcpCorrections[] = { + 0x2922, 0x2923, 0x4dcc, 0x4dcd, 0x51d0, 0x51d1, 0x54e6, 0x54e7, 0x6e8c, 0x6e8d, 0x75de, 0x75df, + 0x07a4, 0x07a5, 0x085a, 0x085b, 0x0f7e, 0x0f7f, 0x1e06, 0x1e07, 0x21a8, 0x21a9, 0x3bde, 0x3bdf, + 0x46e2, 0x46e3, 0x54da, 0x54db, 0x6264, 0x6265, 0x6a42, 0x6a43, 0x6e3a, 0x6fda, 0x6fdb, 0x739b, + 0x73de, 0x73df, 0x0ca2, 0x0ca3, 0x1c74, 0x396e, 0x396f, 0x3df5, 0x4b2a, 0x4e7c, 0x4e7d, 0x5902, + 0x59ab, 0x68c4, 0x705a, 0x705b, 0x772e, 0x796e, 0x796f, 0x7e7d, 0x07a9, 0x14ff, 0x1752, 0x1753, + 0x1a82, 0x1a83, 0x2466, 0x2467, 0x287e, 0x287f, 0x3524, 0x36ef, 0x3cb3, 0x4288, 0x4289, 0x45b7, + 0x5d2a, 0x5d2b, 0x713e, 0x75d5, 0x7ef3, 0x08fb, 0x0a8d, 0x1302, 0x142e, 0x1906, 0x20d9, 0x289f, + 0x2ea5, 0x3f34, 0x4f43, 0x5379, 0x545a, 0x545b, 0x5f1f, 0x61b6, 0x6938, 0x7297, 0x0626, 0x162d, + 0x18cf, 0x22b4, 0x2c20, 0x2f67, 0x3068, 0x32b5, 0x3cdc, 0x3cdd, 0x3ebe, 0x46e1, 0x4c9e, 0x50ab, + 0x576d, 0x58a7, 0x5da0, 0x6123, 0x6258, 0x6259, 0x651e, 0x656e, 0x68ff, 0x6ecb, 0x76dc, 0x02ba, + 0x0396, 0x03ac, 0x03ad, 0x03c3, 0x0927, 0x1372, 0x2031, 0x2125, 0x218f, 0xa2f6, 0x3035, 0x358e, + 0x41e0, 0x5011, 0x56a0, 0x5a6f, 0x617e, 0x63c2, 0x655f, 0x6975, 0x6f45, 0x7649, 0x7742, 0x7952, + 0x051f, 0x074d, 0x14f7, 0x1872, 0x1a55, 0x1c5a, 0x1c5b, 0x2165, 0x2dab, 0x31f3, 0x3364, 0x3d5b, + 0x4368, 0x487f, 0x4953, 0x4e73, 0x51bc, 0x5361, 0xde36, 0x664a, 0x72d2, 0x73a6, 0x7954, 0x7d24, + 0x002a, 0x081e, 0x0e87, 0x3669, 0x3e6b, 0x435b, 0x458a, 0x4aa9, 0x5018, 0x544a, 0x54f5, 0x56f5, + 0x5bbb, 0x6116, 0x61bf, 0x7681, 0x0121, 0x0279, 0x0e62, 0x133b, 0x1b27, 0x2dc8, 0x314d, 0x346b, + 0x3cb1, 0x50a7, 0x5230, 0x73a1, 0x7549, 0x0332, 0x08dd, 0x0927, 0x0c76, 0x1064, 0x11ac, 0x993b, + 0x1998, 0x22eb, 0x2e9d, 0x2fa3, 0x3286, 0x3d68, 0x4808, 0x5ae4, 0x5f6d, 0x6387, 0x6451, 0x69fa, + 0x7098, 0x7a96, 0x0cbe, 0x19ef, 0x22c2, 0x2317, 0x3185, 0x417d, 0x43c8, 0x52ed, 0x5453, 0x603e, + 0x61c4, 0x6688, 0x6d62, 0x6f4d, 0x6fe3, 0x0b36, 0x1293, 0x1afb, 0x1c79, 0x264a, 0x2ed6, 0x342a, + 0x3c5d, 0x414e, 0x4535, 0x5353, 0x539f, 0x5499, 0x5953, 0x6246, 0x6303, 0x686f, 0x6d76, 0x17b5, + 0x1928, 0x1dc4, 0x2a5a, 0x324e, 0x34ec, 0x3683, 0x376a, 0x37b2, 0x3f6f, 0x4775, 0x62a5, 0x6d3c, + 0x73bf, 0x7c6c, 0x00c7, 0x086a, 0x1270, 0x12b5, 0x16cd, 0x20ad, 0x2515, 0x25b0, 0x463c, 0x497b, + 0x4df2, 0x54d0, 0x5e9c, 0x6989, 0x6992, 0x6d83, 0x709a, 0x0f40, 0x122b, 0x1d86, 0x1e34, 0x1ee7, + 0x2897, 0x2956, 0x2cf7, 0x2d24, 0x3343, 0x3665, 0x408f, 0x4488, 0x453c, 0x46bf, 0x4c1c, 0x5ba8, + 0x5c32, 0x5caf, 0x5d84, 0x671f, 0x6b02, 0x6ded, 0x7558, 0x77c4, 0x7b18, 0x0022, 0x14cd, 0x218b, + 0x25e3, 0x298d, 0x2fb9, 0x3b1e, 0x3dc4, 0x4171, 0x4acc, 0x530b, 0x5a1c, 0x5f4d, 0x5fcb, 0x61b2, + 0x621b, 0x6273, 0x692d, 0x6b27, 0x6d14, 0x0d64, 0x1258, 0x18cb, 0x1bb5, 0x26b1, 0x36a2, 0x5ab5, + 0xdaf9, 0x6128, 0x6d04, 0x6eb0, 0x72d1, 0x76b1, 0x7b39, 0x7bc7, 0x2783, 0x296b, 0x2d74, 0x362b, + 0x3e6f, 0x4150, 0x433b, 0xca11, 0x4f37, 0x59ad, 0x5d85, 0x6715, 0x6fab, 0x7a48, 0x098e, 0x181b, + 0x1be0, 0x2c85, 0x34bf, 0x43fc, 0x4d48, 0x5e16, 0x6c66, 0x6e13, 0x71f8, 0x775c, 0x7e3d, 0x1358, + 0x1877, 0x30bd, 0x32b4, 0x3646, 0x3697, 0x452b, 0x4c4d, 0x50bd, 0x5179, 0x5227, 0x56bb, 0x596e, + 0x5a9c, 0x611c, 0x6a78, 0x74f8, 0x020b, 0x0fcb, 0x11a8, 0xb7e3, 0x39d5, 0x3e99, 0x46ab, 0x4cbf, + 0x519d, 0x572d, 0x5d1e, 0x6711, 0x6785, 0x73e6, 0x77b7, 0x7f58, 0x0114, 0x076b, 0x08b3, 0x109f, + 0x1153, 0x148b, 0x1879, 0x19ed, 0x1a69, 0x1eeb, 0x20e7, 0x25c5, 0x2d54, 0x3653, 0x3680, 0xb801, + 0x42c8, 0x46b5, 0x59f2, 0x5a82, 0x64f0, 0x6622, 0x6a5a, 0xf044, 0x0254, 0x042e, 0x08a8, 0x0ca9, + 0x103a, 0x1073, 0x13a5, 0x1741, 0x9908, 0x225a, 0x2ce1, 0x32a1, 0x33c9, 0x33e5, 0x343c, 0x36ab, + 0x38a3, 0x3970, 0x3d91, 0x45c2, 0x4b4c, 0x5325, 0x5a0f, 0x5cc7, 0x606e, 0x671a, 0x6c41, 0x7071, + 0x70b4, 0x7b5a, 0x7e76, 0x0030, 0x0994, 0x18fa, 0x1973, 0x2dfa, 0x34cf, 0x3541, 0x3d47, 0x3d5c, + 0x4257, 0x5abf, 0x6bc4, 0x721d, 0x73c3, 0x743c, 0x787f, 0x7e16, 0x7eaf, 0x09b3, 0x1164, 0x9700, + 0x1726, 0x1775, 0x28d3, 0x32f2, 0x36e3, 0x43f3, 0x5379, 0x5628, 0x5ae4, 0x5dce, 0x60c3, 0x12c4, + 0x267c, 0x29d6, 0x3552, 0x3bf1, 0x50ba, 0x59eb, 0x5af0, 0x5f89, 0x6035, 0x6364, 0x7f97, 0x0052, + 0x0223, 0x0559, 0x128d, 0x3c16, 0x3c21, 0x3d84, 0x4130, 0x468c, 0x4f87, 0x6036, 0x618e, 0x7598, + 0x77c4, 0x780f, 0x7cc6, 0x0411, 0x04ee, 0x09e4, 0x12d2, 0x1898, 0x2c97, 0x2ed6, 0x3722, 0x3cd1, + 0x3dbb, 0x3ecf, 0x48f8, 0x4f60, 0x5931, 0x5b65, 0x5cc0, 0x76dc, 0x77dd, 0x7a9e, 0x7ca7, 0x7ed9, + 0x0058, 0x2115, 0x2745, 0x2d02, 0x3cc6, 0x56e8, 0x572a, 0x586f, 0x5e21, 0x613f, 0xebde, 0x6fc6, + 0x70ca, 0x76ff, 0x7d5f, 0x061a, 0x06cf, 0x073b, 0x0a40, 0x0f65, 0x1234, 0x1bc6, 0x1bd0, 0x2028, + 0x20c3, 0x2bcd, 0x2f98, 0x306e, 0x34c6, 0x357e, 0x37d0, 0x61ae, 0x643d, 0x69d7, 0x6d0e, 0x7871, + 0x7b46, 0x0822, 0x09c6, 0x0a39, 0x0ab3, 0x0c32, 0x0dd1, 0x12e7, 0x1f0b, 0x2d85, 0x2ddc, 0x327f, + 0x444c, 0x4599, 0x4a83, 0x5771, 0x6ae2, 0xeb61, 0x6b68, 0x6cca, 0x76aa, 0x001c, 0x18a7, 0x202f, + 0x2a66, 0x2b99, 0x543d, 0x54a4, 0x62d4, 0x69c4, 0x6ba9, 0x0ba2, 0x0fe7, 0x1268, 0x18a8, 0x1b89, + 0x2491, 0x29ce, 0x3966, 0x51af, 0x5797, 0x5c3e, 0x5f5c, 0x6275, 0x6a0d, 0x72d7, 0x764a, 0x1057, + 0x118a, 0x3126, 0x3792, 0x3ed3, 0x54c3, 0x5a55, 0x04b9, 0x0cef, 0x0de6, 0x17d1, 0x1a98, 0x1f5d, + 0x2d52, 0x2f8a, 0x3aeb, 0x43c6, 0x488f, 0x4caf, 0x56ea, 0x76b4, 0x7a91, 0x2904, 0x2a57, 0x2bbb, + 0x303a, 0x38a6, 0x3de3, 0x3f59, 0x3fb7, 0x5602, 0x683d, 0x008e, 0x0124, 0x0194, 0x11bb, 0x27a1, + 0x2c68, 0x2f1e, 0x3310, 0x392c, 0x396d, 0x39b4, 0x4297, 0x5d12, 0x5e2d, 0x62b7, 0xe9ce, 0x6c2e, + 0x70a9, 0xf18d, 0x767f, 0x7cf7, 0x7e25, 0x0179, 0x8421, 0x0dd9, 0x0dfe, 0x0f04, 0x1133, 0x1b3c, + 0x213b, 0x233b, 0x248e, 0x2ac2, 0x2bcd, 0x2d05, 0x2edd, 0x3da4, 0x446f, 0x4ae2, 0x52a4, 0x5d16, + 0x6669, 0x71d2, 0x7f24, 0x0f9b, 0x9b07, 0x24cb, 0x36fe, 0x3d2f, 0x4ad0, 0x4f4c, 0x4fd4, 0x576e, + 0x5ea2, 0x639a, 0x63a2, 0x65b2, 0x6fc6, 0x7cfe, 0x0000, 0x0004, 0x04a8, 0x0cee, 0x93e9, 0x1a93, + 0x2433, 0x28e6, 0x4317, 0x5337, 0x592c, 0x5f12, 0x66b0, 0x6e8e, 0xf400, 0x7719, 0x0745, 0x1248, + 0x1a4f, 0x9bc1, 0x451d, 0x4930, 0x4e5f, 0xf8de, 0x051a, 0x069f, 0x0abb, 0x0e8e, 0x1231, 0x19a1, + 0x1afb, 0x1fe1, 0x2146, 0x22f7, 0x320c, 0x3a38, 0x4995, 0x4b38, 0x5727, 0x5b42, 0x6494, 0x691a, + 0xec40, 0x7736, 0x1376, 0x2c32, 0x3949, 0x39b1, 0x4370, 0x4a27, 0x52c5, 0x6593, 0x66cb, 0x6f4a, + 0x769e, 0x7e1e, 0x03f9, 0x0e05, 0x217c, 0x2df6, 0x3184, 0x37cb, 0x38ba, 0x4c1d, 0x50b8, 0x5e35, + 0x6c99, 0x7273, 0x76eb, 0x0a3f, 0x16e2, 0x263d, 0x286c, 0x35ff, 0x38b7, 0x3a82, 0x5660, 0x585f, + 0x5f51, 0x62dd, 0x6576, 0x6613, 0x7581, 0x061a, 0x0e54, 0x1038, 0x12df, 0x16b1, 0x229e, 0x2752, + 0x2abb, 0xac81, 0x32db, 0x3a45, 0x3d85, 0x3ecb, 0x41a6, 0x451a, 0x4774, 0x49ea, 0x4cf0, 0x4e39, + 0x5019, 0x54c4, 0x60cf, 0x6551, 0x66c8, 0x7372, 0x73b8, 0x0494, 0x953d, 0x205a, 0x211d, 0x2243, + 0x307c, 0x40ac, 0x46f7, 0x488c, 0x4b5c, 0x5ce7, 0x5d84, 0x5e7b, 0x62a2, 0x6409, 0x684b, 0x731a, + 0x764a, 0x002e, 0x0cd2, 0x1c9d, 0x236c, 0x50c8, 0x57a7, 0x5f1a, 0x0150, 0x0742, 0x0f5d, 0x1b85, + 0x2072, 0x2bb2, 0x2dba, 0x3cae, 0xc371, 0x4689, 0x5adf, 0x74f4, 0x7659, 0x7836, 0x7f3b, 0x8281, + 0x08fd, 0x2191, 0x3c9d, 0x437e, 0x4b57, 0x4f2f, 0x65a8, 0x6cc4, 0x07d5, 0x164a, 0x2f67, 0x3834, + 0x4d1f, 0x5241, 0x66cb, 0x68c5, 0x7d7f, 0x0d3e, 0x1198, 0x188a, 0x1eb6, 0x3d3b, 0x4031, 0x4834, + 0xcf7d, 0x5159, 0x5cdf, 0x63e2, 0x6c50, 0x6cf3, 0xeef7, 0x70fe, 0x71c4, 0x750f, 0x0c36, 0x17ac, + 0x257d, 0x2658, 0x2743, 0x32c8, 0x3d4c, 0x4d3e, 0x5264, 0x5657, 0x5e6e, 0x61ee, 0x6a4b, 0x6ab3, + 0x7098, 0x7116, 0x77e3, 0x00e3, 0x050b, 0x27da, 0x413e, 0x4349, 0x4568, 0x50d1, 0xd4d7, 0x6230, + 0x62a0, 0x6efa, 0x7f6a, 0x1b9f, 0x1d0f, 0x1ff5, 0x28b8, 0x334e, 0x3a10, 0x53f4, 0x5958, 0x59d6, + 0x6336, 0x6967, 0x6e9d, 0x72d7, 0x7548, 0x7849, 0x0012, 0x173f, 0x2b8f, 0x307c, 0x3fc8, 0xc912, + 0x68dc, 0x6ed0, 0x7533, 0x7d89, 0x12fb, 0x2308, 0x2c8d, 0x2e41, 0x3224, 0xbcb8, 0x48fc, 0x4e7d, + 0x68be, 0xe8c1, 0x6e4b, 0x06ef, 0x0b85, 0x4b0b, 0x519c, 0x52bb, 0xe414, 0x6933, 0x1342, 0x2190, + 0xbbf8, 0x463b, 0x4d59, 0x5d2a, 0x690d, 0x6b7a, 0x6e27, 0x787f, 0x046d, 0x16dc, 0x1a6d, 0x2538, + 0x3314, 0x4907, 0x5ee0, 0x6ce4, 0x09b7, 0x0a8d, 0x3367, 0x3e68, 0x4486, 0x4d92, 0x5288, 0x591b, + 0x5fb4, 0x6d01, 0x6ece, 0x79d8, 0x048a, 0x1ea6, 0x3b5d, 0x42da, 0x4b88, 0x5157, 0x678b, 0x016c, + 0x037c, 0x174d, 0x278d, 0x28be, 0x2b3f, 0x325f, 0x3c4c, 0x4536, 0x46f9, 0x49b7, 0x4b46, 0x64b0, + 0x68eb, 0x6c12, 0x7a46, 0xfe8a, 0x7f72, 0x0008, 0x0021, 0x003a, 0x02cf, 0x83f4, 0x053d, 0x0642, + 0x10aa, 0x10e7, 0x1ad4, 0x24b1, 0x2e90, 0x3fa0, 0x6eb0, 0x7097, 0x7b05, 0x0474, 0x0ba9, 0xb671, + 0x47d6, 0x6a30, 0x0a4f, 0x0b8a, 0x1af9, 0x1d2d, 0x29bb, 0x32e1, 0x3d75, 0x4c65, 0x51d9, 0x6f5c, + 0x7f6f, 0x0199, 0x0ac5, 0x0ea3, 0x17b3, 0x1d15, 0x1ed1, 0x1f7a, 0x3455, 0x35ec, 0x39e8, 0x61b6, + 0x6cc1, 0x74bd, 0x7bd0, 0x17bd, 0x1b13, 0x2247, 0xa9af, 0x662a, 0x6a7c, 0x74da, 0x08ca, 0x0ee3, + 0x100f, 0x1be5, 0x1ef7, 0x246d, 0x2ecc, 0x350d, 0x4947, 0x6478, 0xe721, 0x7b14, 0x7c26, 0x04a3, + 0x0e27, 0x20f3, 0x5034, 0x586a, 0x5e58, 0x6efb, 0x7553, 0x7947, 0x7bf9, 0x7f77, 0x0059, 0x0c10, + 0x12e8, 0x177b, 0x17d0, 0x1e85, 0x275f, 0x2e3f, 0x31c9, 0x4736, 0x4dd8, 0x5564, 0x5a4b, 0x675f, + 0x6e3d, 0x6ee6, 0x7212, 0x757a, 0x7c5a, 0x7db1, 0x0008, 0x067f, 0x21bb, 0x3393, 0x4565, 0x485e, + 0x4bc3, 0x4efa, 0x4f6f, 0x5338, 0x7034, 0x7cba, 0x800f, 0x84d7, 0x106e, 0x196d, 0x1d1a, 0x23d4, + 0x29ee, 0x3496, 0x3b78, 0x4f7f, 0x58fe, 0x5a0e, 0x6b90, 0x71e2, 0x75b0, 0x7e67, 0x80c3, 0x0d33, + 0x2302, 0x455f, 0x48c2, 0x696b, 0x69c1, 0xfacc, 0x0345, 0x18aa, 0x22c2, 0x2376, 0x24d4, 0x3575, + 0x3ee8, 0x3f3b, 0x4f66, 0x6aa0, 0x7215, 0x74fc, 0x205e, 0x2e43, 0x3350, 0x3ba6, 0xbcf9, 0x54c8, + 0x57af, 0x5df7, 0x67cb, 0x7697, 0x7965, 0x7ed7, 0x05bc, 0x207b, 0x2f1a, 0x3c96, 0x4c04, 0x55f6, + 0x5dcb, 0x7e80, 0x0f82, 0x19c7, 0x2612, 0x34b4, 0x3c57, 0x44e3, 0x48b9, 0x59c5, 0xdf4a, 0x645b, + 0xe4d7, 0x6aa8, 0x6ecb, 0x7c2b, 0xfef5, 0xa80a, 0x345d, 0x4ab0, 0x50b3, 0x5391, 0x5587, 0x6c7c, + 0x72a9, 0x7d5b, 0x000e, 0x002d, 0x0c68, 0x1972, 0x3231, 0x4ecf, 0x7afa, 0x13c7, 0x26f5, 0x4d6c, + 0x5bcf, 0x1132, 0x91cf, 0x14e8, 0x1f7b, 0x2cdf, 0x39de, 0x46ca, 0x56c8, 0x5b0e, 0x6562, 0x1123, + 0x1b65, 0x1fb2, 0x3882, 0x42c7, 0x4749, 0x68c4, 0x6b66, 0x6d48, 0x73cf, 0xf771, 0x79f2, 0x06ca, + 0x4456, 0x663c, 0x70d5, 0x7add, 0x38af, 0x3b33, 0x3f84, 0x5ea3, 0x64d7, 0x6b72, 0x6e35, 0x74e7, + 0x7a95, 0x043b, 0x0cbf, 0x9845, 0x1a2b, 0x1fd7, 0x23e2, 0x3c8c, 0x408a, 0x487a, 0xcd76, 0xddd3, + 0x5f35, 0x6030, 0x684e, 0x7bd6, 0x7d8b, 0x1105, 0x1568, 0x1660, 0x2c4d, 0x2c9e, 0x2f6a, 0xb355, + 0x373a, 0xc5bc, 0x46dd, 0x4823, 0x4c6c, 0x5250, 0x5e9b, 0xe571, 0xeeed, 0x784f, 0x7949, 0x7a5f, + 0x000e, 0x046a, 0x87f2, 0x1923, 0xbc78, 0x4cd1, 0x7228, 0x7557, 0x008b, 0x061c, 0x181d, 0x292d, + 0x2b07, 0x44b4, 0x56ec, 0x61b7, 0xf4f2, 0x7994, 0x7a3c, 0x820d, 0x0a1b, 0x0e60, 0x149c, 0xa719, + 0x3984, 0x58da, 0x633e, 0x6627, 0x69c4, 0x711a, 0x010d, 0x0727, 0x082c, 0x24f0, 0x315d, 0x457a, + 0x49d7, 0xda85, 0xdccc, 0x5d2b, 0x6792, 0x697d, 0x7e69, 0x01c8, 0x0366, 0x12bb, 0x1c68, 0x309b, + 0x39f7, 0x3a29, 0x071a, 0x0e7d, 0x9c04, 0x1e94, 0xa1d3, 0x23a9, 0x3957, 0x3a24, 0x3bf4, 0x4c94, + 0x5b28, 0x5b5f, 0x6fc5, 0x228e, 0x2d84, 0x38ae, 0x4c71, 0x65d2, 0x6aed, 0x7571, 0x7a9c, 0x152e, + 0x1ac8, 0x254e, 0x2c7e, 0x2f88, 0x383d, 0x4ef6, 0x7a28, 0x206f, 0x2787, 0xa9d5, 0x2f28, 0x32ac, + 0x333c, 0x3a5f, 0x5c9b, 0xde5c, 0x6553, 0x7d2f, 0x7f84, 0x0fe6, 0x2214, 0x48ae, 0x516a, 0x59d1, + 0x5d58, 0x783d, 0x3e5d, 0xe1e0, 0x6e50, 0x02ea, 0x181d, 0x1e27, 0x22ea, 0x3280, 0x35d2, 0x3992, + 0x4726, 0x4c5e, 0xce66, 0x59ba, 0x5ef6, 0x6029, 0x6223, 0x75b5, 0x75ef, 0x1793, 0x9f47, 0x36e3, + 0x3e11, 0x40b0, 0x428f, 0x42b0, 0x4634, 0x49fe, 0x61d1, 0x7249, 0xf586, 0x7725, 0x8520, 0x1a68, + 0x9e25, 0x2257, 0x22c8, 0x2e88, 0x3029, 0x3f13, 0x46ec, 0xcaa6, 0xd5fa, 0xdc60, 0x5ef3, 0x5f99, + 0x66f0, 0x6894, 0x1bef, 0x32b3, 0x3d89, 0x3f1b, 0x4145, 0x44a6, 0x5c5b, 0xe098, 0x717f, 0x76a2, + 0x9e12, 0xc67f, 0xc9a0, 0x4b0c, 0x50ce, 0xd9a5, 0x618e, 0x64f6, 0x6c36, 0x6e71, 0x6f15, 0x7012, + 0xfe03, 0x0000, 0x994f, 0x4156, 0x4330, 0xf4b8, 0x7bc2, 0x9e57, 0x1f3d, 0x2105, 0x25a8, 0xbaf9, + 0x4862, 0x49eb, 0x75b1, 0x1d5f, 0x23eb, 0x448c, 0x561a, 0x5963, 0x619d, 0x89dc, 0x8beb, 0x1469, + 0x2e07, 0x37f2, 0x4bbb, 0x5fcf, 0x67fd, 0xecc0, 0x0b64, 0x8e80, 0xab38, 0x3465, 0x4a7d, 0x4b9d, + 0x0005, 0x0b72, 0x263d, 0x3fd6, 0xd1ba, 0x56ae, 0xd6b1, 0x6138, 0x721b, 0x082c, 0x8bea, 0x1322, + 0x16f8, 0x96f9, 0x21a2, 0x2250, 0xba20, 0x409f, 0xd00f, 0x5acd, 0x5ef2, 0x6cbf, 0x036d, 0x1569, + 0xa02b, 0x270e, 0x2b4d, 0x3621, 0x4d3e, 0xd8fa, 0x5b06, 0x74fd, 0x07b6, 0x1845, 0x9a45, 0x1bea, + 0x3573, 0x38be, 0xbbe9, 0x3d0a, 0x40a4, 0xf927, 0x7a34, 0x7ad6, 0x7f23, 0x1541, 0x1c2d, 0x28c4, + 0x326d, 0x38bf, 0x39bb, 0x407e, 0x8041, 0x012c, 0x81b9, 0x2a59, 0x32c8, 0xb8a2, 0x3a76, 0x3b54, + 0x4530, 0x47af, 0x69e1, 0x7403, 0x0406, 0x91f9, 0x132f, 0x1b5f, 0x1d5e, 0x1db0, 0x27de, 0x281b, + 0xaf5e, 0x4e29, 0x52d6, 0xdab2, 0x7b13, 0x0ab6, 0x17b4, 0x3b31, 0xc1f8, 0x469c, 0xf63e, 0x79a3, + 0x126c, 0x156b, 0x28b5, 0x3426, 0x3788, 0x51c3, 0xe4e1, 0x7794, 0x054f, 0x185f, 0xcd41, 0x52fd, + 0x6794, 0xf69e, 0x8680, 0x0d5d, 0x8ffc, 0x274f, 0x391a, 0xc92e, 0xce08, 0xdb41, 0x6a9a, 0x6b4f, + 0x7b1e, 0x7e57, 0x001f, 0x0288, 0x05a2, 0x078e, 0x9c0d, 0x31b0, 0xb3e1, 0x5693, 0x68fe, 0x7ad7, + 0xfdcf, 0x7ed0, 0x0a56, 0x160b, 0x37e4, 0x43c8, 0x5d62, 0xdfbc, 0xed8d, 0x738c, 0x19fc, 0x2336, + 0x286a, 0x39ca, 0x4bac, 0x5a26, 0x78e4, 0x7f77, 0x240f, 0x3774, 0x4e69, 0x6e34, 0x6e3f, 0x7cfe, + 0x7fbf, 0x2e9f, 0x3715, 0x3a4f, 0x4fcd, 0x6116, 0x7642, 0x075f, 0x190d, 0x1a3f, 0x3382, 0x3dac, + 0x5ce9, 0x63b4, 0x7523, 0xf641, 0x103e, 0x1355, 0x4d33, 0x5259, 0xe6bd, 0x6c65, 0x75c3, 0x788f, + 0x7971, 0x068a, 0x1dd7, 0x1f4b, 0x204f, 0x27d9, 0xb548, 0x386a, 0xe000, 0x6734, 0x7274, 0x768c, + 0x0000, 0x0009, 0x0012, 0x001b, 0x0024, 0x002d, 0x0036, 0x003f, 0x06e2, 0x13a0, 0x2985, 0x5c7a, + 0xe0f2, 0xe3a3, 0xf025, 0x7b49, 0x00d3, 0x0338, 0x16e3, 0x9c72, 0xa176, 0x4c69, 0x516b, 0xda11, + 0x5b72, 0x5cc2, 0x6df1, 0x7571, 0x7f1a, 0x9568, 0x18aa, 0x2cd1, 0x3ce6, 0x4463, 0x450e, 0x4c35, + 0x74af, 0x0460, 0x161d, 0x1fed, 0x2967, 0xabe0, 0xd6c4, 0x5d1d, 0x5e55, 0x6f64, 0x7417, 0x77e0, + 0x7c5a, 0x9419, 0xa554, 0xad92, 0x2ea5, 0xbdf9, 0x4a5e, 0x4eb0, 0x59ca, 0xf103, 0x7df9, 0x7e5a, + 0x0a65, 0x0e69, 0x31ee, 0x3758, 0x3a05, 0x3db3, 0x3f7b, 0x5ee3, 0x62d1, 0x64cf, 0x694d, 0x708a, + 0xf872, 0x78a5, 0x7bd0, 0x7e30, 0x0f24, 0x4945, 0x616c, 0x6fb7, 0x7bcb, 0x0096, 0x00f8, 0x0ade, + 0x1bfc, 0xb094, 0xb465, 0x375e, 0x6539, 0x731a, 0xfb10, 0x0011, 0x887b, 0x1b15, 0x40b4, 0x4352, + 0xcbde, 0x5a2e, 0xf95b, 0x812e, 0x83ea, 0x095a, 0x185e, 0x19ce, 0x1f1d, 0xa806, 0x3d9d, 0x4713, + 0x5b88, 0xf82d, 0x7a9d, 0xfc99, 0x22fd, 0xaac7, 0x3bf4, 0x41cc, 0x44d7, 0x44ea, 0x4cd2, 0x5587, + 0x5a48, 0x6afe, 0xfcb2, 0x7d42, 0xa310, 0x52d7, 0xebf7, 0x7df2, 0xa65e, 0x2c0e, 0x5165, 0x540a, + 0x01ad, 0x2154, 0x321b, 0xd1a0, 0x5d0e, 0x6daa, 0x7545, 0x7778, 0x798e, 0x7a3c, 0x0f4b, 0x8f62, + 0x234d, 0x24da, 0xae83, 0x338f, 0x3a87, 0x41ef, 0x5b47, 0x614e, 0x628d, 0x89c6, 0x126b, 0xa264, + 0x2a7e, 0x375b, 0x444c, 0x6480, 0xeacb, 0x6c4a, 0x722f, 0x0024, 0x1dc5, 0xa182, 0xaf8f, 0x6824, + 0x7859, 0x07c0, 0x2058, 0x33f5, 0x3430, 0x3d1a, 0x56a9, 0x5993, 0x6f4d, 0x00d6, 0x18b5, 0x2eb3, + 0x4779, 0xd036, 0x7606, 0x0c6e, 0x13b3, 0x15e6, 0x2b25, 0x532a, 0x5ea9, 0xe1c0, 0x033e, 0x8632, + 0x078b, 0x2d40, 0xb0a8, 0x3227, 0x3a0f, 0x5353, 0x56ed, 0x67f2, 0x79e0, 0x12c2, 0x1cd0, 0x2f9a, + 0xb817, 0x3e1f, 0x3fc7, 0x699f, 0x8fbe, 0x1033, 0x94f9, 0x15a8, 0x96f3, 0x18cb, 0x34cc, 0x3b00, + 0x3d5d, 0x6743, 0x7e7e, 0x0433, 0x0c33, 0x97e2, 0xbddc, 0x43ec, 0xc604, 0x53ea, 0x6c76, 0x6eac, + 0x0000, 0x0021, 0x89dc, 0x0d0d, 0xa5fc, 0x2ae6, 0x3887, 0xbb30, 0x4fdb, 0x6a06, 0x78da, 0x8d4a, + 0x8ff9, 0x1ef5, 0x276f, 0x6514, 0x705f, 0x7258, 0x8050, 0x1d64, 0x2045, 0x415b, 0x5032, 0x6809, + 0x7a8d, 0x3c25, 0x5422, 0x5703, 0x5a87, 0x5d18, 0x0588, 0x1086, 0x18c4, 0x1e56, 0x2c54, 0xb111, + 0x3b74, 0xe511, 0x0068, 0x096d, 0x0b25, 0x189c, 0x32da, 0x3641, 0x048a, 0x85cd, 0x18f1, 0x20f3, + 0xbfb3, 0x7ddb, 0x06aa, 0x0d8e, 0x1644, 0x1736, 0x1f00, 0x2291, 0xb6bf, 0x776e, 0x0000, 0x0042, + 0x8945, 0x0f82, 0x12c5, 0xae12, 0x3125, 0x3817, 0x5672, 0x57d5, 0xd806, 0x7313, 0x74be, 0xfb59, + 0x03da, 0x2c5a, 0x48f8, 0xe049, 0xede5, 0x6f6e, 0xf6a6, 0xfc20, 0x12b7, 0x61ec, 0x81f9, 0x1bbc, + 0x22d1, 0xbdf1, 0x6345, 0x6e88, 0x7350, 0x7619, 0x015e, 0x8935, 0x106b, 0x2185, 0x2955, 0x3364, + 0xd1ce, 0xd657, 0x12fb, 0x1a31, 0x22fb, 0x38d8, 0x3ac8, 0x6c76, 0x7103, 0x07ed, 0x1b8e, 0x3204, + 0x4cdc, 0x4ff5, 0x57c3, 0xde67, 0xdedf, 0x6d35, 0xee21, 0xeea4, 0x7603, 0x0045, 0x0b12, 0x355c, + 0xbf82, 0x3fab, 0x5a7e, 0x7843, 0xf9dd, 0x0054, 0x86be, 0x0764, 0x314a, 0x38c3, 0xb9a3, 0x4b9d, + 0x92b2, 0x19a1, 0x26d8, 0xab69, 0x411c, 0x4262, 0xd5f9, 0x6147, 0xe5e0, 0x676d, 0x7380, 0x0aae, + 0x122e, 0x186d, 0x403e, 0xcfe3, 0xd660, 0x6107, 0x1e1d, 0xa35d, 0xa6c5, 0x370a, 0x3ad7, 0x3c7c, + 0x4663, 0xcf54, 0xd2b5, 0x7616, 0x0423, 0x8ce4, 0x9183, 0x1656, 0x34cc, 0x43b9, 0x4860, 0x5447, + 0x5a18, 0x5c0e, 0x0997, 0x8a4f, 0x0b7d, 0x2067, 0xb13d, 0x3e9f, 0xd481, 0x5a60, 0x6d5b, 0x1556, + 0x1a58, 0x28cb, 0x2c37, 0xbe19, 0x41fb, 0xc643, 0x6418, 0x0337, 0x2a9c, 0x34c2, 0xd8a4, 0x6239, + 0x0010, 0x079b, 0x30a9, 0x469f, 0x5146, 0x70a4, 0x7687, 0x841c, 0x8a7e, 0xa3d9, 0xa811, 0xb0b8, + 0xc70a, 0x0b62, 0x1fbb, 0xabc1, 0x30a1, 0x517d, 0x67dc, 0x74a7, 0x78d3, 0x019c, 0x9ea4, 0x23dd, + 0x2f3a, 0x3c19, 0x528e, 0x771b, 0x7925, 0x7999, 0x0979, 0x1157, 0x306f, 0x6529, 0xe8f1, 0x73e1, + 0x01d2, 0x8b7e, 0x29cb, 0xb4d6, 0x6c7e, 0x9128, 0x1190, 0x11b9, 0x19b0, 0x206b, 0x2566, 0x3f0c, + 0x4d40, 0x643a, 0x65e4, 0x055c, 0x0979, 0x0f51, 0x1103, 0x97a5, 0x2688, 0x2f3c, 0xb269, 0xb26c, + 0xbcbf, 0x5525, 0x64d9, 0x0007, 0x0048, 0x8991, 0x9090, 0x1fe2, 0xa7b3, 0xdecb, 0x6c18, 0x0f2e, + 0x3a35, 0x5546, 0x6314, 0x6756, 0x754d, 0x1d68, 0x9e85, 0xb329, 0xbc10, 0x3d75, 0xc5b4, 0xdd48, + 0x68f3, 0x741d, 0x77b4, 0x7f81, 0x0641, 0x2b52, 0x3cef, 0x52f2, 0x0742, 0x09e7, 0x14d9, 0x99e0, + 0x2c5d, 0x2e2d, 0xcb11, 0x7020, 0x0118, 0x908e, 0x13cc, 0x13cf, 0x2b77, 0x403b, 0x48d5, 0x512a, + 0x59a4, 0x5aca, 0x7807, 0x7ed8, 0x9265, 0x9817, 0x197f, 0x257f, 0x41ca, 0x543d, 0x6c51, 0x705d, + 0x079a, 0x8efd, 0x212f, 0x3c03, 0xc258, 0xd84e, 0x0000, 0x0031, 0x2f23, 0xaf96, 0x3ff4, 0xc1f0, + 0x538d, 0xe40f, 0xe607, 0x7a42, 0x2388, 0x5162, 0x7d9b, 0x0eee, 0x2a39, 0x3cd8, 0x3d38, 0x4aea, + 0x4ba7, 0x55b9, 0x5730, 0x5d25, 0x6276, 0x6783, 0xf5f9, 0x7c29, 0x0be3, 0x2ba2, 0x5303, 0x5aa6, + 0xee0c, 0x7e86, 0x17ae, 0x9fbb, 0x5d1c, 0x74bf, 0x005a, 0x81b4, 0x9792, 0xa0b1, 0x2202, 0xad87, + 0x5bfd, 0x723b, 0xff61, 0x85b8, 0x18bc, 0x1b30, 0xa148, 0x2cf0, 0x6ec6, 0xf080, 0x018b, 0x2fd8, + 0xb4c5, 0x40e3, 0x45d4, 0x5058, 0xd309, 0x5896, 0x60cb, 0xe5a4, 0xf956, 0xfb48, 0x0015, 0x1477, + 0xa957, 0x2d02, 0xd3fe, 0xd6a7, 0x5876, 0x008b, 0x88c1, 0x8d53, 0x2279, 0x29a7, 0x33ca, 0x38d7, + 0x3a00, 0x5531, 0x8a6b, 0x0dba, 0x2e3e, 0x3397, 0xd562, 0x6030, 0xf4d6, 0xfcc2, 0x0949, 0x968c, + 0x16b1, 0xba02, 0x4f0c, 0x5def, 0x7055, 0x7246, 0x75c4, 0x03ca, 0x9ac3, 0x2b59, 0xb28e, 0xc88f, + 0x4a22, 0x5198, 0x52a8, 0xd565, 0xe00c, 0x77cd, 0x04b2, 0x9a70, 0x207e, 0x2151, 0x323d, 0xb863, + 0x3a73, 0x3ac1, 0xd469, 0x55df, 0x5666, 0xe46f, 0x743b, 0x0f27, 0x8f46, 0xa716, 0xb4a8, 0x3a73, + 0x3c56, 0x45d9, 0xd5d3, 0x6bb2, 0x7ede, 0x13ba, 0x96d7, 0x18c7, 0x2138, 0xa445, 0xab31, 0x32d0, + 0x3cc4, 0x4eea, 0x5326, 0x616d, 0x64c4, 0x6584, 0x666d, 0x73c5, 0x75de, 0x1661, 0x214c, 0x45ef, + 0x55a8, 0x656d, 0xe65c, 0xe7c0, 0x69a3, 0x72b1, 0x792c, 0x1382, 0x37dc, 0xc3e7, 0xc43d, 0x4c8a, + 0x4d36, 0x6555, 0x96ad, 0xc3b9, 0x4f47, 0x5e9f, 0xe37c, 0x6f3f, 0x90d1, 0x1c10, 0x3072, 0x34c8, + 0x63e8, 0x7064, 0x0a4a, 0x16a2, 0x204a, 0x3a3e, 0x3f4e, 0x5372, 0x5d91, 0x5eae, 0x6076, 0x7598, + 0x7a5a, 0x8bf1, 0x9332, 0x9519, 0x2322, 0xab98, 0xbad4, 0x3bdc, 0xfef7, 0x921a, 0x13fe, 0x148f, + 0x2b34, 0xb700, 0x3c20, 0xc8fc, 0xc979, 0x54e6, 0xd579, 0x9224, 0x94d7, 0x474a, 0x5361, 0x6586, + 0x6be1, 0x000e, 0x001f, 0x0030, 0x0041, 0x0052, 0x8e9b, 0x30f6, 0x3615, 0x4013, 0x4964, 0x4c7e, + 0x60cd, 0x645f, 0xfa21, 0x8406, 0x285b, 0x323d, 0x36fb, 0x393d, 0x4dea, 0x546d, 0x57d6, 0x5a64, + 0xe640, 0xe94d, 0x085b, 0x10a6, 0x19bc, 0x2130, 0xb6a0, 0x5581, 0xedec, 0x7954, 0xfdd0, 0x03c0, + 0x0864, 0x1d24, 0x2108, 0x2595, 0xadb9, 0xc676, 0x46a8, 0x4ef6, 0x65eb, 0xe5fe, 0xa5e9, 0x2f77, + 0xcf96, 0x874d, 0x34a6, 0xbe35, 0x404b, 0x5165, 0x5278, 0x7213, 0x7a61, 0x09a9, 0x263c, 0xb77d, + 0x4105, 0x0907, 0x9855, 0x25ca, 0xb152, 0xba05, 0xc330, 0x5785, 0x59a7, 0xfdf2, 0xff55, 0x0000, + 0x0007, 0x0744, 0xccbf, 0xfa5d, 0x0885, 0x1cbc, 0x20d2, 0x289f, 0xad3a, 0x32e9, 0x392b, 0x3bf9, + 0xc958, 0xf155, 0x73ca, 0xf9b5, 0x7a1a, 0x06b4, 0x1ed0, 0xa811, 0x37a6, 0x45bf, 0x57fd, 0x63cf, + 0x68f3, 0xed40, 0xf2c8, 0x7bf3, 0x370a, 0x400a, 0x4abe, 0x4e1a, 0xddc8, 0xde5e, 0x66c2, 0xf280, + 0xf299, 0x1d86, 0x33c8, 0x4ba3, 0xefce, 0x0776, 0x29f7, 0x2b74, 0xaefc, 0x49bd, 0xd84f, 0x65ad, + 0xeb89, 0x7aba, 0x1713, 0x3079, 0xb655, 0x3a63, 0x4f98, 0x5964, 0xd96d, 0xe32f, 0x7351, 0x743e, + 0x07c7, 0x93c7, 0x1605, 0x2863, 0x3212, 0x3ee3, 0x4397, 0x4e6e, 0xcebc, 0x4f0a, 0xdec8, 0x6fc4, + 0xfa13, 0x7c65, 0x8d6c, 0x1ada, 0xa288, 0x3957, 0x4916, 0x562a, 0xf4b4, 0x187e, 0x1e45, 0x2bd7, + 0xc1e8, 0xc726, 0xd5b9, 0x5e68, 0x6c43, 0xf285, 0xf4a3, 0x87db, 0x9a71, 0x1f9b, 0xade6, 0x3702, + 0xc334, 0x68f0, 0xeccb, 0xedba, 0x71d0, 0x07bd, 0x1166, 0x9644, 0x9ad3, 0x217a, 0x4b61, 0xcd21, + 0xd080, 0xd60c, 0x6405, 0x670c, 0x892b, 0x6537, 0xf2ac, 0x08d4, 0x3786, 0x6b1d, 0x1117, 0x2a56, + 0x7130, 0x74a2, 0x7b7b, 0x24ca, 0x29bf, 0x5827, 0x6a93, 0x7da5, 0x0007, 0x8959, 0x268b, 0xa6ba, + 0x31c3, 0xbed4, 0x4f57, 0x50b5, 0x7a4f, 0x865c, 0x2e76, 0x3a68, 0xedb6, 0x110c, 0x17e9, 0x1a7c, + 0xa8f3, 0xc2e8, 0xd014, 0x8f99, 0x12f6, 0x955c, 0x20f4, 0x2e76, 0x408e, 0x4747, 0xe3d0, 0xfa64, + 0x03b6, 0x0d03, 0x15e2, 0xb2d1, 0x33e6, 0x34bc, 0xd353, 0x6174, 0x762d, 0xa2c1, 0x3f24, 0xda8c, + 0x76d9, 0x1466, 0x36d6, 0xc1c1, 0x5fa7, 0xe885, 0x697a, 0x0cce, 0xa509, 0xaf9b, 0x61ee, 0x0000, + 0x001f, 0x003e, 0x18b8, 0xab2c, 0x3237, 0xc52b, 0x64eb, 0xedeb, 0xfb2f, 0x8400, 0x1499, 0xa128, + 0xbbe7, 0x406b, 0xc5e1, 0xd600, 0xedf0, 0x10f4, 0xa371, 0x2818, 0x2a86, 0x3153, 0x4b03, 0x5da2, + 0x6433, 0x0a82, 0x0ab5, 0x149c, 0x159f, 0x4816, 0x4e1f, 0x5d64, 0x0303, 0x0cd0, 0x8d50, 0x8f9b, + 0x2c3c, 0x3f04, 0xc7db, 0xc9e0, 0x4b51, 0x4fb2, 0x6384, 0xf8fa, 0x08e4, 0x0d4f, 0x0da6, 0x2a89, + 0xb20f, 0x67e7, 0x6cf4, 0xede0, 0x8320, 0x14e0, 0x170a, 0x1826, 0x1ff8, 0xaa74, 0xb619, 0x412a, + 0x4481, 0xda2d, 0xe488, 0x7074, 0x0388, 0x069d, 0x1f3c, 0xa1f0, 0xa615, 0x53ff, 0x571e, 0xd720, + 0xf486, +}; + +inline uint32_t amdgpu_rcp_normalized_bits(uint32_t mantissa) { + uint32_t result = amdgpu_rcp_polynomial(mantissa); + const uint32_t bucket = mantissa >> 15; + const uint16_t key = mantissa & 0x7fffu; + uint32_t first = kAmdgpuRcpCorrectionOffsets[bucket]; + uint32_t last = kAmdgpuRcpCorrectionOffsets[bucket + 1]; + const uint32_t end = last; + while (first < last) { + const uint32_t middle = first + (last - first) / 2; + if ((kAmdgpuRcpCorrections[middle] & 0x7fffu) < key) + first = middle + 1; + else + last = middle; + } + if (first != end && (kAmdgpuRcpCorrections[first] & 0x7fffu) == key) + result = (kAmdgpuRcpCorrections[first] & 0x8000u) ? result + 1 : result - 1; + return result; +} + +} // namespace util::detail + +namespace util { + +/// @brief AMD FP32 reciprocal matching physical RDNA3/4 within the ISA 1-ULP bound. +/// @details Flushes input and output subnormals independently of MODE and host rounding. +/// Preserves signs and NaN payloads; the caller selects signaling-NaN quieting. +inline float amdgpu_rcp_f32(float value, bool quiet_snan = true) { + const uint32_t bits = std::bit_cast(value); + const uint32_t sign = bits & 0x80000000u; + const uint32_t magnitude = bits & 0x7fffffffu; + if (magnitude > 0x7f800000u) + return std::bit_cast(bits | (quiet_snan ? 0x00400000u : 0u)); + if (magnitude < 0x00800000u) + return std::bit_cast(sign | 0x7f800000u); + if (magnitude == 0x7f800000u) + return std::bit_cast(sign); + const uint32_t normalized = detail::amdgpu_rcp_normalized_bits(magnitude & 0x7fffffu); + const int exponent = int(normalized >> 23) + 127 - int(magnitude >> 23); + if (exponent <= 0) + return std::bit_cast(sign); + if (exponent >= 255) + return std::bit_cast(sign | 0x7f800000u); + return std::bit_cast(sign | (uint32_t(exponent) << 23) | (normalized & 0x7fffffu)); +} + +} // namespace util diff --git a/emulation/rocjitsu/lib/util/include/util/amdgpu_rsq.h b/emulation/rocjitsu/lib/util/include/util/amdgpu_rsq.h new file mode 100644 index 000000000000..03a956bfc081 --- /dev/null +++ b/emulation/rocjitsu/lib/util/include/util/amdgpu_rsq.h @@ -0,0 +1,392 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +/// @file +/// @brief Shared reciprocal-square-root mapping for AMDGPU instruction execution. + +#include "util/big_int.h" + +#include +#include + +namespace util::detail { + +// A fixed-point cubic and sparse one-ULP residuals reproduce all normalized +// outputs captured on physical gfx1100 and gfx1201. This is an empirical +// mapping, not a claim about the hardware implementation. The complete mapping +// stays within the ISA's one-ULP bound. Coefficients use quarter units of the +// result's FP32 ULP; index combines exponent parity and the input mantissa. +inline uint32_t amdgpu_rsq_polynomial(uint32_t index) { + struct Coefficient { + uint32_t constant; + uint32_t linear; + uint16_t quadratic; + uint16_t cubic; + }; + static constexpr Coefficient coefficients[] = { + {67108864u, 1048572u, 24555u, 606u}, {66084241u, 1001273u, 22738u, 545u}, + {65105161u, 957426u, 21104u, 491u}, {64168347u, 916688u, 19629u, 445u}, + {63270843u, 878759u, 18295u, 403u}, {62409975u, 843376u, 17084u, 367u}, + {61583317u, 810305u, 15983u, 335u}, {60788660u, 779340u, 14978u, 306u}, + {60023992u, 750298u, 14060u, 280u}, {59287474u, 723016u, 13219u, 257u}, + {58577418u, 697349u, 12446u, 237u}, {57892279u, 673165u, 11735u, 218u}, + {57230631u, 650347u, 11080u, 201u}, {56591163u, 628789u, 10475u, 186u}, + {55972661u, 608397u, 9915u, 173u}, {55374006u, 589084u, 9396u, 160u}, + {54794158u, 570771u, 8914u, 149u}, {54232151u, 553388u, 8467u, 139u}, + {53687091u, 536870u, 8050u, 129u}, {53158141u, 521157u, 7661u, 121u}, + {52644524u, 506196u, 7298u, 113u}, {52145512u, 491938u, 6959u, 105u}, + {51660427u, 478336u, 6641u, 99u}, {51188632u, 465350u, 6343u, 93u}, + {50729532u, 452941u, 6064u, 87u}, {50282567u, 441074u, 5802u, 82u}, + {49847212u, 429717u, 5555u, 77u}, {49422973u, 418838u, 5322u, 73u}, + {49009384u, 408411u, 5103u, 68u}, {48606008u, 398409u, 4897u, 65u}, + {48212431u, 388809u, 4702u, 61u}, {47828262u, 379589u, 4517u, 58u}, + {47453132u, 741452u, 17363u, 428u}, {46728614u, 708007u, 16078u, 385u}, + {46036300u, 677002u, 14922u, 347u}, {45373873u, 648196u, 13880u, 314u}, + {44739242u, 621376u, 12936u, 285u}, {44130517u, 596356u, 12080u, 259u}, + {43545981u, 572972u, 11301u, 236u}, {42984073u, 551076u, 10591u, 216u}, + {42443372u, 530541u, 9942u, 198u}, {41922574u, 511250u, 9347u, 182u}, + {41420490u, 493100u, 8801u, 167u}, {40936023u, 475999u, 8298u, 154u}, + {40468167u, 459864u, 7835u, 142u}, {40015995u, 444621u, 7407u, 132u}, + {39578648u, 430202u, 7011u, 122u}, {39155335u, 416545u, 6644u, 113u}, + {38745320u, 403596u, 6303u, 105u}, {38347922u, 391304u, 5987u, 98u}, + {37962506u, 379624u, 5692u, 91u}, {37588482u, 368514u, 5417u, 85u}, + {37225300u, 357935u, 5160u, 80u}, {36872445u, 347852u, 4920u, 74u}, + {36529438u, 338235u, 4696u, 70u}, {36195829u, 329052u, 4485u, 65u}, + {35871196u, 320278u, 4288u, 61u}, {35555144u, 311887u, 4102u, 58u}, + {35247302u, 303855u, 3928u, 54u}, {34947319u, 296163u, 3763u, 51u}, + {34654868u, 288790u, 3609u, 48u}, {34369638u, 281718u, 3462u, 45u}, + {34091337u, 274930u, 3325u, 43u}, {33819688u, 268410u, 3194u, 41u}, + }; + const Coefficient coefficient = coefficients[index >> 18]; + const uint32_t fraction = index & 0x3ffffu; + const uint64_t product = uint64_t{coefficient.linear} * fraction; + const unsigned shift = product >= (uint64_t{1} << 37) ? 14 : 13; + const uint64_t quotient = product >> shift; + const uint64_t remainder = product & ((uint64_t{1} << shift) - 1); + const uint64_t midpoint = uint64_t{1} << (shift - 1); + const uint64_t linear = + (quotient + (remainder > midpoint || (remainder == midpoint && (quotient & 1)))) + << (shift - 13); + const uint64_t inner = + (uint64_t{coefficient.quadratic} << 18) - uint64_t{coefficient.cubic} * fraction; + const uint32_t square_input = fraction & ~1u; + const uint64_t quadratic = + static_cast((uint128_t{inner} * square_input * square_input) >> 51); + return 0x3e800000u + + static_cast( + (uint64_t{coefficient.constant} * 32 - linear + quadratic * 4 + 64) >> 7); +} + +// Each bucket covers fifteen low index bits; the residual high bit selects +// +1 rather than -1 ULP. Both exponent parities have their own buckets. +inline constexpr uint16_t kAmdgpuRsqCorrectionOffsets[] = { + 0x0000, 0x0004, 0x000b, 0x0011, 0x0018, 0x001b, 0x0022, 0x002d, 0x0036, 0x003e, 0x0049, 0x004f, + 0x0055, 0x005d, 0x0064, 0x006a, 0x0071, 0x0074, 0x007b, 0x0088, 0x008f, 0x0093, 0x0098, 0x00a0, + 0x00a9, 0x00b1, 0x00b6, 0x00bd, 0x00c2, 0x00ca, 0x00d2, 0x00d8, 0x00e1, 0x00e6, 0x00e9, 0x00ee, + 0x00f5, 0x00fb, 0x0106, 0x010f, 0x0113, 0x011b, 0x0121, 0x0122, 0x0129, 0x0130, 0x0131, 0x0138, + 0x0145, 0x0148, 0x014c, 0x014f, 0x0157, 0x015f, 0x0163, 0x016d, 0x0176, 0x017d, 0x0181, 0x0186, + 0x018e, 0x0191, 0x0198, 0x019c, 0x01a0, 0x01a5, 0x01ab, 0x01ae, 0x01b5, 0x01b9, 0x01c1, 0x01c8, + 0x01d1, 0x01d6, 0x01dc, 0x01e0, 0x01e4, 0x01e7, 0x01e9, 0x01eb, 0x01f4, 0x01ff, 0x0205, 0x0205, + 0x020a, 0x020c, 0x0211, 0x0219, 0x021d, 0x0221, 0x0227, 0x022c, 0x022f, 0x0233, 0x0237, 0x023b, + 0x0240, 0x0247, 0x024b, 0x0253, 0x0256, 0x0258, 0x025a, 0x025b, 0x0260, 0x0264, 0x026b, 0x026f, + 0x0271, 0x0273, 0x0279, 0x027b, 0x027e, 0x0281, 0x0282, 0x0285, 0x0289, 0x0291, 0x0294, 0x0299, + 0x029e, 0x02a3, 0x02a3, 0x02a8, 0x02aa, 0x02ae, 0x02b2, 0x02b7, 0x02bd, 0x02c8, 0x02cb, 0x02d2, + 0x02d4, 0x02d5, 0x02d7, 0x02d9, 0x02dc, 0x02df, 0x02e4, 0x02e7, 0x02ef, 0x02f5, 0x02fa, 0x02ff, + 0x0304, 0x030a, 0x0310, 0x0312, 0x0314, 0x0315, 0x031b, 0x031e, 0x0322, 0x0325, 0x0326, 0x0329, + 0x032b, 0x032e, 0x0330, 0x0335, 0x033c, 0x0345, 0x034a, 0x034d, 0x0350, 0x0356, 0x035c, 0x035e, + 0x0363, 0x036b, 0x036f, 0x0373, 0x037c, 0x037f, 0x0385, 0x0388, 0x0388, 0x038d, 0x0391, 0x0393, + 0x0394, 0x039a, 0x03a0, 0x03a5, 0x03ab, 0x03ae, 0x03b0, 0x03b2, 0x03b6, 0x03bc, 0x03c1, 0x03c5, + 0x03cb, 0x03d1, 0x03d3, 0x03d8, 0x03de, 0x03e1, 0x03e5, 0x03ef, 0x03f4, 0x03fd, 0x0400, 0x0404, + 0x0408, 0x040b, 0x040e, 0x0411, 0x0419, 0x0421, 0x0426, 0x0427, 0x042e, 0x042f, 0x0431, 0x0434, + 0x0439, 0x043d, 0x0441, 0x0444, 0x0447, 0x0449, 0x044f, 0x0457, 0x045c, 0x0461, 0x0466, 0x0468, + 0x046b, 0x0470, 0x0471, 0x0474, 0x047b, 0x0481, 0x0484, 0x0487, 0x048a, 0x048f, 0x0494, 0x0499, + 0x049e, 0x04a2, 0x04a7, 0x04b1, 0x04b5, 0x04b7, 0x04b9, 0x04c0, 0x04c4, 0x04c7, 0x04c9, 0x04cc, + 0x04d1, 0x04d4, 0x04d7, 0x04da, 0x04dd, 0x04e8, 0x04f1, 0x04f7, 0x04fb, 0x0500, 0x0505, 0x050c, + 0x0514, 0x0517, 0x0519, 0x0521, 0x0528, 0x052e, 0x0537, 0x053f, 0x0544, 0x0549, 0x054c, 0x0550, + 0x055a, 0x055f, 0x0566, 0x0568, 0x0577, 0x057c, 0x0580, 0x0581, 0x0586, 0x058c, 0x058f, 0x0599, + 0x05a6, 0x05b1, 0x05b1, 0x05b4, 0x05bb, 0x05c2, 0x05cd, 0x05d5, 0x05d8, 0x05dd, 0x05e1, 0x05e6, + 0x05e9, 0x05ee, 0x05f4, 0x05fa, 0x0600, 0x0608, 0x0609, 0x060d, 0x060f, 0x0614, 0x0619, 0x0621, + 0x0624, 0x0627, 0x062b, 0x0630, 0x0634, 0x0638, 0x063c, 0x0643, 0x064b, 0x064d, 0x0650, 0x0655, + 0x0655, 0x0658, 0x0660, 0x0666, 0x0669, 0x066e, 0x0672, 0x067a, 0x067d, 0x067f, 0x0681, 0x0685, + 0x0689, 0x0690, 0x0691, 0x0696, 0x0699, 0x069f, 0x06a6, 0x06ab, 0x06b5, 0x06ba, 0x06be, 0x06c1, + 0x06c6, 0x06ca, 0x06ce, 0x06d0, 0x06d3, 0x06d9, 0x06dd, 0x06e2, 0x06e5, 0x06e8, 0x06ec, 0x06ef, + 0x06f3, 0x06fd, 0x0701, 0x0705, 0x0709, 0x070d, 0x0713, 0x0718, 0x071b, 0x071e, 0x0721, 0x0723, + 0x0727, 0x072d, 0x072f, 0x0736, 0x073c, 0x0742, 0x0744, 0x074b, 0x074f, 0x0754, 0x0758, 0x075b, + 0x0762, 0x0766, 0x0769, 0x076e, 0x0776, 0x0778, 0x077b, 0x077f, 0x0785, 0x0790, 0x0794, 0x0796, + 0x079c, 0x07a4, 0x07aa, 0x07ae, 0x07b5, 0x07b8, 0x07be, 0x07c3, 0x07c6, 0x07ca, 0x07cd, 0x07d4, + 0x07d9, 0x07e2, 0x07e6, 0x07ec, 0x07f0, 0x07f3, 0x07fa, 0x07fe, 0x0804, 0x080a, 0x080f, 0x0810, + 0x0817, 0x081c, 0x0820, 0x0829, 0x0832, 0x0835, 0x083b, 0x083c, 0x0840, 0x0843, 0x0845, 0x0849, + 0x084c, 0x084d, 0x0852, 0x0855, 0x0856, 0x0859, 0x0862, 0x086a, 0x0873, 0x087b, 0x087f, 0x0884, + 0x0888, 0x088c, 0x088d, 0x088e, 0x0893, 0x0897, 0x089b, 0x089c, 0x08a1, 0x08a4, 0x08a8, 0x08b0, + 0x08b9, 0x08bf, 0x08c0, 0x08c3, 0x08c5, 0x08c7, 0x08c9, 0x08d0, 0x08d4, 0x08d9, 0x08db, 0x08e1, + 0x08e4, 0x08e7, 0x08ea, 0x08ed, 0x08f0, 0x08f9, 0x08fe, 0x0906, 0x0908, 0x090a, 0x0910, 0x0912, + 0x0915, 0x0919, 0x091b, 0x091d, 0x091f, 0x0923, 0x0926, 0x092c, 0x0933, 0x093b, 0x093e, 0x0943, + 0x0947, 0x094c, 0x094d, 0x0956, 0x0957, 0x0960, 0x0967, 0x096d, 0x096d, 0x0970, 0x0978, 0x097c, + 0x0981, 0x0989, 0x098b, 0x0994, 0x099a, 0x099f, 0x09a7, 0x09ad, 0x09b4, +}; +inline constexpr uint16_t kAmdgpuRsqCorrections[] = { + 0x1bbc, 0x5dd1, 0x65b4, 0x7dce, 0x3ee0, 0x53d1, 0x6c65, 0x7a8d, 0x7ae4, 0x7d47, 0x7df5, 0x19a7, + 0x51b2, 0x51b3, 0x5b8b, 0x5cc6, 0x74ff, 0x3e99, 0x4cf2, 0x552a, 0x5ab0, 0x5ab1, 0x7273, 0x7859, + 0x0eca, 0x1c7e, 0x70cb, 0x0507, 0x15c3, 0x26dc, 0x3b93, 0x3e72, 0x3e73, 0x67ae, 0x01e6, 0x03d7, + 0x0eeb, 0x10d3, 0x1129, 0x2b29, 0x3281, 0x4b66, 0x5fbd, 0x668c, 0x79b1, 0x3007, 0x4295, 0x5ef2, + 0x5ef3, 0x64d4, 0xeb09, 0x6d6b, 0x6f5b, 0x742e, 0x0006, 0x0032, 0x0a31, 0x1eab, 0x4abb, 0x51fe, + 0x5572, 0x61ba, 0x0a96, 0x118b, 0x1546, 0x1ba4, 0x3b70, 0x4547, 0xccb7, 0x539b, 0x69fa, 0x6d2c, + 0x7693, 0x000c, 0x014e, 0x2a1e, 0x3d61, 0x4d46, 0x679e, 0x0caf, 0x0db2, 0x0fb8, 0x438a, 0x56e9, + 0x648d, 0x3693, 0x3c33, 0xc941, 0x4aa9, 0x5149, 0x5a73, 0x68c5, 0x78e4, 0x0b78, 0x0fcd, 0x294d, + 0x30f6, 0x3f4a, 0x5874, 0x6df2, 0x192c, 0x1c7c, 0x2bc5, 0x3902, 0x410c, 0x5c09, 0x05e6, 0x095f, + 0x3286, 0x3e6a, 0x42a0, 0x4727, 0x5890, 0x001a, 0x21ba, 0x35c7, 0x0db6, 0x27c2, 0x2ac4, 0x46a6, + 0x4e6f, 0x5d3c, 0x6d4a, 0x023d, 0x0827, 0x8be3, 0x29d7, 0x2a5b, 0x2db9, 0x3ca5, 0x3d32, 0x4859, + 0x6852, 0x6da8, 0x6eb3, 0xfffb, 0x0f96, 0x38c4, 0x39db, 0x3ad5, 0xc9f4, 0x4d59, 0x6f24, 0x84c0, + 0x1713, 0x275e, 0x4889, 0x0d75, 0x9000, 0x1da9, 0x4ca9, 0x5731, 0x037b, 0x1468, 0x23af, 0x30b4, + 0x3a58, 0x43c2, 0x591e, 0x5d8e, 0x1288, 0x154a, 0x303c, 0x4c9b, 0xe8bd, 0x6915, 0x737a, 0x778f, + 0x7a44, 0x000e, 0x001e, 0x10c1, 0x22e1, 0x25a5, 0xa7e3, 0x49ec, 0x6b4d, 0x0647, 0x19d6, 0x251e, + 0x2553, 0x3fab, 0x83a3, 0x099d, 0x9309, 0x1eee, 0x5afc, 0x5cb5, 0x6764, 0x9201, 0x46cf, 0x4eda, + 0x568d, 0x6cd3, 0x0346, 0x0e32, 0x2f31, 0x35f6, 0x37ac, 0x4075, 0x711e, 0x7785, 0x03c5, 0x0793, + 0x0ead, 0x1fd1, 0x306c, 0x3ce6, 0x69a9, 0x7db0, 0x024e, 0x0458, 0x0a6e, 0x2c8d, 0x4d1c, 0x4e70, + 0x02cb, 0x36f3, 0x4b6f, 0x4ba1, 0x4fad, 0x5951, 0x5d60, 0x5e21, 0x688a, 0x0011, 0x0030, 0x3afd, + 0x3e6b, 0x52b5, 0xa445, 0x5af4, 0xde81, 0x0926, 0x09a1, 0x20e3, 0x22af, 0x7fa2, 0x1263, 0x1b4e, + 0x20e7, 0x2960, 0x2b8c, 0x74fe, 0x7f5d, 0x14df, 0x97ef, 0x5d17, 0x5ead, 0x78ca, 0x7ab0, 0x0858, + 0x1067, 0x15f2, 0x1c5a, 0x20a2, 0x211f, 0x38f2, 0x56e3, 0x5aea, 0x5def, 0x6736, 0x0afd, 0x0b82, + 0x2606, 0x3cc0, 0x3efa, 0x4eb4, 0x53e7, 0x5c76, 0x7bc2, 0x374d, 0x627a, 0x751a, 0x75c8, 0x0009, + 0x1114, 0xbb41, 0x4914, 0x6594, 0x6703, 0x71d7, 0x7a11, 0x0e3c, 0x2029, 0x220c, 0x4924, 0x5738, + 0x6fa7, 0x5059, 0x1df3, 0x32eb, 0xd5c7, 0x56be, 0x7908, 0xf9a4, 0x7bac, 0x9bbd, 0x27ac, 0x2f93, + 0x4205, 0x49ac, 0x5a07, 0x6d45, 0x1d4a, 0x108c, 0x93f9, 0x365e, 0x4628, 0x50dd, 0x512e, 0x5efb, + 0x1567, 0x2c6f, 0x4739, 0x47be, 0x4cbf, 0x50fc, 0x5750, 0x5c2b, 0x5ecb, 0x6a37, 0x6cda, 0x7347, + 0x7523, 0x0020, 0x4086, 0x56f1, 0x0dd6, 0x34ee, 0x7547, 0x7cf4, 0x02a9, 0x3b35, 0x4a78, 0x0e44, + 0x2506, 0x5af5, 0x64fa, 0x65c5, 0xf28a, 0x72c8, 0x79d2, 0x84de, 0x1aa1, 0x278b, 0x480c, 0x4dc9, + 0x5617, 0x6b0e, 0x6fb4, 0x132b, 0x3e38, 0x4f3c, 0x6d0b, 0x0b68, 0x0b90, 0x0bb8, 0x1358, 0x13c8, + 0x2b85, 0xcd61, 0x52bb, 0x581c, 0x5bdc, 0x8f60, 0x1184, 0x1255, 0x14b0, 0x251d, 0xaebe, 0x4336, + 0xf2a4, 0x7c38, 0x0029, 0x25e7, 0xbc03, 0x5fc4, 0xedc6, 0x752b, 0xf875, 0x97f1, 0x713c, 0x7985, + 0xfcf1, 0xb33e, 0x44b5, 0x4513, 0x4562, 0x7f67, 0x83ba, 0x0ce5, 0x9407, 0x9d66, 0x1dc7, 0x244d, + 0x5f9e, 0x65f4, 0x9307, 0x51a8, 0x6a3c, 0x2ced, 0x52bb, 0x6171, 0x6355, 0x6a69, 0x76b9, 0x7ea9, + 0x2818, 0x4f82, 0x6725, 0xf35e, 0x1b04, 0x1fec, 0x2985, 0x3e4e, 0x002c, 0x003a, 0x09a8, 0x1f73, + 0x4d9b, 0x031a, 0x314a, 0x37a0, 0x5bc6, 0x618c, 0x7b7c, 0x01d5, 0x3185, 0xf3bd, 0x02e9, 0x21db, + 0x28da, 0x3ad5, 0x42a5, 0x5ce8, 0x6ee6, 0x358c, 0x5bd0, 0x7455, 0x7deb, 0x11f6, 0x9bfa, 0x1d2b, + 0x21c7, 0x334e, 0x5313, 0x74db, 0x7ef8, 0x13b9, 0x1874, 0x2b2e, 0x40de, 0x7264, 0x797a, 0x7e76, + 0x3642, 0x3fcc, 0x4463, 0xc4c4, 0x5345, 0xdf5c, 0x6536, 0x686d, 0x69e0, 0x0000, 0x001d, 0x003a, + 0x1140, 0x3b32, 0x1ad9, 0x34e6, 0x4a0b, 0x4fbd, 0x51cb, 0x79fe, 0x05eb, 0x077f, 0x917d, 0x5bbb, + 0x0a49, 0x19d6, 0x63c2, 0x6495, 0x2607, 0x3baa, 0xe18d, 0x091e, 0x1710, 0x0cfe, 0xe78f, 0x0d2b, + 0x24a3, 0x2aea, 0x3593, 0x3605, 0x3ecf, 0x5ee8, 0x6731, 0x7b7e, 0x0000, 0x0003, 0x0006, 0x0009, + 0x000c, 0x000f, 0xaf63, 0x4343, 0xd82d, 0x6b4d, 0x72c4, 0x1de2, 0x28fd, 0x4216, 0x583c, 0x5ea8, + 0xe586, 0x24fd, 0x31c5, 0xbccc, 0x5542, 0x709a, 0x25ee, 0x545a, 0x94ce, 0x3190, 0x524b, 0x5b68, + 0x6395, 0x0b5b, 0x2654, 0x2978, 0x38dd, 0x5439, 0x66c8, 0x6e91, 0x7702, 0x1e95, 0x5773, 0xe6d2, + 0x6cb0, 0x003a, 0x077b, 0x08d8, 0xe111, 0x405a, 0x547f, 0xde8b, 0xe901, 0x6bc2, 0xf4c5, 0x00d4, + 0x045a, 0x2e78, 0x5b9d, 0x67a7, 0x21ff, 0xc9ec, 0x7e84, 0xc68f, 0x58b1, 0xdf05, 0x7016, 0x071d, + 0x306d, 0xb73f, 0x6002, 0x88f0, 0x513f, 0x7b79, 0x7c30, 0x2513, 0x3333, 0x463f, 0x49c1, 0xea92, + 0x0002, 0x001f, 0x145a, 0x2de8, 0x36bd, 0x61da, 0x7aa8, 0x2ebc, 0x39d9, 0x5328, 0x797a, 0x031b, + 0x0b8b, 0x22b5, 0x2f40, 0x410b, 0x5e98, 0x78ef, 0x7e44, 0x9722, 0x1aad, 0x60f6, 0x37c5, 0x586a, + 0x0b92, 0x1111, 0x7740, 0x0632, 0x424c, 0xc520, 0xd2e6, 0x5d89, 0x1a01, 0x1e70, 0xdfb8, 0x7858, + 0x15d0, 0x2291, 0x23ae, 0x24cb, 0x491c, 0x583f, 0xdbfe, 0x1f77, 0x4685, 0x5693, 0x6530, 0x1403, + 0x984f, 0x2e99, 0x4ca2, 0x3078, 0x3df9, 0x3fb8, 0x43e4, 0x73ea, 0x78b5, 0x6851, 0x76ed, 0x0d4f, + 0x466b, 0xf7ca, 0x0035, 0x03a4, 0x4042, 0x4ce8, 0x3d95, 0x3dfa, 0xffe9, 0x318e, 0x49b3, 0xde9a, + 0x738f, 0x26d1, 0x2bf1, 0xac5c, 0xc2b8, 0x4315, 0x61ed, 0x678e, 0x7e49, 0x1fa6, 0x45a0, 0x6fcc, + 0x1839, 0x3912, 0x5cbf, 0x6900, 0x7b13, 0x9178, 0x28b6, 0x3a3d, 0x4d0a, 0x747b, 0x0010, 0x0e79, + 0xab40, 0xd20e, 0x5d4d, 0x13cf, 0x3068, 0x491c, 0xdf7a, 0x7b43, 0x45e9, 0xea5e, 0x8b70, 0x521b, + 0x5a7e, 0x7cd7, 0x1ed0, 0xa442, 0x5d63, 0x6e26, 0x895a, 0x0f33, 0x2f16, 0xd2ae, 0x5416, 0x32ff, + 0x3b30, 0x3e69, 0xc867, 0x4ce5, 0x7fbc, 0x8cb8, 0xa1e0, 0x22f2, 0x2e5b, 0x38c7, 0x4cd7, 0xcf4b, + 0x60c9, 0x75cf, 0x7858, 0xfa0d, 0x9735, 0x4d0e, 0x6291, 0x1d26, 0x2c87, 0x39f5, 0x5110, 0x5b0e, + 0x6bf5, 0x6dd8, 0x6a6d, 0x79ee, 0x01a4, 0x01aa, 0x38cf, 0x3273, 0x5058, 0x9540, 0x2bbd, 0x4c32, + 0x003f, 0x1e7d, 0x52c6, 0x0132, 0x82de, 0x15e7, 0xc402, 0x5652, 0x0639, 0x1681, 0x3b05, 0x019c, + 0x07d3, 0x0a44, 0x0c9e, 0x1c33, 0x1e95, 0x5a8a, 0x7a66, 0x2825, 0x4c3e, 0x52fc, 0x616c, 0x6299, + 0x68ee, 0x0b66, 0xa9cd, 0x53a2, 0x6939, 0x79fb, 0xaeb9, 0x43b2, 0xd80b, 0x6048, 0x7c5a, 0x0189, + 0x901e, 0x9243, 0x1f18, 0xa1b4, 0x0014, 0x2331, 0x2ab9, 0x311c, 0x5274, 0x7816, 0x176e, 0x9de0, + 0xbe7f, 0x7405, 0xf78c, 0xfc44, 0x346d, 0x4c31, 0xa1e9, 0x6976, 0x379f, 0x08c2, 0x1715, 0x3699, + 0x4f63, 0x5539, 0x785e, 0xe4f3, 0x72a5, 0xfa51, 0x048d, 0x5f68, 0xecd4, 0x73e6, 0x0056, 0x005a, + 0x9281, 0x6070, 0xaba0, 0x3ce7, 0x633b, 0x55c4, 0x57ce, 0x10f8, 0x4a19, 0x52e8, 0x6104, 0xf308, + 0x20e0, 0x24dd, 0x2ef3, 0x33d3, 0x6064, 0x0052, 0x0dd6, 0x2bef, 0xc4c4, 0x6342, 0x69d9, 0xf725, + 0x0001, 0x001e, 0x003b, 0x0058, 0x11fd, 0x52ab, 0x5dc8, 0xe15c, 0x74c8, 0x088c, 0x3b13, 0x3cc0, + 0x4325, 0xf99c, 0x0203, 0x2316, 0xc1f2, 0x0c34, 0x8dab, 0xb126, 0x0d5f, 0x203d, 0x2341, 0x3a38, + 0x4ad9, 0x6ac3, 0x8ca4, 0x164f, 0x248a, 0xaac4, 0x3b20, 0x7d00, 0xb872, 0x3ebb, 0x2178, 0xa96f, + 0xc9cb, 0x743a, 0x75b1, 0x0021, 0x0786, 0x9760, 0x1951, 0x9ac2, 0x1c22, 0x1eb3, 0x4dae, 0x0b90, + 0x1733, 0x3262, 0xf62e, 0x4afd, 0x4b28, 0x577a, 0x6d8a, 0x0dc4, 0xaf83, 0xc707, 0x4a46, 0x4e79, + 0x631b, 0x6937, 0xeff1, 0x7225, 0x34ca, 0x4069, 0x67da, 0x0137, 0x0251, 0x1a6d, 0x2733, 0x3db4, + 0x67ea, 0x04d6, 0x28c7, 0x56d4, 0x0028, 0x32de, 0x4c7b, 0x5aad, 0x6350, 0x9d41, 0x1fb0, 0x43cf, + 0xc900, 0x1229, 0xc16c, 0x43e2, 0x975b, 0xa844, 0xafa3, 0xc07d, 0x6d8f, 0x7a92, 0x0cad, 0x2516, + 0xcb23, 0x4fa2, 0x7b8a, 0xff4a, 0x9e3b, 0x2f21, 0xbddd, 0x77f0, 0x79ba, 0x8328, 0x07f9, 0x0926, + 0xdb7f, 0xe21c, 0x72ba, 0x0050, 0x0059, 0x39e3, 0xec37, 0x705a, 0x3bb0, 0x4997, 0x0256, 0x1cdd, + 0x2234, 0xf4f3, 0x8cb8, 0x8e41, 0x3fec, 0x5a95, 0xdabc, 0xe72b, 0x074f, 0x12f8, 0xac88, 0x3000, + 0x4476, 0x291d, 0x35a2, 0x640f, 0xe982, 0xa039, 0xa25a, 0x583a, 0x59d1, 0x5e86, 0xfb7e, 0x000f, + 0x129a, 0x19e5, 0xad54, 0xb670, 0x76d1, 0x0d28, 0x6ca2, 0x8aa5, 0x9eaf, 0x4033, 0xe10d, 0x64d2, + 0x9760, 0xab5c, 0x2f4d, 0x682c, 0x742c, 0x7d22, 0x9093, 0x1453, 0x578a, 0x0148, 0x9c8e, 0xa041, + 0x486d, 0x0728, 0x0c2d, 0x1db2, 0x3457, 0xbbb4, 0x4171, 0x575e, 0xf37a, 0x7522, 0x7c8e, 0xd3dd, + 0x5c21, 0xe495, 0x711f, 0xf77b, 0x0029, 0x003c, 0x004f, 0x0aa3, 0x9bb9, 0x26a0, 0x3219, 0xb974, + 0x3a6e, 0x3784, 0x46d2, 0x78dd, 0x13f5, 0xa4f9, 0x2b75, 0xaf01, 0x0e4f, 0x9b01, 0xb7f5, 0x3b71, + 0xa932, 0xdb17, 0xf68b, 0x01b3, 0x3ab6, 0x6d74, 0x8a19, 0x28c1, 0xb2fc, 0x02eb, 0x05bb, 0x897c, + 0xac4d, 0xcfbd, 0x5f19, 0x7350, 0xfe4d, 0x0032, 0x0059, 0x90d5, 0x9fdd, 0xa552, 0xde21, 0x6d07, + 0xf42b, 0x838b, 0x9e9e, 0xa1df, 0x2c02, 0x6ce9, 0xf01a, 0x0a36, 0x935e, 0x9d5d, 0x2298, 0x2e9e, + 0xe3da, 0xeee9, 0x84d8, 0x5805, 0xdaf1, 0x1593, 0x9e2e, 0xb1ed, 0x19f6, 0x1a0a, 0xc693, 0x532a, + 0xff1e, 0x0052, 0x0057, 0x6150, 0x6ff1, 0x0ec4, 0x5143, 0xdc99, 0x677c, 0x09b0, 0x0f0b, 0xc179, + 0xafa2, 0x3c51, 0x7eef, 0x5437, 0xd54c, 0x2afd, 0x51e5, 0x590b, 0x648e, 0xf160, 0x79b7, 0x1ec8, + 0xa329, 0xb87e, 0x394d, 0xd463, 0x58c8, 0x5ea5, 0x72cf, 0x1e54, 0x32d8, 0x6898, 0x74af, 0xf8ce, + 0x0032, 0x0056, 0x08c8, 0x13a3, 0x1ed6, 0x0ad5, 0x0cf2, 0x9226, 0x35f7, 0x7839, 0xdbb8, 0x68dc, + 0xb220, 0xefd6, 0x7c99, 0x2012, 0xa29f, 0xc630, 0xe0af, 0x751e, 0x5977, 0x4260, 0x503c, 0x5887, + 0x1d50, 0x452b, 0x49cf, 0x4a40, 0xd60a, 0xe030, 0x7f66, 0x0019, 0x002e, 0x26bb, 0x3a8e, 0xd84f, + 0x7de1, 0xb9ae, 0xb9f8, 0xd81d, 0x8650, 0x6d14, 0x6ee2, 0x2663, 0xb89e, 0xf940, 0x205d, 0xb756, + 0xd3f1, 0x5524, 0xf609, 0x16af, 0x9b55, 0x281c, 0xbe62, 0xf6db, 0x2768, 0xc40a, 0x443d, 0xd88f, + 0x73f7, 0xd8e5, 0x6108, 0x6aa2, 0x702c, 0x795b, 0x130b, 0x4ab5, 0x57ec, 0x662d, 0x0d4f, 0x6a93, + 0xeb0d, 0x6bd3, 0x790c, 0x1946, 0x2bfc, 0xbd35, 0xc8a1, 0x5263, 0x55ef, 0xdbd4, 0x5e53, 0x7b2f, + 0x7f6a, 0x46a2, 0xdabb, 0x63e2, 0xf173, 0x8f7f, 0x4451, 0x356f, 0xff3e, 0x1e51, 0xa7be, 0xab6a, + 0xc510, 0xd19d, 0x577e, 0xd92e, 0x90c4, 0x3f12, 0x400d, 0xe42c, 0x003a, 0x0045, 0x72bd, 0xc94e, + 0x6080, 0x2c8a, 0xe5f6, 0x759f, 0x2f3d, 0x5bb5, 0xf004, 0x70d0, 0x76b5, 0x504c, 0x6281, 0x647f, + 0x12ed, 0xe7ab, 0xeebd, 0xaeaa, 0xda8f, 0x6c9d, 0x8f01, 0xc214, 0xc498, 0x000c, 0x02c1, 0x9006, + 0x16aa, 0x1aef, 0x2069, 0x3605, 0x41a7, 0x4add, 0x5d5d, 0x7e9c, 0x09b1, 0x10bb, 0x1230, 0xb444, + 0x37a7, 0x4398, 0x6226, 0x74cb, 0x7f5c, 0x350d, 0x4359, 0x5286, 0x6798, 0x6f2d, 0x7c0e, 0x0ebc, + 0x606a, 0x6738, 0xfb23, 0x22b7, 0x3785, 0x5fa6, 0x683b, 0x7f1d, 0x031e, 0x092a, 0x1539, 0x42dc, + 0x50de, 0x0e0a, 0xad2f, 0x2f1c, 0x3e58, 0x4de3, 0xf460, 0xfde0, 0x2f58, 0x315e, 0x35cd, 0x4efc, + 0xd21d, 0x5789, 0x75c8, 0x7f09, 0x0025, 0x3e47, 0x5131, 0x0054, 0x6ca9, 0x175f, 0x3a92, 0x4b32, + 0x5c67, 0x64a4, 0x6e5d, 0x73e1, 0x7fc9, 0x0b31, 0x1889, 0x23d6, 0x2683, 0x2b16, 0xee90, 0x7d00, + 0x2184, 0xa33f, 0xb2b2, 0x6213, 0x630b, 0x68d2, 0x06a2, 0x0945, 0x1c6f, 0x2553, 0x4ca4, 0x554b, + 0x58c8, 0x5e44, 0x70e4, 0x8b11, 0x1aeb, 0x1c3e, 0x3e6f, 0xc79f, 0xd7a4, 0xdc19, 0x5fce, 0x14e7, + 0x3934, 0x3b99, 0x4fee, 0x6b71, 0x118b, 0x38f8, 0x55d5, 0xeb33, 0x7f36, 0x8220, 0x4aa2, 0x5e4b, + 0x247f, 0x34ad, 0x6aa7, 0x75c7, 0x176b, 0x22bb, 0x2835, 0x29cc, 0x34be, 0xb840, 0x3887, 0x3e56, + 0x4f24, 0x6c2d, 0x9b3d, 0x2117, 0x43ca, 0x5b63, 0x69fb, 0x01cc, 0x1aba, 0x1ed1, 0x375c, 0x5eab, + 0x6315, 0x6f06, 0x4e13, 0x7caf, 0x02cb, 0x04b1, 0x9326, 0x14bf, 0x217d, 0x26da, 0x3fcd, 0x4cae, + 0x57b1, 0x7333, 0xf571, 0x7770, 0x781d, 0x788b, 0x7d48, 0x0013, 0x1867, 0x26c4, 0x7198, 0x7c5a, + 0x1db9, 0xaf8f, 0x32b9, 0x5568, 0x7754, 0x5808, 0x5ca2, 0x75e3, 0x769c, 0x78c7, 0x00b4, 0x2a81, + 0x3088, 0x3dbe, 0x4ae2, 0x77bb, 0x1d0f, 0x451b, 0x5ab5, 0x8536, 0x0ae0, 0x1a2f, 0x218d, 0x28f9, + 0x2b21, 0x4622, 0x7339, 0x7561, 0x7622, 0x0f51, 0x1539, 0x1cd8, 0x3b9a, 0x3e51, 0x48ae, 0x53a7, + 0x5ca1, 0x6916, 0x6c06, 0x6d28, 0x74d3, 0x7f25, 0x0000, 0x001b, 0x0036, 0x0a13, 0x103c, 0x9171, + 0x44c5, 0xc63b, 0x5e86, 0x6380, 0x6818, 0x07cd, 0x365d, 0x5cb2, 0x85f9, 0x0db2, 0x1484, 0x2bfe, + 0x5a01, 0x6639, 0x6a6b, 0x8001, 0x0ad9, 0x2d16, 0x3672, 0x49fc, 0x5d1d, 0xf66e, 0x0702, 0x91f0, + 0x159d, 0x1dff, 0x2914, 0x32a9, 0x5011, 0x5c68, 0x5e6e, 0x64de, 0xf7b5, 0x2a4e, 0x2c59, 0x3549, + 0x42bd, 0x5645, 0x6691, 0x6fb9, 0x7bc4, 0x261f, 0x2aa0, 0xe90e, 0x0028, 0x002f, 0x0952, 0x16d5, + 0x4b1b, 0x1f2a, 0xa0aa, 0x64ee, 0x737d, 0x16d3, 0x5d0f, 0x67d0, 0x6b08, 0x7a6f, 0x13b6, 0x40b1, + 0x575a, 0x0f67, 0x332d, 0xbee6, 0x46dc, 0x6439, 0x1a50, 0x345b, 0x3844, 0xd2ff, 0xd5cf, 0x6f98, + 0x8224, 0x1b7d, 0x37ce, 0x4418, 0x679e, 0x7c1f, 0x0974, 0x31eb, 0x60ec, 0x735f, 0x75c2, 0x7e92, + 0x0005, 0x0010, 0x001b, 0x0ffb, 0x34e7, 0x351e, 0x574f, 0x75f8, 0x136a, 0x1c06, 0x4d33, 0x4d8c, + 0x54d5, 0x1a48, 0x59fd, 0x19f2, 0x2abe, 0x4979, 0xf7bd, 0xf8f8, 0x0fa9, 0x2417, 0x38e7, 0x50c6, + 0x7f36, 0x8204, 0x194c, 0x3db5, 0xbe92, 0x5016, 0x5523, 0xece4, 0xf6aa, 0x2785, 0x458a, 0x7276, + 0x0031, 0x3508, 0x4ee3, 0x421e, 0x438e, 0x448b, 0x6d43, 0x058e, 0x8f8c, 0x3565, 0x6d20, 0x7e21, + 0x8935, 0x3165, 0xb845, 0x7272, 0x0c57, 0x91f5, 0x314b, 0x5ce5, 0x1c63, 0x22c2, 0x330d, 0x3ce6, + 0x15e5, 0x21bb, 0x3070, 0x3d4c, 0xc0fc, 0x4b8b, 0x5c60, 0x1cb3, 0xa13b, 0x279c, 0x3247, 0x46d9, + 0x4765, 0x4d01, 0x7c15, 0x4ad3, 0xd381, 0x10ce, 0x4d6d, 0x7dba, 0x19b3, 0x99b9, 0x5203, 0x776e, + 0x7f56, 0x1337, 0x2f3c, 0x5221, 0x049a, 0x054a, 0x1dcb, 0x2a2b, 0x3753, 0xd543, 0x5a60, 0x5b33, + 0x095f, 0x0cbb, 0x0ee0, 0x3bd6, 0x52c6, 0x73d2, 0x0a16, 0x130e, 0x582c, 0x0002, 0xe9c3, 0xe9e4, + 0x7419, 0xf6cb, 0x9304, 0x2038, 0x5ef6, 0x6fb7, 0x9529, 0x1a97, 0x233c, 0xaa7b, 0x41a8, 0x508a, + 0x6f28, 0xf549, 0x063c, 0x15ef, 0x6d98, 0x1a60, 0x619c, 0x5942, 0xfd7f, 0x96bf, 0x2c8c, 0x381c, + 0x3a41, 0x4e6f, 0x58cd, 0x612c, 0x624f, 0x0000, 0x0011, 0x0022, 0x0033, 0x03df, 0xadd9, 0x39ca, + 0xdadb, 0x09c2, 0x1b92, 0x43c8, 0x4994, 0x5d77, 0x2831, 0x44b7, 0xe4d7, 0x2a08, 0x464b, 0x47f0, + 0x6621, 0xe865, 0x726b, 0x182f, 0x2659, 0x292e, 0x3c4e, 0x4d7e, 0xcdd3, 0xe10f, 0x04b5, 0x1d59, + 0x3972, 0x47c6, 0x6277, 0x05f0, 0x13a9, 0xa189, 0x4014, 0x42a3, 0xcd3a, 0x591c, 0xdc81, 0x603c, + 0xea35, 0x0026, 0x003c, 0x4c67, 0xde0d, 0x76d4, 0x0032, 0x285c, 0x6f0c, 0xefb7, 0x0c4c, 0x2b1b, + 0x7373, 0xafff, 0x5cc8, 0x624f, 0x69cc, 0x7387, 0xc225, 0x4aee, 0xddec, 0x7fe8, 0x0d78, 0x131e, + 0x9fa4, 0x260f, 0x243b, 0x7874, 0x3252, 0x6bc4, 0x6fa1, 0x0035, 0x188d, 0x2708, 0x331c, 0x477a, + 0xfd68, 0x16a6, 0x2656, 0x2ef4, 0x651d, 0x4e1b, 0x4e6e, 0x62f4, 0xe7b4, 0x68fe, 0x0a13, 0x4855, + 0x70b5, 0x890c, 0x1781, 0x7bda, 0x1099, 0x16ef, 0xd08e, 0x56d7, 0x3be0, 0xe71a, 0x7f60, 0x0659, + 0x1b42, 0x45ac, 0x5c27, 0x000a, 0x002b, 0x004c, 0x0b22, 0x21aa, 0x401d, 0xc400, 0x624a, 0xe791, + 0x7500, 0x063e, 0x372c, 0xc511, 0x756b, 0x07ae, 0x94de, 0x2c7f, 0x6daa, 0x5a67, 0x675f, 0x6faa, + 0x714b, 0x8001, 0x14d5, 0x3813, 0xf08c, 0x092d, 0x12a9, 0x1754, 0x264c, 0x6ba9, 0x6f27, 0x8215, + 0xaeac, 0x2f92, 0x4358, 0x6ba6, 0x06a1, 0x2622, 0x746c, 0x001c, 0x0043, 0xf980, 0x2d96, 0xe982, + 0xee0e, 0x3745, 0x5945, 0x109c, 0x2bee, 0x3292, 0x3ab3, 0xa740, 0xaec0, 0x43ae, 0x520e, 0x563f, + 0xffad, 0xa78e, 0x2e4c, 0x047c, 0x30ac, 0x34a6, 0x3d60, 0x41c6, 0x41f3, 0x74e6, 0x039e, 0x0e17, + 0x9dd2, 0x225c, 0x3de2, 0x6d92, 0x94d6, 0x46f8, 0x6418, 0x712d, 0x736d, 0xffb7, 0xd39f, 0x5a20, + 0x12dd, 0xa1d0, 0x3968, 0x52a3, 0xd8cf, 0xe847, 0x70d2, 0x15e7, 0x1e8c, 0x59b5, 0x7ef3, 0xa9ed, + 0x3448, 0xd8fa, 0x73c4, 0xfc5c, 0x3847, 0x484c, 0x5fb5, 0x7175, 0x3466, 0x369e, 0xdd67, 0x071b, + 0x1b7f, 0x9cc8, 0xa880, 0x5f00, 0x7849, 0x7c72, 0x003d, 0x0057, 0x1b5e, 0x613e, 0x1b94, 0x4784, + 0x7c73, 0x0085, 0x8f6d, 0x23de, 0x25fa, 0x3d16, 0x1cdf, 0x29b3, 0xb602, 0x5829, 0x60ca, 0xea07, + 0x7130, 0x7ac0, 0x0339, 0x4637, 0x8089, 0x7445, 0xf730, 0x81f8, 0xa5e9, 0x3e09, 0xf084, 0x02b0, + 0x30d0, 0x33e2, 0xb943, 0x3c78, 0xf6fd, 0x0000, 0x004b, 0x1bb0, 0x9fbe, 0x2356, 0x2ae7, 0xcc12, + 0xd3a0, 0x56f9, 0xdb29, 0xfe8d, 0x5479, 0x5e25, 0x7952, 0xfb91, 0x3d53, 0x68de, 0x164a, 0xa716, + 0xbd3a, 0x4792, 0x5b09, 0x7c22, 0x1d59, 0x2ea6, 0xb116, 0xb3d5, 0x372a, 0x4d04, 0xde5a, 0x6b5e, + 0x1dd6, 0x4623, 0xcf21, 0xd9c8, 0xfcc7, 0x7f04, 0x3e0d, 0x5038, 0x5e59, 0x6bc6, 0x8370, 0x92a7, + 0xa301, 0xcc17, 0x5efb, 0xee2c, 0x7367, 0x0000, 0x000b, 0xf689, 0x372e, 0x3ebb, 0x4d8e, 0x665b, + 0x66b4, 0x6c1d, 0x4a38, 0xd32a, 0xda02, 0x7563, 0xfc3a, 0x1115, 0x3fe7, 0x57e5, 0x3098, 0x4f89, + 0x5145, 0x7373, 0x5378, 0x6fc7, 0x7bfa, 0x191d, 0x2ad4, 0x42ad, 0x42be, 0xd0a4, 0x5c00, 0x6353, + 0x197f, 0xb70b, 0x391e, 0xbdf7, 0x4a12, 0x0000, 0x0011, 0x005b, 0x1052, 0x1715, 0xb88b, 0xdb90, + 0xf4d3, 0x7d89, 0x0597, 0x453f, 0x6a94, 0xfcf2, 0x0351, 0xa4ab, 0xac11, 0x49ef, 0xfdaa, 0x7ec1, + 0x2b7c, 0x4b91, 0x553b, 0xd602, 0x1a49, 0x38af, 0x6f8a, 0x85f0, 0x87d7, 0x2763, 0xa84e, 0x3f20, + 0xcb1d, 0x547f, 0x1719, 0xb7ef, 0x3dd7, 0xf837, 0x22c0, 0x22e3, 0xbd1d, 0x3f86, 0x6267, 0x791e, + 0x0013, 0x003c, 0x9176, 0x585e, 0x5f4c, 0x69cf, 0xbf45, 0xd9e7, 0xf752, 0x78b7, 0x7a19, 0x7d6a, + 0x1d60, 0x3e8c, 0x45d9, 0x51a0, 0xd541, 0xf100, 0x7560, 0x05c4, 0xbdab, 0xbe2e, 0x3f31, 0x722a, + 0x0b4e, 0x149b, 0xa8e8, 0x3311, 0x8489, 0x2410, 0x26f1, 0x2a44, 0xbbef, 0xbbfb, 0xc3b6, 0x6f53, + 0xfb9b, 0x01fa, 0x19d7, 0x99e9, 0x2164, 0x3623, 0x376e, 0x7ae3, 0x7e32, 0x7fcc, 0x41ba, 0xd0ad, + 0x70b5, 0x80a5, 0x879e, 0x26a8, 0x29fe, 0x5c7f, 0x7197, 0x784f, 0x8b75, 0x1e36, 0x5108, 0xd4d6, + 0x39b5, 0xd823, 0x6cd1, 0x3abb, 0x6449, 0x9260, 0x359c, 0x66f6, 0x6950, 0x0918, 0x3a16, 0x7d60, + 0xab42, 0x8cb8, 0xbbcb, 0xe3dc, 0x6ebe, 0x7b91, 0x38d3, 0x5a8c, 0x6321, 0x77dc, 0x2fe1, 0xe107, + 0x722e, 0x8cce, 0x12d5, 0x2405, 0x5329, 0xd798, 0xe011, 0x745e, 0xf51f, 0x7788, 0x07e0, 0x0cf2, + 0x1a4e, 0x2434, 0xc071, 0x421d, 0xe49f, 0x703d, 0x0749, 0x153c, 0x2088, 0x25ac, 0xaf4c, 0x506a, + 0x569b, 0xeb4a, 0x6c29, 0x002f, 0x87a4, 0x095f, 0xaf74, 0x334e, 0xcce1, 0xe5ef, 0x7979, 0x1b5b, + 0x33b0, 0xc43d, 0x7ed5, 0x9650, 0x9b14, 0x2944, 0xab56, 0x6ff8, 0x006a, 0x1bea, 0xa75b, 0xccc3, + 0x3abd, 0x4404, 0xee32, 0x72f1, 0x6143, 0x7faf, 0x12d2, 0x982a, 0xcfc3, 0x65c0, 0x6f49, 0x0036, + 0x0588, 0x3bd4, 0x6db0, 0x260f, 0xc8ec, 0x5186, 0xf26b, 0x7d37, 0x13fb, 0x3e8d, 0xd604, 0xf754, + 0x7a45, 0x044d, 0xa48d, 0xc9cd, 0x2377, 0xb7d7, 0xf82c, 0x7d3f, 0x861b, 0x130a, 0x16bf, 0x9846, + 0x2e5b, 0xbffb, 0xdb5b, 0x6a98, 0x230c, 0xa80a, 0xb327, 0xb653, 0xbb26, 0xbbe2, 0x3ec1, 0x55d6, + 0x7030, 0x0005, 0x004f, 0x87fe, 0x10f7, 0x4544, 0x6346, 0x6362, 0x0342, 0x4484, 0x5833, 0xbe27, + 0x4db6, 0x14e6, 0x609e, 0x15b7, 0x233c, 0x21b4, 0xb7d7, 0xba11, 0xd540, 0xda15, 0x5b77, 0xe90a, + 0x8bf8, 0xbb7f, 0x5e0e, 0x7f9b, 0x001f, 0x0064, 0x1c4e, 0x1d77, 0x5d07, 0xbbf4, 0x7149, 0x879a, + 0x0af4, 0x175c, 0xcd16, 0x5b75, 0x5e65, 0x868c, 0xaa11, 0x3fd5, 0xba52, 0x40a7, 0xcc28, 0x0270, + 0x9ad8, 0x61a7, 0x1454, 0x2f26, 0x7ef2, 0x8186, 0xb40b, 0xf892, 0x0036, 0x003d, 0x3d5c, 0x40af, + 0x4311, 0x67fd, 0x68d2, 0x6c6d, 0xed62, 0x0aab, 0x33e8, 0x34e1, 0x73ce, 0xf661, 0x1939, 0x9e39, + 0x2760, 0x2f83, 0xe6d4, 0xe706, 0xe79c, 0xf323, 0x3a66, 0x5e4b, 0x60a0, 0xf978, 0xa776, 0x333c, + 0x3b1d, 0xbda9, 0x517e, 0x7489, 0x5c9a, 0xfbbc, 0x25d8, 0x450c, 0x5c4f, 0x0009, 0xac4c, 0xd158, + 0x697f, 0x9167, 0xa02e, 0x00f6, 0x7b75, 0xad34, 0x3318, 0x2637, 0xb8b8, 0xcd76, 0xd59b, 0x12b6, + 0xb039, 0xee24, 0x0cc6, 0x8dee, 0xb68a, 0x3eb7, 0x4a69, 0x5935, 0x86ce, 0x1786, 0x2668, 0xa8af, + 0xcdaf, 0xe857, 0x7287, 0x001a, 0x005d, 0x0c51, 0x1961, 0xcda0, 0x6220, 0x7620, 0x7832, 0x3fad, + 0x6db6, 0x7de3, 0x848b, 0x899b, 0x43cf, 0xe6a7, 0xed57, 0x8639, 0x3b13, 0x4ee3, 0x633d, 0x07ac, + 0xbfee, 0x50ee, 0x55ea, 0x6668, 0x5cc9, 0x010d, 0x17c6, 0x265b, 0x3042, 0x513e, 0x66f8, 0x7034, + 0x7522, 0x7d4d, 0x74f9, 0x003c, 0x9f66, 0xa49e, 0xa7b8, 0x30a7, 0xd073, 0x5449, 0x5ad3, 0x6dbb, + 0x8274, 0x0ec1, 0xa2ce, 0x32ad, 0x6083, 0xe367, 0x73de, 0x02ea, 0x100c, 0x925f, 0x24af, 0x487e, + 0x692d, 0x14d8, 0xa476, 0xf63f, 0x0553, 0x11b3, 0x9867, 0x9f46, 0x21cd, 0x6364, 0x6aca, 0x782a, + 0x011d, 0xa2ca, 0xa803, 0x5482, 0x1d60, 0xbaf0, 0xbb8c, 0x43da, 0xcfa2, 0x0054, 0x1220, 0x2781, + 0x2bef, 0x33aa, 0xbbc7, 0x5bc6, 0x6101, 0x8802, 0xe1a0, 0x8958, 0x90ee, 0x9b50, 0xa687, 0xaf90, + 0xbeb0, 0xe0b5, 0x60f0, 0x7bd7, 0x1538, 0x3163, 0x46a2, 0xcbb5, 0xd555, 0xd5f3, 0x95ae, 0xaa1c, + 0x315c, 0x4af0, 0xe970, 0x0174, 0x1aa5, 0x1aad, 0x2839, 0x425e, 0x47bc, 0xf9eb, 0xffd2, 0xa380, + 0x282e, 0x3f6a, 0xd2f9, 0xd7f4, 0x6a17, 0x2020, 0xb631, 0xdad5, 0x7add, 0xfb31, 0x7d29, 0xfe59, +}; + +inline uint32_t amdgpu_rsq_normalized_bits(uint32_t index) { + uint32_t result = amdgpu_rsq_polynomial(index); + const uint32_t bucket = index >> 15; + const uint16_t key = index & 0x7fffu; + uint32_t first = kAmdgpuRsqCorrectionOffsets[bucket]; + uint32_t last = kAmdgpuRsqCorrectionOffsets[bucket + 1]; + const uint32_t end = last; + while (first < last) { + const uint32_t middle = first + (last - first) / 2; + if ((kAmdgpuRsqCorrections[middle] & 0x7fffu) < key) + first = middle + 1; + else + last = middle; + } + if (first != end && (kAmdgpuRsqCorrections[first] & 0x7fffu) == key) + result = (kAmdgpuRsqCorrections[first] & 0x8000u) ? result + 1 : result - 1; + return result; +} + +inline uint32_t amdgpu_rsq_bits(uint32_t input, bool quiet_snan = true) { + const uint32_t magnitude = input & 0x7fffffffu; + const uint32_t sign = input & 0x80000000u; + if (magnitude > 0x7f800000u) + return input | (quiet_snan ? 0x00400000u : 0u); + if (magnitude < 0x00800000u) + return sign | 0x7f800000u; + if (sign != 0) + return 0xffc00000u; + if (magnitude == 0x7f800000u) + return 0; + + const int exponent = static_cast(magnitude >> 23) - 127; + const unsigned parity = exponent & 1; + const uint32_t normalized = + amdgpu_rsq_normalized_bits((parity << 23) | (magnitude & 0x007fffffu)); + const int result_exponent = + static_cast(normalized >> 23) - (exponent - static_cast(parity)) / 2; + return (static_cast(result_exponent) << 23) | (normalized & 0x007fffffu); +} +} // namespace util::detail + +namespace util { + +/// @brief AMD single-precision reciprocal square root matching captured RDNA3/4 outputs. +/// @details The empirical mapping satisfies the shared ISA one-ULP bound and flushes input +/// subnormals independently of MODE. Positive normal inputs produce normal outputs. NaN +/// payloads retain their sign, with caller-selected quieting; negative normal inputs and negative +/// infinity produce 0xffc00000. Signed zero and flushed subnormals produce signed infinity; +/// positive infinity produces positive zero. Results are independent of host rounding. +inline float amdgpu_rsq_f32(float value, bool quiet_snan = true) { + return std::bit_cast(detail::amdgpu_rsq_bits(std::bit_cast(value), quiet_snan)); +} + +} // namespace util diff --git a/emulation/rocjitsu/lib/util/include/util/amdgpu_sqrt.h b/emulation/rocjitsu/lib/util/include/util/amdgpu_sqrt.h new file mode 100644 index 000000000000..23716b6d3763 --- /dev/null +++ b/emulation/rocjitsu/lib/util/include/util/amdgpu_sqrt.h @@ -0,0 +1,185 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +/// @file +/// @brief Shared single-precision square-root mapping for AMDGPU instruction execution. + +#include +#include + +namespace util::detail { + +// Staged fixed-point cubic fitted to the exhaustive normalized SQRT captures of +// physical gfx1100 and gfx1201 (byte-identical). The index combines exponent +// parity with the 23-bit input mantissa; 32 segments of 2^19 fractions f each. +// Per segment the result mantissa offset, in units of 2^-24 result ULP, is +// constant * 2^22 (constant in quarter ULP) +// + L * 2^19, L = floor((linear * f + b) / 2^17), b = 2^15, or 2^16 when +// linear * f >= 2^40 +// - Q * 2^12, Q = round(inner * square / 2^23), square = (g * g) >> 14, +// g = f & ~3, inner = quadratic * 2^8 - ceil'(cubic * f / 2^10) +// where ceil' negates, adds a quarter of the 2^10 step, and floors, and the +// product rounds at bit 24 instead of bit 23 once it reaches 2^47. The sum +// rounds half-to-even to a mantissa. Quadratic uses eighth ULP; cubic uses +// quarter ULP. This is an empirical mapping, not a claim about the hardware +// implementation. Sparse one-ULP residuals complete the captured mapping. +namespace sqrt { +struct Coefficient { + uint32_t constant; + uint32_t linear; + uint16_t quadratic; + uint16_t cubic; +}; +inline constexpr Coefficient coefficients[] = { + {0u, 2097143u, 32722u, 474u}, {1032684u, 2034529u, 29882u, 409u}, + {2035417u, 1977208u, 27431u, 356u}, {3010662u, 1924474u, 25297u, 312u}, + {3960563u, 1875745u, 23425u, 275u}, {4886999u, 1830541u, 21774u, 244u}, + {5791627u, 1788454u, 20308u, 218u}, {6675918u, 1749143u, 18999u, 195u}, + {7541186u, 1712315u, 17825u, 176u}, {8388608u, 1677719u, 16767u, 159u}, + {9219243u, 1645139u, 15810u, 145u}, {10034053u, 1614387u, 14940u, 132u}, + {10833909u, 1585296u, 14147u, 120u}, {11619604u, 1557724u, 13422u, 110u}, + {12391866u, 1531542u, 12757u, 102u}, {13151360u, 1506637u, 12145u, 94u}, + {13898700u, 2965809u, 46276u, 670u}, {15359137u, 2877259u, 42260u, 578u}, + {16777216u, 2796194u, 38793u, 503u}, {18156420u, 2721617u, 35775u, 441u}, + {19499783u, 2652705u, 33129u, 389u}, {20809961u, 2588776u, 30793u, 346u}, + {22089298u, 2529256u, 28720u, 308u}, {23339875u, 2473662u, 26869u, 277u}, + {24563549u, 2421579u, 25208u, 249u}, {25761984u, 2372654u, 23712u, 225u}, + {26936680u, 2326579u, 22358u, 205u}, {28088995u, 2283088u, 21129u, 186u}, + {29220162u, 2241948u, 20008u, 171u}, {30331303u, 2202954u, 18982u, 156u}, + {31423446u, 2165927u, 18042u, 144u}, {32497533u, 2130707u, 17176u, 133u}, +}; + +// Sorted residual keys: (index << 1) | 1 selects +1 ULP, otherwise -1 ULP. +inline constexpr uint32_t corrections[] = { + 0x0067350u, 0x00a791au, 0x00c1d1au, 0x00d7428u, 0x00efacau, 0x0180002u, 0x01ca73eu, 0x01eb1a6u, + 0x01ec160u, 0x0241f44u, 0x02a30b4u, 0x02abb44u, 0x02f1d10u, 0x02fd82au, 0x03871a6u, 0x03b8802u, + 0x03b9f4au, 0x03d10e8u, 0x03f1530u, 0x04a9836u, 0x04c3abeu, 0x04c3ea4u, 0x04cec24u, 0x04e9f2au, + 0x0553ac4u, 0x05ab3c8u, 0x05cfba6u, 0x05dd09au, 0x05f089au, 0x05f2246u, 0x05f6e24u, 0x065f474u, + 0x06775c6u, 0x068debau, 0x06a81d0u, 0x06c84c8u, 0x06d4bd4u, 0x06de130u, 0x06e6624u, 0x06f16fcu, + 0x06f94eeu, 0x06fbf06u, 0x06fddaau, 0x06fee94u, 0x0763c3eu, 0x0778aceu, 0x0783472u, 0x079041eu, + 0x07918b6u, 0x07b36f2u, 0x07b95e2u, 0x07c3e74u, 0x07c6838u, 0x07c8ef2u, 0x07d1b22u, 0x07e27b0u, + 0x07e5882u, 0x07e9ee8u, 0x07ec934u, 0x07ee12au, 0x07ee510u, 0x07eefb0u, 0x07f24a6u, 0x07f573cu, + 0x07fd694u, 0x0842278u, 0x088cfbeu, 0x08d66cau, 0x08e77b2u, 0x08ec484u, 0x08f3aecu, 0x08f54d6u, + 0x08f6e26u, 0x08f9b56u, 0x094af02u, 0x094e5a6u, 0x09618e0u, 0x096ea6eu, 0x0995500u, 0x09a165cu, + 0x09ab5f4u, 0x09b55a6u, 0x09c8038u, 0x09cf812u, 0x09dfd00u, 0x09e1c0au, 0x09ec02cu, 0x09fc82cu, + 0x09fdd1au, 0x0a6fabau, 0x0a933e2u, 0x0a98158u, 0x0aa0166u, 0x0ab2166u, 0x0ab576au, 0x0ac2d5eu, + 0x0ac5264u, 0x0acba1eu, 0x0ad63b6u, 0x0adbceeu, 0x0ae2256u, 0x0ae275cu, 0x0ae8c6eu, 0x0aeb076u, + 0x0aebb8cu, 0x0aeea64u, 0x0af21c8u, 0x0af27beu, 0x0af2e50u, 0x0af2f78u, 0x0afe82eu, 0x0aff786u, + 0x0b5497au, 0x0b67d8au, 0x0bac3b8u, 0x0bb5ed0u, 0x0bbf5c6u, 0x0bc4df6u, 0x0bc7e42u, 0x0bd03d6u, + 0x0bd6e34u, 0x0bdc3c6u, 0x0be492cu, 0x0be9b1cu, 0x0bedc16u, 0x0bfafc8u, 0x0bfffb8u, 0x0c191f4u, + 0x0c340f6u, 0x0c45fb0u, 0x0c5ece4u, 0x0c7f2b0u, 0x0c83fb0u, 0x0c87a4au, 0x0c896a0u, 0x0c9df9eu, + 0x0ca50d2u, 0x0cb4b3au, 0x0cd1406u, 0x0cd5bb0u, 0x0ce1bd2u, 0x0ce61b2u, 0x0ce6890u, 0x0cf13a0u, + 0x0cf6216u, 0x0cf92d2u, 0x0cf95f4u, 0x0cfc2e4u, 0x0cfc5a0u, 0x0cff416u, 0x0d569c4u, 0x0d6042au, + 0x0d7eea4u, 0x0d7eecau, 0x0d8e43eu, 0x0da9db4u, 0x0db6864u, 0x0dbacc0u, 0x0dd19c4u, 0x0de56b6u, + 0x0df2864u, 0x0df2eb8u, 0x0df8d1eu, 0x0dfc25au, 0x0e70dfcu, 0x0e8e366u, 0x0e95ba2u, 0x0ea332cu, + 0x0eaaba2u, 0x0ec054au, 0x0ec64bau, 0x0edc480u, 0x0edfaeeu, 0x0ee10f6u, 0x0ee38e2u, 0x0ee41eau, + 0x0eeeeecu, 0x0eefcceu, 0x0ef25d4u, 0x0ef5688u, 0x0efe948u, 0x0f4a7c6u, 0x0f65538u, 0x0f824ccu, + 0x0f8293au, 0x0f8e590u, 0x0f9f04au, 0x0f9fd78u, 0x0faa22au, 0x0fb93c8u, 0x0fbcc60u, 0x0fc88e2u, + 0x0fcd5e8u, 0x0fd64a0u, 0x0fe190cu, 0x0fe1e2au, 0x0feb4e0u, 0x0ff0fdcu, 0x0ff871au, 0x0ffbff0u, + 0x0fffcb8u, 0x10cb3e3u, 0x10d3f75u, 0x10f4b75u, 0x10f5acdu, 0x116ccb3u, 0x11b9a27u, 0x11e2dabu, + 0x139bed8u, 0x13b1320u, 0x13c0b12u, 0x144823au, 0x145ba10u, 0x147709au, 0x1489138u, 0x14c3b76u, + 0x14cd4dcu, 0x14ce90eu, 0x14e310eu, 0x14e8a74u, 0x1562b56u, 0x1580004u, 0x15a0484u, 0x15ae878u, + 0x15e3b62u, 0x166802au, 0x169499eu, 0x169ea94u, 0x16a5abcu, 0x16bd62au, 0x16d779eu, 0x16dac16u, + 0x17aa020u, 0x17af760u, 0x17b6174u, 0x17c1be8u, 0x17de544u, 0x17deba6u, 0x17ed4ceu, 0x17eebbcu, + 0x17f649au, 0x17fc41cu, 0x184ca7cu, 0x18ae6cau, 0x18cfb18u, 0x18d207eu, 0x18d25dcu, 0x18d4766u, + 0x18dca7cu, 0x18e2102u, 0x18eb8c0u, 0x18ec94cu, 0x18f9646u, 0x18fc1a6u, 0x1972f7au, 0x199164eu, + 0x1998574u, 0x199f938u, 0x19a3842u, 0x19ab1dcu, 0x19ad3acu, 0x19b2292u, 0x19b2e2au, 0x19bf558u, + 0x19c8292u, 0x19dd2e4u, 0x19e0ad2u, 0x19e567cu, 0x19e778cu, 0x19e98deu, 0x19f0e18u, 0x19f2ee0u, + 0x19f74c6u, 0x1a50d5cu, 0x1a703a4u, 0x1a7a3b8u, 0x1a97e06u, 0x1ab2278u, 0x1ab3c30u, 0x1acbfdcu, + 0x1ad1b72u, 0x1adb78cu, 0x1adb7fau, 0x1aec3feu, 0x1aedfe6u, 0x1af3da2u, 0x1b4431cu, 0x1b7e7d8u, + 0x1ba06f0u, 0x1ba9424u, 0x1bb16f0u, 0x1bbc774u, 0x1bca2c4u, 0x1bd0538u, 0x1bdabccu, 0x1bdeaaeu, + 0x1bdf3f8u, 0x1beefa0u, 0x1bf24d4u, 0x1bf5c92u, 0x1bfcbacu, 0x1bfd0beu, 0x1bfe3eeu, 0x1c4cbb6u, + 0x1c583e6u, 0x1c5ad72u, 0x1c7c40au, 0x1c8e93cu, 0x1c8f33eu, 0x1c92aa2u, 0x1ca4b26u, 0x1cae5aeu, + 0x1cb3fd4u, 0x1cc4f8cu, 0x1cc7c5eu, 0x1cc90a0u, 0x1cccc6au, 0x1cce930u, 0x1cdfa42u, 0x1ce4c9au, + 0x1ce7e60u, 0x1cf1abau, 0x1cf7e48u, 0x1cfdaf6u, 0x1cfe8b8u, 0x1cff362u, 0x1d49c6eu, 0x1d78e3au, + 0x1d9838eu, 0x1d9b6aeu, 0x1d9f82cu, 0x1da04e4u, 0x1da4dc2u, 0x1da8ebcu, 0x1da91c2u, 0x1db7140u, + 0x1dc1e60u, 0x1dc5cb0u, 0x1dc934cu, 0x1dd64d6u, 0x1dd88b0u, 0x1dd9ba8u, 0x1dde3f8u, 0x1df750au, + 0x1dfbf5au, 0x1e54068u, 0x1e9ee84u, 0x1ea6776u, 0x1eaa586u, 0x1eb2a92u, 0x1eb4094u, 0x1ebc14cu, + 0x1ebdd5au, 0x1ec0412u, 0x1ed5522u, 0x1ed5a22u, 0x1ee173eu, 0x1ee47ccu, 0x1efafa2u, 0x1f83a7eu, + 0x1f8d648u, 0x1f98e28u, 0x1fb593au, 0x1fcbf8au, 0x1fd1a20u, 0x1fd3be0u, 0x1fd87e6u, 0x1fe3880u, + 0x1fe3cd6u, 0x1fe7c68u, 0x1ffab36u, 0x1ffab46u, 0x1ffc73cu, +}; +} // namespace sqrt + +inline uint32_t amdgpu_sqrt_correct(uint32_t index, uint32_t result) { + const uint32_t key = index << 1; + uint32_t first = 0; + uint32_t last = sizeof(sqrt::corrections) / sizeof(sqrt::corrections[0]); + const uint32_t end = last; + while (first < last) { + const uint32_t middle = first + (last - first) / 2; + if ((sqrt::corrections[middle] & ~1u) < key) + first = middle + 1; + else + last = middle; + } + if (first != end && (sqrt::corrections[first] & ~1u) == key) + result = (sqrt::corrections[first] & 1u) ? result + 1 : result - 1; + return result; +} + +inline uint32_t amdgpu_sqrt_normalized_bits(uint32_t index) { + const auto coefficient = sqrt::coefficients[index >> 19]; + const int64_t fraction = index & 0x7ffffu; + const uint64_t product = uint64_t{coefficient.linear} * static_cast(fraction); + const uint64_t bias = product >= (uint64_t{1} << 40) ? uint64_t{1} << 16 : uint64_t{1} << 15; + const int64_t linear = static_cast((product + bias) >> 17); + const int64_t square_input = fraction & ~int64_t{3}; + const int64_t square = (square_input * square_input) >> 14; + const int64_t inner = + (int64_t{coefficient.quadratic} << 8) + ((256 - int64_t{coefficient.cubic} * fraction) >> 10); + const int64_t quadratic = inner * square; + const int64_t negated = quadratic >= (int64_t{1} << 47) + ? (((int64_t{1} << 23) - quadratic) >> 24) * 2 + : ((int64_t{1} << 22) - quadratic) >> 23; + const int64_t sum = (int64_t{coefficient.constant} << 22) + (linear << 19) + (negated * 4096); + const int64_t quotient = sum >> 24; + const int64_t remainder = sum & ((int64_t{1} << 24) - 1); + const int64_t midpoint = int64_t{1} << 23; + const int64_t mantissa = + quotient + (remainder > midpoint || (remainder == midpoint && (quotient & 1))); + const uint32_t result = 0x3f800000u + static_cast(mantissa); + // Exhaustive normalized captures bound every residual to within 1/2048 + // ULP of a rounding midpoint. Other inputs need no correction-table lookup. + if (remainder < midpoint - 8192 || remainder > midpoint + 8192) + return result; + return amdgpu_sqrt_correct(index, result); +} + +inline uint32_t amdgpu_sqrt_bits(uint32_t input, bool quiet_snan = true) { + const uint32_t magnitude = input & 0x7fffffffu; + const uint32_t sign = input & 0x80000000u; + if (magnitude > 0x7f800000u) + return input | (quiet_snan ? 0x00400000u : 0u); + if (magnitude < 0x00800000u) + return sign; + if (sign != 0) + return 0xffc00000u; + if (magnitude == 0x7f800000u) + return input; + + const int exponent = static_cast(magnitude >> 23) - 127; + const unsigned parity = exponent & 1; + const uint32_t normalized = + amdgpu_sqrt_normalized_bits((parity << 23) | (magnitude & 0x007fffffu)); + const int result_exponent = + static_cast(normalized >> 23) + (exponent - static_cast(parity)) / 2; + return (static_cast(result_exponent) << 23) | (normalized & 0x007fffffu); +} +} // namespace util::detail + +namespace util { + +/// @brief AMD single-precision square root matching captured RDNA3/4 outputs. +/// @details The empirical mapping satisfies the ISA one-ULP bound and flushes input +/// subnormals to signed zero independently of MODE. Signed zero is preserved; negative +/// normal inputs and negative infinity produce 0xffc00000; positive infinity is preserved. +/// NaN payloads keep their sign and are quieted unless @p quiet_snan is false. +/// Results are independent of MODE rounding and host rounding. +inline float amdgpu_sqrt_f32(float value, bool quiet_snan = true) { + return std::bit_cast(detail::amdgpu_sqrt_bits(std::bit_cast(value), quiet_snan)); +} + +} // namespace util diff --git a/emulation/rocjitsu/lib/util/include/util/amdgpu_trig.h b/emulation/rocjitsu/lib/util/include/util/amdgpu_trig.h new file mode 100644 index 000000000000..1e54f7f279ab --- /dev/null +++ b/emulation/rocjitsu/lib/util/include/util/amdgpu_trig.h @@ -0,0 +1,226 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +/// @file +/// Integer SIN/COS reduction and staged approximation from RDNA3/4 captures. + +#include "util/big_int.h" + +#include +#include +#include + +namespace util::detail::trig { +using U128 = uint128_t; +struct Coefficient { + int32_t constant; + int32_t linear; + int32_t quadratic; + int32_t cubic; +}; +// Coefficients use units of 2^-26. The first table approximates sine, the +// second cosine, in sixteen intervals covering the first octant. Staged +// integer arithmetic follows captures from gfx1100 and gfx1201; it avoids +// dependence on host rounding, denormal controls, or libm argument reduction. +inline constexpr Coefficient coefficients[2][16] = {{{0, 3294200, 0, -1322}, + {3292874, 3290232, -3968, -1320}, + {6577818, 3278340, -7928, -1312}, + {9846914, 3258548, -11864, -1304}, + {13092288, 3230908, -15778, -1290}, + {16306122, 3195480, -19650, -1274}, + {19480674, 3152356, -23476, -1256}, + {22608294, 3101636, -27246, -1234}, + {25681450, 3043444, -30948, -1210}, + {28692736, 2977924, -34576, -1182}, + {31634898, 2905224, -38122, -1152}, + {34500850, 2825532, -41576, -1118}, + {37283686, 2739032, -44928, -1082}, + {39976702, 2645928, -48176, -1042}, + {42573412, 2546452, -51304, -1002}, + {45067558, 2440844, -54310, -958}}, + {{67108864, 0, -80872, 32}, + {67028028, -161636, -80774, 98}, + {66785716, -322884, -80482, 162}, + {66382510, -483352, -79996, 226}, + {65819384, -642664, -79318, 290}, + {65097694, -800416, -78448, 352}, + {64219178, -956248, -77390, 414}, + {63185952, -1109776, -76144, 476}, + {62000504, -1260628, -74716, 536}, + {60665694, -1408448, -73104, 594}, + {59184734, -1552872, -71320, 652}, + {57561192, -1693552, -69364, 708}, + {55798980, -1830152, -67242, 762}, + {53902344, -1962348, -64956, 814}, + {51875852, -2089812, -62514, 864}, + {49724388, -2212244, -59922, 912}}}; + +inline uint64_t round_even(U128 value, unsigned shift) { + if (!shift) + return static_cast(value); + U128 whole = value >> shift; + U128 rest = value & ((U128{1} << shift) - 1); + U128 midpoint = U128{1} << (shift - 1); + return static_cast(whole + + (rest > midpoint || (rest == midpoint && ((whole & 1) != 0)))); +} +// Convert a positive fixed-point result to FP32 with truncation. +inline uint32_t fixed_to_float(uint64_t value, unsigned fractional) { + if (!value) + return 0; + const unsigned leading = static_cast(std::bit_width(value)) - 1; + const uint32_t mantissa = + static_cast(leading >= 23 ? value >> (leading - 23) : value << (23 - leading)); + const int exponent = static_cast(leading) - static_cast(fractional) + 127; + return (static_cast(exponent) << 23) | (mantissa & 0x7fffff); +} +// Round the linear coordinate while saturating the low twenty bits. +// Reflected intervals approach their right endpoint from below. +inline uint32_t linear_fraction(uint32_t fraction, unsigned shift, bool reflected) { + uint32_t value = fraction - static_cast(reflected); + uint32_t block = value >> (20 + shift); + uint32_t rounded = (value + (1u << (shift - 1))) >> shift; + uint32_t limit = ((block + 1) << 20) - 1; + return std::min(rounded, limit) << shift; +} +// The original small-input region uses this captured FP32 multiplier, +// independently of guest rounding. Integer multiplication also preserves +// subnormal inputs and outputs when requested by MODE. +inline uint32_t small_sine(uint32_t bits, unsigned denorm) { + const uint32_t magnitude = bits & 0x7fffffff; + const uint32_t exponent_bits = magnitude >> 23; + const uint32_t mantissa = (magnitude & 0x7fffff) | (exponent_bits ? 0x800000 : 0); + const int exponent = exponent_bits ? static_cast(exponent_bits) - 127 : -126; + const uint64_t product = uint64_t{mantissa} * 0xc90fd5u; + if (!product) + return bits & 0x80000000u; + const unsigned leading = static_cast(std::bit_width(product)) - 1; + int result_exponent = exponent - 44 + static_cast(leading); + uint32_t result; + if (result_exponent < -126) { + result = static_cast(round_even(product, 21)); + if (!(denorm & 2) && result < 0x800000) + result = 0; + } else { + uint32_t rounded = static_cast(round_even(product, leading - 23)); + if (rounded == 0x1000000) { + rounded >>= 1; + ++result_exponent; + } + result = (uint32_t(result_exponent + 127) << 23) | (rounded & 0x7fffff); + } + return result | (bits & 0x80000000u); +} +inline uint32_t evaluate(uint32_t bits, bool cosine, unsigned denorm = 3, bool quiet_snan = true) { + uint32_t magnitude = bits & 0x7fffffff; + if (magnitude >= 0x7f800000) { + if (magnitude == 0x7f800000) + return 0xffc00000; + return bits | (quiet_snan ? 0x00400000u : 0); + } + if (!magnitude || (magnitude < 0x00800000 && !(denorm & 1))) + return cosine ? 0x3f800000 : bits & 0x80000000u; + if (!cosine && magnitude < 0x39c00000) + return small_sine(bits, denorm); + // Reduce finite inputs in turns before polynomial evaluation. Every FP32 + // value with exponent >= 23 is integral, so no large float-to-int cast is + // needed. The phase retains 31 fractional bits. + int exponent = static_cast(magnitude >> 23) - 127; + if (exponent >= 23) + return cosine ? 0x3f800000 : 0; + uint64_t mantissa = (magnitude & 0x7fffff) | 0x800000; + int shift = exponent + 8; + uint32_t phase = static_cast(shift >= 0 ? (mantissa << shift) & 0x7fffffff + : -shift >= 64 ? 0 + : mantissa >> -shift); + unsigned quadrant = phase >> 29; + uint32_t quarter_phase = phase & 0x1fffffff; + bool reflected = quarter_phase >= 0x10000000; + uint32_t reduced = reflected ? 0x20000000 - quarter_phase : quarter_phase; + unsigned table = unsigned(cosine) ^ (quadrant & 1) ^ unsigned(reflected); + unsigned index = (reduced - unsigned(reflected)) >> 24; + uint32_t fraction = reduced - (index << 24); + const auto c = coefficients[table][index]; + uint32_t result; + if (table && reduced - unsigned(reflected) < 46592) + result = 0x3f800000; + else if (!table && !reduced) + result = 0; + else { + uint32_t square_input = (fraction - unsigned(reflected)) >> 7; + // Every interval has a nonpositive quadratic correction. Keep its + // magnitude unsigned so the wide-integer fallback follows the same path. + // The inner coefficient retains eighteen coordinate bits. Cosine uses + // the ones-complement distance from the right endpoint. Round the + // coefficient to seven fractional bits in its table units (Q33 overall). + uint32_t inner_coordinate = c.cubic <= 0 ? fraction - unsigned(reflected) + : (1u << 24) - fraction + unsigned(reflected) - 1; + inner_coordinate &= ~63u; + uint64_t inner_value = + c.cubic <= 0 ? static_cast(-int64_t{c.quadratic} * (int64_t{1} << 24) - + int64_t{c.cubic} * inner_coordinate) + : static_cast(-int64_t{c.quadratic + c.cubic} * (int64_t{1} << 24) + + int64_t{c.cubic} * inner_coordinate); + uint64_t staged_inner = round_even(inner_value, 17); + // Sine near zero retains more product bits. The zero-origin interval + // normalizes in groups of four bits to cover reduced phases near zero. + if (!table && index < 2) { + unsigned width = static_cast(std::bit_width(fraction)); + unsigned normalization = index ? 0 : 4 * (width < 24 ? (24 - width) / 4 : 0); + unsigned offset_shift = index ? 0 : 4 * (width < 25 ? (25 - width) / 4 : 0); + uint64_t linear_input = index ? uint64_t{linear_fraction(fraction, 2, reflected)} + : uint64_t{fraction} << offset_shift; + // Saturation follows the normalized field, even when the linear + // input uses an additional four offset bits. + if (!index && reflected && ((uint64_t{fraction} << normalization) & 0xfffff) == 0) + --linear_input; + unsigned linear_precision = index ? 30 : 32 + normalization; + unsigned quadratic_precision = 34 + normalization; + int64_t linear = + static_cast((uint64_t{static_cast(c.linear)} * linear_input) >> + (50 + offset_shift - linear_precision)); + // Truncate the square before multiplying by the rounded coefficient. + unsigned square_precision = std::min(34u, 24 + normalization); + uint64_t staged_square = (uint64_t{square_input} * square_input) >> (34 - square_precision); + int64_t quadratic = -static_cast(round_even( + U128{staged_inner} * staged_square, 33 + square_precision - quadratic_precision)); + int64_t sum = (int64_t{c.constant} << (quadratic_precision - 26)) + + (linear << (quadratic_precision - linear_precision)) + quadratic; + result = fixed_to_float(static_cast(sum), quadratic_precision); + } else { + uint32_t linear_input = linear_fraction(fraction, 4, reflected); + int64_t product = int64_t{c.linear} * linear_input; + uint64_t absolute = static_cast(product < 0 ? -product : product); + unsigned precision = absolute >= (uint64_t{1} << 45) ? 28 : 29; + int64_t linear = static_cast(round_even(absolute, 50 - precision)) * + (product < 0 ? -1 : 1) * (precision == 28 ? 2 : 1); + uint64_t staged_square = (uint64_t{square_input} * square_input) >> 10; + uint64_t quadratic_product = staged_inner * staged_square; + // Round the quadratic product to 34 fractional bits, capped at + // 24 significant bits, before flooring the negative correction to Q28. + const uint64_t bias = uint64_t{1} << (quadratic_product >= (uint64_t{1} << 47) ? 23 : 22); + int64_t quadratic = + quadratic_product <= bias + ? 0 + : -static_cast((quadratic_product - bias + (uint64_t{1} << 29) - 1) >> 29); + int64_t sum = int64_t{c.constant} * 8 + linear + quadratic * 2; + result = fixed_to_float(static_cast(sum), 29); + } + } + if (!result) + return 0; + bool negative = (quadrant >> 1) ^ (unsigned(cosine) & quadrant & 1) ^ (!cosine && (bits >> 31)); + return result | (uint32_t{negative} << 31); +} +} // namespace util::detail::trig + +namespace util { + +inline float amdgpu_trig_f32(float value, bool cosine, uint32_t denorm_mode, bool quiet_snan) { + return std::bit_cast( + detail::trig::evaluate(std::bit_cast(value), cosine, denorm_mode, quiet_snan)); +} + +} // namespace util diff --git a/emulation/rocjitsu/lib/util/include/util/data_types.h b/emulation/rocjitsu/lib/util/include/util/data_types.h index e4b5fd7887bd..bfee5d7109c8 100644 --- a/emulation/rocjitsu/lib/util/include/util/data_types.h +++ b/emulation/rocjitsu/lib/util/include/util/data_types.h @@ -90,6 +90,9 @@ inline void f16_to_f32_block(const uint16_t *src, float *dst, size_t n) { for (; i < n; ++i) dst[i] = f16_to_f32(src[i]); } + +/// @brief Round F32 to F16 with nearest-even rounding, preserving retained NaN payload bits. +/// @details If truncation discards the entire payload, keep bit 0 to avoid producing infinity. inline uint16_t f32_to_f16(float val) { uint32_t f = std::bit_cast(val); uint32_t sign = (f >> 16) & 0x8000; @@ -97,8 +100,10 @@ inline uint16_t f32_to_f16(float val) { uint32_t f_mant = f & 0x7FFFFF; if (f_exp == 0xFF) { - if (f_mant) - return static_cast(sign | 0x7C00 | (f_mant >> 13) | 1); + if (f_mant) { + const uint32_t payload = f_mant >> 13; + return static_cast(sign | 0x7C00 | (payload != 0 ? payload : 1)); + } return static_cast(sign | 0x7C00); } @@ -137,11 +142,13 @@ inline uint16_t f32_to_f16(float val) { /// clamped to signed MAX_FP16; true infinities remain infinities. inline uint16_t f32_to_f16_mode(float val, bool fp16_ovfl) { uint16_t result = f32_to_f16(val); - if (fp16_ovfl && std::isfinite(val) && (result & 0x7FFFu) == 0x7C00u) + const bool finite = (std::bit_cast(val) & 0x7fffffffu) < 0x7f800000u; + if (fp16_ovfl && finite && (result & 0x7FFFu) == 0x7C00u) return static_cast((result & 0x8000u) | 0x7BFFu); return result; } +/// @brief Truncate F32 to F16 with the same NaN payload policy as f32_to_f16. inline uint16_t f32_to_f16_rtz(float val) { uint32_t f = std::bit_cast(val); uint32_t sign = (f >> 16) & 0x8000; @@ -149,8 +156,10 @@ inline uint16_t f32_to_f16_rtz(float val) { uint32_t f_mant = f & 0x7FFFFF; if (f_exp == 0xFF) { - if (f_mant) - return static_cast(sign | 0x7C00 | (f_mant >> 13) | 1); + if (f_mant) { + const uint32_t payload = f_mant >> 13; + return static_cast(sign | 0x7C00 | (payload != 0 ? payload : 1)); + } return static_cast(sign | 0x7C00); } diff --git a/emulation/rocjitsu/lib/util/include/util/simd.h b/emulation/rocjitsu/lib/util/include/util/simd.h index ca7a2f9d951e..dd99482dc4cb 100644 --- a/emulation/rocjitsu/lib/util/include/util/simd.h +++ b/emulation/rocjitsu/lib/util/include/util/simd.h @@ -4,6 +4,11 @@ #ifndef UTIL_SIMD_H_ #define UTIL_SIMD_H_ +#include "util/amdgpu_exp.h" +#include "util/amdgpu_log.h" +#include "util/amdgpu_rcp.h" +#include "util/amdgpu_rsq.h" +#include "util/amdgpu_sqrt.h" #include "util/bit.h" #include @@ -610,7 +615,8 @@ inline native f16_to_f32_simd(native v) { // set bit of mant; mant is in 1..1023 (< 2^24) so the int->float cast is // exact and p = floor(log2(mant)) reads straight out of the f32 exponent. const native mf = stdx::static_simd_cast>(mant); - const U p = (std::bit_cast(mf) >> 23) - 127u; + U p = (std::bit_cast(mf) >> 23) - 127u; + stdx::where(mant == 0u, p) = 0u; // Scalar k = 10 - p shifts; exp_final = 1 - k = p - 9; exp field = p + 103. const U dn = sign31 | ((p + 103u) << 23) | (((mant << (10u - p)) & 0x3FFu) << 13); stdx::where(exp == 0u && mant != 0u, bits) = dn; @@ -671,12 +677,10 @@ inline native f32_to_f16_simd(native val) { // Denormal (fe 102..112): m' = fm|0x800000, shift right by sh=126-fe (14..24) // with round-to-nearest-even. const U mm = fm | 0x800000u; - // sh = 126 - fe is in [14,24] for the real denormal range (fe 102..112), but - // lanes with fe < 102 (flushed to zero below) yield sh up to 126. Clamp to 31 - // so the shifts here never hit undefined/masked behaviour; those lanes are - // overwritten by the `fe < 102` flush, so the clamped value is discarded. + // Only denormal-result lanes use these shifts. Keep every lane in [14,24] + // so both sh and sh-1 are valid, including discarded normal/special lanes. U sh = 126u - fe; - stdx::where(sh > 31u, sh) = 31u; + stdx::where(fe < 102u || fe > 112u, sh) = 14u; const U drb = (mm >> (sh - 1u)) & 1u; const U mask_lo = (1u << (sh - 1u)) - 1u; U dsticky(0u); @@ -689,9 +693,11 @@ inline native f32_to_f16_simd(native val) { stdx::where(fe < 102u, out) = sign; stdx::where(fe >= 143u && fe <= 254u, out) = sign | 0x7C00u; - // Inf / NaN (fe == 255): NaN keeps payload MSBs and forces the low bit. + // Inf / NaN (fe == 255): retain payload MSBs, keeping NaNs distinct from infinity. stdx::where(fe == 255u, out) = sign | 0x7C00u; - stdx::where(fe == 255u && fm != 0u, out) = sign | 0x7C00u | (fm >> 13) | 1u; + U payload = fm >> 13; + stdx::where(payload == 0u, payload) = 1u; + stdx::where(fe == 255u && fm != 0u, out) = sign | 0x7C00u | payload; return out; } @@ -726,14 +732,16 @@ inline native f32_to_f16_rtz_simd(native val) { const U mm = fm | 0x800000u; U sh = 126u - fe; - stdx::where(sh > 31u, sh) = 31u; + stdx::where(fe < 102u || fe > 112u, sh) = 14u; stdx::where(fe <= 112u, out) = sign | (mm >> sh); stdx::where(fe < 102u, out) = sign; stdx::where(fe >= 143u && fe <= 254u, out) = sign | 0x7BFFu; stdx::where(fe == 255u, out) = sign | 0x7C00u; - stdx::where(fe == 255u && fm != 0u, out) = sign | 0x7C00u | (fm >> 13) | 1u; + U payload = fm >> 13; + stdx::where(payload == 0u, payload) = 1u; + stdx::where(fe == 255u && fm != 0u, out) = sign | 0x7C00u | payload; return out; } @@ -900,11 +908,7 @@ inline double ceil_scalar(double a) { return quiet_snan_scalar(a, std::ceil(a)); inline float trunc_scalar(float a) { return quiet_snan_scalar(a, std::trunc(a)); } inline double trunc_scalar(double a) { return quiet_snan_scalar(a, std::trunc(a)); } -/// Flush f32 denormals to sign-preserving zero (FTZ). Branchless vector port of -/// `amdgpu::transcendental::flush_denorm_f32`: a lane with biased exponent 0 and -/// nonzero mantissa becomes ±0 (sign preserved); every other lane (normal, Inf, -/// NaN, ±0) passes through unchanged. AMD transcendental micro-ops always run in -/// FTZ mode, so the f32 rcp/rsq/exp/log SIMD ports below funnel through this. +/// Flush F32 subnormals to signed zero without host floating-point arithmetic. inline native flush_denorm_f32_simd(native v) { using U = native; U b = std::bit_cast(v); @@ -912,58 +916,30 @@ inline native flush_denorm_f32_simd(native v) { return std::bit_cast>(b); } -/// Vector ports of `amdgpu::transcendental::*_f32`, mirroring the scalar -/// reference body bit-for-bit so the VOP1 SIMD fast path agrees with the -/// forced-scalar path on every lane. The ±0/Inf special cases fall out of IEEE -/// div/sqrt after the FTZ input flush; only NaN payload preservation with signaling NaNs quieted -/// and the negative-domain canonical qNaN (0x7FC00000) need explicit blends. The 16-bit (f16) ops -/// reuse these on the f16->f32 intermediate, matching the scalar `f32_to_f16(rcp_f32(f16_to_f32))`. -// Canonical positive quiet-NaN (f32), broadcast across the vector. Shared by -// the transcendental fast paths below, which blend it into out-of-domain -// lanes (negative sqrt/rsqrt, log of a negative) to match the scalar refs. -inline const native kQNaN = std::bit_cast>(native(0x7FC00000u)); - -inline native rcp_f32_simd(native a) { - native x = flush_denorm_f32_simd(a); - native r = flush_denorm_f32_simd(native(1.0f) / x); - stdx::where(stdx::isnan(a), r) = - std::bit_cast>(std::bit_cast>(a) | 0x00400000u); - return r; +/// Native-width adapters for the shared integer transcendental mappings. +inline native rcp_f32_simd(native a, bool quiet_snan = true) { + return map_native_convert_scalar( + a, [quiet_snan](float value) { return amdgpu_rcp_f32(value, quiet_snan); }); } -inline native rsq_f32_simd(native a) { - native x = flush_denorm_f32_simd(a); - native r = flush_denorm_f32_simd(native(1.0f) / stdx::sqrt(x)); - stdx::where(x < native(0.0f), r) = kQNaN; // negatives incl -Inf -> qNaN - stdx::where(stdx::isnan(a), r) = - std::bit_cast>(std::bit_cast>(a) | 0x00400000u); - return r; +inline native rsq_f32_simd(native a, bool quiet_snan = true) { + return map_native_convert_scalar( + a, [quiet_snan](float value) { return amdgpu_rsq_f32(value, quiet_snan); }); } -inline native sqrt_f32_simd(native a) { - native x = flush_denorm_f32_simd(a); - native r = stdx::sqrt(x); - stdx::where(x < native(0.0f), r) = kQNaN; - stdx::where(stdx::isnan(a), r) = - std::bit_cast>(std::bit_cast>(a) | 0x00400000u); - return r; +inline native sqrt_f32_simd(native a, bool quiet_snan = true) { + return map_native_convert_scalar( + a, [quiet_snan](float value) { return amdgpu_sqrt_f32(value, quiet_snan); }); } -inline native log_f32_simd(native a) { - native x = flush_denorm_f32_simd(a); - native r = stdx::log2(x); // input-flush only; scalar log_f32 has no out-flush - stdx::where(x < native(0.0f), r) = kQNaN; - stdx::where(stdx::isnan(a), r) = - std::bit_cast>(std::bit_cast>(a) | 0x00400000u); - return r; +inline native log_f32_simd(native a, bool quiet_snan = true) { + return map_native_convert_scalar( + a, [quiet_snan](float value) { return amdgpu_log_f32(value, quiet_snan); }); } -inline native exp_f32_simd(native a) { - native x = flush_denorm_f32_simd(a); - native r = flush_denorm_f32_simd(stdx::exp2(x)); - stdx::where(stdx::isnan(a), r) = - std::bit_cast>(std::bit_cast>(a) | 0x00400000u); - return r; +inline native exp_f32_simd(native a, bool quiet_snan = true) { + return map_native_convert_scalar( + a, [quiet_snan](float value) { return amdgpu_exp_f32(value, quiet_snan); }); } /// Branchless SWAR population count over a uint32 vector. Each lane holds @@ -1117,55 +1093,6 @@ template V ieee_minimum_simd(V a, V b) { } } -/// Cubemap face ops (back v_cube{id,sc,tc}_f32). The scalar bodies select among -/// the three axes by an `else if` cascade on |x|,|y|,|z| with `>=` ties: the X -/// face wins ties, then Y, then Z. The vector forms apply the blends in reverse -/// priority (Z default, Y overwrite, X overwrite last) so X wins ties exactly as -/// scalar. Every branch is a bit-identical value copy / sign flip of the -/// (already abs/neg-applied) inputs — no FP arithmetic — so byte-identical. NaN -/// inputs make every `>=` mask false -> the Z-axis default, matching scalar. -/// (cubema = 2 * fmax(|x|,fmax(|y|,|z|)) is emitted inline at the call site.) -inline native cube_id_f32_simd(native x, native y, native z) { - using F = native; - const F ax = stdx::abs(x), ay = stdx::abs(y), az = stdx::abs(z); - const auto x_face = (ax >= ay) && (ax >= az); - const auto y_face = (ay >= ax) && (ay >= az); - F r = F(5.0f); - stdx::where(z >= F(0.0f), r) = F(4.0f); // Z face: z>=0 ? 4 : 5 - F yv = F(3.0f); - stdx::where(y >= F(0.0f), yv) = F(2.0f); - stdx::where(y_face, r) = yv; - F xv = F(1.0f); - stdx::where(x >= F(0.0f), xv) = F(0.0f); - stdx::where(x_face, r) = xv; - return r; -} -inline native cube_sc_f32_simd(native x, native y, native z) { - using F = native; - const F ax = stdx::abs(x), ay = stdx::abs(y), az = stdx::abs(z); - const auto x_face = (ax >= ay) && (ax >= az); - const auto y_face = (ay >= ax) && (ay >= az); - F r = x; - stdx::where(z >= F(0.0f), r) = -x; // Z face: z>=0 ? -x : x - stdx::where(y_face, r) = x; - F xv = -z; - stdx::where(x >= F(0.0f), xv) = z; // X face: x>=0 ? z : -z - stdx::where(x_face, r) = xv; - return r; -} -inline native cube_tc_f32_simd(native x, native y, native z) { - using F = native; - const F ax = stdx::abs(x), ay = stdx::abs(y), az = stdx::abs(z); - const auto x_face = (ax >= ay) && (ax >= az); - const auto y_face = (ay >= ax) && (ay >= az); - F r = -y; // Z face and X face both return -y - F yv = z; - stdx::where(y >= F(0.0f), yv) = -z; // Y face: y>=0 ? -z : z - stdx::where(y_face, r) = yv; - stdx::where(x_face, r) = -y; // restore -y on ties where the Y mask also fired - return r; -} - /// Round an in-range finite float to the nearest integer, with halfway values /// choosing the even integer. Unlike nearbyint(), this is independent of the /// process floating-point environment. @@ -1189,25 +1116,32 @@ inline native round_to_nearest_even_simd(native value) { return lower; } -/// Normalized f32->int16 / ->uint16 pack-convert lanes (back v_cvt_pk[_]norm_*). -/// Scalar: `isnan(f) ? 0 : static_cast(round_to_nearest_even(clamp(f * K, lo, hi)))`. The -/// NaN->0 blend is done in the FLOAT domain (so the mask type matches) before the int conversion, -/// avoiding any float-mask -> int-mask conversion. The caller masks &0xFFFF when packing. i16: -/// K=32767, clamp -/// [-32768,32767]; u16: K=65535, clamp [0,65535]. +/// Round a normalized value scaled by 32767 or 65535 to the nearest integer, +/// with ties to even. Clamp inputs to [minimum, 1] and convert NaNs to zero. +/// An FP32 product can round onto an integer midpoint. The FMA residual +/// distinguishes these false ties so the conversion rounds only once. +inline native round_normalized_simd(native f, float scale, float minimum) { + using F = native; + stdx::where(f < F(minimum), f) = F(minimum); + stdx::where(f > F(1.0f), f) = F(1.0f); + stdx::where(stdx::isnan(f), f) = F(0.0f); + const F product = f * F(scale); + const F residual = stdx::fma(f, F(scale), -product); + const F lower = stdx::floor(product); + const auto midpoint = product == lower + F(0.5f); + F rounded = rndne_simd(product); + stdx::where(midpoint && (residual < F(0.0f)), rounded) = lower; + stdx::where(midpoint && (residual > F(0.0f)), rounded) = lower + F(1.0f); + return rounded; +} + +/// Convert to signed normalized integers in [-32767, 32767]; NaNs become zero. inline native cvt_pknorm_i16_f32_simd(native f) { - native p = f * native(32767.0f); - stdx::where(p < native(-32768.0f), p) = native(-32768.0f); - stdx::where(p > native(32767.0f), p) = native(32767.0f); - stdx::where(stdx::isnan(f), p) = native(0.0f); - return stdx::static_simd_cast>(round_to_nearest_even_simd(p)); + return stdx::static_simd_cast>(round_normalized_simd(f, 32767.0f, -1.0f)); } +/// Convert to unsigned normalized integers in [0, 65535]; NaNs become zero. inline native cvt_pknorm_u16_f32_simd(native f) { - native p = f * native(65535.0f); - stdx::where(p < native(0.0f), p) = native(0.0f); - stdx::where(p > native(65535.0f), p) = native(65535.0f); - stdx::where(stdx::isnan(f), p) = native(0.0f); - return stdx::static_simd_cast>(round_to_nearest_even_simd(p)); + return stdx::static_simd_cast>(round_normalized_simd(f, 65535.0f, 0.0f)); } /// Vector port of the f32 `std::frexp` mantissa over raw float bits. Returns the diff --git a/emulation/rocjitsu/schemas/simulation_config.fbs b/emulation/rocjitsu/schemas/simulation_config.fbs index 88c4483657ab..3ac51fe185cf 100644 --- a/emulation/rocjitsu/schemas/simulation_config.fbs +++ b/emulation/rocjitsu/schemas/simulation_config.fbs @@ -28,6 +28,7 @@ table CommandProcessorConfig { table ShaderEngineConfig { num_compute_units: uint32; // Number of CUs in this shader engine. compute_unit: ComputeUnitConfig; // Shared CU configuration for all CUs. + cus_per_shader_array: uint32; // Shader-array width; absent/zero means unknown. } /// Configuration for an Accelerator Complex Die (XCD). @@ -43,6 +44,14 @@ table GpuMemoryConfig { memory_side_cache_mb: uint32; // Memory-side cache size in MB (0 = disabled). } +/// Hardware block version advertised through device IP discovery. +table DeviceIpVersion { + hardware_id: uint16; // AMDGPU discovery hardware ID, e.g. GC=11, SDMA0=42, NBIF=108. + major: uint8; + minor: uint8; + revision: uint8; +} + /// KFD device identity and topology properties for sysfs generation. /// These values are written to /sys/class/kfd/kfd/topology/nodes//properties /// and consumed by ROCR, HIP, rocprofiler, and AMD SMI for GPU discovery. @@ -116,6 +125,9 @@ table KfdDeviceInfo { // Appended to preserve existing field IDs. Required to be nonzero when // num_sdma_engines is nonzero. num_sdma_queues_per_engine: uint32; + // Versions for instance zero of each block on die zero. Omitted blocks are + // not advertised; discovery metadata does not enable execution support. + ip_versions: [DeviceIpVersion]; } /// How the GPU presents itself on a PCI bus, for front ends that attach it to a @@ -288,14 +300,14 @@ table SimulationConfig { dbt_guest: DbtGuestConfig; // Optional guest-GPU DBT interposition mode. // Functional CU-dispatch width, including the calling command-processor thread. // 0 or omission in JSON selects the execution-budget policy; a nonzero value - // applies per SoC. Each effective SoC width is capped at its largest per-CP CU - // count. This is host acceleration, + // applies per SoC. The shared pool covers concurrent XCD submissions; each + // submission is limited to its active CUs. This is host acceleration, // not a modeled GPU resource: changing the width must preserve the observable // result of race-free workloads. cpu_dispatch_threads: uint32 = 1; // Total engine + retained dispatch-worker + shared async-helper budget. - // 0/omitted uses affinity, keeping engine/dispatch cost at most 32. - // Preset helpers are additive, up to 48 total. Nonzero overrides the budget. + // 0/omitted uses affinity; the target table determines its allocation ceiling. + // Nonzero overrides the budget. cpu_thread_budget: uint32; // Ordered preferred allocations. Select the largest fitting effective cost. // Omission falls back to serial execution; explicit knobs still apply. diff --git a/emulation/rocjitsu/tests/CMakeLists.txt b/emulation/rocjitsu/tests/CMakeLists.txt index 992cc6384bed..d718d8f1d27f 100644 --- a/emulation/rocjitsu/tests/CMakeLists.txt +++ b/emulation/rocjitsu/tests/CMakeLists.txt @@ -553,6 +553,7 @@ add_executable( gpu_vm_pipeline_test.cpp gpu_vm_service_test.cpp gpu_vm_translation_test.cpp + legacy_page_table_cache_test.cpp config_test.cpp gfx1251_compiled_kernel_test.cpp daemon_api_test.cpp @@ -571,9 +572,9 @@ add_executable( completion_tracker_test.cpp mes_engine_test.cpp aql_packet_processor_test.cpp - pm4_ring_consumer_test.cpp pm4_packet_processor_test.cpp - pm4_queue_controller_test.cpp + packet_processor_contract_test.cpp + pm4_compute_queue_test.cpp sdma_queue_scheduler_test.cpp sdma_packet_processor_test.cpp sdma_ring_consumer_test.cpp @@ -592,6 +593,7 @@ add_executable( simdojo_sim_test.cpp partitioning_test.cpp simulated_kfd_test.cpp + kfd_events_test.cpp decode_smoke_test.cpp readlane_selector_test.cpp decoder_c_api_test.cpp @@ -603,12 +605,14 @@ add_executable( shared_infra_test.cpp scalar_buffer_test.cpp transcendental_test.cpp + comparison_test.cpp amdgpu_elf_test_support.cpp analysis/liveness_test.cpp patch/spill_manager_test.cpp patch/log_buffer_test.cpp patch/instruction_builder_test.cpp patch/vector_builder_test.cpp + patch/smem_builder_test.cpp patch/kernarg_extension_test.cpp patch/sidecar_metadata_test.cpp patch/trampoline_builder_test.cpp @@ -642,6 +646,7 @@ add_executable( vgpr_redispatch_test.cpp instruction_execution_harness_test.cpp vop3_conversion_modifiers_test.cpp + cube_instruction_test.cpp division_test.cpp division_macro_test.cpp valu_floating_policy_test.cpp @@ -653,6 +658,7 @@ add_executable( pseudo_scalar_exec_test.cpp valu_fp_mode_test.cpp scalar_scc_test.cpp + vskip_test.cpp cdna3_predicate_mask_test.cpp scoped_temp_test.cpp wave_debug_test.cpp @@ -665,10 +671,14 @@ add_executable( util_simd_test.cpp v_add_simd_benchmark.cpp mfma_simd_benchmark.cpp + smfmac_simd_benchmark.cpp wmma_simd_benchmark.cpp swmmac_k64_simd_benchmark.cpp + swmmac_k128_simd_benchmark.cpp simd_correctness/simd_coverage_test.cpp simd_correctness/swmmac_k64_simd_exact_test.cpp + simd_correctness/smfmac_simd_exact_test.cpp + simd_correctness/swmmac_k128_simd_exact_test.cpp simd_correctness/vop1_cvt_f64_scalar_correctness_test.cpp simd_correctness/vopc_cmp_class_scalar_correctness_test.cpp simd_correctness/vop2_simd_correctness_test.cpp @@ -720,6 +730,7 @@ add_executable( execution_plugin_test.cpp plugin_config_resolver_test.cpp plugin_loader_test.cpp + race-detector/race_detector_counter_capacity_tests.cpp race-detector/race_detector_event_registry_test.cpp race-detector/race_detector_tests.cpp race-detector/interval_set_tests.cpp @@ -836,7 +847,7 @@ endif() rj_add_gtest_tests( rocjitsu_tests TEST_FILTER - "-*Benchmark*:*RequestExitWakesAllPartitions*:*DispatchedWaitSurvivesAConcurrentFinalClose*" + "-*Benchmark*:*RequestExitWakesAllPartitions*:*DispatchedWaitSurvivesAConcurrentFinalClose*:*EffectiveConfigTest.ReadsNativeLaunchConfigHandoff" ) # RequestExitWakesAllPartitions may require several runs to show regression @@ -1178,35 +1189,20 @@ if(HAS_DEVICE_KERNELS AND HAS_PROBE_FIXTURES) probe_callable_sgpr_probe_gfx950 ) rj_configure_target(probe_fixture_test TEST) - if(RJ_INSTALL_TESTS) - rj_install_test_target(probe_fixture_test) - endif() - rj_add_test( - NAME "ProbeFixture.NopProbeResolvesAndReturns" - COMMAND probe_fixture_test - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - INSTALL_COMMAND "bin/probe_fixture_test" - ) - rj_add_test( - NAME "ProbeFixture.ArgProbeReceivesItsArgumentInV0" - COMMAND - probe_fixture_test - --gtest_filter=ProbeFixture.ArgProbeReceivesItsArgumentInV0 + # Discover each case once instead of combining a whole-binary run with + # additional filtered registrations of the same cases. + rj_add_gtest_tests( + probe_fixture_test WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - INSTALL_COMMAND - "bin/probe_fixture_test" - "--gtest_filter=ProbeFixture.ArgProbeReceivesItsArgumentInV0" - ) - rj_add_test( - NAME "ProbeFixture.CallableSgprUseExceedsKernelDescriptor" - COMMAND - probe_fixture_test - --gtest_filter=ProbeFixture.CallableSgprUseExceedsKernelDescriptor - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - INSTALL_COMMAND - "bin/probe_fixture_test" - "--gtest_filter=ProbeFixture.CallableSgprUseExceedsKernelDescriptor" ) + # The helper installs the fragment without including it. Keep the installed + # probe cases reachable whenever this fixture target is available. + if(RJ_INSTALL_TESTS) + file( + APPEND "${RJ_INSTALL_CTEST_FILE}" + "include(\"probe_fixture_test.cmake\")\n\n" + ) + endif() endif() # Pin the zero-test failure behavior independently of GPU availability. This is @@ -1334,6 +1330,30 @@ get_filename_component( ) set(RJ_CLI "$") +rj_add_test( + NAME "EffectiveConfigTest.ReadsNativeLaunchConfigHandoff" + COMMAND + ${RJ_CLI} + --config + ${CMAKE_SOURCE_DIR}/configs/gfx942_cdna3.json + --cpu-thread-budget + 4 + -- + $ + --gtest_filter=EffectiveConfigTest.ReadsNativeLaunchConfigHandoff + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + TIMEOUT 60 + INSTALL_COMMAND + "\${RJ_INSTALLED_BINDIR}/rocjitsu" + "--config" + "\${RJ_INSTALLED_CONFIG_DIR}/gfx942_cdna3.json" + "--cpu-thread-budget" + "4" + "--" + "bin/rocjitsu_tests" + "--gtest_filter=EffectiveConfigTest.ReadsNativeLaunchConfigHandoff" +) + function(rj_detect_kfd_gfx_target_versions out_var) set(_versions) foreach( @@ -1856,8 +1876,41 @@ add_dependencies(interposer_dup_test rocjitsu_bin rocjitsu_shared) if(RJ_INSTALL_TESTS) rj_install_test_target(interposer_dup_test) endif() +# Present servers pass real DRM descriptors through IPC. A second preload +# supplies that identity on GPU-less runners, beneath rocjitsu's stat wrappers. +add_library(host_drm_preload SHARED host_drm_preload_test.cpp) +rj_configure_target(host_drm_preload TEST) +target_link_libraries(host_drm_preload PRIVATE ${CMAKE_DL_LIBS}) +add_dependencies(interposer_dup_test host_drm_preload) +if(RJ_INSTALL_TESTS) + install( + TARGETS host_drm_preload + LIBRARY DESTINATION ${RJ_INSTALL_TEST_BIN_DIR} + ) +endif() +foreach(_config IN ITEMS gfx1100_w7900 gfx1201_r9700 gfx950_mi355x_kmd_2gpu) + rj_add_test( + NAME "InterposerDrmTest.BorrowedDisplayDescriptorUsesSimulatedIdentity.${_config}" + COMMAND + ${RJ_CLI} + --config "${CMAKE_SOURCE_DIR}/configs/${_config}.json" + -- $ + "--gtest_filter=InterposerDrmTest.BorrowedDisplayDescriptorUsesSimulatedIdentity" + ENVIRONMENT + "LD_PRELOAD=${RJ_SANITIZER_PRELOAD_PREFIX}$" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + INSTALL_COMMAND + "\${RJ_INSTALLED_BINDIR}/rocjitsu" + --config "\${RJ_INSTALLED_CONFIG_DIR}/${_config}.json" + -- bin/interposer_dup_test + "--gtest_filter=InterposerDrmTest.BorrowedDisplayDescriptorUsesSimulatedIdentity" + INSTALL_ENVIRONMENT + "LD_PRELOAD=${RJ_SANITIZER_PRELOAD_PREFIX}bin/${CMAKE_SHARED_LIBRARY_PREFIX}host_drm_preload${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) +endforeach() + function(rj_add_interposer_test_suite _suite) - cmake_parse_arguments(PARSE_ARGV 1 ARG "" "CONFIG" "") + cmake_parse_arguments(PARSE_ARGV 1 ARG "" "CONFIG" "ENVIRONMENT") set(_config "${RJ_CDNA4_KMD_CONFIG}") set(_suffix "") if(ARG_CONFIG) @@ -1876,6 +1929,7 @@ function(rj_add_interposer_test_suite _suite) -- $ "--gtest_filter=${_suite}.${_case}" + ENVIRONMENT ${ARG_ENVIRONMENT} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} SKIP_REGULAR_EXPRESSION "\\[ SKIPPED \\]" INSTALL_COMMAND @@ -1885,6 +1939,7 @@ function(rj_add_interposer_test_suite _suite) "--" "bin/interposer_dup_test" "--gtest_filter=${_suite}.${_case}" + INSTALL_ENVIRONMENT ${ARG_ENVIRONMENT} ) endforeach() endfunction() @@ -1923,13 +1978,38 @@ rj_add_interposer_test_suite(InterposerDupTest ${_rj_interposer_dup_tests}) rj_add_interposer_test_suite( InterposerDrmTest + CanonicalSysfsPathsRetainSimulatedDevice DeviceInfoReportsActiveCuCount OpensWithinCurrentDescriptorLimit + PrimaryNodePathHasCharacterDeviceMetadata RenderNodePathMetadataMatchesDescriptor TimestampUsesAdvertisedNanosecondClock ) set(_rj_interposer_pm4_tests + SparseReplacementPreservesResidentTail + SubmissionFenceQueryPollsAndWaitsForCompletion + SubmissionFenceSequencesAreLocalToContextAndEngine + SubmissionFenceQueryRejectsInvalidIdentityAndReportsFailure + SubmissionFenceQueryRetiresOldHistory + TransferWaitsForSubmissionWithoutWaitingForCompletion + GraphicsRegistersRetainAllPacketForms + ContextMaskedUpdatesAndTranslationPrefetchPreserveOtherState + DispatchInterleaveShadowIsQualifiedForGfx12 + ComputeTranslationPrefetchIgnoresPfpSelector + StreamoutQuerySamplesAllFourMemoryCounterPairs + StreamoutQueryEventMarksZeroCounterSamplesValid + InterleavedDispatchExecutesDirectAndIndirectShaders + ContextRegisterLoadReadsMemoryAndPreservesAdjacentRegisters + ContextRegisterLoadRejectsRangeOverflow + ContextRegisterLoadsRejectComputeEngine + ClearStatePushPopRestoresContextAcrossSubmissions + ClearStateResetUsesRdna3Defaults + InvalidClearStateFailsBeforeFollowingWrite + ShadowRegisterLoadsUseSparseMemoryOffsets + MemoryAtomicsApplyIntegerOperationsAtBothWidths + InvalidMemoryPacketsFailFenceWithoutWritingMemory + MalformedPackedGraphicsRegistersFailFence DependencyDefersMemoryWriteAndCompletion MalformedPacketFailsCompletion FaultedQueueRejectsNewSubmission @@ -1944,7 +2024,9 @@ set(_rj_interposer_pm4_tests TimelineSubmissionWaitsForEarlierQueue RepeatedTimelineTransfersVisitSharedDependenciesOnce GraphicsQueueAcceptsBufferOwnershipFlushEvents + ComputeQueueAcceptsPipelineStatisticsControls MergedSyncFilesRetainPendingFenceAcrossDupAndImport + ThreadDimensionsMaskPartialGroupsForDirectAndIndirectDispatch GridProductOverflowFailsFence UnmappedShaderFailsFence OversizedLdsFailsFence @@ -2007,6 +2089,7 @@ rj_add_interposer_test_suite( rj_add_interposer_test_suite( InterposerGemTest + SparseReservationsPreserveRangesAndFileOwnership GemNamespaceIsPrivateButSharedByDuplicates IndependentDrmFilesRejectOverlappingMappings ReusedDmabufFdMintsDistinctHandles @@ -2037,6 +2120,18 @@ rj_add_interposer_test_suite( InterposerSyncobjTest ${_rj_interposer_syncobj_tests} ) +set(_rj_signal_exit_env) +if(RJ_ENABLE_TSAN) + # Model Wine's deliberate pthread_exit from a signal handler. Forced unwind + # allocates and frees memory, so disable only this test's signal-safety + # diagnostic; retain TSan's race and mutex checks. + set(_rj_signal_exit_env "TSAN_OPTIONS=report_signal_unsafe=0") +endif() +rj_add_interposer_test_suite( + InterposerSyncobjTest + TimelineWaitAllowsThreadExitFromSignal + ENVIRONMENT ${_rj_signal_exit_env} +) # --- rocminfo test (requires rocminfo binary + HSA runtime + CLI launcher) --- # Spawns rocminfo via the rocjitsu CLI and validates output. diff --git a/emulation/rocjitsu/tests/amdgpu_vm_test.cpp b/emulation/rocjitsu/tests/amdgpu_vm_test.cpp index 3119169b125f..5cab5790848c 100644 --- a/emulation/rocjitsu/tests/amdgpu_vm_test.cpp +++ b/emulation/rocjitsu/tests/amdgpu_vm_test.cpp @@ -65,6 +65,7 @@ RJ_DIAGNOSTIC_POP #include #include #include +#include #include #include #include @@ -85,6 +86,22 @@ namespace rocjitsu::amdgpu { using GpuMemoryTestAccess = LegacyAddressSpaceTestAccess; +class CommandProcessorPlacementTestAccess { +public: + static std::pair place(CommandProcessor &cp, DispatchEntry &entry) { + std::lock_guard lock(cp.hw_queue_mutex_); + const auto result = cp.dispatch_workgroups(entry); + return {result.dispatched, result.outcome}; + } + + static void retire(CommandProcessor &cp, const DispatchEntry &entry, uint32_t workgroup) { + for (auto *cu : cp.cus_) + cu->abort_workgroup(entry.dispatch_id, workgroup); + for (auto *spi : cp.spis_) + spi->release_wgp_workgroup(entry.dispatch_id, workgroup); + } +}; + } // namespace rocjitsu::amdgpu namespace { @@ -142,7 +159,8 @@ struct VmFixture { VmFixture(std::string_view arch = "cdna3", uint32_t num_cus = 1, uint32_t num_wf_slots = 10, uint32_t lds_size_kb = 64, uint32_t sgprs_per_wf = 104, uint32_t vgprs_per_wf = 256, - uint32_t num_shader_engines = 1, uint64_t max_ticks = 10000) { + uint32_t num_shader_engines = 1, uint64_t max_ticks = 10000, + bool create_engine = true) { std::string cu_range = "cu[0:" + std::to_string(num_cus) + "]"; std::string shader_engines; std::string links; @@ -195,7 +213,8 @@ struct VmFixture { engine = std::make_unique(loaded.engine_config); engine->topology().set_root(loaded.take_root()); loaded.wire_links(engine->topology()); - engine->create(); + if (create_engine) + engine->create(); } amdgpu::Xcd *xcd(uint32_t idx = 0) { return soc_ptr->xcd(idx); } @@ -256,6 +275,86 @@ struct VmFixture { } }; +TEST(CommandProcessorTest, LargeInternalWorkgroupsKeepCallbackOrderAndReclaimFailedSetup) { + class WorkgroupWavesPlugin final : public ExecutionPlugin { + public: + WorkgroupWavesPlugin() : ExecutionPlugin("workgroup-waves") {} + + std::vector group_waves; + std::vector wave_callbacks; + uint32_t workgroups = 0; + + void onAmdgpuWorkgroupDispatched(uint32_t dispatch, uint32_t group, uint32_t, uint32_t, + std::span waves) override { + ++workgroups; + group_waves.assign(waves.begin(), waves.end()); + wave_callbacks.clear(); + for (uint32_t index = 0; index < waves.size(); ++index) { + EXPECT_EQ(waves[index]->dispatch_id(), dispatch); + EXPECT_EQ(waves[index]->wg_id(), group); + EXPECT_EQ(waves[index]->wave_in_group(), index); + } + } + + void onAmdgpuWavefrontDispatched(amdgpu::Wavefront &wave) override { + wave_callbacks.push_back(&wave); + } + }; + + // CDNA5 permits 64 resident Wave32 contexts. Internal entries can therefore + // exercise larger groups even though PM4 admission limits groups to 1024 threads. + for (uint32_t waves : {32u, 33u}) { + SCOPED_TRACE(waves); + VmFixture f("cdna5", 1, 64); + auto plugin = std::make_unique(); + auto *observed = plugin.get(); + auto plugins = std::make_shared(PluginSinkConfig{}); + ASSERT_TRUE(plugins->add(std::move(plugin))); + f.soc_ptr->set_plugin_group(plugins); + amdgpu::DispatchEntry entry; + entry.kind = amdgpu::DispatchPacketKind::Kernel; + entry.dispatch_id = 1; + entry.kernel_entry_pc = 0x8000; + entry.total_wgs = entry.grid_wgs_x = 1; + entry.grid_yz_valid = true; + entry.group_segment_fixed_size = 2048; + entry.vgprs_per_wf = 24; + entry.kernel_wave_size = 32; + entry.wfs_per_workgroup = waves; + entry.workgroup_size_x = entry.grid_size_x = waves * 32; + + ASSERT_EQ(amdgpu::CommandProcessorPlacementTestAccess::place(*f.cp(), entry), + (std::pair{1u, amdgpu::VmAccessOutcome::Complete})); + ASSERT_EQ(observed->group_waves.size(), waves); + EXPECT_EQ(observed->wave_callbacks, observed->group_waves); + EXPECT_EQ(observed->workgroups, 1u); + amdgpu::CommandProcessorPlacementTestAccess::retire(*f.cp(), entry, 0); + EXPECT_EQ(f.cu()->num_wfs(), 0u); + + // Missing launch metadata fails after all waves have been reserved. None + // may escape to observers or keep resources needed by the following group. + entry.dispatch_id = 2; + entry.dispatched_wgs = 0; + entry.kernel_code_properties = 1; + entry.queue_ptr = 0x1000; + EXPECT_EQ(amdgpu::CommandProcessorPlacementTestAccess::place(*f.cp(), entry), + (std::pair{0u, amdgpu::VmAccessOutcome::Malformed})); + EXPECT_EQ(observed->workgroups, 1u); + EXPECT_EQ(f.cu()->num_wfs(), 0u); + EXPECT_TRUE(f.cu()->can_accept_workgroup(waves, 64 * 1024)); + + entry.dispatch_id = 3; + entry.kernel_code_properties = 0; + entry.queue_ptr = 0; + ASSERT_EQ(amdgpu::CommandProcessorPlacementTestAccess::place(*f.cp(), entry), + (std::pair{1u, amdgpu::VmAccessOutcome::Complete})); + EXPECT_EQ(observed->workgroups, 2u); + EXPECT_EQ(observed->wave_callbacks, observed->group_waves); + amdgpu::CommandProcessorPlacementTestAccess::retire(*f.cp(), entry, 0); + EXPECT_EQ(f.cu()->num_wfs(), 0u); + } +} + TEST(ComputeUnitConfigTest, RejectsWavefrontSlotsAboveIsaMaximum) { EXPECT_THROW((void)VmFixture("cdna3", 1, 33), util::ConfigError); } @@ -679,7 +778,10 @@ TEST(GpuMemoryTest, SinglePageApplicationStoresDoNotDependOnProcMaps) { TEST(GpuMemoryTest, RevokedApplicationPagesFaultInsteadOfKillingTheProcess) { rocjitsu::test::LegacyGpuMemoryFixture mem("guarded_mem"); KfdProcess process(/*process_id=*/322); - mem.register_process(process.process_id(), &process.page_table_, &process.page_table_mutex_); + mem.register_process(process.process_id(), &process.page_table_, &process.page_table_mutex_, + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); + const amdgpu::GpuVmAccessBatchGuard batch; auto *page = static_cast(mmap(nullptr, KfdProcess::kPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); @@ -1055,6 +1157,7 @@ TEST(GpuMemoryTest, HostBackingProbeAcceptsAdjacentSubpageExtents) { TEST(GpuMemoryTest, UnregisterInvalidatesThreadLocalTranslationCaches) { rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); + const amdgpu::GpuVmAccessBatchGuard batch; constexpr uint32_t kPid = 7; constexpr uint64_t kBaseVa = 0x40000000; constexpr uint64_t kOffset = 0x123; @@ -1065,7 +1168,8 @@ TEST(GpuMemoryTest, UnregisterInvalidatesThreadLocalTranslationCaches) { page[kOffset] = 0x5a; process.map_pages(kBaseVa, page.data(), page.size(), amdgpu::Mtype::UC); memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, - process.page_table_generation()); + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); EXPECT_EQ(memory.resolve_host_ptr(kAddr, kPid), page.data() + kOffset); EXPECT_EQ(memory.read8(kAddr, kPid), page[kOffset]); @@ -1079,6 +1183,7 @@ TEST(GpuMemoryTest, UnregisterInvalidatesThreadLocalTranslationCaches) { TEST(GpuMemoryTest, ReregisterProcessInvalidatesThreadLocalTranslationCaches) { rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); + const amdgpu::GpuVmAccessBatchGuard batch; constexpr uint32_t kPid = 7; constexpr uint64_t kBaseVa = 0x40000000; constexpr uint64_t kOffset = 0x123; @@ -1093,14 +1198,18 @@ TEST(GpuMemoryTest, ReregisterProcessInvalidatesThreadLocalTranslationCaches) { first_process.map_pages(kBaseVa, first_page.data(), first_page.size(), amdgpu::Mtype::UC); second_process.map_pages(kBaseVa, second_page.data(), second_page.size(), amdgpu::Mtype::CC); - memory.register_process(kPid, &first_process.page_table_, &first_process.page_table_mutex_, - first_process.page_table_generation()); + memory.register_process( + kPid, &first_process.page_table_, &first_process.page_table_mutex_, + first_process.page_table_generation(), first_process.page_table_request_mutex(), + first_process.page_table_mutation_epoch(), first_process.page_table_cache_state()); ASSERT_EQ(memory.resolve_host_ptr(kAddr, kPid), first_page.data() + kOffset); ASSERT_EQ(memory.read8(kAddr, kPid), first_page[kOffset]); ASSERT_EQ(memory.pte_mtype(kAddr, kPid), amdgpu::Mtype::UC); - memory.register_process(kPid, &second_process.page_table_, &second_process.page_table_mutex_, - second_process.page_table_generation()); + memory.register_process( + kPid, &second_process.page_table_, &second_process.page_table_mutex_, + second_process.page_table_generation(), second_process.page_table_request_mutex(), + second_process.page_table_mutation_epoch(), second_process.page_table_cache_state()); EXPECT_EQ(memory.resolve_host_ptr(kAddr, kPid), second_page.data() + kOffset); EXPECT_EQ(memory.read8(kAddr, kPid), second_page[kOffset]); EXPECT_EQ(memory.pte_mtype(kAddr, kPid), amdgpu::Mtype::CC); @@ -1108,6 +1217,7 @@ TEST(GpuMemoryTest, ReregisterProcessInvalidatesThreadLocalTranslationCaches) { TEST(GpuMemoryTest, PageTableEntryMutationsInvalidateCachedPtes) { rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); + const amdgpu::GpuVmAccessBatchGuard batch; constexpr uint32_t kPid = 7; constexpr uint64_t kBaseVa = 0x40000000; constexpr uint64_t kOffset = 0x123; @@ -1120,7 +1230,8 @@ TEST(GpuMemoryTest, PageTableEntryMutationsInvalidateCachedPtes) { new_page[kOffset] = 0x22; process.map_pages(kBaseVa, old_page.data(), old_page.size(), amdgpu::Mtype::UC); memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, - process.page_table_generation()); + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); ASSERT_EQ(memory.read8(kAddr, kPid), old_page[kOffset]); ASSERT_EQ(memory.pte_mtype(kAddr, kPid), amdgpu::Mtype::UC); @@ -1132,6 +1243,285 @@ TEST(GpuMemoryTest, PageTableEntryMutationsInvalidateCachedPtes) { EXPECT_EQ(memory.pte_mtype(kAddr, kPid), amdgpu::Mtype::CC); } +TEST(GpuMemoryTest, TranslationPolicyCombinesLiveExtentAndPageMtype) { + rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); + constexpr uint32_t kPid = 7; + constexpr uint64_t kAddress = 0x40000040; + KfdProcess process(kPid); + std::array allocation{}; + process.map_pages(kAddress, allocation.data(), allocation.size(), amdgpu::Mtype::UC); + memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, + process.page_table_generation()); + auto access = memory.gpu_vm().snapshot_vmid(kPid); + ASSERT_TRUE(access); + auto translated = access->translate(kAddress + 16, 4, amdgpu::VmAccessKind::Read); + ASSERT_TRUE(translated); + EXPECT_EQ(translated.translation.contiguous_bytes, 112u); + EXPECT_EQ(translated.translation.mtype, amdgpu::Mtype::UC); + + process.set_page_mtype(kAddress, allocation.size(), amdgpu::Mtype::CC); + translated = access->translate(kAddress + 16, 4, amdgpu::VmAccessKind::Read); + ASSERT_TRUE(translated); + EXPECT_EQ(translated.translation.contiguous_bytes, 112u); + EXPECT_EQ(translated.translation.mtype, amdgpu::Mtype::CC); + EXPECT_EQ(access->translate(kAddress + 124, 8, amdgpu::VmAccessKind::Read).outcome, + amdgpu::VmAccessOutcome::Faulted); +} + +TEST(GpuMemoryTest, AccessBatchObservesRemappedPteBeforeQuantumExit) { + // Isolate the writer-waits-for-reader protocol so a lock-order regression is + // a bounded test failure rather than a hung suite. + const pid_t child = fork(); + ASSERT_GE(child, 0); + if (child == 0) { + alarm(10); + rocjitsu::test::LegacyGpuMemoryFixture memory("batched_pte_memory"); + constexpr uint32_t kPid = 17; + constexpr uint64_t kBaseVa = 0x41000000; + constexpr size_t kOffset = 0x123; + constexpr uint64_t kAddr = kBaseVa + kOffset; + + KfdProcess process(kPid); + alignas(KfdProcess::kPageSize) std::array old_page{}; + alignas(KfdProcess::kPageSize) std::array new_page{}; + old_page[kOffset] = 0x11; + new_page[kOffset] = 0x22; + process.map_pages(kBaseVa, old_page.data(), old_page.size(), amdgpu::Mtype::RW, + KfdProcess::HostExtentOwner::Driver); + memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); + + // Populate the copied-PTE cache before the quantum to exercise admission + // on a hit, then require the next operation to see a completed mutation. + if (memory.read8(kAddr, kPid) != old_page[kOffset]) + _exit(1); + + auto state = process.page_table_cache_state(); + std::atomic mutation_complete{false}; + std::thread mutator; + { + const amdgpu::GpuVmAccessBatchGuard batch; + if (memory.read8(kAddr, kPid) != old_page[kOffset]) + _exit(2); + const uint64_t admitted_generation = state->generation(); + mutator = std::thread([&] { + process.remap_page_host_ptrs(kBaseVa, old_page.data(), new_page.data(), new_page.size()); + mutation_complete.store(true, std::memory_order_release); + }); + + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (state->generation() == admitted_generation) { + if (std::chrono::steady_clock::now() >= deadline) + _exit(3); + std::this_thread::yield(); + } + mutator.join(); + if (!mutation_complete.load(std::memory_order_acquire)) + _exit(4); + + // No admission survives the preceding read, so the writer can finish + // before quantum exit and its replacement is visible immediately. + if (memory.read8(kAddr, kPid) != new_page[kOffset]) + _exit(5); + } + + if (!mutation_complete.load(std::memory_order_acquire)) + _exit(6); + if (memory.read8(kAddr, kPid) != new_page[kOffset]) + _exit(7); + memory.unregister_process(kPid); + _exit(0); + } + + int status = 0; + ASSERT_EQ(waitpid(child, &status, 0), child); + ASSERT_TRUE(WIFEXITED(status)) << "child status: " << status; + EXPECT_EQ(WEXITSTATUS(status), 0); +} + +TEST(GpuMemoryTest, AccessBatchDropsAdmissionsBeforeMtypeLookup) { + for (const bool copied_policy : {false, true}) { + SCOPED_TRACE(copied_policy ? "policy snapshot miss" : "direct policy lookup"); + const pid_t child = fork(); + ASSERT_GE(child, 0); + if (child == 0) { + alarm(10); + rocjitsu::test::LegacyGpuMemoryFixture memory("batched_policy_memory"); + constexpr uint32_t kPid = 19; + constexpr uint64_t kAddress = 0x43000000; + KfdProcess process(kPid); + alignas(KfdProcess::kPageSize) std::array page{}; + page[0] = 0x37; + process.map_pages(kAddress, page.data(), page.size(), amdgpu::Mtype::RW, + KfdProcess::HostExtentOwner::Driver); + memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), + process.page_table_cache_state()); + auto access = memory.gpu_vm().snapshot_vmid(kPid); + if (!access || memory.read8(kAddress, kPid) != page[0]) + _exit(1); + auto state = process.page_table_cache_state(); + std::thread mutator; + { + const amdgpu::GpuVmAccessBatchGuard batch; + if (memory.read8(kAddress, kPid) != page[0]) + _exit(2); + const uint64_t generation = state->generation(); + mutator = + std::thread([&] { process.set_page_mtype(kAddress, page.size(), amdgpu::Mtype::CC); }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (state->generation() == generation) { + if (std::chrono::steady_clock::now() >= deadline) + _exit(3); + std::this_thread::yield(); + } + // The earlier hit must leave no admission that would block this + // writer while the policy miss waits for its page-table locks. + amdgpu::VmMtypeCache cache; + const auto policy = + copied_policy ? access->query_mtype(kAddress, cache) : access->query_mtype(kAddress); + if (policy != amdgpu::Mtype::CC) + _exit(4); + mutator.join(); + if (memory.read8(kAddress, kPid) != page[0]) + _exit(5); + } + memory.unregister_process(kPid); + _exit(0); + } + int status = 0; + ASSERT_EQ(waitpid(child, &status, 0), child); + EXPECT_TRUE(WIFEXITED(status)) << "child status: " << status; + if (WIFEXITED(status)) { + EXPECT_EQ(WEXITSTATUS(status), 0); + } + } +} + +TEST(GpuMemoryTest, AccessBatchAllowsMutatingCallbacks) { + for (const int mutation : {0, 1, 2}) { + SCOPED_TRACE(mutation == 0 ? "VM invalidation" : mutation == 1 ? "PTE policy" : "host mapping"); + const pid_t child = fork(); + ASSERT_GE(child, 0); + if (child == 0) { + alarm(5); + rocjitsu::test::LegacyGpuMemoryFixture memory("reentrant_batch_memory"); + constexpr uint32_t kPid = 20; + constexpr uint64_t kAddress = 0x44000000; + KfdProcess process(kPid); + alignas(KfdProcess::kPageSize) std::array page{}; + page[0] = 0x49; + process.map_pages(kAddress, page.data(), page.size(), amdgpu::Mtype::RW, + KfdProcess::HostExtentOwner::Driver); + memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), + process.page_table_cache_state()); + auto access = memory.gpu_vm().snapshot_vmid(kPid); + if (!access) + _exit(1); + { + const amdgpu::GpuVmAccessBatchGuard vm_batch; + std::array value{}; + if (access->read(kAddress, value) != amdgpu::VmAccessOutcome::Complete || + value[0] != std::byte{0x49} || + access->read(kAddress, value) != amdgpu::VmAccessOutcome::Complete) + _exit(2); + const auto callback = [&] { + if (mutation == 0) { + if (!memory.gpu_vm().invalidate(access->cache_namespace().address_space)) + _exit(3); + } else if (mutation == 1) { + process.set_page_mtype(kAddress, page.size(), amdgpu::Mtype::CC); + } else { + auto writer = rocjitsu::host_mapping_lock().lock_exclusive(); + } + }; + callback(); + } + memory.unregister_process(kPid); + _exit(0); + } + int status = 0; + ASSERT_EQ(waitpid(child, &status, 0), child); + EXPECT_TRUE(WIFEXITED(status)) << "child status: " << status; + if (WIFEXITED(status)) { + EXPECT_EQ(WEXITSTATUS(status), 0); + } + } +} + +TEST(GpuMemoryTest, AccessBatchDropsAdmissionsBeforeLockedPteRefill) { + // A mutation may acquire the page-table lock between a cached hit and a + // miss. No admission may survive the hit and block the following refill. + const pid_t child = fork(); + ASSERT_GE(child, 0); + if (child == 0) { + alarm(10); + rocjitsu::test::LegacyGpuMemoryFixture memory("batched_pte_refill_memory"); + constexpr uint32_t kPid = 18; + constexpr uint64_t kFirstVa = 0x42000000; + constexpr uint64_t kSecondVa = kFirstVa + KfdProcess::kPageSize; + constexpr size_t kOffset = 0x45; + + KfdProcess process(kPid); + alignas(KfdProcess::kPageSize) std::array old_page{}; + alignas(KfdProcess::kPageSize) std::array new_page{}; + alignas(KfdProcess::kPageSize) std::array second_page{}; + old_page[kOffset] = 0x31; + new_page[kOffset] = 0x32; + second_page[kOffset] = 0x41; + process.map_pages(kFirstVa, old_page.data(), old_page.size(), amdgpu::Mtype::RW, + KfdProcess::HostExtentOwner::Driver); + process.map_pages(kSecondVa, second_page.data(), second_page.size(), amdgpu::Mtype::RW, + KfdProcess::HostExtentOwner::Driver); + memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); + + if (memory.read8(kFirstVa + kOffset, kPid) != old_page[kOffset]) + _exit(1); + auto state = process.page_table_cache_state(); + std::atomic mutation_complete{false}; + std::thread mutator; + { + const amdgpu::GpuVmAccessBatchGuard batch; + if (memory.read8(kFirstVa + kOffset, kPid) != old_page[kOffset]) + _exit(2); + const uint64_t admitted_generation = state->generation(); + mutator = std::thread([&] { + process.remap_page_host_ptrs(kFirstVa, old_page.data(), new_page.data(), new_page.size()); + mutation_complete.store(true, std::memory_order_release); + }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (state->generation() == admitted_generation) { + if (std::chrono::steady_clock::now() >= deadline) + _exit(3); + std::this_thread::yield(); + } + // This uncached page forces the slow refill while the writer may still + // own the lock. It must make progress and observe stable data. + if (memory.read8(kSecondVa + kOffset, kPid) != second_page[kOffset]) + _exit(5); + mutator.join(); + if (!mutation_complete.load(std::memory_order_acquire)) + _exit(6); + if (memory.read8(kFirstVa + kOffset, kPid) != new_page[kOffset]) + _exit(7); + } + + memory.unregister_process(kPid); + _exit(0); + } + + int status = 0; + ASSERT_EQ(waitpid(child, &status, 0), child); + ASSERT_TRUE(WIFEXITED(status)) << "child status: " << status; + EXPECT_EQ(WEXITSTATUS(status), 0); +} + TEST(GpuMemoryTest, UnalignedMappingUsesGpuPageOffsetForHostTranslation) { rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); constexpr uint32_t kPid = 7; @@ -1166,6 +1556,7 @@ TEST(GpuMemoryTest, UnalignedMappingUsesGpuPageOffsetForHostTranslation) { TEST(GpuMemoryTest, PartialMappedPageReadsZeroFillAndWritesClipToAllocation) { rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); + const amdgpu::GpuVmAccessBatchGuard batch; constexpr uint32_t kPid = 7; constexpr uint64_t kBaseVa = 0x40000000; constexpr size_t kAllocationSize = 24; @@ -1177,7 +1568,8 @@ TEST(GpuMemoryTest, PartialMappedPageReadsZeroFillAndWritesClipToAllocation) { allocation[i] = static_cast(i + 1); process.map_pages(kBaseVa, allocation.data(), allocation.size()); memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, - process.page_table_generation()); + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); EXPECT_EQ(memory.resolve_host_ptr(kBaseVa + kAllocationSize - 1, kPid), allocation.data() + kAllocationSize - 1); @@ -1246,6 +1638,85 @@ TEST(GpuMemoryTest, PartialMappedPageReadsZeroFillAndWritesClipToAllocation) { EXPECT_TRUE(std::ranges::all_of(allocation, [](uint8_t value) { return value == 0xa5; })); } +TEST(GpuMemoryTest, PageTableNormalizationMergesOnlyEqualOwners) { + using Owner = amdgpu::LegacyHostExtentOwner; + constexpr std::array owners{Owner::Application, Owner::Driver, Owner::DriverSealedRam}; + constexpr uint64_t base = 0x40000000; + std::array backing{}; + for (const auto first : owners) { + for (const auto second : owners) { + for (const bool reverse : {false, true}) { + SCOPED_TRACE(static_cast(first)); + SCOPED_TRACE(static_cast(second)); + SCOPED_TRACE(reverse); + KfdProcess process(7); + const auto map_first = [&] { + process.map_pages(base, backing.data(), 16, amdgpu::Mtype::UC, first); + }; + const auto map_second = [&] { + process.map_pages(base + 16, backing.data() + 16, 16, amdgpu::Mtype::UC, second); + }; + if (reverse) { + map_second(); + map_first(); + } else { + map_first(); + map_second(); + } + const auto &extents = process.page_table_.at(base >> KfdProcess::kPageShift).host_extents; + if (first == second) { + EXPECT_EQ(extents, + (std::vector{{backing.data(), 32, 0, first}})); + } else { + EXPECT_EQ(extents, + (std::vector{{backing.data(), 16, 0, first}, + {backing.data() + 16, 16, 16, second}})); + } + } + } + } +} + +TEST(GpuMemoryTest, PageTableReplacementPreservesSurvivingOwners) { + using Owner = amdgpu::LegacyHostExtentOwner; + constexpr std::array owners{Owner::Application, Owner::Driver, Owner::DriverSealedRam}; + constexpr uint64_t base = 0x40000000; + std::array backing{}; + std::array replacement{}; + for (const auto original : owners) { + for (const auto inserted : owners) { + SCOPED_TRACE(static_cast(original)); + SCOPED_TRACE(static_cast(inserted)); + KfdProcess process(7); + process.map_pages(base, backing.data(), backing.size(), amdgpu::Mtype::UC, original); + process.map_pages(base + 16, replacement.data(), replacement.size(), amdgpu::Mtype::UC, + inserted); + const auto &extents = process.page_table_.at(base >> KfdProcess::kPageShift).host_extents; + EXPECT_EQ(extents, + (std::vector{{backing.data(), 16, 0, original}, + {replacement.data(), 16, 16, inserted}, + {backing.data() + 32, 32, 32, original}})); + } + } +} + +TEST(GpuMemoryTest, PageTableUnmapPreservesSurvivingOwners) { + using Owner = amdgpu::LegacyHostExtentOwner; + constexpr uint64_t base = 0x40000000; + std::array backing{}; + for (const auto owner : {Owner::Application, Owner::Driver, Owner::DriverSealedRam}) { + SCOPED_TRACE(static_cast(owner)); + KfdProcess process(7); + process.map_pages(base, backing.data(), backing.size(), amdgpu::Mtype::UC, owner); + process.unmap_pages(base + 16, 16); + const auto &extents = process.page_table_.at(base >> KfdProcess::kPageShift).host_extents; + EXPECT_EQ(extents, (std::vector{ + {backing.data(), 16, 0, owner}, {backing.data() + 32, 32, 32, owner}})); + process.map_pages(base + 16, backing.data() + 16, 16, amdgpu::Mtype::UC, owner); + EXPECT_EQ(extents, (std::vector{{backing.data(), 64, 0, owner}})); + } +} + TEST(GpuMemoryTest, SamePageMappingsRemainIndependent) { rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); constexpr uint32_t kPid = 7; @@ -1526,6 +1997,33 @@ TEST(GpuMemoryTest, SanitizedCacheLineAccessClipsRoundedMapping) { [](uint8_t value) { return value == 0; })); } +TEST(GpuMemoryTest, SanitizedHeapBoundsAreRevalidatedWithoutPteMutation) { + rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); + constexpr uint32_t kPid = 7; + constexpr uint64_t kBaseVa = 0x40000000; + KfdProcess process(kPid); + auto allocation = std::make_unique(24); + allocation[0] = 0x5a; + process.map_pages(kBaseVa, allocation.get(), KfdProcess::kPageSize); + memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); + + size_t queries = 0; + std::function query_hook = [&] { ++queries; }; + memory.set_page_table_unlocked_hook(&query_hook); + const uint64_t generation = *process.page_table_generation(); + EXPECT_EQ(memory.read8(kBaseVa, kPid), 0x5a); + const size_t initial_queries = queries; + EXPECT_GT(initial_queries, 0u); + EXPECT_EQ(memory.read8(kBaseVa, kPid), 0x5a); + // HIP can reuse an allocation inside the same mapped pool without changing + // the PTE. Bounds shortened by allocator metadata need a fresh query. + EXPECT_GT(queries, initial_queries); + EXPECT_EQ(*process.page_table_generation(), generation); + memory.set_page_table_unlocked_hook(nullptr); +} + TEST(GpuMemoryTest, SanitizedMappedExtentTracksCurrentShadowState) { rocjitsu::test::LegacyGpuMemoryFixture memory("memory"); constexpr uint32_t kPid = 7; @@ -1603,7 +2101,8 @@ TEST(GpuMemoryTest, SanitizedMtypeLookupAvoidsAllocatorMetadataReentry) { }; memory.set_page_table_unlocked_hook(&query_hook); { - amdgpu::RequestMtypeResolver request(&memory.gpu_vm(), kPid); + amdgpu::VmMtypeCache policy_cache; + amdgpu::RequestMtypeResolver request(&memory.gpu_vm(), kPid, policy_cache); EXPECT_EQ(request.at(kBaseVa), amdgpu::Mtype::UC); EXPECT_EQ(hook_calls, 0u); } @@ -2177,7 +2676,8 @@ TEST(GpuMemoryThreadingTest, ReadersRemainSafeWhilePagesAreRemapped) { KfdProcess process(kPid); memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, - process.page_table_generation()); + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); auto initial_page = std::make_unique(kValuePrefix); process.map_pages(kAddr, initial_page.get(), sizeof(*initial_page)); @@ -2190,6 +2690,7 @@ TEST(GpuMemoryThreadingTest, ReadersRemainSafeWhilePagesAreRemapped) { readers.reserve(kReaders); for (uint32_t i = 0; i < kReaders; ++i) { readers.emplace_back([&] { + const amdgpu::GpuVmAccessBatchGuard batch; start.arrive_and_wait(); const uint32_t initial_value = memory.read32(kAddr, kPid); if ((initial_value & 0xffff0000) == kValuePrefix) @@ -2745,7 +3246,9 @@ TEST(GpuMemoryTest, FaultsAreReportedOutsideTranslationLocks) { IdentityHostPage reserved; ASSERT_EQ(mprotect(reserved.data, KfdProcess::kPageSize, PROT_NONE), 0); - // Each of these discovers the miss under the VMID lock. + const amdgpu::GpuVmAccessBatchGuard batch; + // Each of these discovers the miss under the VMID lock. The enclosing + // quantum must not retain leases across delivery to the reentering reporter. memory.atomic_rmw(reserved.addr(), sizeof(uint32_t), [](uint8_t *) {}, kPid); EXPECT_EQ(memory.resolve_host_ptr(reserved.addr(), kPid), nullptr); std::vector bytes(8, 0); @@ -3033,7 +3536,8 @@ TEST(GpuMemoryTest, MappedAtomicMakesProgressAgainstPageTableMutation) { KfdProcess process(kPid); process.map_pages(kGpuVa, page.data, KfdProcess::kPageSize); memory.register_process(kPid, &process.page_table_, &process.page_table_mutex_, - process.page_table_generation()); + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_mutation_epoch(), process.page_table_cache_state()); std::atomic stop{false}; std::atomic completed{0}; @@ -3045,6 +3549,7 @@ TEST(GpuMemoryTest, MappedAtomicMakesProgressAgainstPageTableMutation) { }); std::thread atomics([&] { + const amdgpu::GpuVmAccessBatchGuard batch; for (int i = 0; i < kIterations; ++i) { if (memory.atomic_fetch_add64(kGpuVa, 1, kPid) == amdgpu::AccessOutcome::Complete) completed.fetch_add(1, std::memory_order_relaxed); @@ -3587,7 +4092,7 @@ TEST(CommandProcessorTest, KfdQueueRequestsResizesAndReclaimsDynamicScratchBefor fixture.mem()->write64(kQueueSignal + kSignalValueOffset, 0); }); - amdgpu::AqlQueueConfig queue{}; + amdgpu::ComputeQueueConfig queue{}; queue.interrupt_sink = subscription.sink(); queue.process_id = kProcessId; queue.queue_id = kQueueId; @@ -3691,7 +4196,7 @@ TEST(CommandProcessorTest, DynamicScratchRequestBlocksRemovalOnlyUntilDelivery) // metadata untouched, then suspend and destroy the queue below. }); - amdgpu::AqlQueueConfig queue{}; + amdgpu::ComputeQueueConfig queue{}; queue.interrupt_sink = subscription.sink(); queue.process_id = kProcessId; queue.queue_id = kQueueId; @@ -3841,7 +4346,7 @@ TEST(CommandProcessorTest, KfdQueueHonorsAsyncScratchCutoffsAndTracksPerXccUse) fixture.mem()->write64(kQueueSignal + kSignalValueOffset, 0); }); - amdgpu::AqlQueueConfig queue{}; + amdgpu::ComputeQueueConfig queue{}; queue.interrupt_sink = subscription.sink(); queue.process_id = kProcessId; queue.queue_id = kQueueId; @@ -4020,6 +4525,34 @@ TEST(CommandProcessorTest, RetryNotificationsCoalesceWithoutSuppressingDoorbells EXPECT_EQ(cp->doorbell_handle_count_for_test() - passes_before, 3u); } +TEST(CommandProcessorTest, FreshStallPullsPendingRecheckEarlier) { + VmFixture f("cdna4", 1, 2, 64, 104, 256, 1, 100000); + constexpr uint64_t kPersistentSignal = 0x7000; + constexpr uint64_t kFreshSignal = 0x7100; + constexpr uint64_t kFreshReadPointer = 0xf0110000; + constexpr uint32_t kSignalValueOffset = 8; + f.mem()->write64(kPersistentSignal + kSignalValueOffset, 1); + f.mem()->write64(kFreshSignal + kSignalValueOffset, 1); + auto tick = [&] { return f.engine->context(f.cp()->partition_id()).current_tick(); }; + test::AqlQueue blocked(f.mem(), f.cp()); + test::AqlQueue fresh(f.mem(), f.cp(), 0xf0100000, 4096, kFreshReadPointer, 0xf0110008, 0xf0110010, + false, 2); + blocked.barrier_and(kPersistentSignal); + while (tick() < 20000) + ASSERT_TRUE(f.engine->step()); + + const auto start = tick(); + fresh.barrier_and(); + fresh.barrier_and(kFreshSignal); + ASSERT_TRUE(f.engine->step()); + ASSERT_EQ(f.mem()->read64(kFreshReadPointer), 1u); + f.mem()->write64(kFreshSignal + kSignalValueOffset, 0); + for (uint32_t step = 0; step < 16 && f.mem()->read64(kFreshReadPointer) < 2; ++step) + ASSERT_TRUE(f.engine->step()); + EXPECT_EQ(f.mem()->read64(kFreshReadPointer), 2u); + EXPECT_LE(tick() - start, 2u); +} + TEST(CommandProcessorTest, PendingRetryDoesNotSurviveEngineShutdown) { VmFixture f("cdna4", 1, 2); auto *cp = f.cp(); @@ -4532,8 +5065,11 @@ TEST_P(IsaTest, NonKernelBarrierPacketsOrderQueueEntries) { hsa_kernel_dispatch_packet_t barrier{}; barrier.header = packet_type | (header_barrier_bit ? (1 << HSA_PACKET_HEADER_BARRIER) : 0); - if (packet_type == HSA_PACKET_TYPE_VENDOR_SPECIFIC) - barrier.setup = amdgpu::kAmdAqlFormatPm4Ib; + if (packet_type == HSA_PACKET_TYPE_VENDOR_SPECIFIC) { + f.mem()->write32(0x9000, 0x80000000); + barrier = test::make_pm4_ib_packet(0x9000, 1); + barrier.header |= header_barrier_bit ? (1 << HSA_PACKET_HEADER_BARRIER) : 0; + } barrier.completion_signal.handle = kBarrierCompletionSignal; test::AqlQueue queue(f.mem(), f.cp()); @@ -4604,7 +5140,7 @@ TEST(CommandProcessorAqlTest, BlockingBarriersRetireBeforeUnsupportedSuccessorFa } } -TEST(CommandProcessorAqlTest, Pm4IbDoesNotBlockFetchOfUnsupportedSuccessor) { +TEST(CommandProcessorAqlTest, Pm4IbRetiresBeforeUnsupportedSuccessorFaultsQueue) { VmFixture f("cdna5", 1, 8); const uint32_t code[] = {SOPP_S_ENDPGM}; const uint64_t kernel = f.write_kernel(0x1000, code, sizeof(code)); @@ -4613,10 +5149,9 @@ TEST(CommandProcessorAqlTest, Pm4IbDoesNotBlockFetchOfUnsupportedSuccessor) { init_completion_signal(f.mem(), kPriorCompletionSignal); init_completion_signal(f.mem(), kPm4CompletionSignal); - hsa_kernel_dispatch_packet_t pm4_ib{}; - pm4_ib.header = HSA_PACKET_TYPE_VENDOR_SPECIFIC | (1 << HSA_PACKET_HEADER_BARRIER); - pm4_ib.setup = amdgpu::kAmdAqlFormatPm4Ib; - pm4_ib.completion_signal.handle = kPm4CompletionSignal; + f.mem()->write32(0x9000, 0x80000000); + auto pm4_ib = test::make_pm4_ib_packet(0x9000, 1, kPm4CompletionSignal); + pm4_ib.header |= 1 << HSA_PACKET_HEADER_BARRIER; hsa_kernel_dispatch_packet_t unsupported{}; unsupported.header = HSA_PACKET_TYPE_AGENT_DISPATCH; @@ -4627,13 +5162,212 @@ TEST(CommandProcessorAqlTest, Pm4IbDoesNotBlockFetchOfUnsupportedSuccessor) { queue.submit(unsupported); EXPECT_NO_THROW((void)f.engine->step()); - EXPECT_EQ(completion_signal_value(f.mem(), kPriorCompletionSignal), 1); - EXPECT_EQ(completion_signal_value(f.mem(), kPm4CompletionSignal), 1) - << "PM4 IB unexpectedly blocked fetch of the following packet"; + EXPECT_FALSE(f.cp()->queue_faulted_for_test(1, 0)); + for (uint32_t i = 0; i < 100 && !f.cp()->queue_faulted_for_test(1, 0); ++i) + EXPECT_NO_THROW((void)f.engine->step()); + EXPECT_EQ(completion_signal_value(f.mem(), kPriorCompletionSignal), 0); + EXPECT_EQ(completion_signal_value(f.mem(), kPm4CompletionSignal), 0); EXPECT_EQ(f.mem()->read64(test::AqlQueue::DEFAULT_READ_PTR_ADDR), 2u); EXPECT_TRUE(f.cp()->queue_faulted_for_test(1, 0)); } +TEST(CommandProcessorAqlTest, CuIdleExecutesPm4IbBeforeCompletionAndSuccessor) { + for (const auto *arch : {"cdna3", "cdna4"}) { + SCOPED_TRACE(arch); + for (bool blocked : {false, true}) { + SCOPED_TRACE(blocked); + // Retry backoff can exceed the default tick limit while the gate is closed. + VmFixture f(arch, 1, 8, 64, 104, 256, 1, /*max_ticks=*/0); + const uint32_t code[] = {SOPP_S_NOP, SOPP_S_ENDPGM}; + const uint64_t kernel = f.write_kernel(0x1000, code, sizeof(code)); + constexpr uint64_t kKernelSignal = 0x7000; + constexpr uint64_t kPm4Signal = 0x7100; + constexpr uint64_t kSuccessorSignal = 0x7200; + constexpr uint32_t kCounter = 0xb000; + constexpr uint32_t kGate = 0xb008; + constexpr uint32_t kValue = 0xb010; + constexpr uint32_t kObserved = 0xb018; + init_completion_signal(f.mem(), kKernelSignal); + init_completion_signal(f.mem(), kPm4Signal); + init_completion_signal(f.mem(), kSuccessorSignal); + f.mem()->write32(kGate, blocked ? 0 : 1); + // The atomic catches replay when the wait needs another CP turn. + const uint32_t commands[] = { + 0xc0071e00, 15, kCounter, 0, 1, 0, 0, 0, + 0, 0xc0053c00, (1u << 4) | 3, kGate, 0, 1, UINT32_MAX, 4, + 0xc0033700, 5u << 8, kValue, 0, 99, + }; + f.mem()->load_image(reinterpret_cast(commands), sizeof(commands), 0x9000); + // COPY_DATA makes the successor observe the preceding buffer's write. + const uint32_t successor[] = {0xc0044000, (5u << 8) | 1, kValue, 0, kObserved, 0}; + f.mem()->load_image(reinterpret_cast(successor), sizeof(successor), 0xa000); + auto pm4 = test::make_pm4_ib_packet(0x9000, std::size(commands), kPm4Signal); + pm4.header |= 1 << HSA_PACKET_HEADER_BARRIER; + test::AqlQueue queue(f.mem(), f.cp()); + queue.submit(make_dispatch_packet(kernel, kKernelSignal)); + queue.submit(pm4); + queue.submit(test::make_pm4_ib_packet(0xa000, std::size(successor), kSuccessorSignal)); + (void)f.engine->step(); + ASSERT_EQ(f.cu()->num_wfs(), 1u); + ASSERT_EQ(completion_signal_value(f.mem(), kPm4Signal), 1); + for (unsigned i = 0; i < 100 && completion_signal_value(f.mem(), kKernelSignal); ++i) + (void)f.engine->step(); + ASSERT_EQ(completion_signal_value(f.mem(), kKernelSignal), 0); + for (unsigned i = 0; i < 20; ++i) + (void)f.engine->step(); + EXPECT_EQ(f.mem()->read32(kCounter), 1u); + if (blocked) { + EXPECT_EQ(f.mem()->read32(kValue), 0u); + EXPECT_EQ(f.mem()->read32(kObserved), 0u); + EXPECT_EQ(completion_signal_value(f.mem(), kPm4Signal), 1); + EXPECT_EQ(completion_signal_value(f.mem(), kSuccessorSignal), 1); + f.mem()->write32(kGate, 1); + } + for (unsigned i = 0; i < 100 && completion_signal_value(f.mem(), kSuccessorSignal); ++i) + (void)f.engine->step(); + EXPECT_EQ(f.mem()->read32(kCounter), 1u); + EXPECT_EQ(f.mem()->read32(kValue), 99u); + EXPECT_EQ(f.mem()->read32(kObserved), 99u); + EXPECT_EQ(completion_signal_value(f.mem(), kPm4Signal), 0); + EXPECT_EQ(completion_signal_value(f.mem(), kSuccessorSignal), 0); + EXPECT_FALSE(f.cp()->queue_faulted_for_test(1, 0)); + } + } +} + +TEST(CommandProcessorAqlTest, CuIdleDoesNotExecutePm4AfterPeerGridFault) { + for (const auto *arch : {"cdna3", "cdna4"}) { + SCOPED_TRACE(arch); + for (bool blocked_before_fault : {false, true}) { + SCOPED_TRACE(blocked_before_fault); + VmFixture owner(arch), peer(arch); + const std::vector peers{owner.cp(), peer.cp()}; + owner.cp()->set_xcd_topology(0, peers); + peer.cp()->set_xcd_topology(1, peers); + amdgpu::ComputeQueueConfig queue{}; + queue.queue_id = 1; + queue.ring_base_va = test::AqlQueue::DEFAULT_RING_ADDR; + queue.ring_size = test::AqlQueue::DEFAULT_RING_SIZE; + queue.read_ptr_va = test::AqlQueue::DEFAULT_READ_PTR_ADDR; + queue.write_ptr_va = test::AqlQueue::DEFAULT_WRITE_PTR_ADDR; + queue.doorbell_mode = amdgpu::QueueDoorbellMode::Explicit; + const uint64_t owner_registration = owner.cp()->register_queue(std::move(queue)); + ASSERT_NE(owner_registration, 0u); + test::AqlQueue peer_queue(peer.mem(), peer.cp()); + constexpr uint64_t kSignal = 0x7100; + constexpr uint32_t kOutput = 0xb000; + constexpr uint32_t kCounter = 0xb008, kGate = 0xb010; + owner.mem()->write32(kGate, blocked_before_fault ? 0 : 1); + init_completion_signal(owner.mem(), kSignal); + const uint32_t words[] = { + 0xc0053c00, (1u << 4) | 3, + kGate, 0, + 1, UINT32_MAX, + 4, // WAIT_REG_MEM + 0xc0033700, 5u << 8, + kOutput, 0, + 99, // WRITE_DATA + 0xc0071e00, 15, + kCounter, 0, + 1, 0, + 0, 0, + 0, // ATOMIC_MEM add + }; + owner.mem()->load_image(reinterpret_cast(words), sizeof(words), 0x9000); + peer.mem()->write32(0x9000, 0xc000ff00); // Unsupported opcode faults the peer. + peer.mem()->write32(0x9004, 0); + auto grid = std::make_shared(); + grid->grid_wgs = 2; + amdgpu::DispatchEntry entry{}; + entry.dispatch_id = 99; + entry.queue_id = 1; + entry.kind = amdgpu::DispatchPacketKind::NonKernel; + entry.total_wgs = entry.dispatched_wgs = 1; + entry.grid_completion = grid; + entry.blocks_following = true; + entry.aql_pm4_ib_address = 0x9000; + entry.aql_pm4_ib_dwords = std::size(words); + entry.completion_signal = kSignal; + entry.address_space = owner.cp()->queue_address_space_for_test(1, 0); + owner.cp()->accept_fanout_shard(entry); + entry.aql_pm4_ib_dwords = 2; + entry.completion_signal = 0; + entry.address_space = peer.cp()->queue_address_space_for_test(1, 0); + peer.cp()->accept_fanout_shard(entry); + owner.cp()->drain_fanout_inbox_for_test(); + peer.cp()->drain_fanout_inbox_for_test(); + + if (blocked_before_fault) { + (void)owner.cu()->execute_quantum(); + EXPECT_EQ(owner.mem()->read32(kOutput), 0u); + EXPECT_EQ(owner.mem()->read32(kCounter), 0u); + EXPECT_EQ(owner.cp()->prepare_unregister_queue_registration(owner_registration), + amdgpu::QueuePrepareCloseStatus::Busy); + } + + // Drive the idle callbacks directly so the owner's queued fault notification + // cannot run before its CU-idle path observes the shared grid-fault latch. + (void)peer.cu()->execute_quantum(); + ASSERT_TRUE(grid->faulted()); + ASSERT_TRUE(peer.cp()->queue_faulted_for_test(1, 0)); + ASSERT_FALSE(owner.cp()->queue_faulted_for_test(1, 0)); + owner.mem()->write32(kGate, 1); + (void)owner.cu()->execute_quantum(); + EXPECT_EQ(owner.mem()->read32(kOutput), 0u); + EXPECT_EQ(owner.mem()->read32(kCounter), 0u); + EXPECT_EQ(completion_signal_value(owner.mem(), kSignal), 1); + EXPECT_TRUE(owner.cp()->has_dispatch_for_test(1, 0, 99)); + owner.cp()->drain_fanout_inbox_for_test(); + EXPECT_TRUE(owner.cp()->queue_faulted_for_test(1, 0)); + EXPECT_FALSE(owner.cp()->has_dispatch_for_test(1, 0, 99)); + EXPECT_EQ(owner.cp()->prepare_unregister_queue_registration(owner_registration), + amdgpu::QueuePrepareCloseStatus::Ready) + << "the canceled vendor stream must not pin a drained queue"; + } + } +} + +TEST(CommandProcessorAqlTest, InvalidPm4IbFaultsWithoutCompletionOrSuccessorExecution) { + for (const auto *arch : {"cdna3", "cdna4"}) { + SCOPED_TRACE(arch); + // An unknown opcode, a truncated WRITE_DATA, and a forbidden shader dispatch. + for (const std::vector &words : { + std::vector{0xc000ff00, 0}, + std::vector{0xc0033700, 5u << 8}, + std::vector{0xc0031500, 1, 1, 1, 1}, + }) { + SCOPED_TRACE(words[0]); + for (bool preceding_kernel : {false, true}) { + SCOPED_TRACE(preceding_kernel); + VmFixture f(arch, 1, 8); + constexpr uint64_t signal = 0x7100; + init_completion_signal(f.mem(), signal); + f.mem()->load_image(reinterpret_cast(words.data()), words.size() * 4, + 0x9000); + // A later IB would write 99 if the fault incorrectly allowed it to execute. + const uint32_t successor[] = {0xc0033700, 5u << 8, 0xb000, 0, 99}; + f.mem()->load_image(reinterpret_cast(successor), sizeof(successor), + 0xa000); + test::AqlQueue queue(f.mem(), f.cp()); + auto invalid = test::make_pm4_ib_packet(0x9000, words.size(), signal); + if (preceding_kernel) { + const uint32_t code[] = {SOPP_S_NOP, SOPP_S_ENDPGM}; + const uint64_t kernel = f.write_kernel(0x1000, code, sizeof(code)); + queue.dispatch(kernel, 64); + invalid.header |= 1 << HSA_PACKET_HEADER_BARRIER; + } + queue.submit(invalid); + queue.submit(test::make_pm4_ib_packet(0xa000, 5)); + for (unsigned i = 0; i < 100 && !f.cp()->queue_faulted_for_test(1, 0); ++i) + EXPECT_NO_THROW((void)f.engine->step()); + EXPECT_TRUE(f.cp()->queue_faulted_for_test(1, 0)); + EXPECT_EQ(completion_signal_value(f.mem(), signal), 1); + EXPECT_EQ(f.mem()->read32(0xb000), 0u); + } + } + } +} + TEST_P(IsaTest, EmptyCuMaskAllowsNonKernelPackets) { struct PacketCase { uint16_t type; @@ -4660,6 +5394,10 @@ TEST_P(IsaTest, EmptyCuMaskAllowsNonKernelPackets) { hsa_kernel_dispatch_packet_t packet{}; packet.header = test.type; packet.setup = test.format; + if (test.type == HSA_PACKET_TYPE_VENDOR_SPECIFIC && test.format == amdgpu::kAmdAqlFormatPm4Ib) { + f.mem()->write32(0x9000, 0x80000000); + packet = test::make_pm4_ib_packet(0x9000, 1); + } packet.completion_signal.handle = kSignal; queue.submit(packet); queue.dispatch(ko, 64); @@ -5097,6 +5835,17 @@ constexpr uint32_t sopc(uint32_t op, uint32_t ssrc0, uint32_t ssrc1) { } constexpr uint32_t s_cmp_eq_i32(uint32_t s0, uint32_t s1) { return sopc(0, s0, s1); } constexpr uint32_t s_cmp_gt_i32(uint32_t s0, uint32_t s1) { return sopc(2, s0, s1); } +// SOPK: encoding[31:28]=0xB, op[27:23], sdst[22:16], simm16[15:0] +constexpr uint32_t sopk(uint32_t op, uint32_t sdst, uint16_t simm16) { + return (0xBu << 28) | (op << 23) | (sdst << 16) | simm16; +} +// hwreg(id, offset, size) operand of S_GETREG/S_SETREG: id[5:0], offset[10:6], size-1[15:11] +constexpr uint32_t HW_REG_MODE = 1; +constexpr uint16_t hwreg(uint32_t id, uint32_t offset, uint32_t size) { + return static_cast(id | (offset << 6) | ((size - 1) << 11)); +} +// The value to write follows as a 32-bit literal dword. +constexpr uint32_t s_setreg_imm32_b32(uint16_t hwreg) { return sopk(20, 0, hwreg); } // VOP1: encoding[31:25]=0x3F, vdst[24:17], op[16:9], src0[8:0] constexpr uint32_t vop1(uint32_t op, uint32_t vdst, uint32_t src0) { return (0x3Fu << 25) | (vdst << 17) | (op << 9) | src0; @@ -5496,6 +6245,22 @@ TEST(AqlDispatchTest, CompletedPoolBatchRefillsIdleComputeUnits) { EXPECT_EQ(completion_signal_value(f.mem(), signal), 0); } +TEST(AqlDispatchTest, PoolFindsWavesAdmittedBeforeEngineAttachment) { + VmFixture f("cdna4", /*num_cus=*/2, /*num_wf_slots=*/1, /*lds_size_kb=*/64, + /*sgprs_per_wf=*/104, /*vgprs_per_wf=*/256, /*num_shader_engines=*/1, + /*max_ticks=*/10000, /*create_engine=*/false); + f.cp()->set_dispatch_threads(2); + const uint32_t code[] = {SOPP_S_NOP, SOPP_S_ENDPGM}; + f.mem()->load_image(reinterpret_cast(code), sizeof(code), 0x1000); + ASSERT_NE(f.cu(1)->dispatch_wf(/*wg_id=*/0, /*pc=*/0x1000, /*num_sgprs=*/104, + /*num_vgprs=*/256), + nullptr); + f.engine->create(); + f.engine->schedule_event_now(f.cp()->doorbell_event()); + f.engine->run(); + EXPECT_TRUE(f.cu(1)->is_idle()); +} + TEST(AqlDispatchTest, PoolContinuationLetsPeerCommandProcessorSatisfyPollingWave) { const char *json = R"({"max_ticks":10000,"num_threads":1,"vm":{"arch":"cdna4"}, "topology":{"root":{"name":"soc","type":"soc","children":[ @@ -5793,7 +6558,10 @@ TEST_P(IsaTest, VCmpEqF32_SetsVCC) { // Compare v0 (lane index as float-bits) with inline 0 (integer 0). // Lane 0: v0=0, compared with 0 -> equal -> VCC[0]=1. // Lane 1: v0=1, compared with 0 -> not equal -> VCC[1]=0. - fx.load_program({enc::v_cmp_eq_f32(enc::INLINE_CONST(0), 0), SOPP_S_ENDPGM}); + // Bit pattern 1 is an F32 subnormal. Compares flush it to zero unless MODE + // preserves F32 input denormals, so enable them first. + fx.load_program({enc::s_setreg_imm32_b32(enc::hwreg(enc::HW_REG_MODE, 4, 2)), 3u, + enc::v_cmp_eq_f32(enc::INLINE_CONST(0), 0), SOPP_S_ENDPGM}); uint64_t vcc = fx.vcc(); EXPECT_TRUE(vcc & (1ULL << 0)); // lane 0: 0.0 == 0.0 EXPECT_FALSE(vcc & (1ULL << 1)); // lane 1: int 1 as float != 0.0 @@ -5807,7 +6575,9 @@ TEST_P(IsaTest, VCndmaskB32) { // v_cmp_eq_f32 v0, 0 -> VCC[0]=1 (lane 0 = 0 == 0), VCC[1]=0 (1 != 0) // v_mov_b32 v1, inline 99 // v_cndmask_b32 v2, v0, v1 -> lane 0: VCC=1 -> v1=99; lane 1: VCC=0 -> v0=1 - fx.load_program({enc::v_cmp_eq_f32(enc::INLINE_CONST(0), 0), // VCC from v0 == 0 + // Lane 1's bit pattern is an F32 subnormal; preserve F32 input denormals. + fx.load_program({enc::s_setreg_imm32_b32(enc::hwreg(enc::HW_REG_MODE, 4, 2)), 3u, + enc::v_cmp_eq_f32(enc::INLINE_CONST(0), 0), // VCC from v0 == 0 enc::v_mov_b32(1, enc::INLINE_CONST(42)), // v1 = 42 (all lanes) enc::v_cndmask_b32(2, enc::VGPR_SRC(0), 1), // v2 = VCC ? v1 : v0 SOPP_S_ENDPGM}); @@ -6940,6 +7710,30 @@ class LiveConcurrentHotHookPlugin final : public ExecutionPlugin { bool requires_serial_hot_hooks() const override { return false; } }; +class LiveHotHookSubscriptionPlugin final : public ExecutionPlugin { +public: + explicit LiveHotHookSubscriptionPlugin(bool observes) + : ExecutionPlugin(observes ? "live_subscribed_hot_hook" : "live_unsubscribed_hot_hook"), + observes_(observes) {} + + bool observes_hot_hooks_for_wavefront(const amdgpu::Wavefront *) const override { + return observes_; + } + bool observes_before_execute_instruction() const override { return false; } + bool observes_after_execute_instruction() const override { return false; } + bool observes_async_instruction_issued() const override { return false; } + bool observes_memory_instruction_routing() const override { return false; } + bool observes_vgpr_reads() const override { return false; } + bool observes_vgpr_writes() const override { return false; } + bool observes_scalar_register_writes() const override { return false; } + void onAmdgpuReadSgpr(const amdgpu::Wavefront *, uint32_t) override { ++sgpr_reads; } + + uint32_t sgpr_reads = 0; + +private: + bool observes_ = false; +}; + TEST(AqlDispatchTest, DebugPausedPoolWaveDoesNotKeepSchedulingContinuations) { VmFixture f("cdna4", /*num_cus=*/1, /*num_wf_slots=*/1); f.cp()->set_dispatch_threads(2); @@ -7033,6 +7827,48 @@ TEST(AqlDispatchTest, LivePluginReplacementPreservesPoolDuringActiveDispatch) { EXPECT_TRUE(f.cu()->is_idle()); } +TEST(AqlDispatchTest, LivePluginReplacementRefreshesResidentWaveHotHookSubscriptions) { + const auto run_case = [](bool initial_observes, bool replacement_observes) { + SCOPED_TRACE(std::format("{} -> {}", initial_observes, replacement_observes)); + VmFixture f("cdna4", /*num_cus=*/1, /*num_wf_slots=*/1); + f.soc_ptr->set_dispatch_threads(2); + + auto initial_group = std::make_shared(PluginSinkConfig{}); + auto initial = std::make_unique(initial_observes); + auto *initial_ptr = initial.get(); + ASSERT_TRUE(initial_group->add(std::move(initial))); + f.soc_ptr->set_plugin_group(initial_group); + + auto code = make_multi_quantum_nop_kernel(); + const uint64_t kernel = f.write_kernel(0x1000, code.data(), code.size() * sizeof(uint32_t)); + test::AqlQueue queue(f.mem(), f.cp()); + queue.dispatch(kernel, /*grid_size=*/64, /*workgroup_size=*/64); + step_until_first_quantum(f, f.cu()); + + auto *wave = f.cu()->wf(0); + ASSERT_NE(wave, nullptr); + const uint32_t physical_sgpr = wave->sgpr_alloc().base; + EXPECT_EQ(f.cu()->read_sgpr(physical_sgpr), 0u); + EXPECT_EQ(initial_ptr->sgpr_reads, initial_observes ? 1u : 0u); + + auto replacement_group = std::make_shared(PluginSinkConfig{}); + auto replacement = std::make_unique(replacement_observes); + auto *replacement_ptr = replacement.get(); + ASSERT_TRUE(replacement_group->add(std::move(replacement))); + f.soc_ptr->set_plugin_group(replacement_group); + + EXPECT_EQ(f.cu()->read_sgpr(physical_sgpr), 0u); + EXPECT_EQ(replacement_ptr->sgpr_reads, replacement_observes ? 1u : 0u); + EXPECT_EQ(initial_ptr->sgpr_reads, initial_observes ? 1u : 0u); + + f.engine->run(); + EXPECT_TRUE(f.cu()->is_idle()); + }; + + run_case(/*initial_observes=*/false, /*replacement_observes=*/true); + run_case(/*initial_observes=*/true, /*replacement_observes=*/false); +} + uint64_t instructions_visible_at_tick_ten(uint32_t dispatch_threads) { VmFixture f("cdna4", /*num_cus=*/1, /*num_wf_slots=*/1); f.cp()->set_dispatch_threads(dispatch_threads); @@ -7165,9 +8001,9 @@ class BlockingInstructionPlugin final : public ExecutionPlugin { bool released_ = false; }; -amdgpu::AqlQueueConfig make_inert_aql_queue(uint32_t queue_id, uint64_t ring_base, - bool host_accessible = false) { - amdgpu::AqlQueueConfig queue{}; +amdgpu::ComputeQueueConfig make_inert_aql_queue(uint32_t queue_id, uint64_t ring_base, + bool host_accessible = false) { + amdgpu::ComputeQueueConfig queue{}; queue.queue_id = queue_id; queue.ring_base_va = ring_base; queue.ring_size = amdgpu::kAqlPacketBytes; @@ -7196,10 +8032,10 @@ TEST(AqlDispatchTest, WorkerExceptionPropagatesThroughEngineStep) { } TEST(AqlDispatchTest, UnimplementedInstructionReportsFailureThroughEngineStep) { - constexpr uint32_t kSSetvskip = 0xBF100000u; + constexpr uint32_t kSCbranchIFork = 0xB8000000u; VmFixture f("cdna4", /*num_cus=*/2); f.cp()->set_dispatch_threads(2); - uint64_t kernel = f.write_kernel(0x1000, &kSSetvskip, sizeof(kSSetvskip)); + uint64_t kernel = f.write_kernel(0x1000, &kSCbranchIFork, sizeof(kSCbranchIFork)); test::AqlQueue queue(f.mem(), f.cp()); queue.dispatch(kernel, /*grid_size=*/128, /*workgroup_size=*/64); @@ -7208,7 +8044,7 @@ TEST(AqlDispatchTest, UnimplementedInstructionReportsFailureThroughEngineStep) { const auto &exit = f.engine->last_exit(); EXPECT_EQ(exit.reason, simdojo::ExitReason::EXIT_REQUEST); EXPECT_EQ(exit.code, 1); - EXPECT_NE(exit.message.find("s_setvskip"), std::string::npos); + EXPECT_NE(exit.message.find("s_cbranch_i_fork"), std::string::npos); EXPECT_NE(exit.message.find("pc=0x1040"), std::string::npos); EXPECT_NE(exit.message.find("unimplemented instruction"), std::string::npos); EXPECT_TRUE(f.cu(0)->is_idle()); @@ -7247,7 +8083,7 @@ TEST(AqlDispatchTest, ThrowingIssueHooksReclaimDecodedInstruction) { ASSERT_TRUE(group->add(std::make_unique(hook))); f.soc_ptr->set_plugin_group(group); // Halt after rejection, so the hook runs outside execute_instruction(). - const uint32_t code = hook == ThrowingIssuePlugin::Halt ? 0xBF100000u : 0xBE800000u; + const uint32_t code = hook == ThrowingIssuePlugin::Halt ? 0xB8000000u : 0xBE800000u; f.write_kernel(0x1000, &code, sizeof(code)); ASSERT_NE(f.dispatch_scratch_wf(), nullptr); @@ -7315,7 +8151,7 @@ TEST(AqlDispatchTest, QueueMutationWaitsForDispatchWorkerWindow) { const uint32_t code[] = {SOPP_S_NOP, SOPP_S_ENDPGM}; uint64_t kernel = f.write_kernel(0x1000, code, sizeof(code)); test::AqlQueue dispatch_queue(f.mem(), f.cp()); - amdgpu::AqlQueueConfig removable_queue = make_inert_aql_queue(42, 0xF1000000); + amdgpu::ComputeQueueConfig removable_queue = make_inert_aql_queue(42, 0xF1000000); const uint64_t removable_registration = f.cp()->register_queue(removable_queue); ASSERT_NE(removable_registration, 0u); dispatch_queue.dispatch(kernel, /*grid_size=*/128, /*workgroup_size=*/64); @@ -7329,7 +8165,7 @@ TEST(AqlDispatchTest, QueueMutationWaitsForDispatchWorkerWindow) { return; } - amdgpu::AqlQueueConfig added_queue = make_inert_aql_queue(43, 0xF2000000); + amdgpu::ComputeQueueConfig added_queue = make_inert_aql_queue(43, 0xF2000000); std::promise registration_started_promise; auto registration_started = registration_started_promise.get_future(); auto registration = @@ -7382,7 +8218,7 @@ TEST(DoorbellMonitorLifecycle, RetiresAfterLastQueueAndRestartsOnNewQueue) { EXPECT_FALSE(cp->doorbell_monitor_running_for_test()) << "no monitor should run before any host-accessible queue is registered"; - amdgpu::AqlQueueConfig queue = make_inert_aql_queue(7, 0xF1000000, true); + amdgpu::ComputeQueueConfig queue = make_inert_aql_queue(7, 0xF1000000, true); const uint64_t registration = cp->register_queue(queue); ASSERT_NE(registration, 0u); EXPECT_TRUE(wait_for_monitor(true)) << "registering a KFD queue must start the monitor"; @@ -7394,7 +8230,7 @@ TEST(DoorbellMonitorLifecycle, RetiresAfterLastQueueAndRestartsOnNewQueue) { << "the stopped monitor must be joined before queue teardown returns"; // A new queue landing on a CP whose monitor retired must get polling back. - amdgpu::AqlQueueConfig queue2 = make_inert_aql_queue(8, 0xF2000000, true); + amdgpu::ComputeQueueConfig queue2 = make_inert_aql_queue(8, 0xF2000000, true); const uint64_t registration2 = cp->register_queue(queue2); ASSERT_NE(registration2, 0u); EXPECT_TRUE(wait_for_monitor(true)) << "a new KFD queue must restart a retired monitor"; @@ -7411,11 +8247,13 @@ TEST(DoorbellMonitorLifecycle, ConcurrentLastQueueRemovalAndRegistrationKeepsMon amdgpu::CommandProcessor *cp = f.cp(); for (uint32_t iteration = 0; iteration < 50; ++iteration) { - amdgpu::AqlQueueConfig old_queue = make_inert_aql_queue(iteration * 2 + 1, 0xF1000000, true); + amdgpu::ComputeQueueConfig old_queue = + make_inert_aql_queue(iteration * 2 + 1, 0xF1000000, true); const uint64_t old_registration = cp->register_queue(old_queue); ASSERT_NE(old_registration, 0u); - amdgpu::AqlQueueConfig new_queue = make_inert_aql_queue(iteration * 2 + 2, 0xF2000000, true); + amdgpu::ComputeQueueConfig new_queue = + make_inert_aql_queue(iteration * 2 + 2, 0xF2000000, true); std::barrier start(3); std::thread remove_last([&] { @@ -7448,9 +8286,8 @@ TEST(Pm4DispatchTest, BaseCoordinatesThreadgroupInfoAndIndirectZeroOrigin) { for (bool graphics_engine : {false, true}) { VmFixture f(arch, 1, 8, 64, 106, 256, /*num_shader_engines=*/1, /*max_ticks=*/0); auto *snapshots = f.capture_halts(); - amdgpu::Pm4SubmitQueue queue; + amdgpu::ComputeQueueConfig queue; queue.queue_id = 71; - queue.pm4 = std::make_shared(); ASSERT_TRUE(f.cp()->register_drm_queue(std::move(queue))); constexpr uint64_t code = 0x8000, ib = 0x4000, indirect = 0x6010; const bool unpacked_ids = @@ -7552,6 +8389,137 @@ TEST(Pm4DispatchTest, BaseCoordinatesThreadgroupInfoAndIndirectZeroOrigin) { } } +TEST(Pm4DispatchTest, MaskedPredExecDoesNotReplaySkippedNativeRingTail) { + for (const auto *arch : {"cdna3", "cdna4"}) { + SCOPED_TRACE(arch); + for (uint64_t initial_cursor : {0u, 6u}) { + SCOPED_TRACE(initial_cursor); + VmFixture f(arch); + constexpr uint64_t kRing = 0x9000, kReadPointer = 0x8000; + constexpr uint32_t kOutput = 0xb000, kRingDwords = 8; + // XCD 0 skips the write selected only for XCD 1, up to the producer cursor. + const uint32_t words[] = {0xc0002300, (1u << 25) | 5, 0xc0033700, 5u << 8, kOutput, 0, 99}; + for (size_t i = 0; i < std::size(words); ++i) + f.mem()->write32(kRing + (initial_cursor + i) % kRingDwords * 4, words[i]); + f.mem()->write32(kReadPointer, initial_cursor); + const auto registration = f.cp()->register_pm4_queue({ + .address_space = {}, + .ring_base = kRing, + .ring_size_bytes = kRingDwords * 4, + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = initial_cursor, + }); + ASSERT_NE(registration, 0u); + const uint64_t producer = initial_cursor + std::size(words); + ASSERT_EQ(f.cp()->notify_pm4_queue_doorbell(registration, producer), + amdgpu::QueueSubmissionStatus::Accepted); + f.engine->run(); + EXPECT_EQ(f.mem()->read32(kReadPointer), producer % kRingDwords); + EXPECT_EQ(f.mem()->read32(kOutput), 0u); + EXPECT_FALSE(f.cp()->queue_faulted_for_test(0, 0)); + EXPECT_TRUE(f.cp()->unregister_pm4_queue_registration(registration)); + } + } +} + +TEST(Pm4DispatchTest, CondExecDoesNotReplaySkippedNativeRingTail) { + for (const auto *arch : {"rdna3", "rdna4", "cdna3", "cdna4"}) { + SCOPED_TRACE(arch); + for (uint64_t initial_cursor : {0u, 14u}) { + SCOPED_TRACE(initial_cursor); + VmFixture f(arch); + constexpr uint64_t kRing = 0x9000, kReadPointer = 0x8000; + constexpr uint32_t kOutput = 0xb000, kPredicate = 0xb010, kRingDwords = 16; + const uint32_t words[] = {0xc0032200, kPredicate, 0, 0, 5, + 0xc0033700, 5u << 8, kOutput, 0, 99}; + f.mem()->write32(kPredicate, 0); + for (size_t i = 0; i < std::size(words); ++i) + f.mem()->write32(kRing + (initial_cursor + i) % kRingDwords * 4, words[i]); + f.mem()->write32(kReadPointer, initial_cursor); + const auto registration = f.cp()->register_pm4_queue({ + .address_space = {}, + .ring_base = kRing, + .ring_size_bytes = kRingDwords * 4, + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = initial_cursor, + }); + ASSERT_NE(registration, 0u); + const uint64_t producer = initial_cursor + std::size(words); + ASSERT_EQ(f.cp()->notify_pm4_queue_doorbell(registration, producer), + amdgpu::QueueSubmissionStatus::Accepted); + f.engine->run(); + EXPECT_EQ(f.mem()->read32(kReadPointer), producer % kRingDwords); + EXPECT_EQ(f.mem()->read32(kOutput), 0u); + EXPECT_FALSE(f.cp()->queue_faulted_for_test(0, 0)); + EXPECT_TRUE(f.cp()->unregister_pm4_queue_registration(registration)); + } + } +} + +TEST(Pm4DispatchTest, BooleanPredicationOrdersReadsAndOnlySuppressesFlaggedPackets) { + for (const auto *arch : + {"rdna1", "rdna2", "rdna3", "rdna3_5", "rdna4", "cdna1", "cdna2", "cdna3", "cdna4"}) { + SCOPED_TRACE(arch); + VmFixture f(arch, 1, 8, 64, 106, 256, 1, 0); + amdgpu::ComputeQueueConfig queue; + queue.queue_id = 71; + ASSERT_TRUE(f.cp()->register_drm_queue(std::move(queue))); + constexpr uint64_t ib = 0x4000, predicate = 0x6000, output = 0x7000; + std::vector words; + auto packet = [&](amdgpu::Pm4Opcode opcode, std::initializer_list payload, + bool predicated = false) { + words.push_back(0xc0000000 | ((payload.size() - 1) << 16) | (uint32_t(opcode) << 8) | + uint32_t(predicated)); + words.insert(words.end(), payload.begin(), payload.end()); + }; + std::vector expected; + const auto write = [&](bool predicated, bool executed) { + const uint32_t address = output + expected.size() * 4; + packet(amdgpu::Pm4Opcode::WriteData, {5u << 8, address, 0, 0x1234}, predicated); + expected.push_back(executed ? 0x1234 : 0); + }; + for (uint32_t operation : {3u, 4u}) { + for (uint64_t value : {uint64_t{0}, uint64_t{1}, uint64_t{1} << 32}) { + for (bool visible : {false, true}) { + packet(amdgpu::Pm4Opcode::WriteData, + {5u << 8, predicate, 0, uint32_t(value), uint32_t(value >> 32)}); + packet(amdgpu::Pm4Opcode::SetPredication, + {(operation << 16) | (uint32_t(visible) << 8), predicate, 0}); + const bool nonzero = operation == 4 ? uint32_t(value) != 0 : value != 0; + write(true, nonzero == visible); + write(false, true); + packet(amdgpu::Pm4Opcode::SetPredication, {0, 0, 0}); + write(true, true); + // COND_EXEC reads only the low dword, independent of SET_PREDICATION width. + packet(amdgpu::Pm4Opcode::CondExec, {predicate, 0, 0, 5}); + write(false, uint32_t(value) != 0); + write(false, true); + } + } + } + for (uint32_t i = 0; i < words.size(); ++i) + f.mem()->write32(ib + i * 4, words[i]); + for (uint32_t i = 0; i < expected.size(); ++i) + f.mem()->write32(output + i * 4, 0); + bool completed = false, succeeded = false; + amdgpu::Pm4Submission submission; + submission.graphics_engine = true; + submission.buffers.push_back({ib, uint32_t(words.size())}); + f.engine->register_as_primary(); + submission.complete = [&](bool success) { + completed = true; + succeeded = success; + f.engine->primary_release(); + }; + ASSERT_TRUE(f.cp()->submit_pm4(71, 0, std::move(submission))); + f.engine->run(); + ASSERT_TRUE(completed); + ASSERT_TRUE(succeeded); + for (uint32_t i = 0; i < expected.size(); ++i) + EXPECT_EQ(f.mem()->read32(output + i * 4), expected[i]) << i; + } +} + TEST(Pm4DispatchTest, CancellationFailsPendingSubmissionAndReleasesVmBinding) { KfdProcess process(7); VmFixture f("rdna3"); @@ -7563,11 +8531,10 @@ TEST(Pm4DispatchTest, CancellationFailsPendingSubmissionAndReleasesVmBinding) { .request_mutex = process.page_table_request_mutex(), .mutation_epoch = process.page_table_mutation_epoch()}); ASSERT_TRUE(address_space); - amdgpu::Pm4SubmitQueue queue; + amdgpu::ComputeQueueConfig queue; queue.address_space = address_space; queue.queue_id = 71; queue.process_id = 7; - queue.pm4 = std::make_shared(); ASSERT_TRUE(f.cp()->register_drm_queue(std::move(queue))); EXPECT_FALSE(adapter.unregister_address_space(address_space)); amdgpu::Pm4Submission submission; @@ -7597,11 +8564,10 @@ TEST(Pm4DispatchTest, ExactQueueRetirementPreservesOtherQueuesAndReleasesBinding ASSERT_TRUE(address_space); std::array callbacks{}; for (uint32_t i = 0; i < 2; ++i) { - amdgpu::Pm4SubmitQueue queue; + amdgpu::ComputeQueueConfig queue; queue.address_space = address_space; queue.queue_id = 71 + i; queue.process_id = 7; - queue.pm4 = std::make_shared(); ASSERT_TRUE(f.cp()->register_drm_queue(std::move(queue))); amdgpu::Pm4Submission submission; submission.ready = [] { return false; }; @@ -7679,11 +8645,10 @@ TEST(Pm4DispatchTest, OverlappingScratchQueuesSerializeAndPreserveContents) { std::memcpy(code_backing.data(), &end_program, sizeof(end_program)); uint32_t completed = 0; for (uint32_t index = 0; index < 2; ++index) { - amdgpu::Pm4SubmitQueue queue; + amdgpu::ComputeQueueConfig queue; queue.address_space = address_space; queue.queue_id = 71 + index; queue.process_id = pid; - queue.pm4 = std::make_shared(); ASSERT_TRUE(f.cp()->register_drm_queue(std::move(queue))); std::vector words; auto packet = [&](amdgpu::Pm4Opcode op, std::initializer_list payload) { @@ -7747,11 +8712,10 @@ TEST(Pm4DispatchTest, ScratchUsesTargetRegisterLayout) { .mutation_epoch = process.page_table_mutation_epoch()}); ASSERT_TRUE(address_space); auto *snapshots = f.capture_halts(); - amdgpu::Pm4SubmitQueue queue; + amdgpu::ComputeQueueConfig queue; queue.address_space = address_space; queue.queue_id = 71; queue.process_id = pid; - queue.pm4 = std::make_shared(); ASSERT_TRUE(f.cp()->register_drm_queue(std::move(queue))); const uint32_t end_program = legacy ? 0xbf810000 : 0xbfb00000; std::memcpy(code_backing.data(), &end_program, sizeof(end_program)); diff --git a/emulation/rocjitsu/tests/analysis/liveness_test.cpp b/emulation/rocjitsu/tests/analysis/liveness_test.cpp index a9e6ee719317..683860fc3661 100644 --- a/emulation/rocjitsu/tests/analysis/liveness_test.cpp +++ b/emulation/rocjitsu/tests/analysis/liveness_test.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -384,7 +385,396 @@ uint32_t build_s_call_b64(uint16_t sdst, int16_t simm16) { .sdst = static_cast(sdst)})[0]; } -TEST(RegisterSetAnalysis, KeepsRegisterClassesSeparate) { +// Exercise the scalar fallback even when the production type uses AVX2. +template class RegisterSetTyped : public testing::Test {}; +#if defined(__AVX2__) +using RegisterSetTypes = testing::Types, + RegisterSetT>; +#else +using RegisterSetTypes = testing::Types>; +#endif +TYPED_TEST_SUITE(RegisterSetTyped, RegisterSetTypes); + +TYPED_TEST(RegisterSetTyped, WordIterationRangeOperationsMatchModelAtBoundaries) { + using RegisterSet = TypeParam; + constexpr std::array classes{RegClass::SGPR, RegClass::VGPR, RegClass::ACC_VGPR}; + constexpr std::array capacities{REGISTER_SET_MAX_SGPRS, REGISTER_SET_MAX_VGPRS, + REGISTER_SET_MAX_ACC_VGPRS}; + constexpr std::array starts{0, 1, 63, 64, 65, 105, 106, 127, 128, + 255, 256, 257, 511, 512, 1023, 1024, 65535}; + constexpr std::array widths{0, 1, 2, 63, 64, 65, 127, 128, 129, 254, 255}; + using Model = std::bitset; + for (size_t cls = 0; cls < classes.size(); ++cls) { + for (unsigned pattern = 0; pattern < 3; ++pattern) { + RegisterSet original; + Model originalModel; + for (size_t bit = 0; bit < capacities[cls]; ++bit) { + if (pattern == 0 || (pattern == 1 && bit % 3 != 0)) { + original.expand({classes[cls], static_cast(bit), 1}); + originalModel.set(bit); + } + } + for (uint16_t startBit : starts) { + for (uint8_t width : widths) { + SCOPED_TRACE(testing::Message() + << "class=" << cls << " pattern=" << pattern << " startBit=" << startBit + << " width=" << static_cast(width)); + const RegisterRef ref{classes[cls], startBit, width}; + const size_t endBit = startBit + std::max(1, width); + bool contains = true, intersects = false; + Model addedModel = originalModel, removedModel = originalModel; + for (size_t bit = startBit; bit < endBit; ++bit) { + const bool present = bit < capacities[cls] && originalModel.test(bit); + contains &= present; + intersects |= present; + if (bit < capacities[cls]) { + addedModel.set(bit); + removedModel.reset(bit); + } + } + EXPECT_EQ(original.contains(ref), contains); + EXPECT_EQ(original.intersects(ref), intersects); + RegisterSet added = original, removed = original; + added.expand(ref); + removed.erase(ref); + const auto check = [&](const RegisterSet &set, const Model &expected) { + Model actual; + set.for_each_ordinary([&](RegisterRef visited) { + EXPECT_EQ(visited.cls, classes[cls]); + actual.set(visited.index); + }); + EXPECT_EQ(actual, expected); + }; + check(added, addedModel); + check(removed, removedModel); + } + } + } + } +} + +TYPED_TEST(RegisterSetTyped, WordIterationSingleBitHelpersPreserveOtherBits) { + using Word = typename TypeParam::WordType; + using Ops = register_set_detail::WordArrayOps; + const Word fullWord = Ops::word_all_bits(); + for (unsigned bitIndex = 0; bitIndex < Ops::kWordBits; ++bitIndex) { + SCOPED_TRACE(bitIndex); + Word singleBit{}; + Word allOtherBits = fullWord; + Ops::word_set_bit(singleBit, bitIndex); + Ops::word_set_bit(singleBit, bitIndex); // Setting twice is idempotent. + Ops::word_reset_bit(allOtherBits, bitIndex); + Ops::word_reset_bit(allOtherBits, bitIndex); + for (unsigned testBit = 0; testBit < Ops::kWordBits; ++testBit) { + EXPECT_EQ(Ops::word_test_bit(singleBit, testBit), testBit == bitIndex); + EXPECT_EQ(Ops::word_test_bit(allOtherBits, testBit), testBit != bitIndex); + } + Ops::word_reset_bit(singleBit, bitIndex); + Ops::word_set_bit(allOtherBits, bitIndex); + EXPECT_TRUE(Ops::word_none(singleBit)); + EXPECT_TRUE(Ops::word_equal(allOtherBits, fullWord)); + } +} + +TYPED_TEST(RegisterSetTyped, WordIterationCoversEveryBytePattern) { + using RegisterSet = TypeParam; + for (uint16_t base : {0, 8, 56, 64, 120, 128, 240, 248, 256, 504, 1016}) { + for (unsigned pattern = 0; pattern < 256; ++pattern) { + SCOPED_TRACE(base); + SCOPED_TRACE(pattern); + RegisterSet set; + std::vector expected, visited; + for (unsigned bit = 0; bit < 8; ++bit) { + if (pattern & (1u << bit)) { + RegisterRef ref{RegClass::VGPR, static_cast(base + bit), 1}; + set.expand(ref); + expected.push_back(ref); + } + } + set.for_each([&](RegisterRef ref) { visited.push_back(ref); }); + EXPECT_EQ(visited, expected); + } + } +} + +TYPED_TEST(RegisterSetTyped, WordIterationObservesEditsWithinAndBetweenBatches) { + using RegisterSet = TypeParam; + RegisterSet set; + const auto add = [&](uint16_t index) { set.expand({RegClass::VGPR, index, 1}); }; + const auto remove = [&](uint16_t index) { set.erase({RegClass::VGPR, index, 1}); }; + for (uint16_t index : {0, 8, 31, 255, 256}) + add(index); + std::vector visited; + set.for_each([&](RegisterRef ref) { + visited.push_back(ref.index); + switch (ref.index) { + case 0: + remove(8); + for (uint16_t index : {2, 7, 9, 32}) + add(index); + break; + case 2: + remove(7); + add(6); + break; + case 6: + add(1); // Already passed: never revisit this bit. + add(7); + break; + case 7: + remove(9); + add(10); + break; + case 10: + add(9); // Behind the cursor, in the current byte. + add(17); + break; + case 17: + remove(31); + add(30); + break; + case 30: + remove(32); + add(63); + break; + case 63: + add(64); + break; + case 64: + remove(255); + add(254); + break; + case 254: + add(255); + break; + case 255: + remove(256); + add(257); + break; + } + }); + EXPECT_EQ(visited, (std::vector{0, 2, 6, 7, 10, 17, 30, 63, 64, 254, 255, 257})); +} + +TYPED_TEST(RegisterSetTyped, WordIterationHandlesEditsAtEveryPositionInFullBatches) { + using RegisterSet = TypeParam; + for (unsigned trigger = 0; trigger < 8; ++trigger) { + SCOPED_TRACE(trigger); + RegisterSet set; + set.expand({RegClass::VGPR, 248, 16}); + std::vector visited, expected; + for (unsigned index = 248; index <= 248 + trigger; ++index) + expected.push_back(index); + for (unsigned index = 257; index <= 264; ++index) + expected.push_back(index); + set.for_each([&](RegisterRef ref) { + visited.push_back(ref.index); + if (ref.index == 248 + trigger) { + set.erase({RegClass::VGPR, 248, 1}); + if (trigger < 7) + set.erase({RegClass::VGPR, static_cast(249 + trigger), + static_cast(7 - trigger)}); + set.erase({RegClass::VGPR, 256, 1}); + set.expand({RegClass::VGPR, 264, 1}); + } + }); + EXPECT_EQ(visited, expected); + } +} + +TYPED_TEST(RegisterSetTyped, WordIterationHandlesEditsAtEveryPositionInFullLanes) { + using RegisterSet = TypeParam; + for (unsigned trigger = 0; trigger < 64; ++trigger) { + SCOPED_TRACE(trigger); + RegisterSet set; + set.expand({RegClass::VGPR, 192, 128}); + std::vector visited, expected; + for (unsigned index = 192; index <= 192 + trigger; ++index) + expected.push_back(index); + for (unsigned index = 257; index <= 320; ++index) + expected.push_back(index); + set.for_each([&](RegisterRef ref) { + visited.push_back(ref.index); + if (ref.index == 192 + trigger) { + set.erase({RegClass::VGPR, 192, 1}); + if (trigger < 63) + set.erase({RegClass::VGPR, static_cast(193 + trigger), + static_cast(63 - trigger)}); + set.erase({RegClass::VGPR, 256, 1}); + set.expand({RegClass::VGPR, 320, 1}); + } + }); + EXPECT_EQ(visited, expected); + } +} + +TYPED_TEST(RegisterSetTyped, WordIterationMatchesBitsetModelAcrossMutationsAndAlgebra) { + using RegisterSet = TypeParam; + constexpr std::array classes{RegClass::SGPR, RegClass::VGPR, RegClass::ACC_VGPR}; + constexpr std::array capacities{REGISTER_SET_MAX_SGPRS, REGISTER_SET_MAX_VGPRS, + REGISTER_SET_MAX_ACC_VGPRS}; + using Model = std::array, 3>; + const auto check = [&](const RegisterSet &actual, const Model &model) { + std::vector expected, visited; + for (size_t cls = 0; cls < classes.size(); ++cls) + for (size_t index = 0; index < model[cls].size(); ++index) + if (model[cls].test(index)) + expected.push_back({classes[cls], static_cast(index), 1}); + actual.for_each_ordinary([&](RegisterRef ref) { visited.push_back(ref); }); + EXPECT_EQ(visited, expected); + EXPECT_EQ(actual.ordinary_size(), expected.size()); + EXPECT_EQ(actual.none(), expected.empty()); + EXPECT_EQ(actual.size(), expected.size()); + }; + std::array sets; + std::array models; + uint32_t seed = 0x8a021bu; + const auto next = [&]() { return seed = seed * 1664525u + 1013904223u; }; + // Deliberately include word edges, the partial SGPR word, high VGPR banks, + // zero widths, clipped wide tuples, and completely out-of-range tuples. + constexpr std::array bases{0, 1, 62, 63, 64, 65, 104, 105, 106, 127, + 128, 255, 256, 511, 512, 767, 768, 1023, 1024, 65535}; + constexpr std::array widths{0, 1, 2, 32, 255}; + for (unsigned step = 0; step < 300; ++step) { + SCOPED_TRACE(step); + const size_t which = next() % 2; + const size_t cls = next() % classes.size(); + const uint16_t base = bases[next() % bases.size()]; + const uint8_t width = widths[next() % widths.size()]; + const RegisterRef ref{classes[cls], base, width}; + const size_t end = static_cast(base) + std::max(1, width); + const auto operation = next() % 10; + if (operation == 0) { + sets[which].clear_class(classes[cls]); + models[which][cls].reset(); + } else { + const bool insert = operation < 7; + if (insert) + sets[which].expand(ref); + else + sets[which].erase(ref); + for (size_t index = base; index < std::min(end, capacities[cls]); ++index) + models[which][cls].set(index, insert); + } + bool contains = true, intersects = false; + for (size_t index = base; index < end; ++index) { + const bool present = index < capacities[cls] && models[which][cls].test(index); + contains &= present; + intersects |= present; + } + EXPECT_EQ(sets[which].contains(ref), contains); + EXPECT_EQ(sets[which].intersects(ref), intersects); + check(sets[0], models[0]); + check(sets[1], models[1]); + Model united, common, difference; + for (size_t c = 0; c < classes.size(); ++c) { + united[c] = models[0][c] | models[1][c]; + common[c] = models[0][c] & models[1][c]; + difference[c] = models[0][c] & ~models[1][c]; + } + check(sets[0] | sets[1], united); + check(sets[0] & sets[1], common); + check(sets[0] - sets[1], difference); + EXPECT_EQ(sets[0] == sets[1], models[0] == models[1]); + EXPECT_EQ(sets[0].intersects(sets[1]), + std::ranges::any_of(common, [](const auto &bits) { return bits.any(); })); + } +} + +TYPED_TEST(RegisterSetTyped, WordIterationHandlesFullWordsAndPartialClassTails) { + using RegisterSet = TypeParam; + constexpr std::array classes{RegClass::SGPR, RegClass::VGPR, RegClass::ACC_VGPR}; + constexpr std::array capacities{REGISTER_SET_MAX_SGPRS, REGISTER_SET_MAX_VGPRS, + REGISTER_SET_MAX_ACC_VGPRS}; + RegisterSet set; + std::vector expected, visited; + for (size_t cls = 0; cls < classes.size(); ++cls) { + for (size_t base = 0; base < capacities[cls]; base += 64) + set.expand({classes[cls], static_cast(base), 64}); + for (size_t index = 0; index < capacities[cls]; ++index) + if (index != 63 && index != 64) + expected.push_back({classes[cls], static_cast(index), 1}); + } + set.expand({RegClass::EXEC, 0, 1}); + set.for_each_ordinary([&](RegisterRef ref) { + visited.push_back(ref); + if (ref.index == 0) + set.erase({ref.cls, 63, 2}); + }); + EXPECT_EQ(visited, expected); + EXPECT_EQ(set.ordinary_size(), expected.size()); + EXPECT_EQ(set.size(), expected.size() + 1); + EXPECT_TRUE(set.contains({RegClass::EXEC, 0, 1})); +} + +TYPED_TEST(RegisterSetTyped, WordIterationPreservesForwardCallbackEdits) { + using RegisterSet = TypeParam; + RegisterSet set; + for (uint16_t index : {0, 1, 63, 64, 1023}) + set.expand({RegClass::VGPR, index, 1}); + std::vector visited; + set.for_each_ordinary([&](RegisterRef ref) { + visited.push_back(ref.index); + if (ref.index == 0) { + set.erase({RegClass::VGPR, 1, 1}); + set.expand({RegClass::VGPR, 2, 1}); + } else if (ref.index == 63) { + set.erase({RegClass::VGPR, 64, 1}); + set.expand({RegClass::VGPR, 65, 1}); + } + }); + EXPECT_EQ(visited, (std::vector{0, 2, 63, 65, 1023})); +} + +TYPED_TEST(RegisterSetTyped, WordIterationAlternatesUnchangedAndEditedMembership) { + using RegisterSet = TypeParam; + // Compare against the original forward scan, including edits to the current + // and earlier bits, new members in gaps, and changes at word boundaries. + for (auto cls : {RegClass::SGPR, RegClass::VGPR, RegClass::ACC_VGPR}) { + const size_t capacity = cls == RegClass::SGPR ? REGISTER_SET_MAX_SGPRS : REGISTER_SET_MAX_VGPRS; + for (unsigned seed = 0; seed < 16; ++seed) { + RegisterSet set; + std::bitset model; + for (size_t i = 0; i < capacity; ++i) { + if ((i + seed) % 3 == 0 || i % 64 == 63) { + set.expand({cls, static_cast(i), 1}); + model.set(i); + } + } + auto edit = [&](size_t index, auto &&add, auto &&erase) { + if ((index + seed) % 4 == 0) { + erase(index); + if (index != 0) + add(index - 1); // Never revisit members behind the cursor. + if (index + 2 < capacity) + add(index + 2); + } else if ((index + seed) % 4 == 1 && index + 1 < capacity) { + erase(index + 1); + } + }; + std::vector expected, visited; + for (size_t i = 0; i < capacity; ++i) { + if (!model.test(i)) + continue; + expected.push_back(static_cast(i)); + edit(i, [&](size_t index) { model.set(index); }, [&](size_t index) { model.reset(index); }); + } + set.for_each_ordinary([&](RegisterRef ref) { + EXPECT_EQ(ref.cls, cls); + visited.push_back(ref.index); + edit( + ref.index, [&](size_t index) { set.expand({cls, static_cast(index), 1}); }, + [&](size_t index) { set.erase({cls, static_cast(index), 1}); }); + }); + EXPECT_EQ(visited, expected) << "seed=" << seed; + for (size_t i = 0; i < capacity; ++i) + EXPECT_EQ(set.contains({cls, static_cast(i), 1}), model.test(i)); + } + } +} + +TYPED_TEST(RegisterSetTyped, KeepsRegisterClassesSeparate) { + using RegisterSet = TypeParam; RegisterSet set; set.expand({RegClass::SGPR, 4, 1}); @@ -393,7 +783,8 @@ TEST(RegisterSetAnalysis, KeepsRegisterClassesSeparate) { EXPECT_FALSE(set.contains({RegClass::ACC_VGPR, 4, 1})); } -TEST(RegisterSetAnalysis, IntersectsIsTheAnyOfCounterpartToContains) { +TYPED_TEST(RegisterSetTyped, IntersectsIsTheAnyOfCounterpartToContains) { + using RegisterSet = TypeParam; RegisterSet set; set.expand({RegClass::SGPR, 4, 1}); @@ -410,7 +801,8 @@ TEST(RegisterSetAnalysis, IntersectsIsTheAnyOfCounterpartToContains) { EXPECT_FALSE(set.intersects({RegClass::SGPR, 5, 4})); } -TEST(RegisterSetAnalysis, IntersectsKeepsRegisterClassesSeparate) { +TYPED_TEST(RegisterSetTyped, IntersectsKeepsRegisterClassesSeparate) { + using RegisterSet = TypeParam; RegisterSet set; set.expand({RegClass::SGPR, 0, 1}); @@ -419,7 +811,8 @@ TEST(RegisterSetAnalysis, IntersectsKeepsRegisterClassesSeparate) { EXPECT_FALSE(set.intersects({RegClass::ACC_VGPR, 0, 1})); } -TEST(RegisterSetAnalysis, IntersectsAnswersFalseForUntrackedClasses) { +TYPED_TEST(RegisterSetTyped, IntersectsAnswersFalseForUntrackedClasses) { + using RegisterSet = TypeParam; RegisterSet set; set.expand({RegClass::SGPR, 0, 1}); @@ -429,7 +822,8 @@ TEST(RegisterSetAnalysis, IntersectsAnswersFalseForUntrackedClasses) { EXPECT_FALSE(set.intersects({RegClass::VCC, 0, 1})); } -TEST(RegisterSetAnalysis, IntersectsToleratesRunsPastTheTrackedRange) { +TYPED_TEST(RegisterSetTyped, IntersectsToleratesRunsPastTheTrackedRange) { + using RegisterSet = TypeParam; RegisterSet set; set.expand({RegClass::SGPR, 0, 1}); @@ -440,7 +834,8 @@ TEST(RegisterSetAnalysis, IntersectsToleratesRunsPastTheTrackedRange) { {RegClass::SGPR, static_cast(REGISTER_SET_ALLOCATABLE_SGPRS + 8), 4})); } -TEST(RegisterSetAnalysis, TracksGfx1250HighBankVectorRegisters) { +TYPED_TEST(RegisterSetTyped, TracksGfx1250HighBankVectorRegisters) { + using RegisterSet = TypeParam; RegisterSet set; set.expand({RegClass::VGPR, 768, 2}); @@ -569,7 +964,8 @@ TEST(GeneratedInstDefUse, MubufCmpswapReturnUsesElementWidthAndTargetGate) { } } -TEST(RegisterSetAnalysis, SpecialClassesAreHeldButCarryNoOrdinaryLanes) { +TYPED_TEST(RegisterSetTyped, SpecialClassesAreHeldButCarryNoOrdinaryLanes) { + using RegisterSet = TypeParam; RegisterSet set; set.expand({RegClass::EXEC, 0, 2}); set.expand({RegClass::SCC, 0, 1}); @@ -629,7 +1025,7 @@ TEST(RegisterSetAnalysis, Cdna4WritelaneDestinationIsUseAndDef) { // Collect a set's special members in ascending RegClass order, so a test can // assert the *exact* set of special registers an instruction reads or writes, // not just membership. -std::vector specials_in(const RegisterSet &set) { +template std::vector specials_in(const Set &set) { std::vector classes; set.for_each_special([&](RegClass c) { classes.push_back(c); }); return classes; @@ -659,7 +1055,8 @@ TEST(RegisterSetSpecial, ClassifiesSpecialVersusOrdinaryClasses) { // TTMP is a known register class that this set deliberately does not track: // no bitset, no special-mask bit. Adding one never makes the set non-empty and // never reports the register as present, and removal paths are safe no-ops. -TEST(RegisterSetSpecial, TtmpIsUntracked) { +TYPED_TEST(RegisterSetTyped, TtmpIsUntracked) { + using RegisterSet = TypeParam; RegisterSet s; s.expand({RegClass::TTMP, 0, 1}); EXPECT_FALSE(s.contains({RegClass::TTMP, 0, 1})); @@ -672,7 +1069,8 @@ TEST(RegisterSetSpecial, TtmpIsUntracked) { EXPECT_TRUE(s.none()); } -TEST(RegisterSetSpecial, OrdinaryAndSpecialInsertionCoexist) { +TYPED_TEST(RegisterSetTyped, OrdinaryAndSpecialInsertionCoexist) { + using RegisterSet = TypeParam; RegisterSet s; s.expand({RegClass::SGPR, 4, 2}); s.expand({RegClass::EXEC, 0, 1}); @@ -688,7 +1086,8 @@ TEST(RegisterSetSpecial, OrdinaryAndSpecialInsertionCoexist) { EXPECT_EQ(specials_in(s), special_regs({RegClass::EXEC, RegClass::VCC})); } -TEST(RegisterSetSpecial, SpecialMembershipIgnoresIndexAndWidth) { +TYPED_TEST(RegisterSetTyped, SpecialMembershipIgnoresIndexAndWidth) { + using RegisterSet = TypeParam; // Special classes are singletons: any index/width refers to the same member. RegisterSet s; s.expand({RegClass::EXEC, 42, 8}); @@ -701,7 +1100,8 @@ TEST(RegisterSetSpecial, SpecialMembershipIgnoresIndexAndWidth) { EXPECT_TRUE(s.none()); } -TEST(RegisterSetSpecial, FullIterationEmitsCanonicalSpecialRefs) { +TYPED_TEST(RegisterSetTyped, FullIterationEmitsCanonicalSpecialRefs) { + using RegisterSet = TypeParam; RegisterSet s; s.expand({RegClass::VGPR, 1, 1}); s.expand({RegClass::PC, 5, 2}); @@ -720,7 +1120,8 @@ TEST(RegisterSetSpecial, FullIterationEmitsCanonicalSpecialRefs) { EXPECT_EQ(ordinary, (std::vector{{RegClass::VGPR, 1, 1}})); } -TEST(RegisterSetSpecial, OrdinaryOnlyDropsSpecialMembers) { +TYPED_TEST(RegisterSetTyped, OrdinaryOnlyDropsSpecialMembers) { + using RegisterSet = TypeParam; RegisterSet s; s.expand({RegClass::SGPR, 3, 1}); s.expand({RegClass::SCC, 0, 1}); @@ -732,7 +1133,8 @@ TEST(RegisterSetSpecial, OrdinaryOnlyDropsSpecialMembers) { EXPECT_TRUE(s.has_specials()); // source unchanged } -TEST(RegisterSetSpecial, SetAlgebraSpansOrdinaryAndSpecial) { +TYPED_TEST(RegisterSetTyped, SetAlgebraSpansOrdinaryAndSpecial) { + using RegisterSet = TypeParam; RegisterSet a; a.expand({RegClass::SGPR, 0, 1}); a.expand({RegClass::EXEC, 0, 1}); @@ -761,7 +1163,8 @@ TEST(RegisterSetSpecial, SetAlgebraSpansOrdinaryAndSpecial) { EXPECT_EQ(cleared.ordinary_size(), 2u); // clear_class(special) leaves ordinary intact } -TEST(RegisterSetSpecial, EqualityDistinguishesSpecialMembership) { +TYPED_TEST(RegisterSetTyped, EqualityDistinguishesSpecialMembership) { + using RegisterSet = TypeParam; RegisterSet a; a.expand({RegClass::SGPR, 0, 1}); RegisterSet b = a; @@ -770,7 +1173,8 @@ TEST(RegisterSetSpecial, EqualityDistinguishesSpecialMembership) { EXPECT_NE(a, b); // the special mask participates in equality } -TEST(RegisterSetSpecial, HighValuedSpecialMaskBitsRoundTrip) { +TYPED_TEST(RegisterSetTyped, HighValuedSpecialMaskBitsRoundTrip) { + using RegisterSet = TypeParam; // FLAT_SCRATCH and PC are the highest RegClass values, where a mask-width bug // would first surface. RegisterSet s; @@ -7243,5 +7647,109 @@ TEST(GeneratedInstDefUse, Vop3SdstEncDppInvalidSourceCanReadScalarResult) { EXPECT_TRUE(idu.uses.contains({RegClass::SGPR, 8, 2})); } +// explicit_ordinary_sgpr_bound is the cheap whole-scope scan shared by DBT's +// virtual-LDS reservation and DBI's entry-prologue storage selection. It answers +// only "how far up the scalar file does this kernel reach", with no dataflow. + +// A scope with no scalar operands leaves a caller free to place storage at s0. +TEST(ExplicitOrdinarySgprBound, ReportsZeroForAScopeNamingNoScalars) { + std::vector words = {build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4)}; + TestCodeObject co(std::move(words)); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4); + ASSERT_NE(decoder, nullptr); + auto blocks = build_valid_blocks(co, *decoder, ROCJITSU_CODE_ARCH_CDNA4); + ASSERT_FALSE(blocks.empty()); + + std::vector scope; + for (const auto &b : blocks) + scope.push_back(b.get()); + EXPECT_EQ(explicit_ordinary_sgpr_bound(KernelBlockScope(scope)), 0u); +} + +// The bound is one past the highest index, and an operand's width counts: the +// pair write ends at s22 even though its base is lower than the single write. +TEST(ExplicitOrdinarySgprBound, CountsOperandWidthNotJustTheBase) { + std::vector words = { + build_s_mov_b32(/*sdst=*/8, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4), + cdna4::build_sop1(cdna4::kSMovB64Sop1, {.ssrc0 = 128, .sdst = 20})[0], + build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), + }; + TestCodeObject co(std::move(words)); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4); + ASSERT_NE(decoder, nullptr); + auto blocks = build_valid_blocks(co, *decoder, ROCJITSU_CODE_ARCH_CDNA4); + ASSERT_FALSE(blocks.empty()); + + std::vector scope; + for (const auto &b : blocks) + scope.push_back(b.get()); + EXPECT_EQ(explicit_ordinary_sgpr_bound(KernelBlockScope(scope)), 22u); +} + +// RDNA exposes 106 ordinary SGPRs against CDNA's 102, and both consumers bound +// selection by the cross-family minimum. An operand above that bound must not +// push the result out of reach of every caller. +TEST(ExplicitOrdinarySgprBound, IgnoresIndicesPastTheAllocatableMaximum) { + std::vector words = { + build_s_mov_b32(/*sdst=*/104, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_RDNA4), + build_s_mov_b32(/*sdst=*/12, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_RDNA4), + build_s_endpgm(ROCJITSU_CODE_ARCH_RDNA4), + }; + TestCodeObject co(std::move(words)); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4); + ASSERT_NE(decoder, nullptr); + auto blocks = build_valid_blocks(co, *decoder, ROCJITSU_CODE_ARCH_RDNA4); + ASSERT_FALSE(blocks.empty()); + + std::vector scope; + for (const auto &b : blocks) + scope.push_back(b.get()); + EXPECT_EQ(explicit_ordinary_sgpr_bound(KernelBlockScope(scope)), 13u); +} + +// s_and_saveexec_b64 s[0:1], s[0:1] names one pair twice and additionally +// touches EXEC and SCC. Those are special classes, so they cannot raise the +// bound; a scan that folded in implicit operands would still have to filter +// them out by class. +TEST(ExplicitOrdinarySgprBound, SpecialRegisterEffectsDoNotRaiseTheBound) { + std::vector words = { + 0xBE802000u, + build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), + }; + TestCodeObject co(std::move(words)); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4); + ASSERT_NE(decoder, nullptr); + auto blocks = build_valid_blocks(co, *decoder, ROCJITSU_CODE_ARCH_CDNA4); + ASSERT_FALSE(blocks.empty()); + + std::vector scope; + for (const auto &b : blocks) + scope.push_back(b.get()); + EXPECT_EQ(explicit_ordinary_sgpr_bound(KernelBlockScope(scope)), 2u); +} + +// FLAT/GLOBAL saddr appears in both the explicit operand list and +// Flat::implicit_uses. The explicit-only scan must still see it, or a caller +// placing storage above the bound would take a pair the kernel is addressing +// memory through. +TEST(ExplicitOrdinarySgprBound, SeesAnSgprThatIsAlsoAnImplicitUse) { + // global_load_dwordx2 v[0:1], v[2:3], s[10:11] + std::vector words = { + 0xDC548000u, + 0x000A0002u, + build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), + }; + TestCodeObject co(std::move(words)); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4); + ASSERT_NE(decoder, nullptr); + auto blocks = build_valid_blocks(co, *decoder, ROCJITSU_CODE_ARCH_CDNA4); + ASSERT_FALSE(blocks.empty()); + + std::vector scope; + for (const auto &b : blocks) + scope.push_back(b.get()); + EXPECT_EQ(explicit_ordinary_sgpr_bound(KernelBlockScope(scope)), 12u); +} + } // namespace } // namespace rocjitsu diff --git a/emulation/rocjitsu/tests/aql_packet_processor_test.cpp b/emulation/rocjitsu/tests/aql_packet_processor_test.cpp index a2599353b7bc..41aaaeea8283 100644 --- a/emulation/rocjitsu/tests/aql_packet_processor_test.cpp +++ b/emulation/rocjitsu/tests/aql_packet_processor_test.cpp @@ -200,27 +200,43 @@ TEST(AqlPacketProcessorTest, BarrierDependencyMustResolveBeforeAdmission) { EXPECT_TRUE(complete.blocks_following); } -TEST(AqlPacketProcessorTest, Pm4IbDoesNotBlockFollowingPackets) { - AmdExtKernelDispatchPacket packet{}; - packet.header = HSA_PACKET_TYPE_VENDOR_SPECIFIC; - packet.amd_format = kAmdAqlFormatPm4Ib; +TEST(AqlPacketProcessorTest, Pm4IbDecodesAndBlocksFollowingPackets) { + std::array packet{HSA_PACKET_TYPE_VENDOR_SPECIFIC | + (uint32_t{kAmdAqlFormatPm4Ib} << 16), + 0xc0023f00, + 0x8000, + 1, + (1u << 23) | 32, + 10}; const auto bytes = packet_bytes(packet); AqlPreparedPacket admitted{}; + unsigned admissions = 0; AqlPacketProcessor processor({ .load_signal = {}, .admit = [&](const AqlPacketProcessRequest &, AqlPreparedPacket prepared) { admitted = prepared; + ++admissions; return AqlAdmissionResult{.status = AqlAdmissionStatus::Complete}; }, }); - const AqlPacketProcessResult result = processor.process(request_for(bytes)); - EXPECT_EQ(result.packet.status, PacketProcessStatus::Complete); - EXPECT_EQ(admitted.kind, AqlPreparedPacketKind::NonKernel); - EXPECT_FALSE(admitted.blocks_following); - EXPECT_FALSE(result.blocks_following); + EXPECT_EQ(admitted.kind, AqlPreparedPacketKind::Pm4Ib); + EXPECT_EQ(admitted.pm4_ib_address, 0x100008000ull); + EXPECT_EQ(admitted.pm4_ib_dwords, 32u); + EXPECT_TRUE(admitted.blocks_following); + EXPECT_TRUE(result.blocks_following); + // Malformed envelopes must never reach admission or become successful no-ops. + for (const auto &[word, value] : + {std::pair{1u, 0u}, {2u, 0x8001u}, {3u, 0x10000u}, {4u, 32u}, {4u, 1u << 23}, {5u, 9u}}) { + auto bad = packet; + bad[word] = value; + const auto bad_bytes = packet_bytes(bad); + EXPECT_EQ(processor.process(request_for(bad_bytes)).packet.status, + PacketProcessStatus::Malformed); + } + EXPECT_EQ(admissions, 1u); } TEST(AqlPacketProcessorTest, DecodesExtendedDispatchIntoNormalizedKernelPacket) { diff --git a/emulation/rocjitsu/tests/aql_queue.h b/emulation/rocjitsu/tests/aql_queue.h index 2067c34e6a69..5e29014c140f 100644 --- a/emulation/rocjitsu/tests/aql_queue.h +++ b/emulation/rocjitsu/tests/aql_queue.h @@ -18,6 +18,7 @@ RJ_DIAGNOSTIC_IGNORE_PEDANTIC #include "hsa/hsa.h" RJ_DIAGNOSTIC_POP +#include #include #include #include @@ -26,6 +27,24 @@ RJ_DIAGNOSTIC_POP namespace rocjitsu::test { +/// @brief Construct ROCr's amd_aql_pm4_ib vendor envelope for queue tests. +/// @details The embedded type-3 INDIRECT_BUFFER selects the supplied command +/// buffer. The remaining ten dwords pad the envelope to one AQL ring slot. +inline hsa_kernel_dispatch_packet_t make_pm4_ib_packet(uint64_t address, uint32_t dwords, + uint64_t completion_signal = 0) { + std::array words{}; + words[0] = HSA_PACKET_TYPE_VENDOR_SPECIFIC | (uint32_t{amdgpu::kAmdAqlFormatPm4Ib} << 16); + words[1] = 0xc0023f00; + words[2] = uint32_t(address); + words[3] = uint32_t(address >> 32); + words[4] = (1u << 23) | dwords; + words[5] = 10; + hsa_kernel_dispatch_packet_t packet{}; + std::memcpy(&packet, words.data(), sizeof(packet)); + packet.completion_signal.handle = completion_signal; + return packet; +} + /// Simulates what ROCR's user-mode AqlQueue does: manages a ring buffer, /// writes AQL packets, and rings the doorbell. Does NOT load kernels -- /// that's the runtime's job. Tests write kernel descriptors and code @@ -69,7 +88,7 @@ class AqlQueue { memory_->load_image(reinterpret_cast(&zero), 8, write_ptr_addr_); memory_->load_image(reinterpret_cast(&zero), 8, doorbell_addr_); - amdgpu::AqlQueueConfig hw{}; + amdgpu::ComputeQueueConfig hw{}; hw.address_space = address_space; hw.process_id = process_id; hw.queue_id = queue_id; @@ -162,11 +181,9 @@ class AqlQueue { /// to place on every XCD rather than split. /// @param completion_signal_va Signal to decrement when it retires, or 0. void pm4_ib(uint64_t completion_signal_va = 0) { - hsa_kernel_dispatch_packet_t pkt{}; - pkt.header = HSA_PACKET_TYPE_VENDOR_SPECIFIC; - pkt.setup = amdgpu::kAmdAqlFormatPm4Ib; - pkt.completion_signal.handle = completion_signal_va; - submit(pkt); + constexpr uint64_t address = 0xf0020000; + memory_->write32(address, 0x80000000); // One type-2 NOP. + submit(make_pm4_ib_packet(address, 1, completion_signal_va)); } /// Build and submit an AMD extended kernel dispatch packet with cluster shape. diff --git a/emulation/rocjitsu/tests/atomic_semantics_and_decode_test.cpp b/emulation/rocjitsu/tests/atomic_semantics_and_decode_test.cpp index 192d6d978180..d0a9f0366e72 100644 --- a/emulation/rocjitsu/tests/atomic_semantics_and_decode_test.cpp +++ b/emulation/rocjitsu/tests/atomic_semantics_and_decode_test.cpp @@ -628,6 +628,54 @@ class AtomicPolicyExecutionTest : public testing::TestWithParam uint32_t base_ = 0; }; +TEST_P(AtomicPolicyExecutionTest, DirectLdsLoadConvertsAndBroadcastsToActiveQuads) { + const bool gfx12 = GetParam() == ROCJITSU_CODE_ARCH_RDNA4; + if (!gfx12 && GetParam() != ROCJITSU_CODE_ARCH_RDNA3 && GetParam() != ROCJITSU_CODE_ARCH_RDNA3_5) + GTEST_SKIP() << "Direct LDS loads start with RDNA3"; + // Physical GFX1100/1201: M0 addresses one DWORD; any active lane enables + // all four writes in its quad. Preserve the workgroup allocation offset. + wave_->set_lds_base(4096); + wave_->lds().write32(16, 0xfacefeed); + wave_->lds().write32(4096 + 16, 0x918293e4); + const std::array types{0, 1, 2, 4, 5}; + const std::array values{0xe4, 0x93e4, 0x918293e4, 0xffffffe4, 0xffff93e4}; + for (uint32_t i = 0; i < types.size(); ++i) { + for (uint64_t mask : + {uint64_t{0}, uint64_t{0x81810101}, uint64_t{1} << (wave_->wf_size() - 1)}) { + SCOPED_TRACE(testing::Message() << types[i] << ',' << mask); + wave_->set_exec(mask); + wave_->set_m0((types[i] << 16) | 16); + for (uint32_t lane = 0; lane < wave_->wf_size(); ++lane) + compute_unit_->write_vgpr(base_ + 6, lane, 0xdeadbeef); + std::array words; + if (gfx12) + words = rdna4::build_vdsdir(1, {.vdst = 6}); + else + words = rdna3::build_ldsdir(1, {.vdst = 6}); + auto decoded = decoder_->decode(words.data()); + ASSERT_FALSE(decoded.failed()); + auto instruction = std::move(decoded).value(); + ASSERT_TRUE(compute_unit_->execute_instruction(instruction.get(), *wave_).succeeded()); + ASSERT_FALSE(wave_->instruction_execution_failed()); + ASSERT_TRUE(instruction->is_memory_op()); + if (mask) { + ASSERT_NE(instruction->data(), nullptr); + amdgpu::LocalMemPipeline pipeline; + pipeline.issue(instruction.release(), *wave_); + } else { + EXPECT_EQ(instruction->data(), nullptr); + } + for (uint32_t lane = 0; lane < wave_->wf_size(); ++lane) + EXPECT_EQ(compute_unit_->read_vgpr(base_ + 6, lane), + mask & (uint64_t{15} << (lane & ~3u)) ? values[i] : 0xdeadbeefu) + << lane; + EXPECT_EQ(wave_->exec(), mask); + EXPECT_TRUE(wave_->wait_counters().empty()); + } + } + EXPECT_EQ(wave_->lds().read32(16), 0xfacefeedu); +} + TEST_P(AtomicPolicyExecutionTest, DsScalarDenormalModesAndRounding) { for (uint16_t opcode : {uint16_t{21}, ds_add_return_opcode()}) { for (uint32_t denorm_mode = 0; denorm_mode < 4; ++denorm_mode) { @@ -648,7 +696,7 @@ TEST_P(AtomicPolicyExecutionTest, DsScalarDenormalModesAndRounding) { EXPECT_EQ(execute_ds(18, 1, 0x3f800000, 0, 0), modern_policy() ? 0u : 1u); EXPECT_EQ(execute_ds(18, 0, 0x80000000, 0, 0xf0), 0x80000000u); EXPECT_EQ(execute_ds(19, 0x80000000, 0, 0, 0xf0), 0u); - EXPECT_EQ(execute_ds(21, 0x7fc00002, 0x7fc00004, 0, 0), 0x7fc00002u); + EXPECT_EQ(execute_ds(21, 0x7fc00002, 0x7fc00004, 0, 0), 0x7fc00004u); EXPECT_EQ(execute_ds(21, 0x7f800000, 0xff800000, 0, 0), 0xffc00000u); EXPECT_EQ(execute_ds(21, 0xff800000, 0x7f800000, 0, 0), 0xffc00000u); for (uint16_t opcode : {18, 19}) { @@ -932,6 +980,11 @@ TEST_P(AtomicPolicyExecutionTest, MemoryAddDenormalsAndHostRounding) { GTEST_SKIP() << "No F32 atomic add on this ISA"; for (bool buffer : {false, true}) { for (uint32_t mode : {0u, 0xffu}) { + // Physical RDNA3 L2 chooses memory's NaN; RDNA4 chooses the operand. + for (uint32_t old_nan : {0x7f800002u, 0x7fc00002u}) + for (uint32_t source_nan : {0xff800004u, 0xffc00004u}) + EXPECT_EQ(execute_memory(buffer, old_nan, source_nan, mode), + (GetParam() == ROCJITSU_CODE_ARCH_RDNA4 ? source_nan : old_nan) | 0x400000u); EXPECT_EQ(execute_memory(buffer, 1, 1, mode), modern_policy() ? 2u : 0u); EXPECT_EQ(execute_memory(buffer, 0x00800001, 0x80800000, mode), 1u); for (int host_round : {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO}) { diff --git a/emulation/rocjitsu/tests/buffer_format_test.cpp b/emulation/rocjitsu/tests/buffer_format_test.cpp index 8185f9d9a667..007a6e68a4c1 100644 --- a/emulation/rocjitsu/tests/buffer_format_test.cpp +++ b/emulation/rocjitsu/tests/buffer_format_test.cpp @@ -275,10 +275,10 @@ class BufferFormatExecutionTest : public ::testing::TestWithParam - Instruction *legacy(bool load, bool d16, int format = -1) { + Instruction *legacy(bool load, bool d16, int format = -1, uint32_t soffset = 128) { Machine m{}; m.srsrc = 1; - m.soffset = 128; + m.soffset = soffset; m.idxen = 1; m.vdata = 8; if constexpr (requires { m.glc; }) @@ -295,49 +295,56 @@ class BufferFormatExecutionTest : public ::testing::TestWithParam(&m); return d16 ? execute(load, raw) : execute(load, raw); } - void issue_typed(uint32_t format) { + void issue_typed(uint32_t format, uint32_t soffset = 128) { Instruction *inst = nullptr; switch (GetParam()) { case ROCJITSU_CODE_ARCH_RDNA1: inst = legacy(true, false, encoded_format(format)); + rdna1::TbufferStoreFormatD16XyzwMtbuf>(true, false, encoded_format(format), + soffset); break; case ROCJITSU_CODE_ARCH_RDNA2: inst = legacy(true, false, encoded_format(format)); + rdna2::TbufferStoreFormatD16XyzwMtbuf>(true, false, encoded_format(format), + soffset); break; case ROCJITSU_CODE_ARCH_CDNA1: inst = legacy(true, false, encoded_format(format)); + cdna1::TbufferStoreFormatD16XyzwMtbuf>(true, false, encoded_format(format), + soffset); break; case ROCJITSU_CODE_ARCH_CDNA2: inst = legacy(true, false, encoded_format(format)); + cdna2::TbufferStoreFormatD16XyzwMtbuf>(true, false, encoded_format(format), + soffset); break; case ROCJITSU_CODE_ARCH_CDNA3: inst = legacy(true, false, encoded_format(format)); + cdna3::TbufferStoreFormatD16XyzwMtbuf>(true, false, encoded_format(format), + soffset); break; case ROCJITSU_CODE_ARCH_CDNA4: inst = legacy(true, false, encoded_format(format)); + cdna4::TbufferStoreFormatD16XyzwMtbuf>(true, false, encoded_format(format), + soffset); break; case ROCJITSU_CODE_ARCH_RDNA3: inst = legacy(true, false, encoded_format(format)); + rdna3::TbufferStoreD16FormatXyzwMtbuf>(true, false, encoded_format(format), + soffset); break; case ROCJITSU_CODE_ARCH_RDNA3_5: inst = legacy( - true, false, encoded_format(format)); + true, false, encoded_format(format), soffset); break; case ROCJITSU_CODE_ARCH_RDNA4: { rdna4::VbufferMachineInst m{}; @@ -357,7 +364,7 @@ class BufferFormatExecutionTest : public ::testing::TestWithParaml1_vector(), &l2); ASSERT_EQ(pipeline.issue(inst, *wf), amdgpu::VmAccessOutcome::Complete); } - void issue(bool load, bool d16 = false) { + void issue(bool load, bool d16 = false, uint32_t soffset = 128) { Instruction *inst; if (GetParam() == ROCJITSU_CODE_ARCH_RDNA4) { rdna4::VbufferMachineInst m{}; @@ -374,7 +381,7 @@ class BufferFormatExecutionTest : public ::testing::TestWithParam(load, d16); + rdna1::BufferStoreFormatD16XyzwMubuf>(load, d16, -1, soffset); } else if (GetParam() == ROCJITSU_CODE_ARCH_RDNA2) { inst = legacy(load, d16); + rdna2::BufferStoreFormatD16XyzwMubuf>(load, d16, -1, soffset); } else if (GetParam() == ROCJITSU_CODE_ARCH_CDNA1) { inst = legacy(load, d16); + cdna1::BufferStoreFormatD16XyzwMubuf>(load, d16, -1, soffset); } else if (GetParam() == ROCJITSU_CODE_ARCH_CDNA2) { inst = legacy(load, d16); + cdna2::BufferStoreFormatD16XyzwMubuf>(load, d16, -1, soffset); } else if (GetParam() == ROCJITSU_CODE_ARCH_CDNA3) { inst = legacy(load, d16); + cdna3::BufferStoreFormatD16XyzwMubuf>(load, d16, -1, soffset); } else if (GetParam() == ROCJITSU_CODE_ARCH_CDNA4) { inst = legacy(load, d16); + cdna4::BufferStoreFormatD16XyzwMubuf>(load, d16, -1, soffset); } else { rdna3_5::MubufMachineInst m{}; m.srsrc = 1; - m.soffset = 128; + m.soffset = soffset; m.idxen = 1; m.vdata = 8; m.glc = m.slc = 1; @@ -566,6 +573,47 @@ TEST_P(BufferFormatExecutionTest, UnsupportedDescriptorFailsBeforeIssuingMemory) EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base + 8, 0), 0xdeadbeef); } +TEST_P(BufferFormatExecutionTest, InlineScalarOffsetsUseTheirUnsignedBitsForTypedAndUntypedLoads) { + if (GetParam() == ROCJITSU_CODE_ARCH_RDNA4) + GTEST_SKIP() << "VBUFFER uses a seven-bit register selector instead of inline SOFFSET"; + const uint32_t lane = wf->wf_size() - 1; + wf->set_exec(uint64_t{1} << lane); + const uint32_t sb = wf->sgpr_alloc().base, vb = wf->vgpr_alloc().base; + constexpr uint64_t address = 0x200001000ull; + constexpr uint32_t expected = 0x92345678; + memory.write32(address, expected); + cu->write_vgpr(vb, lane, 0); + constexpr uint32_t float_bits[] = {0x3f000000, 0xbf000000, 0x3f800000, 0xbf800000, 0x40000000, + 0xc0000000, 0x40800000, 0xc0800000, 0x3e22f983}; + for (uint32_t selector = 128; selector <= 255; ++selector) { + // Some older ISAs also permit dynamic special sources here. This regression + // covers static constants and universally invalid encodings only. + if ((selector >= 235 && selector <= 239) || (selector >= 251 && selector <= 253)) + continue; + SCOPED_TRACE(selector); + const bool valid = selector <= 208 || (selector >= 240 && selector <= 248); + const uint32_t offset = selector <= 192 ? selector - 128 + : selector <= 208 ? 192u - selector + : valid ? float_bits[selector - 240] + : 0; + descriptor(20, 4); // 32_UINT, index-checked so large unsigned offsets remain in range. + const uint64_t base = address - offset; + cu->write_sgpr(sb + 4, uint32_t(base)); + cu->write_sgpr(sb + 5, uint32_t(base >> 32) | (4u << 16)); + for (bool typed : {false, true}) { + SCOPED_TRACE(typed); + cu->write_vgpr(vb + 8, lane, 0xdeadbeef); + cu->write_vgpr(vb + 8, 0, 0xabcdef01); + if (typed) + issue_typed(20, selector); + else + issue(true, false, selector); + EXPECT_EQ(cu->read_vgpr(vb + 8, lane), valid ? expected : 0); + EXPECT_EQ(cu->read_vgpr(vb + 8, 0), 0xabcdef01); + } + } +} + TEST_P(BufferFormatExecutionTest, TypedFormatOverridesDescriptorAndExecIncludesTheHighestLane) { descriptor(11, 4); // Deliberately incompatible 16_UINT descriptor. const uint32_t lane = wf->wf_size() - 1; diff --git a/emulation/rocjitsu/tests/cdna5_dispatch_memory_test.cpp b/emulation/rocjitsu/tests/cdna5_dispatch_memory_test.cpp index cf3c37fcc810..9d8aebf462f5 100644 --- a/emulation/rocjitsu/tests/cdna5_dispatch_memory_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_dispatch_memory_test.cpp @@ -53,8 +53,9 @@ TEST(Gfx1250SimulationTest, DispatchesEndpgmThroughConfig) { EXPECT_EQ(sim.cu()->num_wfs(), 0u); } -TEST(Gfx1250SimulationTest, DispatchedModeSetterControlsPseudoScalarRounding) { - constexpr uint32_t kExpectedRoundTowardPositive = 0x3FB504F4u; +TEST(Gfx1250SimulationTest, DispatchedPseudoScalarExpIgnoresRoundingMode) { + // The gfx1201-qualified mapping is shared here; physical gfx1250 is not yet qualified. + constexpr uint32_t kExpectedHardwareApproximation = 0x3FB504F3u; const uint32_t code[] = { 0xB9800801u, // s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 2), 1 0x00000001u, @@ -66,7 +67,7 @@ TEST(Gfx1250SimulationTest, DispatchedModeSetterControlsPseudoScalarRounding) { const auto *snapshot = dispatch_one_wave(sim, code, std::size(code)); ASSERT_NE(snapshot, nullptr); - EXPECT_EQ(snapshot->sgpr(4), kExpectedRoundTowardPositive); + EXPECT_EQ(snapshot->sgpr(4), kExpectedHardwareApproximation); } TEST(Gfx1250SimulationTest, MultiWaveDispatchHonorsPackedTidComponentCount) { diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 22098f41918b..cdf75a970c58 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -928,7 +928,7 @@ TEST(FpModePolicyTest, F16ClampUsesSelectedNanPolicy) { const auto fma = [=](uint16_t src0, bool clamp_nan_to_zero) { return amdgpu::fp_mode::fma_f16(src0, kOne, kPositiveZero, false, false, false, false, false, - false, 0, 3, 0, true, false, clamp_nan_to_zero); + false, 0, 3, 0, true, false, clamp_nan_to_zero, true); }; EXPECT_EQ(fma(kQuietNan, false), kQuietNan); diff --git a/emulation/rocjitsu/tests/cdna5_sdma_test.cpp b/emulation/rocjitsu/tests/cdna5_sdma_test.cpp index 6d4b5e113aab..da0cdccae990 100644 --- a/emulation/rocjitsu/tests/cdna5_sdma_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_sdma_test.cpp @@ -4,7 +4,6 @@ #include "cdna5_sim_test_common.h" #include "rocjitsu/kmd/linux/kfd_process.h" #include "rocjitsu/kmd/linux/legacy_gpu_vm.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h" #include "rocjitsu/vm/amdgpu/sdma_queue_binding_factory.h" #include @@ -262,7 +261,7 @@ class TransientAqlQueueForTest { if (!address_space_) throw std::runtime_error("cannot register transient AQL test address space"); - amdgpu::AqlQueueConfig queue{}; + amdgpu::ComputeQueueConfig queue{}; queue.address_space = address_space_; queue.process_id = kProcessId; queue.queue_id = kQueueId; @@ -923,7 +922,7 @@ TEST(CommandProcessorInterruptRoutingTest, QueuesWithTheSameProcessIdKeepDistinc auto register_queue = [&](uint32_t queue_id, uint64_t &exception_status, uint32_t event_id, const amdgpu::InterruptSink &interrupt_sink) { - amdgpu::AqlQueueConfig queue{}; + amdgpu::ComputeQueueConfig queue{}; queue.address_space = address_space; queue.interrupt_sink = interrupt_sink; queue.process_id = kProcessId; @@ -1025,7 +1024,7 @@ TEST(SoCLifetimeTest, ReleasesDirectCommandProcessorQueuesBeforeGpuVm) { sim->soc->gpu_vm().register_translated(kProcessId, backing, backing); ASSERT_TRUE(address_space); - amdgpu::AqlQueueConfig aql{}; + amdgpu::ComputeQueueConfig aql{}; aql.address_space = address_space; aql.process_id = kProcessId; aql.queue_id = 1; diff --git a/emulation/rocjitsu/tests/check_cp_sdma_boundary.cmake b/emulation/rocjitsu/tests/check_cp_sdma_boundary.cmake index 787fd0323d3a..0a9b201bb3bc 100644 --- a/emulation/rocjitsu/tests/check_cp_sdma_boundary.cmake +++ b/emulation/rocjitsu/tests/check_cp_sdma_boundary.cmake @@ -11,19 +11,14 @@ set(CP_SOURCES lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_types.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.cpp - lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h - lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.cpp + lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_queue_binding_factory.h + lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_queue_binding_factory.cpp lib/rocjitsu/src/rocjitsu/vm/amdgpu/dispatch_entry.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/completion_tracker.cpp - lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.h - lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.cpp lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.cpp - lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.h - lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_binding_factory.cpp - lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h - lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.cpp + lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_types.h ) foreach(SOURCE_FILE IN LISTS CP_SOURCES) @@ -40,8 +35,7 @@ set(PACKET_PROCESSOR_SOURCES lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_types.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/aql/aql_packet_processor.cpp - lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h - lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.cpp + lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_types.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.h lib/rocjitsu/src/rocjitsu/vm/amdgpu/sdma_packet_processor.cpp ) @@ -124,14 +118,13 @@ foreach(SOURCE_FILE IN LISTS QUEUE_FRONTEND_SOURCES) IN ITEMS "AqlPacketProcessor" - "Pm4PacketProcessor" + "process_pm4_packets" + "Pm4ExecutionContext" "SdmaPacketProcessor" "PacketProcessResult" - "Pm4RingConsumer" "SdmaRingConsumer" "CircularRingReader" "ConsumerCursorJournal" - "Pm4QueueController" "SdmaExecutor" "SdmaQueueRunner" "SdmaQueueBackend" @@ -146,3 +139,33 @@ foreach(SOURCE_FILE IN LISTS QUEUE_FRONTEND_SOURCES) endif() endforeach() endforeach() + +# CP schedules command streams; PM4 opcode semantics belong to pm4/. +file( + READ + "${ROCJITSU_SOURCE_DIR}/lib/rocjitsu/src/rocjitsu/vm/amdgpu/command_processor.cpp" + CP_CONTENT +) +string(FIND "${CP_CONTENT}" "Pm4Opcode::" PM4_OPCODE_POSITION) +if(NOT PM4_OPCODE_POSITION EQUAL -1) + message( + FATAL_ERROR + "command_processor.cpp contains PM4 opcode interpretation" + ) +endif() +foreach( + SOURCE_FILE + IN + ITEMS + lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h + lib/rocjitsu/src/rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.cpp +) + file(READ "${ROCJITSU_SOURCE_DIR}/${SOURCE_FILE}" SOURCE_CONTENT) + string(FIND "${SOURCE_CONTENT}" "command_processor.h" CP_INCLUDE_POSITION) + if(NOT CP_INCLUDE_POSITION EQUAL -1) + message( + FATAL_ERROR + "${SOURCE_FILE} depends directly on CommandProcessor" + ) + endif() +endforeach() diff --git a/emulation/rocjitsu/tests/check_mes_frontend_boundary.cmake b/emulation/rocjitsu/tests/check_mes_frontend_boundary.cmake index fc5a11152031..50d8beaf6176 100644 --- a/emulation/rocjitsu/tests/check_mes_frontend_boundary.cmake +++ b/emulation/rocjitsu/tests/check_mes_frontend_boundary.cmake @@ -38,14 +38,13 @@ foreach(SOURCE_FILE IN LISTS MES_FRONTEND_SOURCES) ITEMS "GpuQueueRegistry" "AqlPacketProcessor" - "Pm4PacketProcessor" + "process_pm4_packets" + "Pm4ExecutionContext" "SdmaPacketProcessor" "PacketProcessResult" - "Pm4RingConsumer" "SdmaRingConsumer" "CircularRingReader" "ConsumerCursorJournal" - "Pm4QueueController" "SdmaExecutor" "SdmaQueueRunner" "SdmaQueueBackend" diff --git a/emulation/rocjitsu/tests/check_sdma_frontend_boundary.cmake b/emulation/rocjitsu/tests/check_sdma_frontend_boundary.cmake index eeda658cca54..51a8c5335e16 100644 --- a/emulation/rocjitsu/tests/check_sdma_frontend_boundary.cmake +++ b/emulation/rocjitsu/tests/check_sdma_frontend_boundary.cmake @@ -32,6 +32,8 @@ foreach(SOURCE_FILE IN LISTS SDMA_CORE_SOURCES) "AqlPacketProcessor" "aql_queue_binding_factory" "AqlQueueBindingFactory" + "compute_queue_binding_factory" + "ComputeQueueBindingFactory" "dispatch_entry.h" "DispatchEntry" "pm4_" diff --git a/emulation/rocjitsu/tests/code/kernel_descriptor_scan_test.cpp b/emulation/rocjitsu/tests/code/kernel_descriptor_scan_test.cpp index 41eec4d15339..f4136c65a77a 100644 --- a/emulation/rocjitsu/tests/code/kernel_descriptor_scan_test.cpp +++ b/emulation/rocjitsu/tests/code/kernel_descriptor_scan_test.cpp @@ -173,5 +173,107 @@ TEST(KernelDescriptorVgprGranule, Rdna4Wave64ZeroGranulatedCountDeclaresFourVgpr EXPECT_EQ((granulated + 1) * g32, 8u); } +// The kernarg pointer's user-SGPR index is the summed width of whichever of +// private_segment_buffer (4), dispatch_ptr (2) and queue_ptr (2) precede it. +// Every consumer that loads through the pointer needs this index, so a wrong sum +// reads an unrelated register rather than failing. +TEST(KernargSegmentPtr, SlotIsTheWidthOfTheEnabledPredecessors) { + using namespace rocr::llvm::amdhsa; + + EXPECT_EQ(kernarg_segment_ptr_slot(KD{}), 0); + + KD buffer{}; + AMDHSA_BITS_SET(buffer.kernel_code_properties, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, 1); + EXPECT_EQ(kernarg_segment_ptr_slot(buffer), 4); + + KD dispatch{}; + AMDHSA_BITS_SET(dispatch.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, + 1); + EXPECT_EQ(kernarg_segment_ptr_slot(dispatch), 2); + + KD queue{}; + AMDHSA_BITS_SET(queue.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, 1); + EXPECT_EQ(kernarg_segment_ptr_slot(queue), 2); + + KD buffer_and_dispatch = buffer; + AMDHSA_BITS_SET(buffer_and_dispatch.kernel_code_properties, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, 1); + EXPECT_EQ(kernarg_segment_ptr_slot(buffer_and_dispatch), 6); + + KD all_three = buffer_and_dispatch; + AMDHSA_BITS_SET(all_three.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, 1); + EXPECT_EQ(kernarg_segment_ptr_slot(all_three), 8); +} + +// The slot query answers where the pointer would go; the sgpr query answers where +// it is. Keeping them distinct lets a caller inserting the pointer ask for the +// index before setting the bit, without suppressing a nullopt it knows is wrong. +TEST(KernargSegmentPtr, SgprIsEmptyUntilTheDescriptorEnablesThePointer) { + using namespace rocr::llvm::amdhsa; + + KD desc{}; + AMDHSA_BITS_SET(desc.kernel_code_properties, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, 1); + EXPECT_FALSE(has_kernarg_segment_ptr(desc)); + EXPECT_FALSE(kernarg_segment_ptr_sgpr(desc).has_value()); + EXPECT_EQ(kernarg_segment_ptr_slot(desc), 4); // still answers, and with the right slot + + AMDHSA_BITS_SET(desc.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, + 1); + EXPECT_TRUE(has_kernarg_segment_ptr(desc)); + ASSERT_TRUE(kernarg_segment_ptr_sgpr(desc).has_value()); + EXPECT_EQ(*kernarg_segment_ptr_sgpr(desc), 4); +} + +// Length and offset occupy one descriptor field. A decoder that confused them +// would report "preloading is on" for a kernel whose offset is non-zero and +// length zero, which is not a preloading kernel at all. +TEST(KernargPreload, LengthAndOffsetDecodeIndependently) { + KD desc{}; + EXPECT_EQ(kernarg_preload_length(desc), 0u); + EXPECT_EQ(kernarg_preload_offset(desc), 0u); + + AMDHSA_BITS_SET(desc.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, 3); + EXPECT_EQ(kernarg_preload_length(desc), 3u); + EXPECT_EQ(kernarg_preload_offset(desc), 0u); + + AMDHSA_BITS_SET(desc.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_OFFSET, 5); + EXPECT_EQ(kernarg_preload_length(desc), 3u); + EXPECT_EQ(kernarg_preload_offset(desc), 5u); +} + } // namespace + +// The dense system-SGPR block following the user block. DBT uses it as the range +// to repair when it inserts a kernarg pointer; DBI as a floor for framework +// storage. Distinct values per field so a dropped or double-counted term shows. +TEST(KernelDescriptorScan, InitialSgprCountFoldsTheSystemBlockOntoTheUserBlock) { + constexpr rj_code_arch_t kArch = ROCJITSU_CODE_ARCH_CDNA4; + + KD none{}; + set_kernel_descriptor_user_sgpr_count(kArch, none, 6); + EXPECT_EQ(kernel_descriptor_initial_sgpr_count(kArch, none), 6u); + + KD ids = none; + AMDHSA_BITS_SET(ids.compute_pgm_rsrc2, + rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); + AMDHSA_BITS_SET(ids.compute_pgm_rsrc2, + rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, 1); + EXPECT_EQ(kernel_descriptor_initial_sgpr_count(kArch, ids), 8u); + + // WORKGROUP_INFO follows the enabled dimensions and is the term the DBI copy + // of this walk was missing before it was shared. + KD info = ids; + AMDHSA_BITS_SET(info.compute_pgm_rsrc2, + rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, 1); + EXPECT_EQ(kernel_descriptor_initial_sgpr_count(kArch, info), 9u); + + // ENABLE_PRIVATE_SEGMENT is deliberately excluded. + KD priv = info; + AMDHSA_BITS_SET(priv.compute_pgm_rsrc2, + rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, 1); + EXPECT_EQ(kernel_descriptor_initial_sgpr_count(kArch, priv), 9u); +} + } // namespace rocjitsu diff --git a/emulation/rocjitsu/tests/comparison_test.cpp b/emulation/rocjitsu/tests/comparison_test.cpp new file mode 100644 index 000000000000..fec841a04028 --- /dev/null +++ b/emulation/rocjitsu/tests/comparison_test.cpp @@ -0,0 +1,306 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file comparison_test.cpp +/// @brief Unit tests for the staged floating-point VOPC relations. + +#include "rocjitsu/isa/arch/amdgpu/shared/comparison.h" +#include "util/simd.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace cmp = rocjitsu::amdgpu::comparison; + +constexpr cmp::Policy kKeep{false}; +constexpr cmp::Policy kFlush{true}; + +// --------------------------------------------------------------------------- +// Reference: decode to double, flush by hand, compare with host IEEE operators. +// --------------------------------------------------------------------------- + +template double decode(typename Fmt::Lane bits, bool flush) { + bits &= Fmt::kBits; + const bool negative = (bits & Fmt::kSign) != 0; + const uint64_t exponent = (bits & Fmt::kInfinity) >> Fmt::kMantissaBits; + const uint64_t mantissa = bits & ((typename Fmt::Lane{1} << Fmt::kMantissaBits) - 1); + double value; + if (exponent == Fmt::kExponentMax) + value = mantissa ? std::numeric_limits::quiet_NaN() + : std::numeric_limits::infinity(); + else if (exponent == 0) + value = + flush ? 0.0 + : std::ldexp(static_cast(mantissa), 2 - (1 << (Fmt::kExponentBits - 1)) - + static_cast(Fmt::kMantissaBits)); + else + value = std::ldexp(static_cast(mantissa | (uint64_t{1} << Fmt::kMantissaBits)), + static_cast(exponent) - ((1 << (Fmt::kExponentBits - 1)) - 1) - + static_cast(Fmt::kMantissaBits)); + return negative ? -value : value; +} + +// Indexed by VOPC opcode: F LT EQ LE GT LG GE O U NGE NLG NGT NLE NEQ NLT T. +bool reference(unsigned opcode, double a, double b) { + switch (opcode) { + case 0: + return false; + case 1: + return a < b; + case 2: + return a == b; + case 3: + return a <= b; + case 4: + return a > b; + case 5: + return a < b || a > b; + case 6: + return a >= b; + case 7: + return !std::isnan(a) && !std::isnan(b); + default: + return !reference(15 - opcode, a, b); + } +} + +template bool scalar(uint64_t a, uint64_t b, cmp::Policy policy) { + using L = typename Fmt::Lane; + return cmp::evaluate(static_cast(a), static_cast(b), policy); +} + +template using RelationFn = bool (*)(uint64_t, uint64_t, cmp::Policy); + +template std::array, 16> relations() { + return {&scalar, &scalar, &scalar, + &scalar, &scalar, &scalar, + &scalar, &scalar, &scalar, + &scalar, &scalar, &scalar, + &scalar, &scalar, &scalar, + &scalar}; +} + +/// Signed zeros, the subnormal and normal boundaries, infinities and NaNs. +template std::vector specials() { + using L = typename Fmt::Lane; + const L max_subnormal = (L{1} << Fmt::kMantissaBits) - 1; + const L min_normal = L{1} << Fmt::kMantissaBits; + const L quiet = L{1} << (Fmt::kMantissaBits - 1); + const std::vector magnitudes = {0, + 1, + 2, + max_subnormal, + min_normal, + min_normal + 1, + Fmt::kInfinity - 1, + Fmt::kInfinity, + Fmt::kInfinity | 1, + Fmt::kInfinity | quiet, + Fmt::kMagnitude}; + std::vector values; + for (const L magnitude : magnitudes) { + values.push_back(magnitude); + values.push_back(magnitude | Fmt::kSign); + } + return values; +} + +template void expect_matches_reference(uint64_t a, uint64_t b) { + const auto fns = relations(); + for (const bool flush : {false, true}) { + const double da = decode(static_cast(a), flush); + const double db = decode(static_cast(b), flush); + for (unsigned opcode = 0; opcode < 16; ++opcode) + ASSERT_EQ(fns[opcode](a, b, cmp::Policy{flush}), reference(opcode, da, db)) + << "opcode " << opcode << " a 0x" << std::hex << a << " b 0x" << b << " flush " << flush; + } +} + +// --------------------------------------------------------------------------- +// Agreement with IEEE relations. +// --------------------------------------------------------------------------- + +TEST(ComparisonTest, F16EveryEncodingAgainstSpecials) { + for (uint64_t a = 0; a <= 0xffffu; ++a) + for (const uint64_t b : specials()) { + expect_matches_reference(a, b); + expect_matches_reference(b, a); + } +} + +template void check_specials_and_random() { + const auto values = specials(); + for (const uint64_t a : values) + for (const uint64_t b : values) + expect_matches_reference(a, b); + std::mt19937_64 rng(0x5eed); + for (int i = 0; i < 20000; ++i) { + // Clear random exponent bits half the time, so subnormals are common. + uint64_t a = rng() & Fmt::kBits; + uint64_t b = rng() & Fmt::kBits; + if (i & 1) + a &= ~static_cast(Fmt::kInfinity); + if (i & 2) + b &= ~static_cast(Fmt::kInfinity); + expect_matches_reference(a, b); + } +} + +TEST(ComparisonTest, F32SpecialsAndRandom) { check_specials_and_random(); } + +TEST(ComparisonTest, F64SpecialsAndRandom) { check_specials_and_random(); } + +TEST(ComparisonTest, IgnoresBitsAboveTheF16Encoding) { + EXPECT_TRUE((cmp::evaluate(0xabcd3c00u, 0x00003c00u, kKeep))); + EXPECT_TRUE((cmp::evaluate(0xffff0000u, 0x00003c00u, kKeep))); +} + +// --------------------------------------------------------------------------- +// Physical gfx1201 lanes. With input denormals disabled, a subnormal compares +// as a zero of the same sign; with them enabled it compares by value. +// --------------------------------------------------------------------------- + +TEST(ComparisonTest, MatchesGfx1201InputFlush) { + EXPECT_FALSE((cmp::evaluate(0x80000001u, 0x00000000u, kFlush))); + EXPECT_TRUE((cmp::evaluate(0x80000001u, 0x00000000u, kKeep))); + EXPECT_FALSE((cmp::evaluate(0x807fffffu, 0x80000000u, kFlush))); + EXPECT_TRUE((cmp::evaluate(0x807fffffu, 0x80000000u, kKeep))); + EXPECT_FALSE( + (cmp::evaluate(uint64_t{0x8000000000000001}, uint64_t{0}, kFlush))); + EXPECT_TRUE((cmp::evaluate(uint64_t{0x8000000000000001}, uint64_t{0}, kKeep))); + EXPECT_FALSE((cmp::evaluate(0x83ffu, 0x0000u, kFlush))); + EXPECT_TRUE((cmp::evaluate(0x83ffu, 0x0000u, kKeep))); +} + +TEST(ComparisonTest, AppliesModifiersBeforeTheFlush) { + // v_cmp_lt_f16 -a, |b| with a = 0x0001: NEG makes -tiny, the flush makes -0. + EXPECT_FALSE((cmp::evaluate(0x0001u, 0x0000u, 0u, 1u, kFlush))); + EXPECT_TRUE((cmp::evaluate(0x0001u, 0x0000u, 0u, 1u, kKeep))); + // ABS clears the sign before NEG sets it again. + EXPECT_EQ(cmp::modify(0x80000002u, true, true), 0x80000002u); + EXPECT_EQ(cmp::modify(0x80000002u, true, false), 0x00000002u); + // The src1 modifiers come from bit 1 of each field. + EXPECT_FALSE((cmp::evaluate(0x3f800000u, 0x3f800000u, 0u, 1u, kKeep))); + EXPECT_TRUE((cmp::evaluate(0x3f800000u, 0x3f800000u, 0u, 2u, kKeep))); +} + +TEST(ComparisonTest, FlushKeepsSignAndSpecials) { + EXPECT_EQ(cmp::flush_input(0x807fffffu, kFlush), 0x80000000u); + EXPECT_EQ(cmp::flush_input(0x007fffffu, kFlush), 0x00000000u); + EXPECT_EQ(cmp::flush_input(0x00800000u, kFlush), 0x00800000u); + EXPECT_EQ(cmp::flush_input(0x7f800001u, kFlush), 0x7f800001u); + EXPECT_EQ(cmp::flush_input(0x807fffffu, kKeep), 0x807fffffu); + EXPECT_EQ(cmp::flush_input(0x83ffu, kFlush), 0x8000u); + EXPECT_EQ(cmp::flush_input(uint64_t{0x800fffffffffffff}, kFlush), + uint64_t{0x8000000000000000}); +} + +TEST(ComparisonTest, NegatedRelationsAreTrueOnNaN) { + constexpr uint32_t kQuiet = 0x7fc00000u; + constexpr uint32_t kSignaling = 0xff800001u; + EXPECT_FALSE((cmp::evaluate(kQuiet, 0u, kKeep))); + EXPECT_TRUE((cmp::evaluate(kQuiet, 0u, kKeep))); + EXPECT_FALSE((cmp::evaluate(kSignaling, kSignaling, kKeep))); + EXPECT_TRUE((cmp::evaluate(kSignaling, kSignaling, kKeep))); + EXPECT_FALSE((cmp::evaluate(0u, kQuiet, kKeep))); + EXPECT_TRUE((cmp::evaluate(0u, kQuiet, kKeep))); + EXPECT_FALSE((cmp::evaluate(0u, kQuiet, kKeep))); + EXPECT_FALSE((cmp::evaluate(kQuiet, kQuiet, kKeep))); + EXPECT_TRUE((cmp::evaluate(kQuiet, kQuiet, kKeep))); +} + +TEST(ComparisonTest, SignedZerosAreEqual) { + EXPECT_TRUE((cmp::evaluate(0x80000000u, 0x00000000u, kKeep))); + EXPECT_FALSE((cmp::evaluate(0x80000000u, 0x00000000u, kKeep))); + EXPECT_FALSE((cmp::evaluate(0x80000000u, 0x00000000u, kKeep))); +} + +// --------------------------------------------------------------------------- +// MODE gating. +// --------------------------------------------------------------------------- + +TEST(ComparisonTest, FlushesWhenInputDenormalsAreDisabled) { + EXPECT_TRUE(cmp::Policy::make(0u).flush_inputs); + EXPECT_TRUE(cmp::Policy::make(2u).flush_inputs); + EXPECT_FALSE(cmp::Policy::make(1u).flush_inputs); + EXPECT_FALSE(cmp::Policy::make(3u).flush_inputs); +} + +// --------------------------------------------------------------------------- +// SIMD lanes evaluate the same relation as the scalar path. +// --------------------------------------------------------------------------- + +#if __has_include() +template void expect_simd_matches_scalar() { + using L = typename Fmt::Lane; + constexpr std::size_t W = V::size(); + std::vector values; + for (const uint64_t v : specials()) + values.push_back(static_cast(v)); + std::mt19937_64 rng(0xc0ffee); + while (values.size() % W != 0 || values.size() < 4 * W) + values.push_back(static_cast(rng()) & ~Fmt::kInfinity); + for (const cmp::Policy policy : {kKeep, kFlush}) + for (std::size_t rotate = 0; rotate < W; ++rotate) + for (std::size_t base = 0; base < values.size(); base += W) { + alignas(64) std::array a{}; + alignas(64) std::array b{}; + for (std::size_t i = 0; i < W; ++i) { + a[i] = values[base + i]; + b[i] = values[(base + i + rotate * 3 + 1) % values.size()]; + } + const V va(a.data(), util::stdx::element_aligned); + const V vb(b.data(), util::stdx::element_aligned); + const uint64_t bits = util::simd_mask_to_bits(cmp::evaluate(va, vb, policy)); + for (std::size_t i = 0; i < W; ++i) + ASSERT_EQ(((bits >> i) & 1u) != 0, (cmp::evaluate(a[i], b[i], policy))) + << "lane " << i << " a 0x" << std::hex << a[i] << " b 0x" << b[i]; + } +} + +template void expect_every_simd_relation_matches_scalar() { + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); + expect_simd_matches_scalar(); +} +#endif + +TEST(ComparisonTest, SimdMatchesScalar) { + if constexpr (!util::has_stdx_simd) { + GTEST_SKIP() << " unavailable"; + } else { +#if __has_include() + expect_every_simd_relation_matches_scalar>(); + expect_every_simd_relation_matches_scalar>(); +#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS + expect_every_simd_relation_matches_scalar>(); +#else + expect_every_simd_relation_matches_scalar>(); +#endif +#endif + } +} + +} // namespace diff --git a/emulation/rocjitsu/tests/completion_tracker_test.cpp b/emulation/rocjitsu/tests/completion_tracker_test.cpp index 57042dd9420c..ccd18aab913b 100644 --- a/emulation/rocjitsu/tests/completion_tracker_test.cpp +++ b/emulation/rocjitsu/tests/completion_tracker_test.cpp @@ -202,7 +202,7 @@ class CompletionFixture { CompletionTracker tracker; InterruptSubscription subscription; CompletionEndCounter *end_counter = nullptr; - std::vector queues; + std::vector queues; uint32_t retired = 0; uint32_t interrupts = 0; }; @@ -299,7 +299,7 @@ TEST(CompletionTrackerTest, UnavailableQueueDoesNotStarveIndependentQueueRetirem fixture.memory->store(kSecondSignal + CompletionFixture::kMailboxOffset, kSecondMailbox); fixture.memory->store(kSecondSignal + CompletionFixture::kEventOffset, kSecondEvent); - AqlQueueRecord second_queue{}; + ComputeQueueRecord second_queue{}; second_queue.fanout_replica = true; DispatchEntry second_entry{}; second_entry.dispatch_id = 8; diff --git a/emulation/rocjitsu/tests/config_test.cpp b/emulation/rocjitsu/tests/config_test.cpp index 769a95d7bcb5..a6b83da44314 100644 --- a/emulation/rocjitsu/tests/config_test.cpp +++ b/emulation/rocjitsu/tests/config_test.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT #include "aql_queue.h" +#include "decode_test_util.h" #include "halt_snapshot_plugin.h" #include "long_path_handoff.h" #include "scoped_temp.h" @@ -19,13 +20,16 @@ #include "rocjitsu/isa/arch/amdgpu/shared/accvgpr_layout.h" #include "rocjitsu/kmd/linux/amdgpu_properties.h" #include "rocjitsu/kmd/linux/rpc.h" +#include "rocjitsu/vm/amdgpu/hwreg.h" #include "rocjitsu/vm/amdgpu/matrix_coexecution.h" #include "rocjitsu/vm/amdgpu/partitioning.h" #include "rocjitsu/vm/amdgpu/pci/gpu_pci_device_spec.h" +#include "rocjitsu/vm/amdgpu/register_access.h" #include "rocjitsu/vm/plugins/execution_plugin_group.h" #include "rocjitsu/vm/rj_vm.h" #include "rocjitsu/vm/rj_vm_impl.h" #include "rocjitsu/vm/soc.h" +#include "util/diagnostic.h" #include "simdojo/sim/simulation.h" @@ -101,6 +105,26 @@ bool replace_exactly_once(std::string &text, std::string_view from, std::string_ return true; } +TEST(SoCTest, SelectsSdmaPacketDialect) { + const std::array cases = { + std::pair{ROCJITSU_CODE_ARCH_CDNA1, amdgpu::SdmaPacketDialect::Legacy}, + std::pair{ROCJITSU_CODE_ARCH_CDNA2, amdgpu::SdmaPacketDialect::LegacyExtendedCount}, + std::pair{ROCJITSU_CODE_ARCH_CDNA3, amdgpu::SdmaPacketDialect::LegacyExtendedCount}, + std::pair{ROCJITSU_CODE_ARCH_CDNA4, amdgpu::SdmaPacketDialect::LegacyExtendedCount}, + std::pair{ROCJITSU_CODE_ARCH_CDNA5, amdgpu::SdmaPacketDialect::Gfx1250}, + std::pair{ROCJITSU_CODE_ARCH_RDNA1, amdgpu::SdmaPacketDialect::Legacy}, + std::pair{ROCJITSU_CODE_ARCH_RDNA2, amdgpu::SdmaPacketDialect::LegacyExtendedCount}, + std::pair{ROCJITSU_CODE_ARCH_RDNA3, amdgpu::SdmaPacketDialect::Gfx11Plus}, + std::pair{ROCJITSU_CODE_ARCH_RDNA3_5, amdgpu::SdmaPacketDialect::Gfx11Plus}, + std::pair{ROCJITSU_CODE_ARCH_RDNA4, amdgpu::SdmaPacketDialect::Gfx11Plus}, + }; + for (const auto &[arch, expected] : cases) { + SCOPED_TRACE(static_cast(arch)); + SoC soc("soc", nullptr, arch); + EXPECT_EQ(soc.sdma_queue_scheduler().packet_dialect(), expected); + } +} + TEST(ConfigLoaderTest, LoadCdna2Config) { auto loaded = config::load_config(CONFIG_DIR_PATH + "/gfx90a_mi210_kmd.json", rocjitsu::kEmbeddedSchema); @@ -360,6 +384,163 @@ TEST(ConfigLoaderTest, LoadFourGpuMi455xKmdConfig) { EXPECT_NE(dynamic_cast(build.root.get()), nullptr); } +TEST(ConfigLoaderTest, Gfx1250WgpIdsAreLocalToShaderArray) { + auto loaded = config::load_config(CONFIG_DIR_PATH + "/gfx1250_mi455x_kmd_4gpu.json", + rocjitsu::kEmbeddedSchema); + ASSERT_EQ(loaded.device.num_cu_per_sh, 8u); + ASSERT_EQ(loaded.extra_gpu_builds.size(), 3u); + auto check_build = [](const config::TopologyBuildResult &build) { + ASSERT_FALSE(build.xcds.empty()); + for (auto *xcd : build.xcds) { + ASSERT_GT(xcd->num_shader_engines(), 0u); + for (uint32_t se_id = 0; se_id < xcd->num_shader_engines(); ++se_id) { + auto *se = xcd->shader_engine(se_id); + ASSERT_EQ(se->num_compute_units(), 16u); + for (uint32_t cu_id : {0u, 5u, 7u, 8u, 13u, 15u}) { + auto *cu = se->compute_unit(cu_id); + auto *wf = cu->dispatch_wf(0, 0, 32, 8); + ASSERT_NE(wf, nullptr); + uint32_t value = 0; + constexpr uint16_t kWgpId = 23 | (10 << 6) | (3 << 11); + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, kWgpId, value), + amdgpu::HwregAccessResult::Success); + EXPECT_EQ(value, cu_id % 8); + EXPECT_EQ(cu->shader_engine_id(), se_id); + EXPECT_EQ(cu->scratch_scoreboard_base(), cu_id * cu->scratch_slots_per_cu()); + wf->halt(); + } + } + } + }; + check_build(loaded.build_result); + for (const auto &build : loaded.extra_gpu_builds) + check_build(build); +} + +TEST(ConfigLoaderTest, MismatchedShaderArrayGeometryKeepsWgpIdUnsupported) { + for (const char *filename : {"gfx1250_mi455x.json", "gfx1250_mi455x_kmd_4gpu.json"}) { + SCOPED_TRACE(filename); + std::ifstream input(CONFIG_DIR_PATH + "/" + filename); + ASSERT_TRUE(input.is_open()); + const std::string original((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + for (const auto &[from, to] : {std::pair{R"("num_cu_per_sh": 8)", R"("num_cu_per_sh": 4)"}, + std::pair{R"("num_shader_arrays_per_engine": 2)", + R"("num_shader_arrays_per_engine": 1)"}}) { + SCOPED_TRACE(to); + std::string json = original; + ASSERT_TRUE(replace_exactly_once(json, from, to)); + auto loaded = config::load_config_from_string(json, rocjitsu::kEmbeddedSchema); + auto check_build = [](const config::TopologyBuildResult &build) { + for (auto *xcd : build.xcds) { + for (auto *se : xcd->shader_engines()) { + ASSERT_EQ(se->num_compute_units(), 16u); + for (auto *cu : se->compute_units()) { + auto *wf = cu->dispatch_wf(0, 0, 32, 8); + ASSERT_NE(wf, nullptr); + uint32_t value = 0xFFFFFFFFu; + constexpr uint16_t kWgpId = 23 | (10 << 6) | (3 << 11); + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, kWgpId, value), + amdgpu::HwregAccessResult::Unsupported); + EXPECT_EQ(value, 0u); + wf->halt(); + } + } + } + }; + check_build(loaded.build_result); + for (const auto &build : loaded.extra_gpu_builds) + check_build(build); + } + } +} + +TEST(ConfigLoaderTest, LaterShaderEngineMismatchInvalidatesAllShaderArrayWidths) { + for (uint32_t second_engine_cus : {2u, 0u}) { + SCOPED_TRACE(second_engine_cus); + std::string json = R"({ + "num_threads": 1, + "vm": {"arch": "cdna5", "gpu": {"device": { + "gfx_target_version": 120500, "num_sdma_engines": 0, + "num_cu_per_sh": 2, "num_shader_arrays_per_engine": 2 + }}}, + "topology": {"root": {"name": "soc", "type": "soc", "children": [ + {"name": "vram", "type": "gpu_memory"}, + {"name": "xcd0", "type": "xcd", "children": [ + {"name": "l2", "type": "l2_cache"}, + {"name": "cp", "type": "command_processor"}, + {"name": "se0", "type": "shader_engine", "children": [ + {"name": "cu[0:4]", "type": "compute_unit"} + ]}, + {"name": "se1", "type": "shader_engine", "children": []} + ]} + ]}, "links": []} + })"; + if (second_engine_cus) { + ASSERT_TRUE( + replace_exactly_once(json, R"("children": [])", + R"("children": [{"name": "cu[0:2]", "type": "compute_unit"}])")); + } + + testing::internal::CaptureStderr(); + auto loaded = config::load_config_from_string(json, rocjitsu::kEmbeddedSchema); + const std::string warning = testing::internal::GetCapturedStderr(); + EXPECT_THAT(warning, + testing::HasSubstr("soc.xcd0.se1 has " + std::to_string(second_engine_cus) + + " compute units; expected 4")); + EXPECT_EQ(std::count(warning.begin(), warning.end(), '\n'), 1); + ASSERT_EQ(loaded.build_result.xcds.size(), 1u); + auto *xcd = loaded.build_result.xcds[0]; + ASSERT_EQ(xcd->num_shader_engines(), 2u); + ASSERT_EQ(xcd->shader_engine(0)->num_compute_units(), 4u); + ASSERT_EQ(xcd->shader_engine(1)->num_compute_units(), second_engine_cus); + for (auto *se : xcd->shader_engines()) + for (auto *cu : se->compute_units()) + EXPECT_EQ(cu->cus_per_shader_array(), 0u); + } +} + +TEST(ConfigLoaderTest, DirectCuShaderArrayGeometryRequiresMatchingCount) { + for (uint32_t num_cus : {4u, 3u}) { + SCOPED_TRACE(num_cus); + std::string json = R"({ + "num_threads": 1, + "vm": {"arch": "cdna5", "gpu": {"device": { + "gfx_target_version": 120500, "num_sdma_engines": 0, + "num_cu_per_sh": 2, "num_shader_arrays_per_engine": 2 + }}}, + "topology": {"root": {"name": "soc", "type": "soc", "children": [ + {"name": "vram", "type": "gpu_memory"}, + {"name": "xcd0", "type": "xcd", "children": [ + {"name": "l2", "type": "l2_cache"}, + {"name": "cp", "type": "command_processor"}, + {"name": "cu[0:CU_COUNT]", "type": "compute_unit"} + ]} + ]}, "links": []} + })"; + ASSERT_TRUE(replace_exactly_once(json, "CU_COUNT", std::to_string(num_cus))); + testing::internal::CaptureStderr(); + auto loaded = config::load_config_from_string(json, rocjitsu::kEmbeddedSchema); + const std::string warning = testing::internal::GetCapturedStderr(); + if (num_cus == 4) { + EXPECT_TRUE(warning.empty()); + } else { + EXPECT_THAT(warning, testing::HasSubstr("soc.xcd0 has 3 compute units; expected 4")); + } + ASSERT_EQ(loaded.build_result.xcds.size(), 1u); + auto *cp = loaded.build_result.xcds[0]->command_processor(); + ASSERT_NE(cp, nullptr); + const auto &cus = cp->compute_units(); + ASSERT_EQ(cus.size(), num_cus); + for (uint32_t cu_index = 0; cu_index < num_cus; ++cu_index) { + EXPECT_EQ(cus[cu_index]->cus_per_shader_array(), num_cus == 4 ? 2u : 0u); + if (num_cus == 4) { + EXPECT_EQ(cus[cu_index]->shader_array_cu_id(), cu_index % 2); + } + } + } +} + TEST(ConfigLoaderTest, DispatchPoolBudgetIsSharedAcrossProductionTopology) { auto loaded = config::load_config(CONFIG_DIR_PATH + "/gfx950_mi355x.json", rocjitsu::kEmbeddedSchema); @@ -514,6 +695,60 @@ TEST(ConfigLoaderTest, LoadRdnaKmdConfigs) { amdgpu::SdmaPacketDialect::Gfx11Plus); } +TEST(ConfigLoaderTest, Gfx1151HighSgprsAndVccStayWithinTheirWave) { + auto loaded = config::load_config(CONFIG_DIR_PATH + "/gfx1151.json", rocjitsu::kEmbeddedSchema); + auto *cu = loaded.soc()->xcd(0)->shader_engine(0)->compute_unit(0); + ASSERT_NE(cu, nullptr); + // RDNA3.5 ISA section 3.3.1: s0-s105 are general-purpose registers; + // selectors 106/107 name VCC, which has separate simulator storage. + ASSERT_EQ(cu->config().sgprs_per_wf, 106u); + auto *wf = cu->dispatch_wf(0, 0, 106, 8, 32); + auto *neighbor = cu->dispatch_wf(1, 0, 106, 8, 32); + ASSERT_NE(wf, nullptr); + ASSERT_NE(neighbor, nullptr); + const uint32_t base = wf->sgpr_alloc().base; + const uint32_t neighbor_base = neighbor->sgpr_alloc().base; + ASSERT_EQ(neighbor_base, base + 106u); + constexpr uint32_t sentinel = 0x12345678u; + cu->write_sgpr(neighbor_base, sentinel); + + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3_5); + ASSERT_NE(decoder, nullptr); + // LLVM MC gfx1151 encodings exercise both the destination and source + // selectors that the undersized 104-register profile could not back. + constexpr std::array words{ + 0xBEE80081u, // s_mov_b32 s104, 1 + 0xBEE90082u, // s_mov_b32 s105, 2 + 0xBE840068u, // s_mov_b32 s4, s104 + 0xBE850069u, // s_mov_b32 s5, s105 + 0xBEEA0083u, // s_mov_b32 vcc_lo, 3 + 0xBE86006Au, // s_mov_b32 s6, vcc_lo + }; + for (uint32_t word : words) { + const std::array encoding{word, 0}; + std::unique_ptr inst(decode_valid(*decoder, encoding.data())); + ASSERT_NE(inst, nullptr); + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + } + EXPECT_EQ(cu->read_sgpr(base + 104), 1u); + EXPECT_EQ(cu->read_sgpr(base + 105), 2u); + EXPECT_EQ(cu->read_sgpr(base + 4), 1u); + EXPECT_EQ(cu->read_sgpr(base + 5), 2u); + EXPECT_EQ(cu->read_sgpr(base + 6), 3u); + EXPECT_EQ(wf->vcc(), 3u); + EXPECT_EQ(cu->read_sgpr(neighbor_base), sentinel); + EXPECT_EQ(neighbor->vcc(), 0u); + + EXPECT_TRUE(cu->owns_sgpr_range(*wf, base + 105, 1)); + EXPECT_FALSE(cu->owns_sgpr_range(*wf, base + 106, 1)); + amdgpu::RegisterAccess registers(*wf); + EXPECT_EQ(registers.read_sgpr(neighbor_base), 0u); + registers.write_sgpr(neighbor_base, 0xDEADBEEFu); + EXPECT_EQ(cu->read_sgpr(neighbor_base), sentinel); + wf->halt(); + neighbor->halt(); +} + TEST(ConfigLoaderTest, BuildFromJsonString) { const char *json = R"({ "max_ticks": 5000, @@ -1178,7 +1413,9 @@ TEST(EffectiveConfigTest, ReplacesTheBudgetAndLeavesEveryOtherFieldAsWritten) { "num_threads": 2 })"; - EXPECT_EQ(config::json_with_cpu_thread_budget(json, 4), R"({ + FailureOr rewritten = config::json_with_cpu_thread_budget(json, 4); + ASSERT_TRUE(rewritten.succeeded()); + EXPECT_EQ(rewritten.value(), R"({ "max_ticks": 4, "cpu_thread_budget": 4, "num_threads": 2 @@ -1186,26 +1423,70 @@ TEST(EffectiveConfigTest, ReplacesTheBudgetAndLeavesEveryOtherFieldAsWritten) { } TEST(EffectiveConfigTest, AddsTheBudgetToConfigsThatDoNotAskForOne) { - EXPECT_EQ(config::json_with_cpu_thread_budget("{}", 8), R"({"cpu_thread_budget": 8})"); - EXPECT_EQ(config::json_with_cpu_thread_budget(R"({"num_threads": 2})", 8), - R"({"cpu_thread_budget": 8,"num_threads": 2})"); + FailureOr inserted = config::json_with_cpu_thread_budget("{}", 8); + ASSERT_TRUE(inserted.succeeded()); + EXPECT_EQ(inserted.value(), R"({"cpu_thread_budget": 8})"); + + FailureOr prefixed = config::json_with_cpu_thread_budget(R"({"num_threads": 2})", 8); + ASSERT_TRUE(prefixed.succeeded()); + EXPECT_EQ(prefixed.value(), R"({"cpu_thread_budget": 8,"num_threads": 2})"); } // The nested name is an unknown field the config parser discards, so only the scanner // can tell the two apart -- and picking the wrong one would silently launch under the // config's own budget instead of the requested one. TEST(EffectiveConfigTest, SetsTheTopLevelBudgetAndNotANestedFieldOfTheSameName) { - EXPECT_EQ(config::json_with_cpu_thread_budget(R"({"vm": {"cpu_thread_budget": 1}})", 6), - R"({"cpu_thread_budget": 6,"vm": {"cpu_thread_budget": 1}})"); + FailureOr rewritten = + config::json_with_cpu_thread_budget(R"({"vm": {"cpu_thread_budget": 1}})", 6); + ASSERT_TRUE(rewritten.succeeded()); + EXPECT_EQ(rewritten.value(), R"({"cpu_thread_budget": 6,"vm": {"cpu_thread_budget": 1}})"); } TEST(EffectiveConfigTest, AcceptsTheCommentsAndBareKeysTheConfigParserAccepts) { - EXPECT_EQ(config::json_with_cpu_thread_budget("{\n // ceiling\n cpu_thread_budget: 32\n}", 2), - "{\n // ceiling\n cpu_thread_budget: 2\n}"); + FailureOr rewritten = + config::json_with_cpu_thread_budget("{\n // ceiling\n cpu_thread_budget: 32\n}", 2); + ASSERT_TRUE(rewritten.succeeded()); + EXPECT_EQ(rewritten.value(), "{\n // ceiling\n cpu_thread_budget: 2\n}"); +} + +TEST(EffectiveConfigTest, PreservesCommentsAdjacentToScalarValues) { + FailureOr block_comment = + config::json_with_cpu_thread_budget(R"({"cpu_thread_budget":32/*,}*/})", 4); + ASSERT_TRUE(block_comment.succeeded()); + EXPECT_EQ(block_comment.value(), R"({"cpu_thread_budget":4/*,}*/})"); + + FailureOr line_comment = + config::json_with_cpu_thread_budget("{\"cpu_thread_budget\":32// ceiling\n}", 4); + ASSERT_TRUE(line_comment.succeeded()); + EXPECT_EQ(line_comment.value(), "{\"cpu_thread_budget\":4// ceiling\n}"); +} + +TEST(EffectiveConfigTest, MatchesSingleQuotedAndEscapedQuotedFieldNames) { + FailureOr single_quoted = + config::json_with_cpu_thread_budget(R"({'cpu_thread_budget':32})", 4); + ASSERT_TRUE(single_quoted.succeeded()); + EXPECT_EQ(single_quoted.value(), R"({'cpu_thread_budget':4})"); + + FailureOr escaped = + config::json_with_cpu_thread_budget(R"({"cpu_thread_budge\u0074":32})", 4); + ASSERT_TRUE(escaped.succeeded()); + EXPECT_EQ(escaped.value(), R"({"cpu_thread_budge\u0074":4})"); +} + +TEST(EffectiveConfigTest, ReportsAnUnknownFieldNameEscape) { + util::StringDiagnostic diagnostic; + FailureOr rewritten = + config::json_with_cpu_thread_budget(R"({"cpu_thread_budget\q": 1})", 4, diagnostic.emitter()); + EXPECT_TRUE(rewritten.failed()); + EXPECT_EQ(diagnostic.message(), "unknown escape code in string constant"); } TEST(EffectiveConfigTest, RejectsInputThatIsNotASimulationConfigObject) { - EXPECT_THROW((void)config::json_with_cpu_thread_budget("[1]", 1), std::runtime_error); + util::StringDiagnostic diagnostic; + FailureOr rewritten = + config::json_with_cpu_thread_budget("[1]", 1, diagnostic.emitter()); + EXPECT_TRUE(rewritten.failed()); + EXPECT_EQ(diagnostic.message(), "simulation config must be a JSON object"); } TEST(EffectiveConfigTest, WritesTheLaunchCopyBesideTheInvocationHandoff) { @@ -1214,11 +1495,14 @@ TEST(EffectiveConfigTest, WritesTheLaunchCopyBesideTheInvocationHandoff) { const std::string source = test::config_path("gfx942_cdna3.json"); const std::string source_before = config::read_config_file(source); - const std::string copy = config::write_effective_config(source, 12, getpid()); + FailureOr copy = config::write_effective_config(source, 12, getpid()); + ASSERT_TRUE(copy.succeeded()); - EXPECT_EQ(copy, rocjitsu::rpc_invocation_runtime_dir(getpid()) + "/effective_config.json"); + EXPECT_EQ(copy.value(), + rocjitsu::rpc_invocation_runtime_dir(getpid()) + "/effective_config.json"); EXPECT_EQ(config::read_config_file(source), source_before); - EXPECT_EQ(config::load_execution_thread_settings(copy, rocjitsu::kEmbeddedSchema).request.budget, + EXPECT_EQ(config::load_execution_thread_settings(copy.value(), rocjitsu::kEmbeddedSchema) + .request.budget, 12u); } @@ -1228,9 +1512,45 @@ TEST(EffectiveConfigTest, ReportsAnUnwritableRuntimeDirectory) { std::ofstream(blocked_root) << "not a directory"; test::ScopedEnvironmentVariable runtime_dir("ROCJITSU_RUNTIME_DIR", blocked_root.string()); - EXPECT_THROW( - (void)config::write_effective_config(test::config_path("gfx942_cdna3.json"), 12, getpid()), - std::runtime_error); + util::StringDiagnostic diagnostic; + FailureOr written = config::write_effective_config( + test::config_path("gfx942_cdna3.json"), 12, getpid(), diagnostic.emitter()); + EXPECT_TRUE(written.failed()); + EXPECT_THAT(diagnostic.message(), testing::HasSubstr("cannot create runtime directory")); +} + +// This case is registered as a native CLI test because it expects the launcher +// to have created the invocation handoff before this child process starts. +TEST(EffectiveConfigTest, ReadsNativeLaunchConfigHandoff) { + const char *invocation_dir = std::getenv(rocjitsu::kRpcInvocationDirEnv); + ASSERT_NE(invocation_dir, nullptr); + ASSERT_NE(*invocation_dir, '\0'); + + const std::filesystem::path invocation_path(invocation_dir); + std::ifstream handoff(invocation_path / "config_path"); + ASSERT_TRUE(handoff.is_open()); + + std::string effective_path; + ASSERT_TRUE(std::getline(handoff, effective_path)); + ASSERT_FALSE(effective_path.empty()); + + EXPECT_EQ( + std::filesystem::absolute(effective_path).lexically_normal(), + std::filesystem::absolute(invocation_path / config::kEffectiveConfigName).lexically_normal()); + ASSERT_TRUE(std::filesystem::exists(effective_path)); + + const std::string source_path = test::config_path("gfx942_cdna3.json"); + const std::string source_before = config::read_config_file(source_path); + const auto source_settings = + config::load_execution_thread_settings(source_path, rocjitsu::kEmbeddedSchema); + // Default config json does not specify a budget, so 0 is the pre-launch value. + EXPECT_EQ(source_settings.request.budget, 0u); + + const auto effective_settings = + config::load_execution_thread_settings(effective_path, rocjitsu::kEmbeddedSchema); + + EXPECT_EQ(effective_settings.request.budget, 4u); + EXPECT_EQ(config::read_config_file(source_path), source_before); } TEST(ConfigLoaderTest, RejectsUnresolvedAutomaticDbtHandoffWrite) { @@ -1686,9 +2006,10 @@ TEST(CheckpointTest, LegacyAbsentFunctionalQuantumUsesNativeDefault) { EXPECT_EQ(cu->scratch_slots_per_cu(), cu->num_wf_slots()); } -TEST(CheckpointTest, RoundTripsCdna5ScratchCapacity) { +TEST(CheckpointTest, RoundTripsCdna5ScratchAndDispatchCapacity) { auto source = - config::load_config(CONFIG_DIR_PATH + "/gfx1250_mi455x.json", rocjitsu::kEmbeddedSchema); + config::load_config(CONFIG_DIR_PATH + "/gfx1250_mi455x.json", rocjitsu::kEmbeddedSchema, 48); + ASSERT_EQ(source.execution_threads.dispatch, (std::vector{33})); auto *source_cu = source.soc()->xcd(0)->shader_engine(0)->compute_unit(15); ASSERT_EQ(source_cu->num_wf_slots(), 64u); ASSERT_EQ(source_cu->scratch_slots_per_cu(), 32u); @@ -1696,7 +2017,9 @@ TEST(CheckpointTest, RoundTripsCdna5ScratchCapacity) { test::ScopedTempFile checkpoint_file("rocjitsu-cdna5-scratch-checkpoint-"); config::save_checkpoint(checkpoint_file.path(), *source.soc(), 0, source.engine_config, - source.cpu_dispatch_threads); + source.cpu_dispatch_threads, /*cpu_thread_budget=*/48, + source.thread_allocations, source.legacy_auto_dispatch, + source.async_helper_threads); auto bytes = read_binary_file(checkpoint_file.path()); const auto *stored_cu = fb::GetSimulationCheckpoint(bytes.data()) ->config() @@ -1709,6 +2032,10 @@ TEST(CheckpointTest, RoundTripsCdna5ScratchCapacity) { EXPECT_EQ(stored_cu->scratch_slots_per_cu(), 32u); auto restored = config::restore_checkpoint(checkpoint_file.path()); + EXPECT_EQ(restored.execution_threads.dispatch, source.execution_threads.dispatch); + EXPECT_EQ(restored.execution_threads.helpers, source.execution_threads.helpers); + restored.apply_cpu_dispatch_threads(); + EXPECT_EQ(restored.soc()->dispatch_threads(), 33u); ASSERT_EQ(restored.soc()->num_xcds(), 8u); for (auto *xcd : restored.soc()->xcds()) { auto *cp = xcd->command_processor(); @@ -1724,6 +2051,95 @@ TEST(CheckpointTest, RoundTripsCdna5ScratchCapacity) { } } +TEST(CheckpointTest, RoundTripsCdna5WgpIds) { + auto source = + config::load_config(CONFIG_DIR_PATH + "/gfx1250_mi455x.json", rocjitsu::kEmbeddedSchema); + ASSERT_EQ(source.device.num_cu_per_sh, 8u); + constexpr uint16_t kWgpId = 23 | (10 << 6) | ((4 - 1) << 11); + constexpr std::array kCuIndices{0, 5, 7, 8, 13, 15}; + ASSERT_GT(source.soc()->num_xcds(), 0u); + auto *source_xcd = source.soc()->xcd(0); + ASSERT_GT(source_xcd->num_shader_engines(), 0u); + for (auto *se : source_xcd->shader_engines()) { + ASSERT_EQ(se->num_compute_units(), 16u); + for (uint32_t cu_index : kCuIndices) { + auto *cu = se->compute_unit(cu_index); + auto *wf = cu->dispatch_wf(0, 0, 32, 8); + ASSERT_NE(wf, nullptr); + uint32_t value = 0; + ASSERT_EQ(amdgpu::read_hwreg_field(*wf, kWgpId, value), amdgpu::HwregAccessResult::Success); + ASSERT_EQ(value, cu_index % 8); + } + } + + test::ScopedTempFile checkpoint_file("rocjitsu-cdna5-wgp-checkpoint-"); + config::save_checkpoint(checkpoint_file.path(), *source.soc(), 0, source.engine_config, + source.cpu_dispatch_threads); + auto restored = config::restore_checkpoint(checkpoint_file.path()); + ASSERT_EQ(restored.soc()->num_xcds(), source.soc()->num_xcds()); + auto *restored_xcd = restored.soc()->xcd(0); + ASSERT_EQ(restored_xcd->num_shader_engines(), source_xcd->num_shader_engines()); + for (auto *se : restored_xcd->shader_engines()) { + ASSERT_EQ(se->num_compute_units(), 16u); + for (uint32_t cu_index : kCuIndices) { + SCOPED_TRACE(se->full_path() + ".cu" + std::to_string(cu_index)); + auto *cu = se->compute_unit(cu_index); + auto *wf = cu->wf(0); + ASSERT_NE(wf, nullptr); + ASSERT_FALSE(wf->is_halted()); + uint32_t value = 0; + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, kWgpId, value), amdgpu::HwregAccessResult::Success); + EXPECT_EQ(value, cu_index % 8); + wf->halt(); + } + } +} + +TEST(CheckpointTest, LegacyAbsentShaderArrayWidthKeepsWgpIdUnsupportedAcrossResaves) { + flatbuffers::FlatBufferBuilder builder; + auto arch = builder.CreateString("cdna5"); + auto cu_config = fb::CreateComputeUnitConfig(builder, 1, 128, 256, 64); + // Use only the fields in the legacy schema, which cannot distinguish two + // arrays of eight WGPs from one array of sixteen. + auto se_config = fb::CreateShaderEngineConfig(builder, 16, cu_config); + auto xcd_config = fb::CreateXcdConfig(builder, 1, se_config); + auto gpu_config = fb::CreateAmdgpuConfig(builder, 1, 0, xcd_config); + auto vm_config = fb::CreateVirtualMachineConfig(builder, arch, gpu_config); + auto simulation_config = fb::CreateSimulationConfig(builder, 0, 1, 0, vm_config); + auto checkpoint = fb::CreateSimulationCheckpoint(builder, 0, simulation_config); + builder.Finish(checkpoint); + const auto *stored_se = fb::GetSimulationCheckpoint(builder.GetBufferPointer()) + ->config() + ->vm() + ->gpu() + ->xcd() + ->shader_engine(); + EXPECT_FALSE( + flatbuffers::IsFieldPresent(stored_se, fb::ShaderEngineConfig::VT_CUS_PER_SHADER_ARRAY)); + + test::ScopedTempFile checkpoint_file("rocjitsu-legacy-wgp-checkpoint-"); + checkpoint_file.write(std::string_view(reinterpret_cast(builder.GetBufferPointer()), + builder.GetSize())); + auto restored = config::restore_checkpoint(checkpoint_file.path()); + auto *cu = restored.soc()->xcd(0)->shader_engine(0)->compute_unit(13); + auto *wf = cu->dispatch_wf(0, 0, 32, 8); + ASSERT_NE(wf, nullptr); + constexpr uint16_t kWgpId = 23 | (10 << 6) | ((4 - 1) << 11); + uint32_t value = 0xFFFFFFFFu; + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, kWgpId, value), amdgpu::HwregAccessResult::Unsupported); + EXPECT_EQ(value, 0u); + config::save_checkpoint(checkpoint_file.path(), *restored.soc(), 0, restored.engine_config, + restored.cpu_dispatch_threads); + auto resaved = config::restore_checkpoint(checkpoint_file.path()); + auto *resaved_wf = resaved.soc()->xcd(0)->shader_engine(0)->compute_unit(13)->wf(0); + ASSERT_NE(resaved_wf, nullptr); + ASSERT_FALSE(resaved_wf->is_halted()); + value = 0xFFFFFFFFu; + EXPECT_EQ(amdgpu::read_hwreg_field(*resaved_wf, kWgpId, value), + amdgpu::HwregAccessResult::Unsupported); + EXPECT_EQ(value, 0u); +} + TEST(CheckpointTest, LegacyAbsentCpuDispatchThreadsStaysSerial) { auto checkpoint_file = write_legacy_quantum_checkpoint(); auto bytes = read_binary_file(checkpoint_file.path()); @@ -2411,6 +2827,10 @@ TEST(CApiTest, FunctionalDispatchThreadsPropagateExplicitAndAutoValues) { }; run_case(/*configured=*/7, /*expected=*/7); + // Two seven-CU XCDs give inclusive shared width 1 + 2 * (7 - 1) = 13. + // The shared pool must be wider than either individual XCD when requested. + run_case(/*configured=*/11, /*expected=*/11); + run_case(/*configured=*/99, /*expected=*/13); const auto loaded = config::load_config_from_string(functional_dispatch_threads_config(0), rocjitsu::kEmbeddedSchema); run_case(/*configured=*/0, /*expected=*/loaded.execution_threads.dispatch.front()); @@ -2616,9 +3036,10 @@ TEST(CApiTest, CheckpointRoundTripPreservesAutomaticFunctionalDispatch) { std::string json = functional_dispatch_threads_config(/*threads=*/0); json.replace(json.find("\"num_threads\":1"), std::string("\"num_threads\":1").size(), "\"num_threads\":0"); - json.insert(json.find('{') + 1, R"("cpu_thread_budget":8,"thread_allocations":[ + // Two seven-CU XCDs can share a pool wider than either XCD alone. + json.insert(json.find('{') + 1, R"("cpu_thread_budget":12,"thread_allocations":[ {"num_threads":1,"cpu_dispatch_threads":1}, - {"num_threads":2,"cpu_dispatch_threads":7}],)"); + {"num_threads":2,"cpu_dispatch_threads":11}],)"); rj_vm_t *raw_source = nullptr; ASSERT_EQ(rj_vm_create_from_string(json.c_str(), RJ_VM_MODE_DEFAULT, &raw_source), ROCJITSU_STATUS_SUCCESS); @@ -2626,7 +3047,7 @@ TEST(CApiTest, CheckpointRoundTripPreservesAutomaticFunctionalDispatch) { std::unique_ptr source(raw_source, &rj_vm_destroy); ASSERT_EQ(source->loaded.cpu_dispatch_threads, 0u); const uint32_t effective_threads = source->soc->dispatch_threads(); - ASSERT_EQ(effective_threads, 7u); + ASSERT_EQ(effective_threads, 11u); ASSERT_EQ(source->engine_config.num_threads, 2u); test::ScopedTempFile checkpoint_file("rocjitsu-auto-dispatch-checkpoint-"); @@ -2640,7 +3061,7 @@ TEST(CApiTest, CheckpointRoundTripPreservesAutomaticFunctionalDispatch) { EXPECT_TRUE(flatbuffers::IsFieldPresent(checkpoint->config(), fb::SimulationConfig::VT_CPU_DISPATCH_THREADS)); EXPECT_EQ(checkpoint->config()->cpu_dispatch_threads(), 0u); - EXPECT_EQ(checkpoint->config()->cpu_thread_budget(), 8u); + EXPECT_EQ(checkpoint->config()->cpu_thread_budget(), 12u); EXPECT_EQ(checkpoint->config()->num_threads(), 0u); ASSERT_NE(checkpoint->config()->thread_allocations(), nullptr); EXPECT_EQ(checkpoint->config()->thread_allocations()->size(), 2u); @@ -2677,7 +3098,7 @@ TEST(CApiTest, CheckpointRoundTripPreservesAutomaticFunctionalDispatch) { TEST(CApiTest, CheckpointRoundTripPreservesFunctionalDispatchControls) { const char *json = R"({ "max_ticks":10000,"num_threads":1,"exec_mode":"functional", - "cpu_dispatch_threads":3, + "cpu_dispatch_threads":5, "vm":{"arch":"cdna3"}, "topology":{"root":{"name":"soc","type":"soc","children":[ {"name":"vram","type":"gpu_memory"}, @@ -2703,7 +3124,7 @@ TEST(CApiTest, CheckpointRoundTripPreservesFunctionalDispatchControls) { ASSERT_NE(raw_source, nullptr); std::unique_ptr source(raw_source, &rj_vm_destroy); - EXPECT_EQ(source->soc->dispatch_threads(), 3u); + EXPECT_EQ(source->soc->dispatch_threads(), 5u); EXPECT_EQ(source->soc->xcd(0)->shader_engine(0)->compute_unit(0)->config().functional_quantum, 37u); @@ -2717,7 +3138,7 @@ TEST(CApiTest, CheckpointRoundTripPreservesFunctionalDispatchControls) { ASSERT_NE(raw_restored, nullptr); std::unique_ptr restored(raw_restored, &rj_vm_destroy); - EXPECT_EQ(restored->soc->dispatch_threads(), 3u); + EXPECT_EQ(restored->soc->dispatch_threads(), 5u); EXPECT_EQ(restored->soc->xcd(0)->shader_engine(0)->compute_unit(0)->config().functional_quantum, 37u); } diff --git a/emulation/rocjitsu/tests/corpus/README.md b/emulation/rocjitsu/tests/corpus/README.md index ac52297b0d1e..2c981bf4ee4a 100644 --- a/emulation/rocjitsu/tests/corpus/README.md +++ b/emulation/rocjitsu/tests/corpus/README.md @@ -73,6 +73,70 @@ by the corpus SDK nightly. `rocblas_sgemm::square_64x64` is excluded because the pinned ROCm SDK kernel reads a scalar-memory result after a nonzero KMCNT wait. +## Vulkan compute and buffer CTS + +The release configuration runs the corpus's opt-in `vulkan` suite separately on +`gfx1100` (RDNA3) and `gfx1201` (RDNA4). It installs Mesa RADV and libdrm in +the job container and builds an unmodified pinned headless Vulkan CTS. CI selects 735 +compute and 2,543 texel-buffer cases, leaving twelve expensive +maximum-workgroup/LDS stress cases available for local runs. Both targets run +in the main corpus workflow's release configuration, after the existing HIP +and DBT corpus steps. Vulkan setup and execution use their own prerequisite +checks, so a Vulkan setup failure cannot suppress that earlier coverage and +Vulkan can still run after a HIP/DBT test failure. Any CI coverage including +the stress cases must use a separate workflow. Graphics and image sampling are +out of scope. Exact hardware-baseline unsupported cases are reported as skips; +unexpected unsupported results and incomplete logs fail. +Four cases that initialize images through graphics commands are excluded from +the original compute selection and recorded in `corpus/vulkan/excluded-graphics.txt`. + +The Vulkan corpus has its own commit pin and checkout so the existing HIP and +DBT corpus, including its translation baseline, retain their current inputs. +For a coordinated corpus change, publish its commit first, then update both +`ROCJITSU_VULKAN_CORPUS_REF` and the `vulkan_corpus_ref` dispatch default. +A manual workflow dispatch can override the Vulkan corpus ref independently. + +For local testing, install Mesa 25.2.8+ and libdrm 2.4.125+, activate a Python +environment, and build CTS once from the corpus checkout: + +```sh +cd /path/to/rocjitsu-test-corpus +uv pip install -r corpus/vulkan/requirements.txt +bash scripts/build_vulkan_cts.sh +export ROCJITSU_CORPUS_DIR="$PWD" +export ROCJITSU_BUILD_DIR=/path/to/rocjitsu/build +bash /path/to/rocm-systems/emulation/rocjitsu/tests/corpus/run-vulkan-cts.sh all --case smoke +``` + +The RocJITsu build should use `CMAKE_BUILD_TYPE=Release`. An explicit +`ROCJITSU_BUILD_DIR` selects its launcher; otherwise the wrapper looks for +`rocjitsu` on `PATH`, then under `emulation/rocjitsu/build`. Set +`ROCJITSU_SOURCE_DIR` to override the source directory used for target configs. +Use `--case ci` to reproduce the release workflow, omit `--case smoke` for +all 3,290 cases, or use +`--case stress` for just the twelve expensive cases. Replace `all` with one +target to narrow the run. The wrapper bounds pytest to sixteen workers by default +and divides `VULKAN_CTS_CPU_BUDGET` among them through RocJITsu's +`--cpu-thread-budget` option. By default the total budget is the smaller of 64 +and the available CPU count, so a host with at least 64 CPUs uses sixteen +processes with four execution threads each. Set +`VULKAN_CTS_WORKERS` and `VULKAN_CTS_CPU_BUDGET` to compare other allocations; +runtime service threads and Mesa compilation are outside this budget. +Each launch uses the shipped target config and passes +`--cpu-thread-budget floor(VULKAN_CTS_CPU_BUDGET / VULKAN_CTS_WORKERS)`. +For example, budget 64 with sixteen workers gives each simulator four execution +threads. Budget three with two workers leaves one execution thread unused. +The wrapper selects the system RADV ICD, avoiding SDK library overrides. +CTS verifies the requested device and driver in each result. Logs +and JUnit reports go to `.pytest-artifacts/vulkan-` in the corpus and +are uploaded even after test failures. Set `VULKAN_CTS_ARTIFACT_ROOT` to a +different repo-local prefix to retain separate configurations and logs when +comparing runs. See the corpus's +`corpus/vulkan/README.md` for dependencies, individual case selection and +existing CTS binary overrides. That guide also documents direct hardware runs; +the corpus adapter has no simulator dependency. Simulator configuration, +thread budgets and launcher selection belong to this wrapper. + ## Sanitizer simulator coverage The Clang and GCC ASan+UBSan lanes run the same target-qualified corpus as the diff --git a/emulation/rocjitsu/tests/corpus/gfx1201_skip_tests.json b/emulation/rocjitsu/tests/corpus/gfx1201_skip_tests.json index d89f6f582776..681128dba879 100644 --- a/emulation/rocjitsu/tests/corpus/gfx1201_skip_tests.json +++ b/emulation/rocjitsu/tests/corpus/gfx1201_skip_tests.json @@ -7,35 +7,9 @@ "cts.gfx1201.fpsan.fpsan_amdgcn_math_test", "cts.gfx1201.fpsan.fpsan_amdgcn_ldexp_test", "cts.gfx1201.fpsan.fpsan_wave_gfx12_w64_test", - "cts.gfx1201.fpsan.fpsan_amdgcn_math_extra_test", - "cts.gfx1201.int_isa.int_isa_rdna4_integer_dot_test", - "cts.gfx1201.int_isa.int_isa_rdna4_swmmac_i32_16x16x64_iu4_w64_test", - "cts.gfx1201.int_isa.int_isa_rdna4_swmmac_i32_16x16x32_iu4_w64_test", - "cts.gfx1201.int_isa.int_isa_rdna4_swmmac_i32_16x16x64_iu4_test", - "cts.gfx1201.int_isa.int_isa_rdna4_swmmac_i32_16x16x32_iu4_test", - "cts.gfx1201.int_isa.int_isa_rdna4_wmma_i32_16x16x16_iu4_w64_test", - "cts.gfx1201.int_isa.int_isa_rdna4_wmma_i32_16x16x32_iu4_w64_test" + "cts.gfx1201.fpsan.fpsan_amdgcn_math_extra_test" ], "kernels": [ - "hip_matmul_matvec::m256_n1_k1024", - "llama_mul_mat_vec_q::default", - "llama_rms_norm::default", - "rocblas_sgemm::square_16x16", - "rocblas_sgemm::rect_64x1x64", - "rocblas_sgemm::fuzz_iter2", - "rocblas_sgemm::rect_1x64x1", - "rocblas_sgemm::tiny_2x2x3", - "rocblas_sgemm::fuzz_iter1", - "rocblas_sgemm::square_4x4", - "rocblas_sgemm::alpha_beta", - "rocblas_sgemm::rect_16x32x8", - "rocblas_sgemm::square_32x32", - "rocblas_sgemm::rect_7x11x13", - "rocblas_sgemm::beta_zero", - "rocblas_sgemm::fuzz_iter0", - "rocblas_sgemm::fuzz_iter4", - "rocblas_sgemm::large_2048x2048", - "rocblas_sgemm::fuzz_iter3", - "rocblas_sgemm::square_8x8" + "hip_matmul_matvec::m256_n1_k1024" ] } diff --git a/emulation/rocjitsu/tests/corpus/run-vulkan-cts.sh b/emulation/rocjitsu/tests/corpus/run-vulkan-cts.sh new file mode 100755 index 000000000000..2108c6a580db --- /dev/null +++ b/emulation/rocjitsu/tests/corpus/run-vulkan-cts.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Run the pinned compute/buffer CTS on the RDNA3 and RDNA4 simulators. +# Usage: ROCJITSU_CORPUS_DIR=/path/to/corpus bash run-vulkan-cts.sh \ +# [all|gfx1100|gfx1201] [pytest options, e.g. --case smoke] +# Optional: ROCJITSU_SOURCE_DIR, ROCJITSU_BUILD_DIR, +# VULKAN_CTS_WORKERS (default up to 16), VULKAN_CTS_CPU_BUDGET +# (total RJ execution threads, default up to 64), VK_DRIVER_FILES, +# VULKAN_CTS_ARTIFACT_ROOT (default .pytest-artifacts/vulkan). +# Adapter options: VULKAN_CTS_BINARY, VULKAN_CTS_TIMEOUT, VULKAN_CTS_DATA_DIR. +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source_dir="$(realpath "${ROCJITSU_SOURCE_DIR:-${script_dir}/../..}")" +if [[ -n "${ROCJITSU_BUILD_DIR:-}" ]]; then + launcher="${ROCJITSU_BUILD_DIR}/tools/rocjitsu/rocjitsu" +else + launcher="$(command -v rocjitsu || true)" + launcher="${launcher:-${source_dir}/build/tools/rocjitsu/rocjitsu}" +fi +corpus_dir="${ROCJITSU_CORPUS_DIR:-${PWD}}" +selection="${1:-all}" +if (( $# )); then shift; fi +case "${selection}" in + all) targets=(gfx1100 gfx1201) ;; + gfx1100|gfx1201) targets=("${selection}") ;; + *) echo "Usage: $0 [all|gfx1100|gfx1201] [pytest options]" >&2; exit 2 ;; +esac +available_cpus="$(nproc)" +default_budget=$((available_cpus < 64 ? available_cpus : 64)) +cpu_budget="${VULKAN_CTS_CPU_BUDGET:-${default_budget}}" +if [[ ! "${cpu_budget}" =~ ^[1-9][0-9]*$ ]]; then + echo "VULKAN_CTS_CPU_BUDGET must be a positive integer" >&2 + exit 2 +fi +default_workers=$((cpu_budget < 16 ? cpu_budget : 16)) +workers="${VULKAN_CTS_WORKERS:-${default_workers}}" +if [[ ! "${workers}" =~ ^[1-9][0-9]*$ ]]; then + echo "VULKAN_CTS_WORKERS must be a positive integer" >&2 + exit 2 +fi +if (( workers > cpu_budget )); then + echo "VULKAN_CTS_CPU_BUDGET must be at least VULKAN_CTS_WORKERS" >&2 + exit 2 +fi +rj_budget=$((cpu_budget / workers)) +echo "CPU budget: ${workers} pytest workers, up to ${rj_budget} RJ execution threads each" +if [[ ! -x "${launcher}" ]]; then + echo "Missing executable RocJITsu launcher: ${launcher}. Build RocJITsu or set ROCJITSU_BUILD_DIR." >&2 + exit 1 +fi +launcher="$(realpath "${launcher}")" +cd "${corpus_dir}" +if [[ ! -f tests/test_suites/vulkan.py ]]; then + echo "Missing Vulkan corpus adapter in ${PWD}. Set ROCJITSU_CORPUS_DIR to the Vulkan corpus checkout." >&2 + exit 1 +fi + +if [[ -z "${VK_DRIVER_FILES:-}" ]]; then + for manifest in /usr/share/vulkan/icd.d/radeon_icd.x86_64.json \ + /usr/share/vulkan/icd.d/radeon_icd.json; do + if [[ -f "${manifest}" ]]; then + export VK_DRIVER_FILES="${manifest}" + break + fi + done +fi +: "${VK_DRIVER_FILES:?Install mesa-vulkan-drivers or set VK_DRIVER_FILES to the RADV ICD manifest}" +python3 "${script_dir}/vulkan-cts-utils.py" "${VK_DRIVER_FILES}" + +# Select system Mesa/libdrm instead of SDK copies. Do not allow an inherited +# physical-device selector or ICD override to redirect the simulated workload. +run_wrapper_prefix=( + env + -u LD_LIBRARY_PATH + -u LD_PRELOAD + -u VK_ICD_FILENAMES + -u VK_ADD_DRIVER_FILES + -u VK_INSTANCE_LAYERS + -u VK_LOADER_DRIVERS_SELECT + -u VK_LOADER_DRIVERS_DISABLE + -u RADV_FORCE_FAMILY + "VK_DRIVER_FILES=$(realpath "${VK_DRIVER_FILES}")" + DRI_PRIME= +) +status=0 +for target in "${targets[@]}"; do + case "${target}" in + gfx1100) config=gfx1100_w7900.json ;; + gfx1201) config=gfx1201_r9700.json ;; + esac + artifact_dir="${VULKAN_CTS_ARTIFACT_ROOT:-.pytest-artifacts/vulkan}-${target}" + mkdir -p "${artifact_dir}" + # Divide the budget among independent processes while retaining the shipped + # target's allocation table and engine/dispatch policy. + printf -v wrapper '%q ' "${run_wrapper_prefix[@]}" "${launcher}" \ + --config "${source_dir}/configs/${config}" --cpu-thread-budget "${rj_budget}" -- + python3 -m pytest tests/test_corpus.py --suite vulkan --target "${target}" \ + --run-wrapper "${wrapper}" --artifact-directory "${artifact_dir}" \ + --junitxml "${artifact_dir}/junit.xml" -n "${workers}" \ + -p no:timeout -q --tb=short "$@" || status=1 +done +exit "${status}" diff --git a/emulation/rocjitsu/tests/corpus/vulkan-cts-utils.py b/emulation/rocjitsu/tests/corpus/vulkan-cts-utils.py new file mode 100644 index 000000000000..a27a97a4f929 --- /dev/null +++ b/emulation/rocjitsu/tests/corpus/vulkan-cts-utils.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Validate RADV selection for the Vulkan CTS runner.""" + +import argparse +import json +from pathlib import Path + + +def check_icd(manifest: Path) -> None: + manifest = manifest.resolve() + driver = json.loads(manifest.read_text())["ICD"]["library_path"] + if "libvulkan_radeon.so" not in driver: + raise SystemExit(f"Expected a Mesa RADV manifest, got {manifest}: {driver}") + print(f"RADV ICD: {manifest} ({driver})") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path, help="Mesa RADV ICD manifest") + args = parser.parse_args() + check_icd(args.manifest) + + +if __name__ == "__main__": + main() diff --git a/emulation/rocjitsu/tests/cpu_dispatch_pool_test.cpp b/emulation/rocjitsu/tests/cpu_dispatch_pool_test.cpp index f0f0e1abad04..7e4413e61d4a 100644 --- a/emulation/rocjitsu/tests/cpu_dispatch_pool_test.cpp +++ b/emulation/rocjitsu/tests/cpu_dispatch_pool_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include namespace rocjitsu::amdgpu { @@ -30,6 +31,25 @@ class CpuDispatchPoolTestAccess { static void construct_with_failure(uint32_t threads, uint32_t fail_after) { CpuDispatchPool pool(threads, fail_after); } + + // The test keeps the submitting caller gated for the lifetime of this view. + static CpuDispatchPool::Submission *pending_submission(CpuDispatchPool &pool, + std::span tasks) { + std::lock_guard lock(pool.mutex_); + for (auto *submission = pool.ready_head_; submission; submission = submission->next) + if (submission->tasks.data() == tasks.data()) + return submission; + return static_cast(nullptr); + } + + struct Progress { + size_t next_task; + uint32_t active_workers; + }; + static Progress progress(CpuDispatchPool &pool, const CpuDispatchPool::Submission *submission) { + std::lock_guard lock(pool.mutex_); + return {submission->next_task.load(std::memory_order_relaxed), submission->active_workers}; + } }; } // namespace rocjitsu::amdgpu @@ -40,7 +60,7 @@ using namespace rocjitsu; constexpr uint32_t kSNop = 0xBF800000u; constexpr uint32_t kSMovB32 = 0xBE800000u; // s_mov_b32 s0, s0 -constexpr uint32_t kSSetvskip = 0xBF100000u; +constexpr uint32_t kSCbranchIFork = 0xB8000000u; constexpr uint64_t kProgramBase = 0x100000; struct DispatchPoolFixture { @@ -121,6 +141,135 @@ GatedInstructionPlugin *gate_fixture(DispatchPoolFixture &fixture) { return gate; } +// Hold the caller and one assigned worker while the other worker drains all +// remaining CUs. The caller identity is published before run() queues work. +class DrainingWorkerGate final : public ExecutionPlugin { +public: + DrainingWorkerGate() : ExecutionPlugin("draining_worker_gate") {} + void set_caller() { + std::lock_guard lock(mutex_); + caller_ = std::this_thread::get_id(); + } + void onAmdgpuReadSgpr(const amdgpu::Wavefront *, uint32_t) override { + std::unique_lock lock(mutex_); + const auto id = std::this_thread::get_id(); + if (id == caller_) { + caller_entered_ = true; + changed_.notify_all(); + changed_.wait(lock, [&] { return caller_released_; }); + return; + } + changed_.wait(lock, [&] { return caller_entered_ || all_released_; }); + if (worker_ == std::thread::id{}) + worker_ = id; + if (id == worker_) { + worker_entered_ = true; + changed_.notify_all(); + changed_.wait(lock, [&] { return worker_released_; }); + } else { + draining_entered_ = true; + changed_.notify_all(); + changed_.wait(lock, [&] { return draining_released_; }); + } + } + bool wait_for_all_roles() { + std::unique_lock lock(mutex_); + return changed_.wait_for(lock, std::chrono::seconds(2), [&] { + return caller_entered_ && worker_entered_ && draining_entered_; + }); + } + void release_draining() { release(draining_released_); } + void release_caller() { release(caller_released_); } + void release_worker() { release(worker_released_); } + void release_all() { + std::lock_guard lock(mutex_); + caller_released_ = worker_released_ = draining_released_ = all_released_ = true; + changed_.notify_all(); + } + +private: + void release(bool &flag) { + std::lock_guard lock(mutex_); + flag = true; + changed_.notify_all(); + } + std::mutex mutex_; + std::condition_variable changed_; + std::thread::id caller_; + std::thread::id worker_; + bool caller_entered_ = false; + bool worker_entered_ = false; + bool draining_entered_ = false; + bool caller_released_ = false; + bool worker_released_ = false; + bool draining_released_ = false; + bool all_released_ = false; +}; + +TEST(CpuDispatchPoolTest, WorkerCancelsUnusedTicketsWhileCallerAndPeerAreGated) { + using Access = amdgpu::CpuDispatchPoolTestAccess; + amdgpu::CpuDispatchPool pool(/*threads=*/5); + DispatchPoolFixture foreign(/*cu_count=*/3), target(/*cu_count=*/4), probe(/*cu_count=*/2); + auto *foreign_gate = gate_fixture(foreign); + auto *probe_gate = gate_fixture(probe); + auto group = std::make_shared(PluginSinkConfig{}); + auto plugin = std::make_unique(); + auto *target_gate = plugin.get(); + ASSERT_TRUE(group->add(std::move(plugin))); + target.memory.write32(kProgramBase, kSMovB32); + for (auto &cu : target.cus) + cu->set_plugin_group(group); + + // Occupy two of the four workers, leaving fewer free workers than the target + // batch's three tickets. Its caller and two workers each enter a known gate. + auto foreign_run = std::async(std::launch::async, [&] { pool.run(foreign.tasks, 3); }); + const bool occupied = foreign_gate->wait_for_entries(3); + EXPECT_TRUE(occupied); + if (!occupied) { + foreign_gate->release(); + foreign_run.get(); + return; + } + auto target_run = std::async(std::launch::async, [&] { + target_gate->set_caller(); + pool.run(target.tasks, 4); + }); + const bool held = target_gate->wait_for_all_roles(); + EXPECT_TRUE(held); + auto *submission = held ? Access::pending_submission(pool, target.tasks) : nullptr; + EXPECT_NE(submission, nullptr); + if (submission) { + target_gate->release_draining(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (Access::progress(pool, submission).active_workers != 1 && + std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + EXPECT_EQ(Access::progress(pool, submission).active_workers, 1u); + + // The only available worker must reach this later submission. Without + // worker-side cancellation it first claims the target's leftover ticket, + // advancing next_task again despite having no CU left to execute. + auto probe_run = std::async(std::launch::async, [&] { pool.run(probe.tasks, 2); }); + EXPECT_TRUE(probe_gate->wait_for_entries(2)); + EXPECT_EQ(Access::progress(pool, submission).next_task, target.tasks.size() + 1); + target_gate->release_caller(); + EXPECT_EQ(target_run.wait_for(std::chrono::milliseconds(20)), std::future_status::timeout); + target_gate->release_worker(); + EXPECT_EQ(target_run.wait_for(std::chrono::seconds(2)), std::future_status::ready); + probe_gate->release(); + EXPECT_NO_THROW(probe_run.get()); + for (const auto *wf : probe.wfs) + EXPECT_EQ(wf->pc, kProgramBase + sizeof(uint32_t)); + } + target_gate->release_all(); + foreign_gate->release(); + EXPECT_NO_THROW(target_run.get()); + EXPECT_NO_THROW(foreign_run.get()); + for (const auto *fixture : {&foreign, &target}) + for (const auto *wf : fixture->wfs) + EXPECT_EQ(wf->pc, kProgramBase + sizeof(uint32_t)); +} + TEST(CpuDispatchPoolTest, ConcurrentSubmissionsUseWorkersAndCompleteIndependently) { DispatchPoolFixture first(/*cu_count=*/2), second(/*cu_count=*/2); auto *first_gate = gate_fixture(first); @@ -318,7 +467,7 @@ TEST(CpuDispatchPoolTest, OneThreadFinishesBatchBeforeRethrowing) { TEST(CpuDispatchPoolTest, UnimplementedInstructionsHaltWithoutThrowing) { DispatchPoolFixture fixture(/*cu_count=*/8); amdgpu::CpuDispatchPool pool(/*threads=*/4); - fixture.memory.write32(kProgramBase, kSSetvskip); + fixture.memory.write32(kProgramBase, kSCbranchIFork); EXPECT_NO_THROW(pool.run(std::span(fixture.tasks), /*threads=*/4)); for (const auto &cu : fixture.cus) diff --git a/emulation/rocjitsu/tests/cube_instruction_test.cpp b/emulation/rocjitsu/tests/cube_instruction_test.cpp new file mode 100644 index 000000000000..aff5b8fab1e5 --- /dev/null +++ b/emulation/rocjitsu/tests/cube_instruction_test.cpp @@ -0,0 +1,545 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "decode_test_util.h" +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" +#include "rocjitsu/vm/amdgpu/compute_unit.h" +#include "rocjitsu/vm/amdgpu/gpu_memory.h" +#include "rocjitsu/vm/amdgpu/l2_cache.h" +#include "rocjitsu/vm/amdgpu/wavefront.h" +#include "util/simd.h" +#include "util/simd_test_hooks.h" + +#include + +#include +#include +#include + +namespace { +using namespace rocjitsu; + +struct ForceScalarGuard { + explicit ForceScalarGuard(bool force_scalar) : old_force_scalar(util::force_scalar()) { + util::set_force_scalar_for_testing(force_scalar); + } + ~ForceScalarGuard() { util::set_force_scalar_for_testing(old_force_scalar); } + bool old_force_scalar; +}; + +struct CubeCase { + std::array input; + // Physical gfx1100 outputs, IEEE=0, RNE: ID, SC, TC, MA. + std::array output; + // Physical CUBEMA outputs for round-up, round-down and round-to-zero. + std::array directed_major; +}; + +// Captured from actual cube opcodes on gfx1100 and cross-checked on gfx1201. +// Face ties, all signs, signed zeros, subnormals, overflow, infinities and NaNs. +constexpr CubeCase kCases[] = { + {{0x40000000u, 0x3f800000u, 0x3f000000u}, + {0x00000000u, 0xbf000000u, 0xbf800000u, 0x40800000u}, + {0x40800000u, 0x40800000u, 0x40800000u}}, + {{0xc0000000u, 0x3f800000u, 0x3f000000u}, + {0x3f800000u, 0x3f000000u, 0xbf800000u, 0xc0800000u}, + {0xc0800000u, 0xc0800000u, 0xc0800000u}}, + {{0x3f800000u, 0x40000000u, 0x3f000000u}, + {0x40000000u, 0x3f800000u, 0x3f000000u, 0x40800000u}, + {0x40800000u, 0x40800000u, 0x40800000u}}, + {{0x3f800000u, 0xc0000000u, 0x3f000000u}, + {0x40400000u, 0x3f800000u, 0xbf000000u, 0xc0800000u}, + {0xc0800000u, 0xc0800000u, 0xc0800000u}}, + {{0x3f800000u, 0x3f000000u, 0x40000000u}, + {0x40800000u, 0x3f800000u, 0xbf000000u, 0x40800000u}, + {0x40800000u, 0x40800000u, 0x40800000u}}, + {{0x3f800000u, 0x3f000000u, 0xc0000000u}, + {0x40a00000u, 0xbf800000u, 0xbf000000u, 0xc0800000u}, + {0xc0800000u, 0xc0800000u, 0xc0800000u}}, + {{0x3f800000u, 0x3f800000u, 0x00000000u}, + {0x40000000u, 0x3f800000u, 0x00000000u, 0x40000000u}, + {0x40000000u, 0x40000000u, 0x40000000u}}, + {{0x3f800000u, 0x3f800000u, 0x3f800000u}, + {0x40800000u, 0x3f800000u, 0xbf800000u, 0x40000000u}, + {0x40000000u, 0x40000000u, 0x40000000u}}, + {{0xbf800000u, 0xbf800000u, 0xbf800000u}, + {0x40a00000u, 0x3f800000u, 0x3f800000u, 0xc0000000u}, + {0xc0000000u, 0xc0000000u, 0xc0000000u}}, + {{0x40000000u, 0x40000000u, 0x3f800000u}, + {0x40000000u, 0x40000000u, 0x3f800000u, 0x40800000u}, + {0x40800000u, 0x40800000u, 0x40800000u}}, + {{0x3f800000u, 0x40000000u, 0x40000000u}, + {0x40800000u, 0x3f800000u, 0xc0000000u, 0x40800000u}, + {0x40800000u, 0x40800000u, 0x40800000u}}, + {{0x40000000u, 0x3f800000u, 0x40000000u}, + {0x40800000u, 0x40000000u, 0xbf800000u, 0x40800000u}, + {0x40800000u, 0x40800000u, 0x40800000u}}, + {{0x00000000u, 0x00000000u, 0x00000000u}, + {0x40800000u, 0x00000000u, 0x80000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u}}, + {{0x80000000u, 0x80000000u, 0x80000000u}, + {0x40800000u, 0x80000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x80000000u, 0x00000000u}}, + {{0x80000000u, 0x00000000u, 0x80000000u}, + {0x40800000u, 0x80000000u, 0x80000000u, 0x00000000u}, + {0x00000000u, 0x80000000u, 0x00000000u}}, + {{0x00000001u, 0x80000001u, 0x007fffffu}, + {0x40800000u, 0x00000001u, 0x00000001u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u}}, + {{0x807fffffu, 0x007fffffu, 0x80000001u}, + {0x40800000u, 0x807fffffu, 0x807fffffu, 0x00000000u}, + {0x00000000u, 0x80000000u, 0x00000000u}}, + {{0x00800000u, 0x80800000u, 0x00000000u}, + {0x40400000u, 0x00800000u, 0x80000000u, 0x81000000u}, + {0x81000000u, 0x81000000u, 0x81000000u}}, + {{0x00000001u, 0x00800000u, 0x80000001u}, + {0x40000000u, 0x00000001u, 0x80000001u, 0x01000000u}, + {0x01000000u, 0x01000000u, 0x01000000u}}, + {{0x80000001u, 0x00000001u, 0x80800000u}, + {0x40a00000u, 0x00000001u, 0x80000001u, 0x81000000u}, + {0x81000000u, 0x81000000u, 0x81000000u}}, + {{0x7f7fffffu, 0x3f800000u, 0x3f000000u}, + {0x00000000u, 0xbf000000u, 0xbf800000u, 0x7f800000u}, + {0x7f800000u, 0x7f7fffffu, 0x7f7fffffu}}, + {{0xff7fffffu, 0x3f800000u, 0x3f000000u}, + {0x3f800000u, 0x3f000000u, 0xbf800000u, 0xff800000u}, + {0xff7fffffu, 0xff800000u, 0xff7fffffu}}, + {{0x3f800000u, 0x7f000000u, 0x3f000000u}, + {0x40000000u, 0x3f800000u, 0x3f000000u, 0x7f800000u}, + {0x7f800000u, 0x7f7fffffu, 0x7f7fffffu}}, + {{0x3f800000u, 0xff000000u, 0x3f000000u}, + {0x40400000u, 0x3f800000u, 0xbf000000u, 0xff800000u}, + {0xff7fffffu, 0xff800000u, 0xff7fffffu}}, + {{0x3f800000u, 0x3f000000u, 0x7f800000u}, + {0x40800000u, 0x3f800000u, 0xbf000000u, 0x7f800000u}, + {0x7f800000u, 0x7f800000u, 0x7f800000u}}, + {{0x3f800000u, 0x3f000000u, 0xff800000u}, + {0x40a00000u, 0xbf800000u, 0xbf000000u, 0xff800000u}, + {0xff800000u, 0xff800000u, 0xff800000u}}, + {{0x7fc12345u, 0x3f800000u, 0x40000000u}, + {0x00000000u, 0xc0000000u, 0xbf800000u, 0x7fc12345u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u}}, + {{0x3f800000u, 0x7fc12345u, 0x40000000u}, + {0x00000000u, 0xc0000000u, 0xffc12345u, 0x40000000u}, + {0x40000000u, 0x40000000u, 0x40000000u}}, + {{0x3f800000u, 0x40000000u, 0x7fc12345u}, + {0x40000000u, 0x3f800000u, 0x7fc12345u, 0x40800000u}, + {0x40800000u, 0x40800000u, 0x40800000u}}, + {{0x7f800001u, 0x3f800000u, 0x40000000u}, + {0x00000000u, 0xc0000000u, 0xbf800000u, 0x7f800001u}, + {0x7f800001u, 0x7f800001u, 0x7f800001u}}, + {{0x3f800000u, 0x7f800001u, 0x40000000u}, + {0x00000000u, 0xc0000000u, 0xff800001u, 0x40000000u}, + {0x40000000u, 0x40000000u, 0x40000000u}}, + {{0x3f800000u, 0x40000000u, 0x7f800001u}, + {0x40000000u, 0x3f800000u, 0x7f800001u, 0x40800000u}, + {0x40800000u, 0x40800000u, 0x40800000u}}, + {{0xff800001u, 0x7f800001u, 0xff800001u}, + {0x00000000u, 0x7f800001u, 0xff800001u, 0xff800001u}, + {0xff800001u, 0xff800001u, 0xff800001u}}, + {{0x7f800001u, 0xff800001u, 0x7f800001u}, + {0x00000000u, 0xff800001u, 0x7f800001u, 0x7f800001u}, + {0x7f800001u, 0x7f800001u, 0x7f800001u}}, + {{0x7f800000u, 0xff800000u, 0x7f800000u}, + {0x40800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u}, + {0x7f800000u, 0x7f800000u, 0x7f800000u}}, + {{0xff800000u, 0x7f800000u, 0xff800000u}, + {0x40a00000u, 0x7f800000u, 0xff800000u, 0xff800000u}, + {0xff800000u, 0xff800000u, 0xff800000u}}, + {{0x00000001u, 0x3f800000u, 0x00000000u}, + {0x40000000u, 0x00000001u, 0x00000000u, 0x40000000u}, + {0x40000000u, 0x40000000u, 0x40000000u}}, + {{0x3f800000u, 0x00000001u, 0x00000000u}, + {0x00000000u, 0x80000000u, 0x80000001u, 0x40000000u}, + {0x40000000u, 0x40000000u, 0x40000000u}}, + {{0x00000000u, 0x80000001u, 0x3f800000u}, + {0x40800000u, 0x00000000u, 0x00000001u, 0x40000000u}, + {0x40000000u, 0x40000000u, 0x40000000u}}, + {{0x807fffffu, 0x3f800000u, 0x00000000u}, + {0x40000000u, 0x807fffffu, 0x00000000u, 0x40000000u}, + {0x40000000u, 0x40000000u, 0x40000000u}}, + {{0x00000000u, 0x807fffffu, 0xbf800000u}, + {0x40a00000u, 0x80000000u, 0x007fffffu, 0xc0000000u}, + {0xc0000000u, 0xc0000000u, 0xc0000000u}}, + {{0x00000000u, 0x00000000u, 0xff800001u}, + {0x40000000u, 0x00000000u, 0xff800001u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u}}, +}; + +class CubeInstructionTest : public ::testing::TestWithParam { +protected: + bool gfx9() const { + const auto arch = GetParam(); + return arch == ROCJITSU_CODE_ARCH_CDNA1 || arch == ROCJITSU_CODE_ARCH_CDNA2 || + arch == ROCJITSU_CODE_ARCH_CDNA3 || arch == ROCJITSU_CODE_ARCH_CDNA4; + } + uint32_t first_opcode() const { + const auto arch = GetParam(); + const bool gfx10 = arch == ROCJITSU_CODE_ARCH_RDNA1 || arch == ROCJITSU_CODE_ARCH_RDNA2; + return gfx9() ? 452 : gfx10 ? 324 : 524; + } + amdgpu::ComputeUnitCore::Config config() const { + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = GetParam(); + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = gfx9() ? 102 : 106; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + return cfg; + } +}; + +INSTANTIATE_TEST_SUITE_P(AllArchitectures, CubeInstructionTest, + ::testing::Values(ROCJITSU_CODE_ARCH_RDNA1, ROCJITSU_CODE_ARCH_RDNA2, + ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA1, + ROCJITSU_CODE_ARCH_CDNA2, ROCJITSU_CODE_ARCH_CDNA3, + ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA5)); + +TEST_P(CubeInstructionTest, FaceCoordinatesAndModeMatchPhysicalWitnesses) { + const auto arch = GetParam(); + for (bool scalar : {false, true}) { + ForceScalarGuard guard(scalar); + amdgpu::GpuMemory memory("cube_memory"); + amdgpu::L2Cache l2("cube_l2"); + const auto cfg = config(); + auto cu = amdgpu::ComputeUnitCore::create("cube_cu", cfg, &memory, &l2); + auto decoder = Decoder::create(arch); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + ASSERT_NE(wf, nullptr); + const uint32_t vb = wf->vgpr_alloc().base; + for (uint32_t op = 0; op < 4; ++op) { + const uint32_t words[] = {(gfx9() ? 0xd0000003u : 0xd4000003u) | + ((first_opcode() + op) << 16), + 256u | (257u << 9) | (258u << 18)}; + std::unique_ptr inst(decode_valid(*decoder, words)); + ASSERT_NE(inst, nullptr); + for (uint32_t mode = 0; mode < 32; ++mode) { + const uint32_t round = mode & 3u; + const bool ieee = mode & 16u; + wf->set_mode_raw(round | (((mode >> 2) & 3u) << 4) | (ieee ? 512u : 0u)); + for (uint64_t exec : {wf->wf_size() == 64 ? ~0ull : 0xffffffffull, 0x55555555ull}) { + wf->set_exec(exec); + for (uint32_t start = 0; start < std::size(kCases); start += wf->wf_size()) { + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const auto &c = kCases[(start + lane) % std::size(kCases)]; + for (uint32_t source = 0; source < 3; ++source) + cu->write_vgpr(vb + source, lane, c.input[source]); + cu->write_vgpr(vb + 3, lane, 0xdeadbeefu); + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const auto &c = kCases[(start + lane) % std::size(kCases)]; + uint32_t expected = op == 3 && round ? c.directed_major[round - 1] : c.output[op]; + // RDNA4/CDNA5 always quiet sNaNs; earlier targets honor IEEE. + if ((ieee || arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5) && + (expected & 0x7fffffffu) > 0x7f800000u) + expected |= 0x00400000u; + if (!(exec & (1ull << lane))) + expected = 0xdeadbeefu; + EXPECT_EQ(cu->read_vgpr(vb + 3, lane), expected) + << "case=" << (start + lane) % std::size(kCases) << " op=" << op + << " mode=" << mode << " scalar=" << scalar << " exec=" << exec; + } + } + } + } + } + wf->halt(); + } +} + +struct CubeModifierCase { + std::array input; + uint32_t mode; + uint32_t absolute; + uint32_t negate; + uint32_t omod; + uint32_t clamp; + std::array legacy_output; + std::array modern_output; +}; + +// Raw gfx1100/gfx1201 captures: source modifiers, scaling, directed overflow, +// underflow, IEEE OMOD suppression and DX10_CLAMP signaling-NaN behavior. +constexpr CubeModifierCase kModifierCases[] = { + {{0xc0000000u, 0x3f800001u, 0x40000000u}, + 0u, + 0u, + 0u, + 0u, + 0u, + {0x40800000u, 0xc0000000u, 0xbf800001u, 0x40800000u}, + {0x40800000u, 0xc0000000u, 0xbf800001u, 0x40800000u}}, + {{0x3f7fffffu, 0xff800001u, 0xff800001u}, + 0u, + 1u, + 0u, + 0u, + 0u, + {0x00000000u, 0x7f800001u, 0x7f800001u, 0x3fffffffu}, + {0x00000000u, 0x7fc00001u, 0x7fc00001u, 0x3fffffffu}}, + {{0x80000001u, 0x00800000u, 0x7f800000u}, + 0u, + 2u, + 0u, + 0u, + 0u, + {0x40800000u, 0x80000001u, 0x80800000u, 0x7f800000u}, + {0x40800000u, 0x80000001u, 0x80800000u, 0x7f800000u}}, + {{0x7f800000u, 0x80000000u, 0x807fffffu}, + 0u, + 4u, + 0u, + 0u, + 0u, + {0x00000000u, 0x807fffffu, 0x00000000u, 0x7f800000u}, + {0x00000000u, 0x807fffffu, 0x00000000u, 0x7f800000u}}, + {{0x80000000u, 0x00000001u, 0x00800000u}, + 0u, + 0u, + 1u, + 0u, + 0u, + {0x40800000u, 0x00000000u, 0x80000001u, 0x01000000u}, + {0x40800000u, 0x00000000u, 0x80000001u, 0x01000000u}}, + {{0x3e800000u, 0xbf800000u, 0xc0400000u}, + 0u, + 0u, + 2u, + 0u, + 0u, + {0x40a00000u, 0xbe800000u, 0xbf800000u, 0xc0c00000u}, + {0x40a00000u, 0xbe800000u, 0xbf800000u, 0xc0c00000u}}, + {{0xbf800001u, 0x007fffffu, 0x3f000000u}, + 0u, + 0u, + 4u, + 0u, + 0u, + {0x3f800000u, 0xbf000000u, 0x807fffffu, 0xc0000001u}, + {0x3f800000u, 0xbf000000u, 0x807fffffu, 0xc0000001u}}, + {{0x7f000000u, 0xc0000000u, 0x80000000u}, + 0u, + 7u, + 7u, + 0u, + 0u, + {0x3f800000u, 0x80000000u, 0x40000000u, 0xff800000u}, + {0x3f800000u, 0x80000000u, 0x40000000u, 0xff800000u}}, + {{0x3f000000u, 0x807fffffu, 0x80800000u}, + 0u, + 0u, + 0u, + 1u, + 0u, + {0x00000000u, 0x01000000u, 0x00000000u, 0x40000000u}, + {0x00000000u, 0x01000000u, 0x00000000u, 0x40000000u}}, + {{0xbf000000u, 0x7fc12345u, 0x7f800001u}, + 0u, + 0u, + 0u, + 1u, + 0u, + {0x40000000u, 0x7f800001u, 0xffc12345u, 0xc0000000u}, + {0x40000000u, 0x7fc00001u, 0xffc12345u, 0xc0000000u}}, + {{0x3e800000u, 0xbf800000u, 0xc0400000u}, + 32u, + 0u, + 0u, + 1u, + 0u, + {0x41200000u, 0xbf000000u, 0x40000000u, 0xc1400000u}, + {0x41200000u, 0xbf000000u, 0x40000000u, 0xc1400000u}}, + {{0x3e800000u, 0xbf800000u, 0xc0400000u}, + 512u, + 0u, + 0u, + 1u, + 0u, + {0x40a00000u, 0xbe800000u, 0x3f800000u, 0xc0c00000u}, + {0x41200000u, 0xbf000000u, 0x40000000u, 0xc1400000u}}, + {{0x7f7fffffu, 0x80000001u, 0xff800000u}, + 1u, + 0u, + 0u, + 1u, + 0u, + {0x41200000u, 0xff7fffffu, 0x00000000u, 0xff800000u}, + {0x41200000u, 0xff7fffffu, 0x00000000u, 0xff800000u}}, + {{0x7f7fffffu, 0x80000001u, 0xff800000u}, + 2u, + 0u, + 0u, + 1u, + 0u, + {0x41200000u, 0xff800000u, 0x00000000u, 0xff800000u}, + {0x41200000u, 0xff800000u, 0x00000000u, 0xff800000u}}, + {{0x7f7fffffu, 0x80000001u, 0xff800000u}, + 3u, + 0u, + 0u, + 1u, + 0u, + {0x41200000u, 0xff7fffffu, 0x00000000u, 0xff800000u}, + {0x41200000u, 0xff7fffffu, 0x00000000u, 0xff800000u}}, + {{0x40000000u, 0xbf000000u, 0xff000000u}, + 3u, + 0u, + 0u, + 2u, + 0u, + {0x41a00000u, 0xc1000000u, 0x40000000u, 0xff7fffffu}, + {0x41a00000u, 0xc1000000u, 0x40000000u, 0xff7fffffu}}, + {{0x00800000u, 0xff800000u, 0x80000001u}, + 0u, + 0u, + 0u, + 3u, + 0u, + {0x3fc00000u, 0x00000000u, 0x00000000u, 0xff800000u}, + {0x3fc00000u, 0x00000000u, 0x00000000u, 0xff800000u}}, + {{0x80800000u, 0x3f800000u, 0x00000000u}, + 0u, + 0u, + 0u, + 3u, + 0u, + {0x3f800000u, 0x80000000u, 0x00000000u, 0x3f800000u}, + {0x3f800000u, 0x80000000u, 0x00000000u, 0x3f800000u}}, + {{0x80000000u, 0x00000001u, 0x00800000u}, + 2u, + 0u, + 0u, + 1u, + 0u, + {0x41000000u, 0x00000000u, 0x00000000u, 0x01800000u}, + {0x41000000u, 0x00000000u, 0x00000000u, 0x01800000u}}, + {{0x80000000u, 0x00000001u, 0x00800000u}, + 0u, + 0u, + 0u, + 1u, + 0u, + {0x41000000u, 0x00000000u, 0x00000000u, 0x01800000u}, + {0x41000000u, 0x00000000u, 0x00000000u, 0x01800000u}}, + {{0xbf000000u, 0x7fc12345u, 0x7f800001u}, + 0u, + 0u, + 0u, + 1u, + 1u, + {0x3f800000u, 0x7f800001u, 0xffc12345u, 0x00000000u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {{0xbf000000u, 0x7fc12345u, 0x7f800001u}, + 256u, + 0u, + 0u, + 1u, + 1u, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {{0xbf000000u, 0x7fc12345u, 0x7f800001u}, + 512u, + 0u, + 0u, + 1u, + 1u, + {0x3f800000u, 0x7fc00001u, 0xffc12345u, 0x00000000u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {{0xbf000000u, 0x7fc12345u, 0x7f800001u}, + 768u, + 0u, + 0u, + 1u, + 1u, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {{0x80800000u, 0x3f800000u, 0x00000000u}, + 0u, + 0u, + 0u, + 3u, + 1u, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}}, + {{0xff800001u, 0x40000000u, 0x7f000000u}, + 2u, + 0u, + 0u, + 2u, + 0u, + {0x00000000u, 0xff800000u, 0xc1000000u, 0xff800001u}, + {0x00000000u, 0xff800000u, 0xc1000000u, 0xffc00001u}}, + {{0xff800001u, 0x40000000u, 0x7f000000u}, + 0u, + 7u, + 7u, + 1u, + 0u, + {0x00000000u, 0x7f800000u, 0x40800000u, 0xff800001u}, + {0x00000000u, 0x7f800000u, 0x40800000u, 0xffc00001u}}, + {{0x7f7fffffu, 0x80000001u, 0xff800000u}, + 1u, + 7u, + 7u, + 2u, + 0u, + {0x41a00000u, 0x7f800000u, 0x00000000u, 0xff800000u}, + {0x41a00000u, 0x7f800000u, 0x00000000u, 0xff800000u}}, + {{0xff800001u, 0x40000000u, 0x7f000000u}, + 768u, + 7u, + 7u, + 3u, + 1u, + {0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u}, + {0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u}}, +}; + +TEST_P(CubeInstructionTest, SourceAndOutputModifiersMatchPhysicalWitnesses) { + const auto arch = GetParam(); + const bool modern = arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + for (bool scalar : {false, true}) { + ForceScalarGuard guard(scalar); + amdgpu::GpuMemory memory("cube_modifier_memory"); + amdgpu::L2Cache l2("cube_modifier_l2"); + const auto cfg = config(); + auto cu = amdgpu::ComputeUnitCore::create("cube_modifier_cu", cfg, &memory, &l2); + auto decoder = Decoder::create(arch); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + ASSERT_NE(wf, nullptr); + const uint32_t vb = wf->vgpr_alloc().base; + for (uint32_t index = 0; index < std::size(kModifierCases); ++index) { + const auto &c = kModifierCases[index]; + wf->set_mode_raw(c.mode); + for (uint32_t op = 0; op < 4; ++op) { + const uint32_t words[] = { + (gfx9() ? 0xd0000003u : 0xd4000003u) | ((first_opcode() + op) << 16) | + (c.absolute << 8) | (c.clamp << 15), + 256u | (257u << 9) | (258u << 18) | (c.omod << 27) | (c.negate << 29)}; + std::unique_ptr inst(decode_valid(*decoder, words)); + ASSERT_NE(inst, nullptr); + for (uint64_t exec : {wf->wf_size() == 64 ? ~0ull : 0xffffffffull, 0x55555555ull}) { + wf->set_exec(exec); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + for (uint32_t source = 0; source < 3; ++source) + cu->write_vgpr(vb + source, lane, c.input[source]); + cu->write_vgpr(vb + 3, lane, 0xdeadbeefu); + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const uint32_t expected = exec & (1ull << lane) + ? (modern ? c.modern_output : c.legacy_output)[op] + : 0xdeadbeefu; + EXPECT_EQ(cu->read_vgpr(vb + 3, lane), expected) + << "case=" << index << " op=" << op << " scalar=" << scalar << " exec=" << exec; + } + } + } + } + wf->halt(); + } +} +} // namespace diff --git a/emulation/rocjitsu/tests/data_types_test.cpp b/emulation/rocjitsu/tests/data_types_test.cpp index dfbbb5a240ac..130ac8059b8b 100644 --- a/emulation/rocjitsu/tests/data_types_test.cpp +++ b/emulation/rocjitsu/tests/data_types_test.cpp @@ -657,6 +657,25 @@ TEST(Fp16, RoundTrip) { EXPECT_TRUE(std::signbit(rt(-0.0f))); } +TEST(Fp16, PreservesEveryEncodingThroughF32) { + for (uint32_t bits = 0; bits < 65536; ++bits) { + const float promoted = util::f16_to_f32(static_cast(bits)); + ASSERT_EQ(util::f32_to_f16(promoted), bits) << bits; + ASSERT_EQ(util::f32_to_f16_rtz(promoted), bits) << bits; + } +} + +TEST(Fp16, NarrowNaNPayloadDoesNotBecomeInfinity) { + for (uint32_t sign : {0u, 0x80000000u}) + for (uint32_t payload : {1u, 0x1fffu, 0x4000u, 0x400001u, 0x7fffffu}) { + const float value = std::bit_cast(sign | 0x7f800000u | payload); + const uint16_t expected = + static_cast((sign >> 16) | 0x7c00u | (payload < 0x2000u ? 1u : payload >> 13)); + EXPECT_EQ(util::f32_to_f16(value), expected); + EXPECT_EQ(util::f32_to_f16_rtz(value), expected); + } +} + // ---- BF16 ---- TEST(Bf16, RoundTrip) { diff --git a/emulation/rocjitsu/tests/dbt/translate_gfx1250_test.cpp b/emulation/rocjitsu/tests/dbt/translate_gfx1250_test.cpp index 5d5f1a6e9808..6d85e909b886 100644 --- a/emulation/rocjitsu/tests/dbt/translate_gfx1250_test.cpp +++ b/emulation/rocjitsu/tests/dbt/translate_gfx1250_test.cpp @@ -332,23 +332,27 @@ expected_cdna3_buffer_load_lds_sequence(uint16_t mubuf_op, uint16_t ds_op, }; } -std::vector expected_cdna3_permlane32_swap_sequence() { +std::vector expected_cdna3_permlane_swap_sequence(uint8_t width) { + const auto low_mask = + width == 16 ? expect_sop2(cdna3::kSPackLlB32B16Sop2) : expect_sop1(cdna3::kSMovB32); + const auto high_mask = + width == 16 ? expect_sop2(cdna3::kSPackHhB32B16Sop2) : expect_sop1(cdna3::kSMovB32); return { expect_sop1(cdna3::kSMovB64), // s_mov_b64 save EXEC. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_lo, -1. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_hi, -1. expect_vop3(cdna3::kVMbcntLoU32B32Vop3), // v_mbcnt_lo_u32_b32 expect_vop3(cdna3::kVMbcntHiU32B32Vop3), // v_mbcnt_hi_u32_b32 - expect_vop3(cdna3::kVXorB32Vop3), // v_xor_b32 lane, 32. + expect_vop3(cdna3::kVXorB32Vop3), // v_xor_b32 lane, width. expect_vop3(cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 byte address. expect_ds(cdna3::kDsBpermuteB32Ds), // ds_bpermute_b32 from old vdst high half. expect_ds(cdna3::kDsBpermuteB32Ds), // ds_bpermute_b32 from old src low half. expect_sopp(cdna3::kSWaitcntSopp), // s_waitcnt lgkmcnt(0). - expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_lo, low-half mask. - expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_hi, low-half mask. + low_mask, // exec_lo = saved_exec_lo & low-group mask. + low_mask, // exec_hi = saved_exec_hi & low-group mask. expect_vop3(cdna3::kVMovB32Vop3), // v_mov_b32 src <- old vdst high. - expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_lo, high-half mask. - expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_hi, high-half mask. + high_mask, // exec_lo = saved_exec_lo & high-group mask. + high_mask, // exec_hi = saved_exec_hi & high-group mask. expect_vop3(cdna3::kVMovB32Vop3), // v_mov_b32 vdst <- old src low. expect_sop1(cdna3::kSMovB64), // s_mov_b64 restore EXEC. }; @@ -1018,28 +1022,28 @@ std::vector cdna4_to_cdna3_semantic_rule_cases() { /*op_sel_hi_2=*/1, /*clamp=*/true)}, {"VPermlane16SwapB32E32", cdna4::encoding::kVop1, cdna4::kVPermlane16SwapB32Vop1, make_cdna4_permlane16_swap_b32_words(cdna4::encoding::kVop1), - expected_cdna3_permlane32_swap_sequence(), 1}, + expected_cdna3_permlane_swap_sequence(16), 1}, {"VPermlane16SwapB32E32Hi1", cdna4::encoding::kVop1Hi1, cdna4::kVPermlane16SwapB32Vop1, make_cdna4_permlane16_swap_b32_words(cdna4::encoding::kVop1Hi1), - expected_cdna3_permlane32_swap_sequence(), 1}, + expected_cdna3_permlane_swap_sequence(16), 1}, {"VPermlane16SwapB32E32Hi2", cdna4::encoding::kVop1Hi2, cdna4::kVPermlane16SwapB32Vop1, make_cdna4_permlane16_swap_b32_words(cdna4::encoding::kVop1Hi2), - expected_cdna3_permlane32_swap_sequence(), 1}, + expected_cdna3_permlane_swap_sequence(16), 1}, {"VPermlane16SwapB32E32Hi3", cdna4::encoding::kVop1Hi3, cdna4::kVPermlane16SwapB32Vop1, make_cdna4_permlane16_swap_b32_words(cdna4::encoding::kVop1Hi3), - expected_cdna3_permlane32_swap_sequence(), 1}, + expected_cdna3_permlane_swap_sequence(16), 1}, {"VPermlane32SwapB32E32", cdna4::encoding::kVop1, cdna4::kVPermlane32SwapB32Vop1, make_cdna4_permlane32_swap_b32_words(cdna4::encoding::kVop1), - expected_cdna3_permlane32_swap_sequence(), 1}, + expected_cdna3_permlane_swap_sequence(32), 1}, {"VPermlane32SwapB32E32Hi1", cdna4::encoding::kVop1Hi1, cdna4::kVPermlane32SwapB32Vop1, make_cdna4_permlane32_swap_b32_words(cdna4::encoding::kVop1Hi1), - expected_cdna3_permlane32_swap_sequence(), 1}, + expected_cdna3_permlane_swap_sequence(32), 1}, {"VPermlane32SwapB32E32Hi2", cdna4::encoding::kVop1Hi2, cdna4::kVPermlane32SwapB32Vop1, make_cdna4_permlane32_swap_b32_words(cdna4::encoding::kVop1Hi2), - expected_cdna3_permlane32_swap_sequence(), 1}, + expected_cdna3_permlane_swap_sequence(32), 1}, {"VPermlane32SwapB32E32Hi3", cdna4::encoding::kVop1Hi3, cdna4::kVPermlane32SwapB32Vop1, make_cdna4_permlane32_swap_b32_words(cdna4::encoding::kVop1Hi3), - expected_cdna3_permlane32_swap_sequence(), 1}, + expected_cdna3_permlane_swap_sequence(32), 1}, {"MfmaF32_16x16x32Bf16", cdna4::encoding::kVop3p, cdna4::kVMfmaF3216x16x32Bf16Vop3pMfma, make_cdna4_mfma_words(cdna4::kVMfmaF3216x16x32Bf16Vop3pMfma, 0, 256, 260), expected_cdna3_mfma_sequence(cdna3::kVMfmaF3216x16x16Bf16Vop3pMfma)}, @@ -6382,42 +6386,101 @@ TEST(BinaryTranslatorE2E, Cdna4ToCdna3PermlaneExecSaveReservesSpecialSgprTail) { expect_cdna3_translated_descriptor_sgprs_eq(result.elf_bytes, 80); } -TEST(BinaryTranslatorE2E, Cdna4ToCdna3Permlane16SwapWritesBothRowPairs) { - constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; - const auto permlane = - make_cdna4_permlane16_swap_b32_words(/*encoding_id=*/cdna4::encoding::kVop1); - auto image = rocjitsu::test_support::make_minimal_amdgpu_elf_with_descriptor_after_text( - {permlane[0], permlane[1], kCdna4SEndpgm}); - - rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); - ASSERT_TRUE(source.is_valid()); - - rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); - auto result = translator.translate(source); - ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" - : result.diagnostics.front().message); - ASSERT_FALSE(result.elf_bytes.empty()); - - rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); - ASSERT_TRUE(translated.is_valid()); - ASSERT_FALSE(translated.text_sections().empty()); - const auto *target_words = - reinterpret_cast(translated.text_sections()[0]->data()); - const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); - - // V_PERMLANE16_SWAP_B32 swaps both 16-lane row pairs: lanes 0..15 with - // 16..31, and lanes 32..47 with 48..63. The lowering therefore emits the - // same low/high row masks into EXEC_LO and EXEC_HI. - constexpr uint8_t kExecLoSgpr = 126; - constexpr uint8_t kExecHiSgpr = 127; - EXPECT_EQ( - count_cdna3_s_mov_b32_literal(target_words, target_word_count, kExecLoSgpr, 0x0000ffffu), 1u); - EXPECT_EQ( - count_cdna3_s_mov_b32_literal(target_words, target_word_count, kExecLoSgpr, 0xffff0000u), 1u); - EXPECT_EQ( - count_cdna3_s_mov_b32_literal(target_words, target_word_count, kExecHiSgpr, 0x0000ffffu), 1u); - EXPECT_EQ( - count_cdna3_s_mov_b32_literal(target_words, target_word_count, kExecHiSgpr, 0xffff0000u), 1u); +TEST(BinaryTranslatorE2E, Cdna4ToCdna3PermlaneSwapPreservesInactiveLanes) { + // Include asymmetric masks to check inactive destinations without prescribing + // the value read from an inactive source peer. Symmetric masks also verify + // that both row pairs gather their original values before either write. + constexpr std::array exec_masks = {0, + UINT64_MAX, + 1, + uint64_t{1} << 16, + uint64_t{1} << 32, + uint64_t{1} << 48, + uint64_t{1} << 63, + 0x0001000100010001ull, + 0x8001800180018001ull, + 0xaaaaaaaaaaaaaaaaull}; + for (const uint8_t width : {16, 32}) { + for (const bool alias : {false, true}) { + SCOPED_TRACE(::testing::Message() << "width=" << unsigned(width) << " alias=" << alias); + constexpr uint8_t vdst = 0; + const uint8_t vsrc = alias ? vdst : 1; + const auto swap = cdna4::build_vop1( + width == 16 ? cdna4::kVPermlane16SwapB32Vop1 : cdna4::kVPermlane32SwapB32Vop1, + {.src0 = static_cast(256 + vsrc), .vdst = vdst}); + auto image = rocjitsu::test_support::make_minimal_amdgpu_elf_with_descriptor_after_text( + {swap[0], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4)}); + rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); + ASSERT_TRUE(source.is_valid()); + rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); + const auto result = translator.translate(source); + ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" + : result.diagnostics.front().message); + rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); + ASSERT_TRUE(translated.is_valid()); + ASSERT_FALSE(translated.text_sections().empty()); + const auto *text = reinterpret_cast(translated.text_sections()[0]->data()); + const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); + auto decoder = rocjitsu::Decoder::create(ROCJITSU_CODE_ARCH_CDNA3); + ASSERT_NE(decoder, nullptr); + + rocjitsu::amdgpu::GpuMemory memory("permlane_mem"); + rocjitsu::amdgpu::L2Cache l2("permlane_l2"); + rocjitsu::amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA3; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 102; + cfg.vgprs_per_wf = 256; + cfg.lds_size_kb = 64; + auto cu = rocjitsu::amdgpu::ComputeUnitCore::create("permlane", cfg, &memory, &l2); + ASSERT_NE(cu, nullptr); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + ASSERT_NE(wf, nullptr); + const uint32_t vb = wf->vgpr_alloc().base; + const auto initial = [](uint8_t reg, uint32_t lane) { return 0x1000u * (reg + 1) + lane; }; + for (const uint64_t exec : exec_masks) { + for (const bool scc : {false, true}) { + SCOPED_TRACE(::testing::Message() << "exec=" << exec << " scc=" << scc); + for (uint8_t reg = 0; reg < 2; ++reg) + for (uint32_t lane = 0; lane < 64; ++lane) + cu->write_vgpr(vb + reg, lane, initial(reg, lane)); + wf->set_exec(exec); + wf->write_scc(scc); + constexpr uint64_t vcc = 0x123456789abcdef0ull; + wf->set_vcc(vcc); + bool reached_end = false; + for (size_t offset = 0; offset < word_count;) { + std::unique_ptr inst(decode_valid(*decoder, text + offset)); + ASSERT_NE(inst, nullptr); + if (std::string_view(inst->mnemonic()) == "s_endpgm") { + reached_end = true; + break; + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()) << inst->mnemonic(); + offset += static_cast(inst->size()) / sizeof(uint32_t); + } + ASSERT_TRUE(reached_end); + EXPECT_EQ(wf->exec(), exec); + EXPECT_EQ(wf->read_scc(), scc); + EXPECT_EQ(wf->vcc(), vcc); + for (uint8_t reg = 0; reg < (alias ? 1 : 2); ++reg) { + for (uint32_t lane = 0; lane < 64; ++lane) { + const bool high = (lane & width) != 0; + const bool output = high ? reg == vdst : reg == vsrc; + const bool written = output && ((exec >> lane) & 1u); + const uint32_t peer = lane ^ width; + if (written && ((exec >> peer) & 1u) == 0) + continue; + const uint32_t expected = + written ? initial(high ? vsrc : vdst, peer) : initial(reg, lane); + EXPECT_EQ(cu->read_vgpr(vb + reg, lane), expected) + << "v" << unsigned(reg) << " lane=" << lane; + } + } + } + } + } + } } TEST(BinaryTranslatorE2E, RelocatedKernelCompactsReachableBodyAndPatchesBranches) { diff --git a/emulation/rocjitsu/tests/decode_smoke_test.cpp b/emulation/rocjitsu/tests/decode_smoke_test.cpp index 72a1f6b93f76..18c1e3fc1834 100644 --- a/emulation/rocjitsu/tests/decode_smoke_test.cpp +++ b/emulation/rocjitsu/tests/decode_smoke_test.cpp @@ -896,6 +896,29 @@ TEST(Rdna35FuzzDecodeTest, PreservesRoundTripSignificantSyntax) { } } +TEST(RdnaMimgDecodeTest, NsaStorePreservesExtensionAndNextInstruction) { + // RADV's image clear shader uses three nonconsecutive address registers. + // Both GFX11 families encode them with one additional instruction DWORD. + for (auto arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5}) { + auto decoder = Decoder::create(arch); + ASSERT_NE(decoder, nullptr); + std::array words{0xf0180f95, 0x00040402, 0x00000803, S_NOP}; + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + ASSERT_EQ(inst->size(), 12); + EXPECT_EQ(inst->mnemonic(), "image_store"); + words[2] = 0; + EXPECT_EQ(inst->raw_encoding()[2], 0x803u); + std::unique_ptr next(decode_valid(*decoder, words.data() + inst->size() / 4)); + ASSERT_NE(next, nullptr); + EXPECT_EQ(next->mnemonic(), "s_nop"); + words[0] &= ~1u; + inst.reset(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + EXPECT_EQ(inst->size(), 8); + } +} + TEST(Rdna35MimgDecodeTest, PartialNsaUsesOneExtensionDword) { struct TestCase { std::array words; diff --git a/emulation/rocjitsu/tests/execution_plugin_test.cpp b/emulation/rocjitsu/tests/execution_plugin_test.cpp index 331327591807..a7442825c620 100644 --- a/emulation/rocjitsu/tests/execution_plugin_test.cpp +++ b/emulation/rocjitsu/tests/execution_plugin_test.cpp @@ -38,6 +38,8 @@ #include "rocjitsu/isa/arch/amdgpu/generated/rdna1/opcodes.h" #include "rocjitsu/isa/arch/amdgpu/generated/rdna2/builders.h" #include "rocjitsu/isa/arch/amdgpu/generated/rdna2/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3_5/opcodes.h" #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/builders.h" #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/execution_backend.h" #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/machine_insts.h" @@ -61,6 +63,7 @@ #include "rocjitsu/vm/amdgpu/l2_cache.h" #include "rocjitsu/vm/amdgpu/lds.h" #include "rocjitsu/vm/amdgpu/memory_pipeline.h" +#include "rocjitsu/vm/amdgpu/memory_wait_scoreboard.h" #include "rocjitsu/vm/soc.h" #include "scoped_temp.h" #include "util/simd.h" @@ -233,6 +236,7 @@ class TestMemoryInstruction : public Instruction { class CounterObservingPipeline : public MemoryPipeline { public: CounterObservingPipeline() : MemoryPipeline(WaitCounterType::VMCNT) {} + using MemoryPipeline::issue_counters; WaitCounters counters_at_access{}; @@ -365,6 +369,7 @@ struct CapturedAccess { std::vector addresses; std::vector pre_routing_addresses; std::vector secondary_addresses; + std::vector issued_counters; }; /// @brief Records both memory hooks, so a test can compare what routing was @@ -379,6 +384,14 @@ class MemoryObservationPlugin final : public ExecutionPlugin { bool observes_memory_routing() const override { return wants_hook_; } + void onAmdgpuMemoryAccessRouted(const MemoryAccessObservation &access, const Instruction &inst, + amdgpu::Wavefront &) override { + onAmdgpuMemoryAccessRouted(access); + const auto counters = CounterObservingPipeline{}.issue_counters(inst); + accesses.back().issued_counters.assign(counters.types.begin(), + counters.types.begin() + counters.size); + } + void onAmdgpuRouteMemoryInstruction(const Instruction &inst, amdgpu::Wavefront &wf) override { before_tag.push_back(inst.data()->tag()); // Only the two vector tags carry a VectorMemState. An instruction no @@ -932,7 +945,8 @@ class MfmaRacePlugin : public ExecutionPlugin { detector_ = std::make_unique( static_cast(wavefronts.size()), static_cast(physical_vgpr_count), static_cast(physical_sgpr_count), Dim3d(static_cast(wg_id)), - [this](RaceViolation v) { violations.push_back(v); }); + [this](RaceViolation v) { violations.push_back(v); }, + counterCapacitiesForArch(wavefronts.front()->cu().arch())); wf_ = wavefronts.front(); state_ = &detector_->getWaveRaceState(0); } @@ -2853,6 +2867,65 @@ TEST(ExecutionPluginTest, ValuSimdReadObservationUsesActiveExecMask) { } } +TEST(ExecutionPluginTest, MultiplyClassificationPreservesActiveReadsAndExceptionFlags) { + const std::pair targets[] = { + {ROCJITSU_CODE_ARCH_CDNA1, cdna1::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_CDNA2, cdna2::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_CDNA3, cdna3::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_CDNA4, cdna4::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_CDNA5, cdna5::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_RDNA1, rdna1::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_RDNA2, rdna2::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_RDNA3, rdna3::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_RDNA3_5, rdna3_5::kVMulF32Vop2}, + {ROCJITSU_CODE_ARCH_RDNA4, rdna4::kVMulF32Vop2}, + }; + constexpr uint64_t kExec = (1u << 1) | (1u << 7) | (1u << 21); + for (const auto &[arch, opcode] : targets) { + SCOPED_TRACE(static_cast(arch)); + Wave32PluginFixture f(arch); + auto *plugin = f.attach_ordering_plugin(); + auto *wf = f.cu->dispatch_wf(0, 0, 104, 32); + ASSERT_NE(wf, nullptr); + const uint32_t base = wf->vgpr_alloc().base; + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const uint32_t value = + std::bit_cast((kExec & (uint64_t{1} << lane)) ? 1.1f : 1e38f); + f.cu->write_vgpr(base, lane, value); + f.cu->write_vgpr(base + 1, lane, value); + } + f.cu->write_sgpr(wf->sgpr_alloc().base + 4, std::bit_cast(1.1f)); + auto decoder = Decoder::create(arch); + for (uint64_t exec : {uint64_t{0}, kExec}) { + // Exercise vector, SGPR, literal, and inline-constant sources. + for (uint32_t source : {256u, 4u, 255u, 240u}) { + SCOPED_TRACE(testing::Message() << "exec " << exec << " source " << source); + wf->set_exec(exec); + wf->set_trapsts(0); + uint32_t words[] = {vop2_encode(opcode, 2, 1, source), std::bit_cast(1.1f)}; + std::unique_ptr inst(decode_valid(*decoder, words)); + ASSERT_NE(inst, nullptr); + plugin->events.clear(); + ASSERT_TRUE(f.cu->execute_instruction(inst.get(), *wf).succeeded()); + EXPECT_EQ(wf->trapsts() & 0x7fu, exec && source != 240 ? 1u << 5 : 0u); + std::map reads; + for (const auto &event : vgpr_read_events(*plugin)) { + EXPECT_EQ(event.lane_mask & ~exec, 0u); + EXPECT_EQ(event.byte_mask, ExecutionPlugin::kFullByteMask); + reads[event.physical_reg - base] |= event.lane_mask; + } + std::map expected; + if (exec) { + expected[1] = exec; + if (source == 256) + expected[0] = exec; + } + EXPECT_EQ(reads, expected); + } + } + } +} + TEST(ExecutionPluginTest, ValuSimdWriteObservationUsesActiveExecMask) { if constexpr (!util::has_stdx_simd) { GTEST_SKIP() << " unavailable"; @@ -4282,7 +4355,7 @@ TEST(ExecutionPluginTest, FlatStoreDwordx4ReportsEverySourceRegisterAndActiveLan } } -TEST(ExecutionPluginTest, MemoryPipelineHoldsBothGenericFlatCountersUntilCompletion) { +TEST(ExecutionPluginTest, MemoryPipelineHoldsDecodedCountersUntilCompletion) { PluginFixture f(/*num_wf_slots=*/1); auto *wf = f.cu()->dispatch_wf(0, 0, /*sgprs=*/104, /*vgprs=*/256); ASSERT_NE(wf, nullptr); @@ -4299,7 +4372,7 @@ TEST(ExecutionPluginTest, MemoryPipelineHoldsBothGenericFlatCountersUntilComplet EXPECT_EQ(wf->wait_counters().lgkmcnt, 0); } -TEST(ExecutionPluginTest, MemoryPipelineHoldsAllGenericFlatStoreCountersUntilCompletion) { +TEST(ExecutionPluginTest, MemoryPipelineHoldsDecodedStoreCountersUntilCompletion) { PluginFixture f(/*num_wf_slots=*/1); auto *wf = f.cu()->dispatch_wf(0, 0, /*sgprs=*/104, /*vgprs=*/256); ASSERT_NE(wf, nullptr); @@ -5571,8 +5644,8 @@ TEST(RaceDetectorPluginTest, NamedVmcntWaitRetiresMonolithicAndSplitLoadEvents) EXPECT_TRUE(run(WaitCounterType::LOADCNT)); } -TEST(RaceDetectorPluginTest, GenericFlatLoadRequiresBothWaitCounterDomains) { - auto reports_race_after_wait = [](uint8_t vmcnt, uint8_t lgkmcnt) { +TEST(RaceDetectorPluginTest, UnresolvedOrMixedFlatWholeResultRequiresBothWaitCounterDomains) { + auto reports_race_after_wait = [](MemoryRoute route, uint8_t vmcnt, uint8_t lgkmcnt) { PluginFixture f(/*num_wf_slots=*/1); PluginSinkConfig sink_config; StringSink &sink = sink_config.emplace(); @@ -5585,30 +5658,605 @@ TEST(RaceDetectorPluginTest, GenericFlatLoadRequiresBothWaitCounterDomains) { EXPECT_NE(wf, nullptr); if (!wf) return false; - wf->set_exec(1u); + wf->set_exec(3u); std::array waves{wf}; f.plugin_group_->onAmdgpuWorkgroupDispatched( /*dispatch_id=*/1, /*wg_id=*/0, /*physical_vgpr_count=*/256, /*physical_sgpr_count=*/104, waves); - auto state = std::make_unique(GLOBAL_MEM); + auto state = + std::make_unique(route == MemoryRoute::LOCAL ? LOCAL_MEM : GLOBAL_MEM); state->is_load = true; + state->elem_size = 4; state->num_elems = 1; state->dst_reg_base = wf->vgpr_alloc().base; - state->exec_mask = 1; - TestMemoryInstruction flat(std::move(state), "test_mem", WaitCounterType::LGKMCNT); - f.plugin_group_->onAmdgpuMemoryAccessRouted({}, flat, *wf); + state->exec_mask = state->lane_mask = 3; + state->wf_size = 64; + TestMemoryInstruction flat(std::move(state), + {{WaitCounterType::VMCNT, MemoryCompletionClass::UNORDERED}, + {WaitCounterType::LGKMCNT, MemoryCompletionClass::UNORDERED}}); + // Exercise the conservative fallback without executing unsupported mixed + // LDS/global routing (#11456), including either choice of the first lane. + MemoryAccessObservation access; + if (route != MemoryRoute::UNKNOWN) { + access.route = route; + access.decoded_space = DecodedMemorySpace::FLAT; + access.request_lane_mask = access.active_lane_mask = 3; + access.flat_local_lane_mask = route == MemoryRoute::LOCAL ? 1 : 2; + } + f.plugin_group_->onAmdgpuMemoryAccessRouted(access, flat, *wf); wf->set_wait_target(vmcnt, lgkmcnt, WaitCounters::EXPCNT_MAX); TestWaitcntInstruction wait; f.plugin_group_->onAmdgpuAfterExecuteInstruction(/*pc=*/4, wait, *wf); - f.plugin_group_->onAmdgpuReadVgprLanes(wf, wf->vgpr_alloc().base, /*lane_mask=*/1); + f.plugin_group_->onAmdgpuReadVgprLanes(wf, wf->vgpr_alloc().base, /*lane_mask=*/3); return sink.str().find("RACE ") != std::string::npos; }; - EXPECT_TRUE(reports_race_after_wait(/*vmcnt=*/0, /*lgkmcnt=*/15)); - EXPECT_TRUE(reports_race_after_wait(/*vmcnt=*/63, /*lgkmcnt=*/0)); - EXPECT_FALSE(reports_race_after_wait(/*vmcnt=*/0, /*lgkmcnt=*/0)); + for (const auto route : {MemoryRoute::UNKNOWN, MemoryRoute::GLOBAL, MemoryRoute::LOCAL}) { + SCOPED_TRACE(static_cast(route)); + EXPECT_TRUE(reports_race_after_wait(route, /*vmcnt=*/0, /*lgkmcnt=*/15)); + EXPECT_TRUE(reports_race_after_wait(route, /*vmcnt=*/63, /*lgkmcnt=*/0)); + EXPECT_FALSE(reports_race_after_wait(route, /*vmcnt=*/0, /*lgkmcnt=*/0)); + } +} + +TEST(RaceDetectorPluginTest, FlatLoadReadyLanesKnownFalsePositives) { + enum class Wait { None, Result, Both }; + for (const bool is_rdna4 : {false, true}) { + for (const uint64_t shared_lanes : {0u, 1u, 2u, 3u}) { + for (const bool wait_lds : {false, true}) { + const uint64_t consumer_lanes = wait_lds ? shared_lanes : 3 & ~shared_lanes; + if (!consumer_lanes) + continue; + for (const auto waited : {Wait::None, Wait::Result, Wait::Both}) { + SCOPED_TRACE(is_rdna4 ? "rdna4" : "cdna4"); + SCOPED_TRACE(shared_lanes); + SCOPED_TRACE(consumer_lanes); + SCOPED_TRACE(static_cast(waited)); + const uint32_t wave_size = is_rdna4 ? 32 : 64; + const uint32_t sgprs = is_rdna4 ? 128 : 104; + PluginFixture f(1, is_rdna4 ? "rdna4" : "cdna4", wave_size, sgprs); + PluginSinkConfig sink_config; + auto &sink = sink_config.emplace(); + f.plugin_group_ = std::make_shared(std::move(sink_config)); + ASSERT_TRUE(f.plugin_group_->add(std::make_unique())); + f.soc->set_plugin_group(f.plugin_group_); + f.plugin_group_->onInit(); + auto *cu = f.cu(); + auto *wf = cu->dispatch_wf(0, 0x100, sgprs, 256, wave_size); + ASSERT_NE(wf, nullptr); + wf->set_exec(3); + std::array waves{wf}; + f.plugin_group_->onAmdgpuWorkgroupDispatched(1, 0, 256, sgprs, waves); + + auto decoder = Decoder::create(cu->arch()); + std::unique_ptr load; + if (is_rdna4) { + const auto words = rdna4::build_vflat( + rdna4::kFlatLoadB32Vflat, {.saddr = amdgpu::kModernNullSelector, .vdst = 8}); + load.reset(decode_valid(*decoder, words.data())); + } else { + const auto words = + cdna4::build_flat(cdna4::kFlatLoadDwordFlat, {.saddr = 0x7F, .vdst = 8}); + load.reset(decode_valid(*decoder, words.data())); + } + ASSERT_NE(load, nullptr); + + // Feed the decoded load and resolved masks into both checkers without + // executing the unsupported mixed LDS/global memory path: + // https://github.com/ROCm/rocm-systems/issues/11456 + // The first requesting lane still selects the whole instruction's route. + const bool local_route = shared_lanes & 1; + auto data = std::make_unique(local_route ? LOCAL_MEM : GLOBAL_MEM); + data->is_load = true; + data->exec_mask = data->lane_mask = 3; + data->wf_size = wave_size; + data->elem_size = 4; + data->num_elems = 1; + data->dst_reg_base = wf->vgpr_alloc().base + 8; + load->set_data(std::move(data)); + MemoryAccessObservation access; + access.route = local_route ? MemoryRoute::LOCAL : MemoryRoute::GLOBAL; + access.decoded_space = DecodedMemorySpace::FLAT; + access.request_lane_mask = access.active_lane_mask = 3; + access.flat_local_lane_mask = shared_lanes; + f.plugin_group_->onAmdgpuMemoryAccessRouted(access, *load, *wf); + cu->track_memory_wait(*load, *wf, shared_lanes); + auto &core = wf->ensure_memory_wait_scoreboard(); + unsigned core_reports = 0; + core.bind(0x200, &core_reports, + [](void *p, const auto &) { ++*static_cast(p); }); + + const auto wait_zero = [&](bool lds) { + if (is_rdna4) { + if (lds) + wf->set_wait_target_dscnt(0); + else + wf->set_wait_target_loadcnt(0); + TestWaitcntInstruction wait(lds ? "s_wait_dscnt" : "s_wait_loadcnt"); + f.plugin_group_->onAmdgpuAfterExecuteInstruction(0x200, wait, *wf); + } else { + wf->set_wait_target(lds ? 63 : 0, lds ? 0 : 15, 7); + TestWaitcntInstruction wait; + f.plugin_group_->onAmdgpuAfterExecuteInstruction(0x200, wait, *wf); + } + core.wait(lds ? WaitCounterKind::Ds : WaitCounterKind::Load, 0); + }; + if (waited != Wait::None) + wait_zero(wait_lds); + if (waited == Wait::Both) + wait_zero(!wait_lds); + core.access({RegClass::VGPR, 8, 1}, consumer_lanes, 0xF, false); + f.plugin_group_->onAmdgpuReadVgprLanes(wf, wf->vgpr_alloc().base + 8, consumer_lanes); + + const bool missing_wait = waited == Wait::None; + EXPECT_EQ(core_reports != 0, missing_wait); + // TODO(newling): https://github.com/ROCm/rocm-systems/issues/12237 + // Expect only missing_wait here after the fix. A zero wait + // suffices for the consumed lane group, but the plugin still reports + // ready lanes for mixed CDNA4 and RDNA4 loads. + // Track the producing counter separately for each consumed lane group. + const bool known_false_positive = + waited == Wait::Result && shared_lanes != 0 && shared_lanes != 3; + EXPECT_EQ(sink.str().find("RACE ") != std::string::npos, + missing_wait || known_false_positive); + } + } + } + } +} + +TEST(RaceDetectorPluginTest, FlatResultsFollowTheirResolvedCounter) { + enum class Space { Global, Lds, Scratch, GlobalAndScratch }; + for (const std::string_view arch : {"cdna2", "cdna4", "rdna2", "rdna4", "cdna5"}) { + const bool cdna_legacy = arch == "cdna2" || arch == "cdna4"; + const bool split = arch == "rdna4" || arch == "cdna5"; + const uint32_t wave_size = cdna_legacy ? 64 : 32; + const uint32_t sgprs = cdna_legacy ? 104 : 128; + for (const auto space : {Space::Global, Space::Lds, Space::Scratch, Space::GlobalAndScratch}) { + // RDNA2's FLAT address calculator does not yet translate private pointers. + if (arch == "rdna2" && (space == Space::Scratch || space == Space::GlobalAndScratch)) + continue; + for (const unsigned width : {1u, 4u}) { + for (const uint64_t mask : {uint64_t{0}, uint64_t{0xA000'0000'A000'0008}, ~uint64_t{0}}) { + const uint64_t exec = mask & (~uint64_t{0} >> (64 - wave_size)); + // None, VM zero, LGKM zero, both zero, and a nonzero wait on both. + for (const unsigned waits : {0u, 1u, 2u, 3u, 4u}) { + SCOPED_TRACE(static_cast(space)); + SCOPED_TRACE(arch); + SCOPED_TRACE(width); + SCOPED_TRACE(exec); + SCOPED_TRACE(waits); + PluginFixture f(1, std::string(arch), wave_size, sgprs); + PluginSinkConfig sink_config; + auto &sink = sink_config.emplace(); + auto plugin = std::make_unique(); + auto *plugin_ptr = plugin.get(); + f.plugin_group_ = std::make_shared(std::move(sink_config)); + ASSERT_TRUE(f.plugin_group_->add(std::move(plugin))); + f.soc->set_plugin_group(f.plugin_group_); + f.plugin_group_->onInit(); + + constexpr uint64_t kShared = 0x0001'0000'0000'0000ULL; + constexpr uint64_t kPrivate = 0x0002'0000'0000'0000ULL; + constexpr uint64_t kGlobal = 0x400000; + constexpr uint64_t kScratch = 0x800000; + constexpr uint32_t kCanary = 0xBAD0BAD0; + auto *cu = f.cu(); + cu->set_apertures(kShared, kShared + 0xFFFF, kPrivate, kPrivate + 0xFFFF); + auto *wf = cu->dispatch_wf(0, 0x100, sgprs, 256, wave_size); + ASSERT_NE(wf, nullptr); + wf->set_exec(exec); + wf->set_scratch_base(kScratch); + wf->set_scratch_lane_size(256); + std::array waves{wf}; + f.plugin_group_->onAmdgpuWorkgroupDispatched(1, 0, 256, sgprs, waves); + const uint32_t vb = wf->vgpr_alloc().base; + for (uint32_t lane = 0; lane < wave_size; ++lane) { + const bool scratch = + space == Space::Scratch || (space == Space::GlobalAndScratch && (lane & 1)); + // CDNA5 embeds the lane in the pointer; older targets use the + // private aperture and obtain the lane from the issuing thread. + const uint64_t private_address = + arch == "cdna5" ? kScratch + (uint64_t{lane} << 52) + 0x20 : kPrivate + 0x20; + const uint64_t address = space == Space::Lds ? kShared + lane * 16 + : scratch ? private_address + : kGlobal + lane * 16; + cu->write_vgpr(vb, lane, static_cast(address)); + cu->write_vgpr(vb + 1, lane, static_cast(address >> 32)); + for (unsigned part = 0; part < width; ++part) { + const uint32_t value = 1000 + lane * 4 + part; + cu->write_vgpr(vb + 8 + part, lane, kCanary); + if (space == Space::Lds) { + cu->lds().write32(wf->lds_base() + lane * 16 + part * 4, value); + } else { + const uint64_t physical = !scratch ? address + part * 4 + : arch == "rdna4" + ? kScratch + lane * 256 + 0x20 + part * 4 + : kScratch + (8 + part) * wave_size * 4 + lane * 4; + f.mem->load_image(reinterpret_cast(&value), 4, physical); + } + } + } + + auto decoder = Decoder::create(cu->arch()); + std::unique_ptr load; + const auto decode = [&](const auto &words) { + load.reset(decode_valid(*decoder, words.data())); + }; + if (arch == "cdna2") + decode(cdna2::build_flat(width == 1 ? cdna2::kFlatLoadDwordFlat + : cdna2::kFlatLoadDwordx4Flat, + {.addr = 0, .saddr = 0x7F, .vdst = 8})); + else if (arch == "cdna4") + decode(cdna4::build_flat(width == 1 ? cdna4::kFlatLoadDwordFlat + : cdna4::kFlatLoadDwordx4Flat, + {.addr = 0, .saddr = 0x7F, .vdst = 8})); + else if (arch == "rdna2") + decode(rdna2::build_flat(width == 1 ? rdna2::kFlatLoadDwordFlat + : rdna2::kFlatLoadDwordx4Flat, + {.addr = 0, .saddr = 0x7F, .vdst = 8})); + else if (arch == "rdna4") + decode(rdna4::build_vflat( + width == 1 ? rdna4::kFlatLoadB32Vflat : rdna4::kFlatLoadB128Vflat, + {.saddr = amdgpu::kModernNullSelector, .vdst = 8, .vaddr = 0})); + else + decode(cdna5::build_vflat( + width == 1 ? cdna5::kFlatLoadB32Vflat : cdna5::kFlatLoadB128Vflat, + {.saddr = amdgpu::kModernNullSelector, .vdst = 8, .vaddr = 0})); + ASSERT_NE(load, nullptr); + ASSERT_TRUE(cu->execute_instruction(load.get(), *wf).succeeded()); + test::ComputeUnitTestAccess::route_memory_inst(*cu, load.release(), *wf); + // Inspect eager values without making these assertions instruction consumers. + for (unsigned part = 0; part < width; ++part) + for (uint32_t lane = 0; lane < wave_size; ++lane) + EXPECT_EQ(cu->read_vgpr_storage(vb + 8 + part, lane), + exec & (uint64_t{1} << lane) ? 1000 + lane * 4 + part : kCanary); + + auto *plugin_state = + static_cast(wf->plugin_state(plugin_ptr->slot_index())); + ASSERT_NE(plugin_state, nullptr); + auto &race = *plugin_state->race_state; + auto &core = wf->ensure_memory_wait_scoreboard(); + std::vector hazards; + core.bind(0x200, &hazards, [](void *p, const auto &hazard) { + static_cast(p)->push_back(hazard); + }); + const uint8_t vmcnt = waits == 4 ? 1 : (waits & 1) ? 0 : 63; + const uint8_t ds_max = cdna_legacy ? 15 : 63; + const uint8_t lgkmcnt = waits == 4 ? 1 : (waits & 2) ? 0 : ds_max; + if (split) { + wf->set_wait_target_loadcnt(vmcnt); + TestWaitcntInstruction vm_wait("s_wait_loadcnt"); + f.plugin_group_->onAmdgpuAfterExecuteInstruction(0x200, vm_wait, *wf); + wf->set_wait_target_dscnt(lgkmcnt); + TestWaitcntInstruction ds_wait("s_wait_dscnt"); + f.plugin_group_->onAmdgpuAfterExecuteInstruction(0x204, ds_wait, *wf); + } else { + wf->set_wait_target(vmcnt, lgkmcnt, 7); + TestWaitcntInstruction wait; + f.plugin_group_->onAmdgpuAfterExecuteInstruction(0x200, wait, *wf); + } + if (vmcnt != 63) + core.wait(WaitCounterKind::Load, vmcnt); + if (lgkmcnt != ds_max) + core.wait(WaitCounterKind::Ds, lgkmcnt); + + // Inactive consumers do not acquire a dependency on active lanes. + race.checkVgprReadLanes(8, ~exec, 0xF); + core.access({RegClass::VGPR, 8, 1}, ~exec, 0xF, false); + EXPECT_EQ(sink.str().find("RACE "), std::string::npos); + EXPECT_TRUE(hazards.empty()); + for (unsigned part = 0; part < width; ++part) { + race.checkVgprReadLanes(8 + part, exec, 0xF); + core.access({RegClass::VGPR, static_cast(8 + part), 1}, exec, 0xF, false); + } + const bool missing = exec && (space == Space::Lds ? lgkmcnt != 0 : vmcnt != 0); + EXPECT_EQ(sink.str().find("RACE ") != std::string::npos, missing); + EXPECT_EQ(!hazards.empty(), missing); + } + } + } + } + } +} + +TEST(RaceDetectorPluginTest, Gfx950FlatEmptyPortionDoesNotOrderAnOlderResult) { + for (const bool lds : {false, true}) { + // Exercise both a partial wait and enough issues to trigger capacity + // constraints if the checker retains empty FLAT portions. + for (const int flat_count : {1, lds ? 64 : 16}) { + SCOPED_TRACE(lds); + SCOPED_TRACE(flat_count); + PluginFixture f(/*num_wf_slots=*/1); + PluginSinkConfig sink_config; + auto &sink = sink_config.emplace(); + auto plugin = std::make_unique(); + auto *plugin_ptr = plugin.get(); + f.plugin_group_ = std::make_shared(std::move(sink_config)); + ASSERT_TRUE(f.plugin_group_->add(std::move(plugin))); + f.soc->set_plugin_group(f.plugin_group_); + f.plugin_group_->onInit(); + auto *wf = f.cu()->dispatch_wf(0, 0x100, 104, 256); + ASSERT_NE(wf, nullptr); + wf->set_exec(1); + std::array waves{wf}; + f.plugin_group_->onAmdgpuWorkgroupDispatched(1, 0, 256, 104, waves); + auto *plugin_state = + static_cast(wf->plugin_state(plugin_ptr->slot_index())); + ASSERT_NE(plugin_state, nullptr); + auto &race = *plugin_state->race_state; + const auto empty_counter = lds ? WaitCounterType::VMCNT : WaitCounterType::LGKMCNT; + + // ds_read v9; flat_load(global) v8; vmcnt(0); lgkmcnt(1) + // and the converse with global_load v9 followed by FLAT-to-LDS. + // The empty FLAT portion cannot be used as a younger ordered operation + // to prove v9 ready, even after many FLAT loads have completed their results. + race.registerEvent(0x80, lds ? MemoryEventType::GLOBAL_TO_VGPR : MemoryEventType::LDS_TO_VGPR, + {9}, 1, 0xF, empty_counter, + lds ? MemoryOrderClass::VMEM : MemoryOrderClass::LDS); + auto state = std::make_unique(lds ? LOCAL_MEM : GLOBAL_MEM); + state->is_load = true; + state->elem_size = 4; + state->num_elems = 1; + state->dst_reg_base = wf->vgpr_alloc().base + 8; + state->exec_mask = state->lane_mask = 1; + state->wf_size = 64; + TestMemoryInstruction flat(std::move(state), + {{WaitCounterType::VMCNT, MemoryCompletionClass::UNORDERED}, + {WaitCounterType::LGKMCNT, MemoryCompletionClass::UNORDERED}}); + MemoryAccessObservation access; + access.route = lds ? MemoryRoute::LOCAL : MemoryRoute::GLOBAL; + access.decoded_space = DecodedMemorySpace::FLAT; + access.request_lane_mask = access.active_lane_mask = 1; + access.flat_local_lane_mask = lds ? 1 : 0; + + const auto apply_wait = [&](uint8_t vmcnt, uint8_t lgkmcnt) { + wf->set_wait_target(vmcnt, lgkmcnt, 7); + TestWaitcntInstruction wait; + f.plugin_group_->onAmdgpuAfterExecuteInstruction(0x200, wait, *wf); + }; + for (int i = 0; i < flat_count; ++i) { + f.plugin_group_->onAmdgpuMemoryAccessRouted(access, flat, *wf); + apply_wait(lds ? 63 : 0, lds ? 0 : 15); + } + EXPECT_TRUE(race.getVgprMemoryEvents(8).empty()); + + apply_wait(lds ? 1 : 63, lds ? 15 : 1); + race.checkVgprReadLanes(8, 1, 0xF); + EXPECT_EQ(sink.str().find("RACE "), std::string::npos); + race.checkVgprReadLanes(9, 1, 0xF); + EXPECT_NE(sink.str().find("RACE "), std::string::npos); + apply_wait(0, 0); + EXPECT_TRUE(race.getWaveMemoryEvents().empty()); + EXPECT_TRUE(race.getVgprMemoryEvents(9).empty()); + } + } +} + +TEST(RaceDetectorPluginTest, FlatCounterCapacityRequiresResolvedDomain) { + using namespace waitcheck_detail; + enum class Phase { BeforeRouting, Routed, UnknownObservation }; + for (const bool requests : {false, true}) { + for (const bool full_lds : {false, true}) { + for (const bool flat_lds : {false, true}) { + for (const auto phase : {Phase::BeforeRouting, Phase::Routed, Phase::UnknownObservation}) { + const bool before_routing = phase == Phase::BeforeRouting; + SCOPED_TRACE(full_lds); + SCOPED_TRACE(requests); + SCOPED_TRACE(flat_lds); + SCOPED_TRACE(static_cast(phase)); + PluginFixture f(1); + PluginSinkConfig sink_config; + auto &sink = sink_config.emplace(); + auto plugin = std::make_unique(); + auto *plugin_ptr = plugin.get(); + f.plugin_group_ = std::make_shared(std::move(sink_config)); + ASSERT_TRUE(f.plugin_group_->add(std::move(plugin))); + f.soc->set_plugin_group(f.plugin_group_); + f.plugin_group_->onInit(); + auto *wf = f.cu()->dispatch_wf(0, 0x100, 104, 256); + ASSERT_NE(wf, nullptr); + wf->set_exec(1); + std::array waves{wf}; + f.plugin_group_->onAmdgpuWorkgroupDispatched(1, 0, 256, 104, waves); + auto &race = + *static_cast(wf->plugin_state(plugin_ptr->slot_index())) + ->race_state; + auto &core = wf->ensure_memory_wait_scoreboard(); + unsigned core_reports = 0; + core.bind(0x200, &core_reports, + [](void *p, const auto &) { ++*static_cast(p); }); + const auto counter = full_lds ? WaitCounterKind::Ds : WaitCounterKind::Load; + const ClassifiedEvent older{counter, full_lds ? WaitEventKind::Ds + : WaitEventKind::VmemNoSamplerLoad}; + const unsigned capacity = full_lds ? 15 : 63; + for (unsigned i = 0; i < capacity; ++i) { + const uint32_t reg = 16 + i; + race.registerEvent(0x80 + i * 4, + full_lds ? MemoryEventType::LDS_TO_VGPR + : MemoryEventType::GLOBAL_TO_VGPR, + {reg}, 1, 0xF); + const auto sequence = core.issue(older, ROCJITSU_CODE_ARCH_CDNA4); + core.add({sequence, + 0x80 + i * 4, + 1, + {RegClass::VGPR, static_cast(reg), 1}, + counter, + 0xF}); + } + + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4); + const auto words = + cdna4::build_flat(cdna4::kFlatLoadDwordFlat, {.addr = 0, .saddr = 0x7F, .vdst = 8}); + std::unique_ptr flat(decode_valid(*decoder, words.data())); + ASSERT_NE(flat, nullptr); + f.plugin_group_->onAmdgpuBeforeExecuteInstruction(0x100, *flat, *wf); + core.before(*flat, ROCJITSU_CODE_ARCH_CDNA4); + if (!before_routing) { + auto data = std::make_unique(flat_lds ? LOCAL_MEM : GLOBAL_MEM); + data->exec_mask = 1; + data->lane_mask = requests ? 1 : 0; + data->elem_size = 4; + data->num_elems = 1; + data->dst_reg_base = wf->vgpr_alloc().base + 8; + flat->set_data(std::move(data)); + MemoryAccessObservation access; + access.route = phase == Phase::UnknownObservation ? MemoryRoute::UNKNOWN + : flat_lds ? MemoryRoute::LOCAL + : MemoryRoute::GLOBAL; + access.decoded_space = DecodedMemorySpace::FLAT; + access.active_lane_mask = 1; + access.request_lane_mask = requests ? 1 : 0; + access.flat_local_lane_mask = flat_lds && requests ? 1 : 0; + f.plugin_group_->onAmdgpuMemoryAccessRouted(access, *flat, *wf); + f.cu()->track_memory_wait(*flat, *wf, access.flat_local_lane_mask); + } + + // Before routing, FLAT proves neither domain completed an old request. + // After routing, only the domain actually used needs a free counter slot. + core.bind(0x200, &core_reports, + [](void *p, const auto &) { ++*static_cast(p); }); + if (phase == Phase::Routed && !requests) { + // A rejected request can zero-fill an active destination without + // adding an asynchronous memory dependency or capacity pressure. + race.checkVgprReadLanes(8, 1, 0xF); + core.access({RegClass::VGPR, 8, 1}, 1, 0xF, false); + EXPECT_EQ(sink.str().find("RACE "), std::string::npos); + EXPECT_EQ(core_reports, 0u); + } + race.checkVgprReadLanes(16, 1, 0xF); + core.access({RegClass::VGPR, 16, 1}, 1, 0xF, false); + const bool missing_wait = before_routing || !requests || full_lds != flat_lds; + // The core has the actual masks; an unresolved plugin observation + // must not infer admission in either domain from decoded metadata. + EXPECT_EQ(sink.str().find("RACE ") != std::string::npos, + missing_wait || phase == Phase::UnknownObservation); + EXPECT_EQ(core_reports != 0, missing_wait); + } + } + } + } +} + +TEST(ExecutionPluginTest, MemoryPipelineFlatCountersFollowRoutedRequests) { + enum class Operation { Load, Store, ReturningAtomic, Atomic }; + enum class Space { Global, Lds, Scratch, GlobalAndScratch }; + for (const std::string_view arch : {"cdna2", "cdna4", "rdna2", "rdna4", "cdna5"}) { + for (const auto operation : + {Operation::Load, Operation::Store, Operation::ReturningAtomic, Operation::Atomic}) { + for (const auto space : + {Space::Global, Space::Lds, Space::Scratch, Space::GlobalAndScratch}) { + // RDNA2's FLAT address calculator does not yet translate private pointers. + if (arch == "rdna2" && (space == Space::Scratch || space == Space::GlobalAndScratch)) + continue; + for (const uint64_t exec : {0u, 2u, 3u}) { + SCOPED_TRACE(arch); + SCOPED_TRACE(static_cast(operation)); + SCOPED_TRACE(static_cast(space)); + SCOPED_TRACE(exec); + const bool cdna_legacy = arch == "cdna2" || arch == "cdna4"; + const bool split = arch == "rdna4" || arch == "cdna5"; + const uint32_t wave_size = cdna_legacy ? 64 : 32; + PluginFixture f(1, std::string(arch), wave_size, cdna_legacy ? 104 : 128); + auto *observer = f.attach_memory_observation_plugin(); + auto *cu = f.cu(); + constexpr uint64_t shared = 0x0001'0000'0000'0000ULL; + constexpr uint64_t private_base = 0x0002'0000'0000'0000ULL; + constexpr uint64_t global = 0x400000; + constexpr uint64_t scratch = 0x800000; + cu->set_apertures(shared, shared + 0xFFFF, private_base, private_base + 0xFFFF); + auto *wf = cu->dispatch_wf(0, 0x100, cdna_legacy ? 104 : 128, 256, wave_size); + ASSERT_NE(wf, nullptr); + wf->set_exec(exec); + wf->set_scratch_base(scratch); + wf->set_scratch_lane_size(256); + const uint32_t vb = wf->vgpr_alloc().base; + for (uint32_t lane = 0; lane < 2; ++lane) { + const bool private_lane = + space == Space::Scratch || (space == Space::GlobalAndScratch && lane == 1); + const uint64_t private_address = + arch == "cdna5" ? scratch + (uint64_t{lane} << 52) + 0x20 : private_base + 0x20; + const uint64_t address = space == Space::Lds ? shared + lane * 4 + : private_lane ? private_address + : global + lane * 4; + cu->write_vgpr(vb, lane, static_cast(address)); + cu->write_vgpr(vb + 1, lane, address >> 32); + cu->write_vgpr(vb + 4, lane, 1); + const uint32_t value = 17; + cu->lds().write32(wf->lds_base() + lane * 4, value); + const uint64_t backing = !private_lane ? global + lane * 4 + : arch == "rdna4" ? scratch + lane * 256 + 0x20 + : scratch + 8 * wave_size * 4 + lane * 4; + f.mem->load_image(reinterpret_cast(&value), 4, backing); + } + auto decoder = Decoder::create(cu->arch()); + std::unique_ptr inst; + const bool load = operation == Operation::Load; + const bool store = operation == Operation::Store; + const bool returns = operation == Operation::ReturningAtomic; + const auto decode = [&](const auto &words) { + inst.reset(decode_valid(*decoder, words.data())); + }; + if (arch == "cdna2") + decode(cdna2::build_flat( + load ? cdna2::kFlatLoadDwordFlat + : store ? cdna2::kFlatStoreDwordFlat + : cdna2::kFlatAtomicAddFlat, + {.glc = returns, .addr = 0, .data = 4, .saddr = 0x7F, .vdst = 8})); + else if (arch == "cdna4") + decode(cdna4::build_flat( + load ? cdna4::kFlatLoadDwordFlat + : store ? cdna4::kFlatStoreDwordFlat + : cdna4::kFlatAtomicAddFlat, + {.sc0 = returns, .addr = 0, .data = 4, .saddr = 0x7F, .vdst = 8})); + else if (arch == "rdna2") + decode(rdna2::build_flat( + load ? rdna2::kFlatLoadDwordFlat + : store ? rdna2::kFlatStoreDwordFlat + : rdna2::kFlatAtomicAddFlat, + {.glc = returns, .addr = 0, .data = 4, .saddr = 0x7F, .vdst = 8})); + else if (arch == "rdna4") + decode(rdna4::build_vflat(load ? rdna4::kFlatLoadB32Vflat + : store ? rdna4::kFlatStoreB32Vflat + : rdna4::kFlatAtomicAddU32Vflat, + {.saddr = amdgpu::kModernNullSelector, + .vdst = 8, + .th = returns, + .vsrc = 4, + .vaddr = 0})); + else + decode(cdna5::build_vflat(load ? cdna5::kFlatLoadB32Vflat + : store ? cdna5::kFlatStoreB32Vflat + : cdna5::kFlatAtomicAddU32Vflat, + {.saddr = amdgpu::kModernNullSelector, + .vdst = 8, + .th = returns, + .vsrc = 4, + .vaddr = 0})); + ASSERT_NE(inst, nullptr); + unsigned exports = 0; + for (const auto obligation : inst->amdgpu_memory_issue_info()->counter_obligations()) + if (obligation.wait_counter_type() == WaitCounterType::EXPCNT) + exports += obligation.counter_increment(); + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + test::ComputeUnitTestAccess::route_memory_inst(*cu, inst.release(), *wf); + ASSERT_EQ(observer->accesses.size(), 1u); + const auto &counters = observer->accesses.front().issued_counters; + const bool result = load || returns; + const auto vm_counter = + split ? (result ? WaitCounterType::LOADCNT : WaitCounterType::STORECNT) + : arch == "rdna2" && !result ? WaitCounterType::VSCNT + : WaitCounterType::VMCNT; + const auto ds_counter = split ? WaitCounterType::DSCNT : WaitCounterType::LGKMCNT; + EXPECT_EQ(std::ranges::count(counters, vm_counter), exec && space != Space::Lds ? 1 : 0); + EXPECT_EQ(std::ranges::count(counters, ds_counter), exec && space == Space::Lds ? 1 : 0); + EXPECT_EQ(std::ranges::count(counters, WaitCounterType::EXPCNT), exports); + } + } + } + } } TEST(RaceDetectorPluginTest, NamedLgkmcntWaitRetiresSplitScalarEvent) { @@ -6179,7 +6827,7 @@ TEST(ExecutionPluginTest, DispatchPacketNameResolvesForVmidMappedCodeObject) { std::memcpy(ring.data(), &packet, sizeof(packet)); constexpr uint32_t queue_id = 7; - amdgpu::AqlQueueConfig queue{}; + amdgpu::ComputeQueueConfig queue{}; queue.address_space = address_space; queue.queue_id = queue_id; queue.process_id = process_id; @@ -6692,8 +7340,9 @@ TEST(HookOrderingTest, ParallelWorkgroupLifecycleRunsOnCommandProcessorAfterWork // -- Race trace tests -------------------------------------------------------- TEST(FindConflictTest, UsesRecordedConflictingEvent) { - RaceDetector detector(/*nWaves=*/1, /*vgprCount=*/4, /*sgprCount=*/4, Dim3d(0), - [](RaceViolation) {}); + RaceDetector detector(/*nWaves=*/ + 1, /*vgprCount=*/4, /*sgprCount=*/4, Dim3d(0), [](RaceViolation) {}, + counterCapacitiesForArch(ROCJITSU_CODE_ARCH_CDNA4)); EventId first = detector.allocateEventId(WaveId{0}, /*pc=*/0x100, MemoryEventType::GLOBAL_TO_VGPR, {2}, /*execMask=*/1); EventId second = detector.allocateEventId(WaveId{0}, /*pc=*/0x200, @@ -6707,16 +7356,18 @@ TEST(FindConflictTest, UsesRecordedConflictingEvent) { } TEST(FindConflictTest, RejectsUnavailableConflictingEvent) { - RaceDetector detector(/*nWaves=*/1, /*vgprCount=*/4, /*sgprCount=*/4, Dim3d(0), - [](RaceViolation) {}); + RaceDetector detector(/*nWaves=*/ + 1, /*vgprCount=*/4, /*sgprCount=*/4, Dim3d(0), [](RaceViolation) {}, + counterCapacitiesForArch(ROCJITSU_CODE_ARCH_CDNA4)); RaceViolation violation{RaceViolation::Space::VGPR, 2, 0, 0, true, Dim3d(0), EventId{}}; EXPECT_THROW(findConflict(violation, detector), std::out_of_range); } TEST(DecorateExceptionTest, UsesRecordedConflictingEvent) { - RaceDetector detector(/*nWaves=*/1, /*vgprCount=*/4, /*sgprCount=*/4, Dim3d(0), - [](RaceViolation) {}); + RaceDetector detector(/*nWaves=*/ + 1, /*vgprCount=*/4, /*sgprCount=*/4, Dim3d(0), [](RaceViolation) {}, + counterCapacitiesForArch(ROCJITSU_CODE_ARCH_CDNA4)); EventId first = detector.allocateEventId(WaveId{0}, /*pc=*/10, MemoryEventType::GLOBAL_TO_VGPR, {2}, /*execMask=*/1); EventId second = detector.allocateEventId(WaveId{0}, /*pc=*/20, MemoryEventType::GLOBAL_TO_VGPR, @@ -7347,6 +7998,92 @@ TEST(RoutedMemoryObservationTest, AnExplicitGlobalAccessIgnoresTheSharedAperture EXPECT_EQ(access.addresses[0], kSharedBase + 0x20); } +TEST(RoutedMemoryObservationTest, DecodedDirectLdsLoadsReportBroadcastMaskAndWidth) { + for (auto arch : {"rdna3", "rdna3_5", "rdna4"}) { + PluginFixture fixture(1, arch, 32); + auto *plugin = fixture.attach_memory_observation_plugin(); + auto *cu = fixture.cu(); + // Seed the whole stream before the CU fills its instruction cache. + for (uint32_t i = 0; i < 10; ++i) + fixture.mem->write32(0x1000 + 4 * i, 0xce100006); // direct LDS load to v6 + auto *wave = cu->dispatch_wf(0, 0x1000, 104, 256, 32); + ASSERT_NE(wave, nullptr); + wave->set_lds_base(4096); + wave->lds().write32(4096 + 16, 0x918293e4); + for (uint32_t type : {0u, 1u, 2u, 4u, 5u}) { + for (uint64_t mask : {uint64_t{0}, uint64_t{0x101}}) { + SCOPED_TRACE(testing::Message() << arch << ',' << type << ',' << mask); + const auto pc = wave->pc; + wave->set_exec(mask); + wave->set_m0((type << 16) | 16); + plugin->accesses.clear(); + cu->step(); + ASSERT_EQ(wave->pc, pc + 4); + ASSERT_EQ(plugin->accesses.size(), mask ? 1u : 0u); + EXPECT_EQ(wave->exec(), mask); + EXPECT_TRUE(wave->wait_counters().empty()); + if (!mask) + continue; + const auto &access = plugin->accesses.front(); + EXPECT_EQ(access.route, MemoryRoute::LOCAL); + EXPECT_EQ(access.wait_counter, WaitCounterType::EXPCNT); + EXPECT_TRUE(access.is_load); + EXPECT_EQ(access.elements_per_lane, 1u); + EXPECT_EQ(access.element_size_bytes, type == 2 ? 4u : (type & 1 ? 2u : 1u)); + EXPECT_EQ(access.architectural_exec_lane_mask, mask); + EXPECT_EQ(access.active_lane_mask, 0xf0fu); + EXPECT_EQ(access.valid_lane_mask, 0xf0fu); + EXPECT_EQ(access.request_lane_mask, 0xf0fu); + EXPECT_EQ(access.addresses[0], 4096u + 16); + EXPECT_EQ(access.addresses[11], 4096u + 16); + } + } + } +} + +TEST(RaceDetectorPluginTest, DecodedDirectLdsLoadsDetectConflictingWaveStores) { + for (auto arch : {"rdna3", "rdna3_5", "rdna4"}) { + for (uint32_t store_address : {16u, 20u}) { + SCOPED_TRACE(testing::Message() << arch << ',' << store_address); + PluginFixture fixture(2, arch, 32); + PluginSinkConfig sink_config; + auto &sink = sink_config.emplace(); + fixture.plugin_group_ = std::make_shared(std::move(sink_config)); + ASSERT_TRUE(fixture.plugin_group_->add(std::make_unique())); + fixture.soc->set_plugin_group(fixture.plugin_group_); + fixture.plugin_group_->onInit(); + auto *cu = fixture.cu(); + auto *writer = cu->dispatch_wf(0, 0, 104, 256, 32); + auto *reader = cu->dispatch_wf(0, 0x1000, 104, 256, 32); + ASSERT_NE(writer, nullptr); + ASSERT_NE(reader, nullptr); + writer->set_exec(1); + reader->set_exec(1); + reader->set_m0((2u << 16) | 16); + std::array waves{writer, reader}; + fixture.plugin_group_->onAmdgpuWorkgroupDispatched(1, 0, 512, 208, waves); + auto state = std::make_unique(LOCAL_MEM); + state->wf_size = 32; + state->elem_size = 4; + state->num_elems = 1; + state->is_load = false; + state->exec_mask = state->lane_mask = 1; + state->per_lane_addr[0] = store_address; + TestMemoryInstruction store(std::move(state)); + fixture.plugin_group_->onAmdgpuMemoryAccessRouted({}, store, *writer); + const uint32_t word = 0xce100006; + auto decoder = Decoder::create(cu->arch()); + std::unique_ptr load(decode_valid(*decoder, &word)); + ASSERT_NE(load, nullptr); + ASSERT_TRUE(load->is_memory_op()); + ASSERT_TRUE(cu->execute_instruction(load.get(), *reader).succeeded()); + ASSERT_NE(load->data(), nullptr); + test::ComputeUnitTestAccess::route_memory_inst(*cu, load.release(), *reader); + EXPECT_EQ(sink.str().find("RACE ") != std::string::npos, store_address == 16); + } + } +} + TEST(RoutedMemoryObservationTest, AnLdsAccessThatWasAlwaysLdsIsNotMarkedNormalized) { // The counterpart to the case above: a consumer separating "traffic that is // really LDS" from "traffic rewritten into LDS" needs the two to differ. diff --git a/emulation/rocjitsu/tests/gpu_vm_pipeline_test.cpp b/emulation/rocjitsu/tests/gpu_vm_pipeline_test.cpp index 858522717b04..8b65a722a334 100644 --- a/emulation/rocjitsu/tests/gpu_vm_pipeline_test.cpp +++ b/emulation/rocjitsu/tests/gpu_vm_pipeline_test.cpp @@ -15,10 +15,12 @@ #include #include #include +#include #include #include #include #include +#include #include namespace { @@ -184,6 +186,18 @@ class TestExternalAddressSpace final : public amdgpu::AddressSpaceTranslator, .outcome = amdgpu::VmAccessOutcome::Complete, .observed = observed, .exchanged = exchanged}; } + amdgpu::VmAccessOutcome atomic_modify(amdgpu::VmMemoryDomain domain, uint64_t address, + uint32_t width, const AtomicMutation &mutation) override { + std::lock_guard lock(mutex_); + ++atomic_modify_calls; + if (atomic_modify_calls == unavailable_atomic_modify_call) + return amdgpu::VmAccessOutcome::Unavailable; + if (!valid_atomic(domain, address, width)) + return amdgpu::VmAccessOutcome::Faulted; + mutation(std::span(bytes_).subspan(address, width)); + return amdgpu::VmAccessOutcome::Complete; + } + template void store(uint64_t address, T value) { std::lock_guard lock(mutex_); ASSERT_TRUE(contains(address, sizeof(value))); @@ -208,6 +222,8 @@ class TestExternalAddressSpace final : public amdgpu::AddressSpaceTranslator, uint32_t read_calls = 0; uint32_t write_calls = 0; uint32_t atomic_load_calls = 0; + uint32_t atomic_modify_calls = 0; + uint32_t unavailable_atomic_modify_call = 0; uint32_t atomic_store_calls = 0; uint32_t unavailable_atomic_store_hits = 0; uint32_t compare_exchange_calls = 0; @@ -241,8 +257,9 @@ class TranslatedPipelineContext { amdgpu::Wavefront *wf = nullptr; amdgpu::AddressSpaceHandle address_space; - TranslatedPipelineContext() { - address_space = sim.soc->gpu_vm().register_translated(77, external, external); + explicit TranslatedPipelineContext(bool legacy_cache_compatible = false) { + address_space = sim.soc->gpu_vm().register_address_space(77, external, external, {}, + legacy_cache_compatible); cu->set_gpu_vm(&sim.soc->gpu_vm()); wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); if (wf != nullptr) { @@ -253,6 +270,178 @@ class TranslatedPipelineContext { } }; +class InstructionVmPlugin final : public ExecutionPlugin { +public: + InstructionVmPlugin() : ExecutionPlugin("instruction-vm") {} + void onAmdgpuBeforeExecuteInstruction(uint64_t, const Instruction &, + amdgpu::Wavefront &wf) override { + if (before_instruction) + before_instruction(wf); + } + void onAmdgpuAfterExecuteInstruction(uint64_t, const Instruction &inst, + amdgpu::Wavefront &wf) override { + if (inst.mnemonic() == "s_nop") + ++nops; + if (after_instruction) + after_instruction(wf); + } + uint32_t nops = 0; + std::function before_instruction; + std::function after_instruction; +}; + +TEST(GpuVmPipeline, NestedStepPreservesOuterInstructionSnapshot) { + TranslatedPipelineContext context; + ASSERT_NE(context.wf, nullptr); + constexpr uint64_t kCode = 0x1000, kData = 0x2000; + constexpr uint32_t kValue = 0x12345678, kDestination = 1; + constexpr auto load = cdna5::build_vglobal( + cdna5::kGlobalLoadB32Vglobal, {.saddr = 0, .vdst = kDestination, .vaddr = 0, .ioffset = 0}); + context.external->store(kCode, std::array{load[0], load[1], load[2], S_ENDPGM_GFX12}); + context.external->store(kData, kValue); + // A nested fetch fault must not replace the snapshot used to probe the + // outer instruction's data access after its plugin callback returns. + auto nested_backing = std::make_shared(kData); + const auto nested_handle = + context.sim.soc->gpu_vm().register_translated(88, nested_backing, nested_backing); + auto *nested = context.cu->dispatch_wf(1, kData, kGfx1250ScalarSlots, 32); + ASSERT_NE(nested, nullptr); + nested->set_dispatch_id(1); + nested->set_process_id(88); + nested->set_address_space(nested_handle); + auto group = std::make_shared(PluginSinkConfig{}); + auto plugin = std::make_unique(); + bool entered = false; + plugin->before_instruction = [&](amdgpu::Wavefront &wf) { + if (&wf != context.wf || std::exchange(entered, true)) + return; + wf.set_debug_halted(true); + static_cast(context.cu->step()); + wf.set_debug_halted(false); + }; + plugin->after_instruction = [&](amdgpu::Wavefront &wf) { + if (&wf == context.wf) + context.cu->request_functional_yield(); + }; + ASSERT_TRUE(group->add(std::move(plugin))); + context.sim.soc->set_plugin_group(group); + uint32_t faults = 0; + context.cu->set_debug_active(true); + context.cu->set_memory_violation_handler([&](amdgpu::Wavefront &wf, uint64_t, bool) { + if (&wf != context.wf) + return false; + ++faults; + return true; + }); + context.wf->pc = kCode; + context.wf->set_exec(1); + context.cu->write_sgpr(context.wf->sgpr_alloc().base, kData); + context.cu->write_sgpr(context.wf->sgpr_alloc().base + 1, 0); + context.cu->write_vgpr(context.wf->vgpr_alloc().base, 0, 0); + context.cu->set_functional_quantum(1); + static_cast(context.cu->run_quantum()); + EXPECT_TRUE(entered); + EXPECT_EQ(faults, 0u); + EXPECT_EQ(context.cu->read_vgpr_storage(context.wf->vgpr_alloc().base + kDestination, 0), kValue); +} + +TEST(GpuVmPipeline, QuantumInstructionFetchTracksAddressSpaceChanges) { + enum class Change { ReplaceRoot, Invalidate, Unregister, ExplicitHandle, NumericVmid }; + for (Change change : {Change::ReplaceRoot, Change::Invalidate, Change::Unregister, + Change::ExplicitHandle, Change::NumericVmid}) { + SCOPED_TRACE(static_cast(change)); + TranslatedPipelineContext context; + ASSERT_NE(context.wf, nullptr); + auto &vm = context.sim.soc->gpu_vm(); + constexpr uint64_t kCode = 0x1000; + constexpr uint32_t kNop = 0xBF800000u; + context.external->store(kCode, std::array{kNop, kNop, kNop, S_ENDPGM_GFX12}); + auto replacement = std::make_shared(); + replacement->store(kCode, std::array{kNop, S_ENDPGM_GFX12, kNop, kNop}); + // The unrouted handle deliberately shares the routed binding's VMID. + const auto other = vm.register_unrouted_address_space(77, replacement, replacement); + ASSERT_TRUE(other); + auto group = std::make_shared(PluginSinkConfig{}); + auto plugin = std::make_unique(); + auto *events = plugin.get(); + bool changed = false; + events->after_instruction = [&](amdgpu::Wavefront &wf) { + if (std::exchange(changed, true)) + return; + switch (change) { + case Change::ReplaceRoot: + EXPECT_TRUE(vm.replace_translated(context.address_space, replacement, replacement)); + break; + case Change::Invalidate: + context.external->store(kCode + 4, S_ENDPGM_GFX12); + EXPECT_TRUE(vm.invalidate(context.address_space)); + break; + case Change::Unregister: + EXPECT_TRUE(vm.unregister_address_space(context.address_space)); + break; + case Change::ExplicitHandle: + wf.set_address_space(other); + break; + case Change::NumericVmid: + // Start on the unrouted handle, then resolve the same numeric VMID. + context.external->store(kCode + 4, S_ENDPGM_GFX12); + wf.set_address_space({}); + break; + } + }; + ASSERT_TRUE(group->add(std::move(plugin))); + context.sim.soc->set_plugin_group(group); + context.wf->pc = kCode; + if (change == Change::NumericVmid) { + replacement->store(kCode, std::array{kNop, kNop, kNop, S_ENDPGM_GFX12}); + context.wf->set_address_space(other); + } + context.cu->set_functional_quantum(16); + EXPECT_TRUE(context.cu->run_quantum().ran); + EXPECT_EQ(events->nops, 1u); + EXPECT_FALSE(context.cu->has_active_wfs()); + } +} + +TEST(GpuVmPipeline, QuantumReleasesInstructionSnapshotOnReturnAndException) { + for (bool throw_from_plugin : {false, true}) { + SCOPED_TRACE(throw_from_plugin); + TranslatedPipelineContext context; + ASSERT_NE(context.wf, nullptr); + constexpr uint64_t kCode = 0x1000; + context.external->store(kCode, std::array{0xBF800000u, S_ENDPGM_GFX12, 0, 0}); + auto group = std::make_shared(PluginSinkConfig{}); + auto plugin = std::make_unique(); + auto *events = plugin.get(); + events->after_instruction = [&](amdgpu::Wavefront &) { + if (throw_from_plugin) + throw std::runtime_error("instruction callback"); + }; + ASSERT_TRUE(group->add(std::move(plugin))); + context.sim.soc->set_plugin_group(group); + context.wf->pc = kCode; + context.cu->set_functional_quantum(1); + if (throw_from_plugin) + EXPECT_THROW(context.cu->run_quantum(), std::runtime_error); + else + EXPECT_TRUE(context.cu->run_quantum().ran); + + std::weak_ptr old_backing = context.external; + auto &vm = context.sim.soc->gpu_vm(); + ASSERT_TRUE(vm.unregister_address_space(context.address_space)); + context.external.reset(); + EXPECT_TRUE(old_backing.expired()); + + auto replacement = std::make_shared(); + replacement->store(kCode, std::array{S_ENDPGM_GFX12, 0, 0, 0}); + context.wf->set_address_space(vm.register_translated(77, replacement, replacement)); + context.wf->pc = kCode; + events->after_instruction = {}; + static_cast(context.cu->step()); + EXPECT_FALSE(context.cu->has_active_wfs()); + } +} + TEST(GpuVmPipeline, TranslatedScalarLoadAndStoreUseExternalBacking) { TranslatedPipelineContext context; ASSERT_TRUE(context.address_space); @@ -410,7 +599,7 @@ TEST(GpuVmPipeline, CompletionRetryOnOneQueueDoesNotStarveAnotherQueue) { external->store(kDoorbellAddress, uint64_t{1}); external->store(kSignalAddress + 8, uint64_t{1}); - amdgpu::AqlQueueConfig stalled_queue{}; + amdgpu::ComputeQueueConfig stalled_queue{}; stalled_queue.address_space = address_space; stalled_queue.process_id = kProcessId; stalled_queue.queue_id = kStalledQueueId; @@ -759,13 +948,18 @@ TEST(GpuVmPipeline, TranslatedAtomicsUseStrongBackingOperationsForBothWidths) { uint64_t source; uint64_t compare; uint64_t expected; + bool source_nan_first = false; }; - constexpr std::array kCases = {{ + constexpr std::array kCases = {{ {amdgpu::AtomicOp::ADD, 4, 0x300, 7, 5, 0, 12}, {amdgpu::AtomicOp::ADD, 8, 0x308, 0x1'0000'0000ULL, 9, 0, 0x1'0000'0009ULL}, {amdgpu::AtomicOp::CMPSWAP, 4, 0x310, 0x11223344, 0xaabbccdd, 0x11223344, 0xaabbccdd}, {amdgpu::AtomicOp::CMPSWAP, 8, 0x318, 0x1122334455667788ULL, 0xaabbccddeeff0011ULL, 0x1122334455667788ULL, 0xaabbccddeeff0011ULL}, + {amdgpu::AtomicOp::PK_ADD_F16, 4, 0x320, 0x3c004000, 0x42004400, 0, 0x44004600}, + {amdgpu::AtomicOp::PK_ADD_BF16, 4, 0x324, 0x3f804000, 0x40404080, 0, 0x408040c0}, + {amdgpu::AtomicOp::FADD, 4, 0x328, 0x7f800002, 0xff800004, 0, 0x7fc00002}, + {amdgpu::AtomicOp::FADD, 4, 0x32c, 0x7f800002, 0xff800004, 0, 0xffc00004, true}, }}; amdgpu::GlobalMemPipeline pipeline(&context.cu->l1_vector(), context.cu->l2()); @@ -781,6 +975,7 @@ TEST(GpuVmPipeline, TranslatedAtomicsUseStrongBackingOperationsForBothWidths) { state->num_elems = 1; state->is_load = true; state->atomic_op = test.operation; + state->atomic_source_nan_first = test.source_nan_first; state->wf_size = context.wf->wf_size(); state->exec_mask = 1; state->lane_mask = 1; @@ -853,6 +1048,63 @@ TEST(GpuVmPipeline, UnavailableAtomicDoesNotCompleteArchitecturalState) { EXPECT_TRUE(context.wf->wait_counters().empty()); } +TEST(GpuVmPipeline, LegacyAtomicBatchPreservesLaneResultsOnRetry) { + for (uint64_t mask : {uint64_t{0}, uint64_t{0xb}}) { + for (bool retry : {false, true}) { + SCOPED_TRACE(mask); + SCOPED_TRACE(retry); + TranslatedPipelineContext context(/*legacy_cache_compatible=*/true); + ASSERT_NE(context.wf, nullptr); + constexpr uint64_t kAddress = 0x420; + constexpr uint32_t kInitial = 10, kDestination = 30, kSentinel = 0xdeadbeef; + constexpr std::array kSource = {1, 2, 3, 4}; + context.external->store(kAddress, kInitial); + context.external->unavailable_atomic_modify_call = retry ? 2 : 0; + context.wf->set_exec(mask); + auto state = std::make_unique(amdgpu::GLOBAL_MEM); + state->elem_size = sizeof(uint32_t); + state->num_elems = 1; + state->is_load = true; + state->atomic_op = amdgpu::AtomicOp::ADD; + state->wf_size = context.wf->wf_size(); + state->exec_mask = mask; + state->lane_mask = mask; + state->dst_reg_base = context.wf->vgpr_alloc().base + kDestination; + state->store_data.resize(state->wf_size * sizeof(uint32_t)); + std::memcpy(state->store_data.data(), kSource.data(), sizeof(kSource)); + for (uint32_t lane = 0; lane < kSource.size(); ++lane) { + state->per_lane_addr[lane] = kAddress; + context.cu->write_vgpr(state->dst_reg_base, lane, kSentinel); + } + const auto &coherence = context.cu->l2()->coherence_domain(); + const auto epoch = coherence->current_epoch(); + amdgpu::GlobalMemPipeline pipeline(&context.cu->l1_vector(), context.cu->l2()); + EXPECT_EQ(pipeline.issue_deferred(new TestMemoryInstruction(std::move(state)), *context.wf), + amdgpu::VmAccessOutcome::Complete); + if (mask && retry) { + ASSERT_EQ(context.wf->state(), amdgpu::WfState::VM_RETRY); + EXPECT_EQ(context.external->load(kAddress), kInitial + kSource[0]); + // Retire the first attempt's snapshot after lane 0 has committed. The + // retry must acquire a new generation without repeating that lane. + ASSERT_TRUE(context.sim.soc->gpu_vm().invalidate(context.address_space)); + pipeline.tick(); + } + EXPECT_TRUE(context.wf->wait_counters().empty()); + EXPECT_EQ(coherence->current_epoch() - epoch, mask ? (retry ? 2u : 1u) : 0u); + uint32_t expected = kInitial; + for (uint32_t lane = 0; lane < kSource.size(); ++lane) { + const bool active = (mask & (uint64_t{1} << lane)) != 0; + EXPECT_EQ(context.cu->read_vgpr_storage(context.wf->vgpr_alloc().base + kDestination, lane), + active ? expected : kSentinel); + if (active) + expected += kSource[lane]; + } + EXPECT_EQ(context.external->load(kAddress), expected); + EXPECT_EQ(context.external->atomic_modify_calls, mask ? (retry ? 4u : 3u) : 0u); + } + } +} + TEST(GpuVmPipeline, TranslatedAtomicRetrySkipsCompletedLanes) { TranslatedPipelineContext context; ASSERT_TRUE(context.address_space); diff --git a/emulation/rocjitsu/tests/gpu_vm_service_test.cpp b/emulation/rocjitsu/tests/gpu_vm_service_test.cpp index 2ff167b09963..a48f4beadaf4 100644 --- a/emulation/rocjitsu/tests/gpu_vm_service_test.cpp +++ b/emulation/rocjitsu/tests/gpu_vm_service_test.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: MIT #include "rocjitsu/vm/amdgpu/aql/aql_packet_types.h" -#include "rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h" #include "rocjitsu/vm/amdgpu/command_processor.h" +#include "rocjitsu/vm/amdgpu/compute_queue_binding_factory.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" #include "rocjitsu/vm/amdgpu/gpu_memory_access.h" #include "rocjitsu/vm/amdgpu/gpu_queue_registry.h" @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -47,27 +48,27 @@ class CommandProcessorCloseTestAccess { public: static bool fetch_first_queue(CommandProcessor &command_processor) { std::lock_guard lock(command_processor.hw_queue_mutex_); - if (command_processor.aql_queues_.size() != 1) + if (command_processor.compute_queues_.size() != 1) return false; - command_processor.fetch_from_queue(command_processor.aql_queues_.front(), 0); + command_processor.fetch_from_queue(command_processor.compute_queues_.front(), 0); return true; } static bool retain_cursor_publication(CommandProcessor &command_processor, GpuVmAccess access, uint64_t cursor) { std::lock_guard lock(command_processor.hw_queue_mutex_); - if (command_processor.aql_queues_.size() != 1) + if (command_processor.compute_queues_.size() != 1) return false; - command_processor.aql_queues_.front().read_pointer_journal.retire(cursor, cursor, - std::move(access)); + command_processor.compute_queues_.front().read_pointer_journal.retire(cursor, cursor, + std::move(access)); return true; } static bool retry_cursor_publication(CommandProcessor &command_processor) { std::lock_guard lock(command_processor.hw_queue_mutex_); - if (command_processor.aql_queues_.size() != 1) + if (command_processor.compute_queues_.size() != 1) return false; - AqlQueueRecord &queue = command_processor.aql_queues_.front(); + ComputeQueueRecord &queue = command_processor.compute_queues_.front(); queue.faulted = true; command_processor.fetch_from_queue(queue, 0); return !queue.read_pointer_journal.publication_pending(); @@ -75,9 +76,9 @@ class CommandProcessorCloseTestAccess { static bool retain_idle_completion_publication(CommandProcessor &command_processor) { std::lock_guard lock(command_processor.hw_queue_mutex_); - if (command_processor.aql_queues_.size() != 1) + if (command_processor.compute_queues_.size() != 1) return false; - AqlQueueRecord &queue = command_processor.aql_queues_.front(); + ComputeQueueRecord &queue = command_processor.compute_queues_.front(); queue.idle_publication.phase = QueueIdlePublicationPhase::StoreMailbox; queue.publication_retry_pending = true; return true; @@ -85,9 +86,9 @@ class CommandProcessorCloseTestAccess { static bool clear_idle_completion_publication(CommandProcessor &command_processor) { std::lock_guard lock(command_processor.hw_queue_mutex_); - if (command_processor.aql_queues_.size() != 1) + if (command_processor.compute_queues_.size() != 1) return false; - AqlQueueRecord &queue = command_processor.aql_queues_.front(); + ComputeQueueRecord &queue = command_processor.compute_queues_.front(); queue.idle_publication.reset(); queue.publication_retry_pending = false; return true; @@ -95,9 +96,9 @@ class CommandProcessorCloseTestAccess { static bool mark_publication_faulted(CommandProcessor &command_processor) { std::lock_guard lock(command_processor.hw_queue_mutex_); - if (command_processor.aql_queues_.size() != 1) + if (command_processor.compute_queues_.size() != 1) return false; - command_processor.aql_queues_.front().publication_faulted = true; + command_processor.compute_queues_.front().publication_faulted = true; return true; } }; @@ -320,7 +321,7 @@ QueueRegistrationRequest queue_request(AddressSpaceHandle address_space, QueueRegistrationRequest queue_request(AddressSpaceHandle address_space, CommandProcessor &owner, uint32_t queue_id) { - return queue_request(address_space, make_aql_queue_binding_factory(owner), queue_id); + return queue_request(address_space, make_compute_queue_binding_factory(owner), queue_id); } enum class BlockedQueueOperation : uint8_t { @@ -562,7 +563,7 @@ TEST(GpuVmService, TwoQueuesRetainOneAddressSpace) { AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x2a); command_processor.set_gpu_vm(&gpu_vm); const std::shared_ptr binding_factory = - make_aql_queue_binding_factory(command_processor); + make_compute_queue_binding_factory(command_processor); QueueHandle first = queues.register_queue(queue_request(address_space, binding_factory, 1)); QueueHandle second = queues.register_queue(queue_request(address_space, binding_factory, 2)); @@ -632,14 +633,14 @@ TEST(CommandProcessorDoorbell, TransportNotificationDoesNotWaitForQueueExecution CommandProcessor command_processor("cp"); const AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x11); command_processor.set_gpu_vm(&gpu_vm); - AqlQueueConfig queue{.address_space = address_space, - .process_id = 7, - .queue_id = 1, - .ring_base_va = 0x100, - .ring_size = 4096, - .read_ptr_va = 0x80, - .write_ptr_va = 0x88, - .last_doorbell = 4}; + ComputeQueueConfig queue{.address_space = address_space, + .process_id = 7, + .queue_id = 1, + .ring_base_va = 0x100, + .ring_size = 4096, + .read_ptr_va = 0x80, + .write_ptr_va = 0x88, + .last_doorbell = 4}; const uint64_t registration = command_processor.register_queue(queue); std::promise locked; @@ -671,14 +672,14 @@ TEST(CommandProcessorDoorbell, StaleRegistrationCannotNotifyReusedQueueIdentity) CommandProcessor command_processor("cp"); const AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x11); command_processor.set_gpu_vm(&gpu_vm); - AqlQueueConfig queue{.address_space = address_space, - .process_id = 7, - .queue_id = 1, - .ring_base_va = 0x100, - .ring_size = 4096, - .read_ptr_va = 0x80, - .write_ptr_va = 0x88, - .last_doorbell = 4}; + ComputeQueueConfig queue{.address_space = address_space, + .process_id = 7, + .queue_id = 1, + .ring_base_va = 0x100, + .ring_size = 4096, + .read_ptr_va = 0x80, + .write_ptr_va = 0x88, + .last_doorbell = 4}; const uint64_t stale = command_processor.register_queue(queue); command_processor.unregister_queue(queue.queue_id, queue.process_id); @@ -799,14 +800,14 @@ TEST(CommandProcessorVmFault, FaultsEveryFanoutReplicaAndDropsPresentAndFutureWo constexpr uint32_t kProcessId = 7; constexpr uint32_t kQueueId = 41; constexpr uint32_t kDispatchId = 19; - AqlQueueConfig queue{.address_space = address_space, - .process_id = kProcessId, - .queue_id = kQueueId, - .ring_base_va = 0x100, - .ring_size = 4096, - .read_ptr_va = 0x80, - .write_ptr_va = 0x88, - .xcd_fanout = true}; + ComputeQueueConfig queue{.address_space = address_space, + .process_id = kProcessId, + .queue_id = kQueueId, + .ring_base_va = 0x100, + .ring_size = 4096, + .read_ptr_va = 0x80, + .write_ptr_va = 0x88, + .xcd_fanout = true}; (void)owner.register_queue(queue); auto grid = std::make_shared(); @@ -951,6 +952,7 @@ TEST(GpuVmService, AccessSnapshotIsRevokedAndNamespacesItsTranslationEpoch) { const std::optional old_access = gpu_vm.snapshot(address_space); ASSERT_TRUE(old_access); + EXPECT_TRUE(old_access->is_current()); const VmCacheNamespace old_namespace = old_access->cache_namespace(); EXPECT_EQ(old_namespace.address_space, address_space); @@ -958,6 +960,8 @@ TEST(GpuVmService, AccessSnapshotIsRevokedAndNamespacesItsTranslationEpoch) { ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); const std::optional new_access = gpu_vm.snapshot(address_space); ASSERT_TRUE(new_access); + EXPECT_FALSE(old_access->is_current()); + EXPECT_TRUE(new_access->is_current()); EXPECT_NE(new_access->cache_namespace(), old_namespace); EXPECT_EQ(new_access->cache_namespace().address_space, address_space); @@ -969,12 +973,293 @@ TEST(GpuVmService, AccessSnapshotIsRevokedAndNamespacesItsTranslationEpoch) { EXPECT_TRUE(gpu_vm.unregister_address_space(address_space)); EXPECT_FALSE(gpu_vm.snapshot(address_space)); + EXPECT_FALSE(new_access->is_current()); value[0] = std::byte{0x5a}; EXPECT_EQ(old_access->read(0, value), VmAccessOutcome::Unavailable); EXPECT_EQ(value[0], std::byte{0x5a}); EXPECT_EQ(new_access->read(0, value), VmAccessOutcome::Unavailable); } +TEST(GpuVmService, ExplicitInvalidationRevokesTheAccessSnapshot) { + GpuVm gpu_vm; + const auto handle = register_byte_address_space(gpu_vm, 7, 0x11); + ASSERT_TRUE(handle); + const auto old_access = gpu_vm.snapshot(handle); + ASSERT_TRUE(old_access); + ASSERT_TRUE(old_access->is_current()); + + ASSERT_TRUE(gpu_vm.invalidate(handle)); + EXPECT_FALSE(old_access->is_current()); + const auto refreshed = gpu_vm.snapshot(handle); + ASSERT_TRUE(refreshed); + EXPECT_TRUE(refreshed->is_current()); + EXPECT_NE(old_access->cache_namespace(), refreshed->cache_namespace()); + std::array value{std::byte{0x5a}}; + EXPECT_EQ(old_access->read(0, value), VmAccessOutcome::Unavailable); + EXPECT_EQ(value[0], std::byte{0x5a}); + EXPECT_EQ(refreshed->read(0, value), VmAccessOutcome::Complete); + EXPECT_EQ(value[0], std::byte{0x11}); +} + +TEST(GpuVmService, PolicyCacheDoesNotRetainSnapshotBacking) { + GpuVm vm; + auto backing = std::make_shared(0x11); + std::weak_ptr weak_backing = backing; + const auto handle = vm.register_translated(77, backing, backing); + ASSERT_TRUE(handle); + auto snapshot = vm.snapshot(handle); + ASSERT_TRUE(snapshot); + VmMtypeCache cache; + EXPECT_EQ(snapshot->query_mtype(0, cache), Mtype::RW); + ASSERT_TRUE(vm.unregister_address_space(handle)); + EXPECT_FALSE(snapshot->is_current()); + EXPECT_FALSE(snapshot->query_mtype(0, cache)); + backing.reset(); + EXPECT_FALSE(weak_backing.expired()); + snapshot.reset(); + EXPECT_TRUE(weak_backing.expired()); +} + +TEST(GpuVmService, PolicyCacheReusesInterleavedPagesAndRevalidatesEveryEntry) { + class PagePolicy final : public AddressSpaceTranslator { + public: + VmTranslationResult translate(uint64_t, std::size_t, VmAccessKind) const override { return {}; } + + VmMtypeSnapshot snapshot_mtype(uint64_t address) const override { + ++queries; + return {.mtype = mtype, + .begin = address & ~uint64_t{4095}, + .size = 4096, + .mutation_epoch = epoch, + .captured_epoch = epoch->load()}; + } + + std::shared_ptr> epoch = std::make_shared>(0); + Mtype mtype = Mtype::RW; + mutable uint32_t queries = 0; + }; + + GpuVm vm; + auto policy = std::make_shared(); + auto backing = std::make_shared(0); + const auto handle = vm.register_translated(77, policy, backing); + auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + VmMtypeCache cache; + for (uint32_t repeat = 0; repeat < 3; ++repeat) + for (uint64_t page = 0; page < 8; ++page) + EXPECT_EQ(access->query_mtype(page * 4096, cache), Mtype::RW); + EXPECT_EQ(policy->queries, 8u); + + policy->epoch->fetch_add(1); + policy->mtype = Mtype::UC; + for (uint64_t page = 0; page < 8; ++page) + EXPECT_EQ(access->query_mtype(page * 4096, cache), Mtype::UC); + EXPECT_EQ(policy->queries, 16u); + + // A different VM generation may reuse the same epoch token. Entries from + // every page must be invalidated even when the token itself is unchanged. + auto replacement = std::make_shared(); + replacement->epoch = policy->epoch; + ASSERT_TRUE(vm.replace_translated(handle, replacement, backing)); + auto new_access = vm.snapshot(handle); + ASSERT_TRUE(new_access); + for (uint64_t page = 0; page < 8; ++page) { + EXPECT_FALSE(access->query_mtype(page * 4096, cache)); + EXPECT_EQ(new_access->query_mtype(page * 4096, cache), Mtype::RW); + } + EXPECT_EQ(replacement->queries, 8u); +} + +TEST(GpuVmService, GenerationsShareFaultReporterWithoutCopyingItsTarget) { + class Reporter { + public: + Reporter(uint32_t &copies, uint32_t &calls) : copies_(copies), calls_(calls) {} + Reporter(const Reporter &other) : copies_(other.copies_), calls_(other.calls_) { ++copies_; } + void operator()(uint64_t, VmAccessKind) const { ++calls_; } + + private: + uint32_t &copies_; + uint32_t &calls_; + }; + + uint32_t copies = 0, calls = 0; + GpuVm vm; + auto backing = std::make_shared(0x11); + const AddressSpaceHandle handle = + vm.register_address_space(7, backing, backing, Reporter(copies, calls)); + ASSERT_TRUE(handle); + copies = 0; + auto fault = [](const std::optional &access) { + ASSERT_TRUE(access); + EXPECT_EQ(access->probe(4096, 1, VmAccessKind::Read), VmAccessOutcome::Faulted); + }; + fault(vm.snapshot(handle)); + auto pinned = vm.snapshot_pinned(handle); + fault(pinned); + EXPECT_TRUE(vm.invalidate(handle)); + fault(vm.snapshot(handle)); + EXPECT_TRUE(vm.replace_translated(handle, backing, backing)); + fault(vm.snapshot(handle)); + EXPECT_EQ(calls, 4u); + // A callback can be active in another snapshot while a new generation is + // created. Copying its possibly mutable target would race that invocation. + EXPECT_EQ(copies, 0u); +} + +TEST(GpuVmService, FaultReporterSerializesDistinctSnapshotsAndGenerations) { + class Reporter { + public: + Reporter(std::atomic &overlapped, std::atomic &calls) + : overlapped_(overlapped), calls_(calls) {} + Reporter(const Reporter &other) : overlapped_(other.overlapped_), calls_(other.calls_) {} + + void operator()(uint64_t, VmAccessKind) { + if (active_.fetch_add(1) != 0) + overlapped_ = true; + std::this_thread::yield(); + ++calls_; + --active_; + } + + private: + // This state belongs to the callable itself, unlike the shared observations. + // Copying a callable gives each snapshot its own state; sharing it requires + // serialization before invoking a potentially mutable target. + std::atomic active_{0}; + std::atomic &overlapped_; + std::atomic &calls_; + }; + + for (bool routed : {false, true}) { + SCOPED_TRACE(routed); + GpuVm vm; + auto backing = std::make_shared(0x11); + std::atomic overlapped{false}; + std::atomic calls{0}; + const auto handle = + routed + ? vm.register_address_space(7, backing, backing, Reporter(overlapped, calls)) + : vm.register_unrouted_address_space(7, backing, backing, Reporter(overlapped, calls)); + ASSERT_TRUE(handle); + auto first = vm.snapshot_pinned(handle); + auto second = vm.snapshot_pinned(handle); + ASSERT_TRUE(vm.replace_translated(handle, backing, backing)); + auto third = vm.snapshot(handle); + auto fourth = vm.snapshot(handle); + ASSERT_TRUE(first && second && third && fourth); + std::array snapshots{*first, *second, *third, *fourth}; + constexpr unsigned kIterations = 128; + std::barrier start(snapshots.size()); + std::vector threads; + for (unsigned index = 0; index < snapshots.size(); ++index) { + threads.emplace_back([&, index] { + std::array bytes{}; + for (unsigned iteration = 0; iteration < kIterations; ++iteration) { + start.arrive_and_wait(); + const auto &snapshot = snapshots[index]; + if (index == 0) + EXPECT_EQ(snapshot.probe(4096, 1, VmAccessKind::Read), VmAccessOutcome::Faulted); + else if (index == 1) + EXPECT_EQ(snapshot.read(4096, bytes), VmAccessOutcome::Faulted); + else if (index == 2) + EXPECT_EQ(snapshot.write(4096, bytes), VmAccessOutcome::Faulted); + else + EXPECT_EQ(snapshot.atomic_load(0, 3).outcome, VmAccessOutcome::Malformed); + } + }); + } + threads.clear(); + EXPECT_FALSE(overlapped); + EXPECT_EQ(calls, snapshots.size() * kIterations); + } +} + +TEST(GpuVmService, FaultReporterAllowsNestedFaultsAndReleasesAfterException) { + GpuVm vm; + auto backing = std::make_shared(0x11); + AddressSpaceHandle handle; + unsigned calls = 0; + handle = vm.register_address_space(7, backing, backing, [&](uint64_t address, VmAccessKind) { + ++calls; + if (address == 4096) { + auto nested = vm.snapshot_pinned(handle); + ASSERT_TRUE(nested); + EXPECT_EQ(nested->probe(4097, 1, VmAccessKind::Read), VmAccessOutcome::Faulted); + } + if (address == 4098) + throw std::runtime_error("fault reporter"); + }); + auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + EXPECT_EQ(access->probe(4096, 1, VmAccessKind::Read), VmAccessOutcome::Faulted); + EXPECT_EQ(calls, 2u); + EXPECT_THROW((void)access->probe(4098, 1, VmAccessKind::Read), std::runtime_error); + EXPECT_EQ(calls, 3u); + // A different thread must be able to enter after the throwing callback. + std::jthread reader( + [&] { EXPECT_EQ(access->probe(4097, 1, VmAccessKind::Read), VmAccessOutcome::Faulted); }); + reader.join(); + EXPECT_EQ(calls, 4u); +} + +TEST(GpuVmService, ConcurrentSnapshotsPreserveRootEpochAndRetirement) { + GpuVm gpu_vm; + const auto handle = register_byte_address_space(gpu_vm, 7, 1); + ASSERT_TRUE(handle); + constexpr unsigned kReaders = 8; + std::barrier phase(kReaders + 1); + std::array, kReaders> pinned; + std::vector readers; + for (unsigned i = 0; i < kReaders; ++i) { + readers.emplace_back([&, i] { + for (unsigned epoch = 2; epoch <= 65; ++epoch) { + phase.arrive_and_wait(); + // These reads race with root replacement. Each returned snapshot must + // pair its captured epoch with the corresponding immutable backing. + auto access = i % 2 ? gpu_vm.snapshot(handle) : gpu_vm.snapshot_vmid(7); + EXPECT_TRUE(access); + if (access) { + EXPECT_EQ(access->cache_namespace().address_space, handle); + std::array value{}; + const auto outcome = access->read(0, value); + if (outcome == VmAccessOutcome::Complete) + EXPECT_EQ(value[0], std::byte(access->info().translation_epoch & 255)); + else + EXPECT_EQ(outcome, VmAccessOutcome::Unavailable); + } + EXPECT_EQ(gpu_vm.find_vmid(7), handle); + EXPECT_TRUE(gpu_vm.lookup(handle)->ready); + EXPECT_EQ(gpu_vm.active_address_spaces(), 1u); + // Pinned transactions survive root replacement, until explicit retirement. + pinned[i] = gpu_vm.snapshot_pinned(handle); + EXPECT_TRUE(pinned[i]); + if (pinned[i]) { + std::array value{}; + EXPECT_EQ(pinned[i]->read(0, value), VmAccessOutcome::Complete); + EXPECT_EQ(value[0], std::byte(pinned[i]->info().translation_epoch & 255)); + } + phase.arrive_and_wait(); + } + }); + } + for (unsigned epoch = 2; epoch <= 65; ++epoch) { + auto replacement = std::make_shared(epoch); + phase.arrive_and_wait(); + EXPECT_TRUE(gpu_vm.replace_translated(handle, replacement, replacement)); + phase.arrive_and_wait(); + } + readers.clear(); + EXPECT_TRUE(gpu_vm.unregister_address_space(handle)); + EXPECT_FALSE(gpu_vm.snapshot_vmid(7)); + for (const auto &access : pinned) { + ASSERT_TRUE(access); + std::array value{std::byte{0xff}}; + EXPECT_EQ(access->read(0, value), VmAccessOutcome::Unavailable); + EXPECT_EQ(value[0], std::byte{0xff}); + } +} + TEST(GpuVmService, ClearingGartPreservesItsIdentityAndUnrelatedAddressSpaces) { GpuMemory memory("memory"); GpuVm gpu_vm; @@ -1274,7 +1559,7 @@ TEST(GpuVmService, FailedSecondAqlBindingCreationLeavesTheOriginalQueueUsable) { AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x11); command_processor.set_gpu_vm(&gpu_vm); const std::shared_ptr binding_factory = - make_aql_queue_binding_factory(command_processor); + make_compute_queue_binding_factory(command_processor); const QueueRegistrationRequest request = queue_request(address_space, binding_factory, 1); const QueueHandle original = queues.register_queue(request); @@ -1299,7 +1584,7 @@ TEST(GpuVmService, AqlBindingFactoryRejectsInvalidRingLayoutsBeforeRegistration) CommandProcessor command_processor("cp"); const AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x11); const std::shared_ptr binding_factory = - make_aql_queue_binding_factory(command_processor); + make_compute_queue_binding_factory(command_processor); const QueueRingLayout valid = queue_request(address_space, binding_factory, 1).ring; std::array invalid{}; @@ -1335,7 +1620,7 @@ TEST(GpuVmService, ExplicitDoorbellAqlBindingPreservesKfdQueueAbi) { command_processor.set_gpu_vm(&gpu_vm); const AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x11); QueueRegistrationRequest request = - queue_request(address_space, make_aql_queue_binding_factory(command_processor), 1); + queue_request(address_space, make_compute_queue_binding_factory(command_processor), 1); request.doorbell = {.mode = QueueDoorbellMode::Explicit, .offset = 0x40}; request.abi = QueueAbi::KfdAql; @@ -1353,15 +1638,15 @@ TEST(CommandProcessorQueueRegistration, RejectsInvalidAqlRingLayoutsAtDirectBoun CommandProcessor command_processor("cp"); const AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x11); command_processor.set_gpu_vm(&gpu_vm); - AqlQueueConfig valid{.address_space = address_space, - .process_id = 7, - .queue_id = 1, - .ring_base_va = 0x100, - .ring_size = 64, - .read_ptr_va = 0x80, - .write_ptr_va = 0x88}; - - std::array invalid{}; + ComputeQueueConfig valid{.address_space = address_space, + .process_id = 7, + .queue_id = 1, + .ring_base_va = 0x100, + .ring_size = 64, + .read_ptr_va = 0x80, + .write_ptr_va = 0x88}; + + std::array invalid{}; invalid.fill(valid); invalid[0].ring_base_va = 0; invalid[1].ring_base_va += 1; @@ -1374,7 +1659,7 @@ TEST(CommandProcessorQueueRegistration, RejectsInvalidAqlRingLayoutsAtDirectBoun invalid[8].write_ptr_va += 1; invalid[9].doorbell_mode = QueueDoorbellMode::VmPolled; - for (AqlQueueConfig &config : invalid) + for (ComputeQueueConfig &config : invalid) EXPECT_EQ(command_processor.register_queue(std::move(config)), 0u); EXPECT_EQ(command_processor.registered_queue_count_for_test(), 0u); @@ -1390,13 +1675,13 @@ TEST(CommandProcessorQueueRegistration, RejectsInvalidAqlRingLayoutsAtDirectBoun } TEST(CommandProcessorQueueRegistration, RequiresAnAddressSpaceAndUsesTheConfiguredInternalOne) { - AqlQueueConfig queue{.address_space = {}, - .process_id = 0, - .queue_id = 1, - .ring_base_va = 0x100, - .ring_size = 64, - .read_ptr_va = 0x80, - .write_ptr_va = 0x88}; + ComputeQueueConfig queue{.address_space = {}, + .process_id = 0, + .queue_id = 1, + .ring_base_va = 0x100, + .ring_size = 64, + .read_ptr_va = 0x80, + .write_ptr_va = 0x88}; CommandProcessor unbound("unbound"); EXPECT_EQ(unbound.register_queue(queue), 0u); @@ -1505,7 +1790,7 @@ TEST(GpuVmService, AqlReconfigureRejectsInvalidRingGeometry) { const AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x11); command_processor.set_gpu_vm(&gpu_vm); const std::shared_ptr binding_factory = - make_aql_queue_binding_factory(command_processor); + make_compute_queue_binding_factory(command_processor); const QueueHandle queue = queues.register_queue(queue_request(address_space, binding_factory, 1)); ASSERT_TRUE(queue); @@ -1616,7 +1901,7 @@ TEST(GpuVmService, StaleAqlBindingCannotChangeOrRemoveReusedQueueIdentity) { const AddressSpaceHandle address_space = register_byte_address_space(gpu_vm, 7, 0x11); command_processor.set_gpu_vm(&gpu_vm); const std::shared_ptr binding_factory = - make_aql_queue_binding_factory(command_processor); + make_compute_queue_binding_factory(command_processor); const QueueRegistrationRequest request = queue_request(address_space, binding_factory, 1); QueueBindingCreateResult stale = binding_factory->create_binding(request); @@ -1784,7 +2069,7 @@ TEST(CommandProcessorQueueRegistration, FailedFanoutRollsBackOnlyNewReplicas) { rejecting_peer.set_gpu_vm(&gpu_vm); owner.set_xcd_topology(0, {&owner, &first_peer, &rejecting_peer}); - AqlQueueConfig existing{}; + ComputeQueueConfig existing{}; existing.address_space = address_space; existing.process_id = 7; existing.queue_id = 11; @@ -1794,7 +2079,7 @@ TEST(CommandProcessorQueueRegistration, FailedFanoutRollsBackOnlyNewReplicas) { existing.write_ptr_va = 0x88; rejecting_peer.register_queue(existing); - AqlQueueConfig fanout = existing; + ComputeQueueConfig fanout = existing; fanout.xcd_fanout = true; EXPECT_EQ(owner.register_queue(fanout), 0u); diff --git a/emulation/rocjitsu/tests/gpu_vm_translation_test.cpp b/emulation/rocjitsu/tests/gpu_vm_translation_test.cpp index 89e6a87b7fad..0dd2e664105c 100644 --- a/emulation/rocjitsu/tests/gpu_vm_translation_test.cpp +++ b/emulation/rocjitsu/tests/gpu_vm_translation_test.cpp @@ -1,6 +1,7 @@ // Copyright (c) 2026 Advanced Micro Devices, Inc. // SPDX-License-Identifier: MIT +#include "legacy_gpu_memory_fixture.h" #include "rocjitsu/kmd/linux/kfd_process.h" #include "rocjitsu/kmd/linux/legacy_gpu_vm.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" @@ -10,6 +11,7 @@ #include #include +#include #include #include #include @@ -21,12 +23,28 @@ #include #include #include +#include +#include #include #include +#if defined(__SANITIZE_ADDRESS__) +#define RJ_GPU_VM_TRANSLATION_TEST_WITH_ASAN 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) +#define RJ_GPU_VM_TRANSLATION_TEST_WITH_ASAN 1 +#endif +#endif + namespace rocjitsu::amdgpu { namespace { +#if defined(RJ_GPU_VM_TRANSLATION_TEST_WITH_ASAN) +constexpr bool kSanitizedVmMetadata = true; +#else +constexpr bool kSanitizedVmMetadata = false; +#endif + class TestPhysicalMemory final : public PhysicalMemoryAccess { public: VmAccessOutcome read(VmMemoryDomain domain, uint64_t address, @@ -129,24 +147,386 @@ class RecordingFaultReporter final : public MemoryFaultReporter { class HostPage { public: - HostPage() { - void *mapping = mmap(nullptr, KfdProcess::kPageSize, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + explicit HostPage(size_t size = KfdProcess::kPageSize) : size_(size) { + void *mapping = + mmap(nullptr, size_, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); data_ = mapping == MAP_FAILED ? nullptr : static_cast(mapping); } HostPage(const HostPage &) = delete; HostPage &operator=(const HostPage &) = delete; ~HostPage() { if (data_ != nullptr) - munmap(data_, KfdProcess::kPageSize); + munmap(data_, size_); } uint8_t *data() const { return data_; } private: + size_t size_; uint8_t *data_ = nullptr; }; +AddressSpaceHandle register_step_process(LegacyGpuVmAdapter &adapter, KfdProcess &process, + MemoryFaultReporter *reporter = nullptr) { + return adapter.register_address_space(7, + {.page_table = &process.page_table_, + .page_table_mutex = &process.page_table_mutex_, + .page_table_generation = process.page_table_generation(), + .request_mutex = process.page_table_request_mutex(), + .mutation_epoch = process.page_table_mutation_epoch(), + .page_table_cache_state = process.page_table_cache_state(), + .fault_reporter = reporter}); +} + +TEST(GpuVmTranslation, DefaultStepsPreserveCustomTransportProgressAndFaultOrigin) { + class Translator final : public AddressSpaceTranslator { + public: + VmTranslationResult translate(uint64_t address, size_t, VmAccessKind) const override { + addresses.push_back(address); + if (address == fault_address) + return {.outcome = VmAccessOutcome::Faulted, .translation = {}}; + return {.outcome = VmAccessOutcome::Complete, + .translation = {.domain = VmMemoryDomain::System, + .address = address + 0x100, + .contiguous_bytes = 2, + .permissions = {.readable = true, .writable = true}}}; + } + mutable std::vector addresses; + uint64_t fault_address = 0x104; + }; + GpuVm vm; + auto translator = std::make_shared(); + auto transport = std::make_shared(); + std::vector faults; + const auto handle = vm.register_address_space( + 7, translator, transport, [&](uint64_t address, VmAccessKind) { faults.push_back(address); }); + ASSERT_TRUE(handle); + const auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + std::array bytes; + bytes.fill(std::byte{0x55}); + ASSERT_EQ(transport->write(VmMemoryDomain::System, 0x200, bytes), VmAccessOutcome::Complete); + bytes.fill(std::byte{0xaa}); + size_t completed = 0; + EXPECT_EQ(access->read(0x100, bytes, completed), VmAccessOutcome::Faulted); + EXPECT_EQ(completed, 4u); + EXPECT_EQ(translator->addresses, (std::vector{0x100, 0x102, 0x104})); + EXPECT_EQ(faults, (std::vector{0x104})); + EXPECT_EQ(transport->read_calls, 2u); + EXPECT_EQ(bytes, (std::array{std::byte{0x55}, std::byte{0x55}, std::byte{0x55}, + std::byte{0x55}, std::byte{0xaa}, std::byte{0xaa}})); + translator->fault_address = 0; + translator->addresses.clear(); + faults.clear(); + transport->system.erase(0x204); + EXPECT_EQ(access->read(0x100, bytes, completed), VmAccessOutcome::Unavailable); + EXPECT_EQ(completed, 4u); + EXPECT_TRUE(faults.empty()); + EXPECT_EQ(translator->addresses, (std::vector{0x104})); + EXPECT_EQ(bytes[4], std::byte{0xaa}); + EXPECT_EQ(bytes[5], std::byte{0xaa}); + transport->system[0x204] = std::byte{0x66}; + EXPECT_EQ(access->read(0x100, bytes, completed), VmAccessOutcome::Complete); + EXPECT_EQ(completed, bytes.size()); + EXPECT_EQ(bytes[4], std::byte{0x66}); + EXPECT_EQ(bytes[5], std::byte{0x55}); + + translator->fault_address = 0x104; + translator->addresses.clear(); + bytes.fill(std::byte{0x99}); + completed = 0; + EXPECT_EQ(access->write(0x100, bytes, completed), VmAccessOutcome::Faulted); + EXPECT_EQ(completed, 4u); + EXPECT_EQ(faults, (std::vector{0x104})); + EXPECT_EQ(transport->system[0x203], std::byte{0x99}); + EXPECT_EQ(transport->system[0x204], std::byte{0x66}); +} + +TEST(GpuVmTranslation, LegacyStepsKeepExtentBoundariesAndRejectOverlappingCoverage) { + for (const auto owner : {LegacyHostExtentOwner::Driver, LegacyHostExtentOwner::Application}) { + for (const auto second_offset : {8u, 12u}) { + for (const auto first_size : {8u, 16u}) { + SCOPED_TRACE(static_cast(owner)); + SCOPED_TRACE(second_offset); + SCOPED_TRACE(first_size); + GpuMemory memory("memory"); + KfdProcess process(7); + HostPage backing; + ASSERT_NE(backing.data(), nullptr); + std::memset(backing.data(), 0x55, 32); + LegacyPageTableEntry pte; + pte.host_extents = {{backing.data(), first_size, 0, owner}, + {backing.data() + 16, 8, second_offset, owner}}; + process.page_table_[4] = std::move(pte); + RecordingFaultReporter reporter; + GpuVm vm; + LegacyGpuVmAdapter adapter(vm, &memory); + const auto handle = register_step_process(adapter, process, &reporter); + ASSERT_TRUE(handle); + const auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + std::array bytes; + bytes.fill(std::byte{0xaa}); + size_t completed = 0; + EXPECT_EQ(access->read(0x4000, bytes, completed), VmAccessOutcome::Faulted); + EXPECT_EQ(completed, 0u); + EXPECT_TRUE(std::ranges::all_of(bytes, [](auto byte) { return byte == std::byte{0xaa}; })); + EXPECT_EQ(access->write(0x4000, bytes, completed), VmAccessOutcome::Faulted); + EXPECT_EQ(completed, 0u); + EXPECT_TRUE(std::ranges::all_of(std::span(backing.data(), 32), + [](auto byte) { return byte == 0x55; })); + EXPECT_EQ(reporter.addresses, (std::vector{0x4000, 0x4000})); + } + } + } +} + +TEST(GpuVmTranslation, LegacyStepsKeepPageProgressAndResumeWithoutReplayingPrefix) { + for (const auto owner : {LegacyHostExtentOwner::Driver, LegacyHostExtentOwner::Application}) { + GpuMemory memory("memory"); + KfdProcess process(7); + HostPage first, second; + ASSERT_NE(first.data(), nullptr); + ASSERT_NE(second.data(), nullptr); + std::memset(first.data(), 0x55, KfdProcess::kPageSize); + std::memset(second.data(), 0x66, KfdProcess::kPageSize); + process.map_pages(0x4000, first.data(), KfdProcess::kPageSize, Mtype::RW, owner); + GpuVm vm; + LegacyGpuVmAdapter adapter(vm, &memory); + const auto handle = register_step_process(adapter, process); + ASSERT_TRUE(handle); + const auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + std::array bytes; + bytes.fill(std::byte{0xaa}); + size_t completed = 0; + EXPECT_EQ(access->read(0x4ff0, bytes, completed), VmAccessOutcome::Unavailable); + EXPECT_EQ(completed, 16u); + for (size_t i = 0; i < bytes.size(); ++i) + EXPECT_EQ(bytes[i], i < 16 ? std::byte{0x55} : std::byte{0xaa}); + std::memset(first.data() + KfdProcess::kPageSize - 16, 0x77, 16); + process.map_pages(0x5000, second.data(), KfdProcess::kPageSize, Mtype::CC, owner); + EXPECT_EQ(access->read(0x4ff0, bytes, completed), VmAccessOutcome::Complete); + EXPECT_EQ(completed, bytes.size()); + EXPECT_EQ(bytes[0], std::byte{0x55}); + EXPECT_EQ(bytes[16], std::byte{0x66}); + process.unmap_pages(0x5000, KfdProcess::kPageSize); + bytes.fill(std::byte{0x88}); + completed = 0; + EXPECT_EQ(access->write(0x4ff0, bytes, completed), VmAccessOutcome::Unavailable); + EXPECT_EQ(completed, 16u); + std::memset(first.data() + KfdProcess::kPageSize - 16, 0x99, 16); + process.map_pages(0x5000, second.data(), KfdProcess::kPageSize, Mtype::CC, owner); + EXPECT_EQ(access->write(0x4ff0, bytes, completed), VmAccessOutcome::Complete); + EXPECT_EQ(first.data()[KfdProcess::kPageSize - 1], 0x99); + EXPECT_EQ(second.data()[0], 0x88); + } +} + +TEST(GpuVmTranslation, LegacyApplicationStepPrevalidatesCrossHostPageWrites) { + GpuMemory memory("memory"); + KfdProcess process(7); + HostPage backing(2 * KfdProcess::kPageSize); + ASSERT_NE(backing.data(), nullptr); + auto *first = backing.data() + KfdProcess::kPageSize - 16; + std::memset(first, 0x55, 32); + process.map_pages(0x4000, first, 32, Mtype::RW, LegacyHostExtentOwner::Application); + RecordingFaultReporter reporter; + GpuVm vm; + LegacyGpuVmAdapter adapter(vm, &memory); + const auto handle = register_step_process(adapter, process, &reporter); + ASSERT_TRUE(handle); + const auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + ASSERT_EQ(mprotect(backing.data() + KfdProcess::kPageSize, KfdProcess::kPageSize, PROT_READ), 0); + std::array bytes; + bytes.fill(std::byte{0xaa}); + size_t completed = 0; + EXPECT_EQ(access->write(0x4000, bytes, completed), VmAccessOutcome::Faulted); + EXPECT_EQ(completed, 0u); + EXPECT_TRUE(std::ranges::all_of(std::span(first, 32), [](auto byte) { return byte == 0x55; })); + EXPECT_EQ(reporter.addresses, (std::vector{0x4000})); + ASSERT_EQ(mprotect(backing.data() + KfdProcess::kPageSize, KfdProcess::kPageSize, PROT_NONE), 0); + EXPECT_EQ(access->read(0x4000, bytes, completed), VmAccessOutcome::Faulted); + EXPECT_EQ(completed, 0u); + EXPECT_TRUE(std::ranges::all_of(bytes, [](auto byte) { return byte == std::byte{0xaa}; })); + EXPECT_EQ(reporter.addresses, (std::vector{0x4000, 0x4000})); + ASSERT_EQ(mprotect(backing.data() + KfdProcess::kPageSize, KfdProcess::kPageSize, + PROT_READ | PROT_WRITE), + 0); +} + +TEST(GpuVmTranslation, LegacyStepsPreserveClientBackingAndTerminalFaults) { + GpuMemory memory("memory"); + KfdProcess process(7); + HostPage backing; + ASSERT_NE(backing.data(), nullptr); + backing.data()[0] = 0x55; + RecordingFaultReporter reporter; + GpuVm vm; + LegacyGpuVmAdapter adapter(vm, &memory); + const auto handle = register_step_process(adapter, process, &reporter); + ASSERT_TRUE(handle); + ASSERT_TRUE(adapter.set_client_pid(handle, getpid())); + const auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + const auto address = reinterpret_cast(backing.data()); + std::array bytes{std::byte{0xaa}}; + EXPECT_EQ(access->read(address, bytes), VmAccessOutcome::Complete); + EXPECT_EQ(bytes[0], std::byte{0x55}); + bytes[0] = std::byte{0x66}; + EXPECT_EQ(access->write(address, bytes), VmAccessOutcome::Complete); + EXPECT_EQ(backing.data()[0], 0x66); + EXPECT_TRUE(reporter.addresses.empty()); + ASSERT_EQ(mprotect(backing.data(), KfdProcess::kPageSize, PROT_NONE), 0); + bytes[0] = std::byte{0xaa}; + EXPECT_EQ(access->read(address, bytes), VmAccessOutcome::Faulted); + EXPECT_EQ(bytes[0], std::byte{0xaa}); + EXPECT_EQ(access->write(address, bytes), VmAccessOutcome::Faulted); + EXPECT_EQ(reporter.addresses, (std::vector{address, address})); + ASSERT_EQ(mprotect(backing.data(), KfdProcess::kPageSize, PROT_READ | PROT_WRITE), 0); + EXPECT_EQ(backing.data()[0], 0x66); +} + +TEST(GpuVmTranslation, LegacyStepFaultCallbacksCanMutatePolicyAndHostMappings) { + EXPECT_EXIT( + ([] { + alarm(10); + for (const auto owner : + {LegacyHostExtentOwner::Driver, LegacyHostExtentOwner::Application}) { + GpuMemory memory("memory"); + KfdProcess process(7); + HostPage backing; + if (!backing.data()) + _exit(1); + backing.data()[0] = 0x55; + process.map_pages(0x4000, backing.data(), KfdProcess::kPageSize, Mtype::RW, owner); + class Reporter final : public MemoryFaultReporter { + public: + explicit Reporter(KfdProcess &process) : process(process) {} + void report_memory_fault(uint32_t, uint64_t address, MemoryFaultCause) override { + if (address != 0x4000) + _exit(2); + ++calls; + process.set_page_mtype(0x4000, KfdProcess::kPageSize, Mtype::CC); + const auto mapping = rocjitsu::host_mapping_lock().lock_exclusive(); + } + KfdProcess &process; + unsigned calls = 0; + } reporter(process); + GpuVm vm; + LegacyGpuVmAdapter adapter(vm, &memory); + const auto handle = register_step_process(adapter, process, &reporter); + const auto access = vm.snapshot(handle); + std::array bytes{std::byte{0xaa}}; + if (!access || access->read(0x4000, bytes) != VmAccessOutcome::Complete) + _exit(3); + if (mprotect(backing.data(), KfdProcess::kPageSize, PROT_NONE) != 0) + _exit(4); + bytes[0] = std::byte{0xaa}; + if (access->read(0x4000, bytes) != VmAccessOutcome::Faulted || reporter.calls != 1 || + bytes[0] != std::byte{0xaa}) + _exit(5); + if (access->write(0x4000, bytes) != VmAccessOutcome::Faulted || reporter.calls != 2) + _exit(6); + if (mprotect(backing.data(), KfdProcess::kPageSize, PROT_READ | PROT_WRITE) != 0 || + backing.data()[0] != 0x55) + _exit(7); + } + _exit(0); + }()), + ::testing::ExitedWithCode(0), ""); +} + +TEST(GpuVmTranslation, LegacyStepKeepsOneMappedSpanStableDuringConcurrentRemap) { + EXPECT_EXIT( + ([] { + alarm(10); + for (const auto owner : + {LegacyHostExtentOwner::Driver, LegacyHostExtentOwner::Application}) { + GpuMemory memory("memory"); + KfdProcess process(7); + HostPage first, second; + if (!first.data() || !second.data()) + _exit(1); + std::memset(first.data(), 0x55, KfdProcess::kPageSize); + std::memset(second.data(), 0xaa, KfdProcess::kPageSize); + process.map_pages(0x4000, first.data(), KfdProcess::kPageSize, Mtype::RW, owner); + GpuVm vm; + LegacyGpuVmAdapter adapter(vm, &memory); + const auto handle = register_step_process(adapter, process); + const auto access = vm.snapshot(handle); + if (!access) + _exit(2); + std::atomic started{false}; + std::thread writer([&] { + while (!started.load(std::memory_order_acquire)) + std::this_thread::yield(); + for (unsigned i = 0; i < 2000; ++i) + process.map_pages(0x4000, i % 2 ? first.data() : second.data(), KfdProcess::kPageSize, + i % 2 ? Mtype::RW : Mtype::CC, owner); + }); + started.store(true, std::memory_order_release); + for (unsigned i = 0; i < 2000; ++i) { + std::array bytes; + bytes.fill(std::byte{0xcc}); + const auto outcome = access->read(0x4000, bytes); + // Continuous remapping may exhaust the bounded metadata + // retry budget. That existing fail-closed result must not + // expose any partial read. + if (kSanitizedVmMetadata && outcome == VmAccessOutcome::Faulted && + std::ranges::all_of(bytes, [](auto byte) { return byte == std::byte{0xcc}; })) + continue; + if (outcome != VmAccessOutcome::Complete || + (bytes[0] != std::byte{0x55} && bytes[0] != std::byte{0xaa}) || + !std::ranges::all_of(bytes, [&](auto byte) { return byte == bytes[0]; })) + _exit(3); + } + writer.join(); + process.unmap_pages(0x4000, KfdProcess::kPageSize); + std::array bytes{std::byte{0xcc}}; + if (access->read(0x4000, bytes) != VmAccessOutcome::Unavailable || + bytes[0] != std::byte{0xcc}) + _exit(4); + } + _exit(0); + }()), + ::testing::ExitedWithCode(0), ""); +} + +#if defined(RJ_GPU_VM_TRANSLATION_TEST_WITH_ASAN) +TEST(GpuVmTranslation, LegacyStepRevalidatesAfterUnlockedAllocatorMetadataQuery) { + GpuMemory memory("memory"); + KfdProcess process(7); + HostPage first, second; + ASSERT_NE(first.data(), nullptr); + ASSERT_NE(second.data(), nullptr); + first.data()[0] = 0x55; + second.data()[0] = 0xaa; + process.map_pages(0x4000, first.data(), KfdProcess::kPageSize); + GpuVm vm; + LegacyGpuVmAdapter adapter(vm, &memory); + const auto handle = register_step_process(adapter, process); + ASSERT_TRUE(handle); + const auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + unsigned calls = 0; + std::function remap = [&] { + if (++calls == 1) + process.map_pages(0x4000, second.data(), KfdProcess::kPageSize); + }; + auto *address_space = adapter.address_space(7); + ASSERT_NE(address_space, nullptr); + LegacyAddressSpaceTestAccess::set_page_table_unlocked_hook(*address_space, &remap); + std::array bytes{}; + const auto outcome = access->read(0x4000, bytes); + LegacyAddressSpaceTestAccess::set_page_table_unlocked_hook(*address_space, nullptr); + EXPECT_EQ(outcome, VmAccessOutcome::Complete); + EXPECT_GE(calls, 2u); + EXPECT_EQ(bytes[0], std::byte{0xaa}); +} +#endif + constexpr uint64_t kValid = uint64_t{1} << 0; constexpr uint64_t kSystem = uint64_t{1} << 1; constexpr uint64_t kSnooped = uint64_t{1} << 2; @@ -683,6 +1063,255 @@ TEST(GpuVmTranslation, InvalidationDrainsAnInFlightAccessAndRevokesOldSnapshots) EXPECT_EQ(current_access->read(0, stale_value), VmAccessOutcome::Complete); } +TEST(GpuVmTranslation, AccessBatchRefreshesRevokedSnapshots) { + // A callback between operations must be able to invalidate the binding while + // its quantum cache remains alive. Keep this regression bounded. + const pid_t child = fork(); + ASSERT_GE(child, 0); + if (child == 0) { + alarm(10); + GpuVm gpu_vm; + auto physical = std::make_shared(); + auto translator = std::make_shared(); + physical->system[0] = std::byte{0x2a}; + const AddressSpaceHandle handle = gpu_vm.register_translated(7, translator, physical); + if (!handle) + _exit(1); + + const GpuVmAccessBatchGuard batch; + const GpuVmAccess *first = gpu_vm.borrow_snapshot_vmid(7); + if (first == nullptr) + _exit(2); + std::array value{}; + if (first->read(0, value) != VmAccessOutcome::Complete || value[0] != std::byte{0x2a}) + _exit(3); + auto invalidation = std::async(std::launch::async, [&] { return gpu_vm.invalidate(handle); }); + if (!invalidation.get()) + _exit(4); + + const GpuVmAccess *repeated = gpu_vm.borrow_snapshot_vmid(7); + if (!repeated || repeated == first || repeated->cache_namespace() == first->cache_namespace() || + repeated->read(0, value) != VmAccessOutcome::Complete || value[0] != std::byte{0x2a}) + _exit(5); + // The old pointer stays alive but loses authority to access backing. + value[0] = std::byte{0x5a}; + if (first->read(0, value) != VmAccessOutcome::Unavailable || value[0] != std::byte{0x5a}) + _exit(6); + if (!gpu_vm.invalidate(handle)) + _exit(7); + const GpuVmAccess *by_handle = gpu_vm.borrow_snapshot(handle); + if (!by_handle || by_handle == repeated || !by_handle->is_current() || repeated->is_current()) + _exit(8); + _exit(0); + } + + int status = 0; + ASSERT_EQ(waitpid(child, &status, 0), child); + ASSERT_TRUE(WIFEXITED(status)) << "child status: " << status; + EXPECT_EQ(WEXITSTATUS(status), 0); +} + +TEST(GpuVmTranslation, AccessBatchRefreshesReusedVmidAndKeepsHandleRoutingDistinct) { + GpuVm gpu_vm; + auto first_physical = std::make_shared(); + auto second_physical = std::make_shared(); + auto translator = std::make_shared(); + first_physical->system[0] = std::byte{0x11}; + second_physical->system[0] = std::byte{0x22}; + const AddressSpaceHandle routed = gpu_vm.register_translated(7, translator, first_physical); + const AddressSpaceHandle unrouted = + gpu_vm.register_unrouted_address_space(7, translator, second_physical); + ASSERT_TRUE(routed); + ASSERT_TRUE(unrouted); + + const GpuVmAccessBatchGuard batch; + const GpuVmAccess *explicit_handle = gpu_vm.borrow_snapshot(unrouted); + ASSERT_NE(explicit_handle, nullptr); + const GpuVmAccess *first = gpu_vm.borrow_snapshot_vmid(7); + ASSERT_NE(first, nullptr); + EXPECT_NE(first, explicit_handle); + std::array value{}; + EXPECT_EQ(first->read(0, value), VmAccessOutcome::Complete); + EXPECT_EQ(value[0], std::byte{0x11}); + EXPECT_EQ(explicit_handle->read(0, value), VmAccessOutcome::Complete); + EXPECT_EQ(value[0], std::byte{0x22}); + + ASSERT_TRUE(gpu_vm.unregister_address_space(routed)); + EXPECT_EQ(gpu_vm.borrow_snapshot_vmid(7), nullptr); + const AddressSpaceHandle replacement = gpu_vm.register_translated(7, translator, second_physical); + ASSERT_TRUE(replacement); + const GpuVmAccess *repeated = gpu_vm.borrow_snapshot_vmid(7); + ASSERT_NE(repeated, nullptr); + EXPECT_NE(repeated, first); + EXPECT_EQ(repeated->cache_namespace().address_space, replacement); + EXPECT_FALSE(first->is_current()); + EXPECT_EQ(repeated->read(0, value), VmAccessOutcome::Complete); + EXPECT_EQ(value[0], std::byte{0x22}); + { + const GpuVmAccessBatchGuard nested; + EXPECT_EQ(gpu_vm.borrow_snapshot_vmid(7), repeated); + } + EXPECT_EQ(gpu_vm.borrow_snapshot_vmid(7), repeated); +} + +TEST(GpuVmTranslation, AccessBatchDistinguishesReusedVmStorage) { + const GpuVmAccessBatchGuard batch; + std::optional gpu_vm(std::in_place); + auto first_physical = std::make_shared(); + auto second_physical = std::make_shared(); + auto translator = std::make_shared(); + first_physical->system[0] = std::byte{0x11}; + second_physical->system[0] = std::byte{0x22}; + const AddressSpaceHandle first_handle = + gpu_vm->register_translated(7, translator, first_physical); + ASSERT_TRUE(first_handle); + const GpuVmAccess *first = gpu_vm->borrow_snapshot_vmid(7); + ASSERT_NE(first, nullptr); + + gpu_vm.emplace(); + const AddressSpaceHandle replacement = + gpu_vm->register_translated(7, translator, second_physical); + ASSERT_TRUE(replacement); + ASSERT_EQ(replacement, first_handle); + const GpuVmAccess *repeated = gpu_vm->borrow_snapshot_vmid(7); + ASSERT_NE(repeated, nullptr); + EXPECT_NE(repeated, first); + std::array value{}; + EXPECT_EQ(repeated->read(0, value), VmAccessOutcome::Complete); + EXPECT_EQ(value[0], std::byte{0x22}); + const GpuVmAccess *by_handle = gpu_vm->borrow_snapshot(replacement); + ASSERT_NE(by_handle, nullptr); + EXPECT_EQ(by_handle->read(0, value), VmAccessOutcome::Complete); + EXPECT_EQ(value[0], std::byte{0x22}); +} + +TEST(GpuVmTranslation, ScopedVmidOperationBorrowsTheQuantumSnapshot) { + GpuVm gpu_vm; + auto physical = std::make_shared(); + auto translator = std::make_shared(); + physical->system[0] = std::byte{0x2a}; + const AddressSpaceHandle handle = gpu_vm.register_translated(7, translator, physical); + ASSERT_TRUE(handle); + + const GpuVmAccess *first = nullptr; + { + const GpuVmAccessBatchGuard batch; + EXPECT_TRUE(gpu_vm.with_vmid_snapshot(7, [&](const GpuVmAccess *access) { + first = access; + std::array value{}; + return access != nullptr && access->read(0, value) == VmAccessOutcome::Complete && + value[0] == std::byte{0x2a}; + })); + EXPECT_TRUE( + gpu_vm.with_vmid_snapshot(7, [&](const GpuVmAccess *access) { return access == first; })); + } + + EXPECT_TRUE( + gpu_vm.with_vmid_snapshot(7, [](const GpuVmAccess *access) { return access != nullptr; })); +} + +TEST(GpuVmTranslation, PublicSnapshotsRefreshQueueMetadataInsideAccessBatch) { + GpuVm gpu_vm; + const AddressSpaceHandle handle = gpu_vm.register_translated( + 7, std::make_shared(), std::make_shared()); + ASSERT_TRUE(handle); + const GpuVmAccessBatchGuard batch; + const GpuVmAccess *borrowed_handle = gpu_vm.borrow_snapshot(handle); + const GpuVmAccess *borrowed_vmid = gpu_vm.borrow_snapshot_vmid(7); + ASSERT_NE(borrowed_handle, nullptr); + ASSERT_NE(borrowed_vmid, nullptr); + const auto initial_handle = gpu_vm.snapshot(handle); + const auto initial_vmid = gpu_vm.snapshot_vmid(7); + ASSERT_TRUE(initial_handle); + ASSERT_TRUE(initial_vmid); + EXPECT_EQ(initial_handle->info().queue_references, 0u); + EXPECT_EQ(initial_vmid->info().queue_references, 0u); + + ASSERT_TRUE(gpu_vm.retain_queue(handle)); + const auto retained_handle = gpu_vm.snapshot(handle); + const auto retained_vmid = gpu_vm.snapshot_vmid(7); + ASSERT_TRUE(retained_handle); + ASSERT_TRUE(retained_vmid); + EXPECT_EQ(retained_handle->info().queue_references, 1u); + EXPECT_EQ(retained_vmid->info().queue_references, 1u); + EXPECT_EQ(initial_handle->info().queue_references, 0u); + EXPECT_EQ(initial_vmid->info().queue_references, 0u); + + ASSERT_TRUE(gpu_vm.release_queue(handle)); + const auto released_handle = gpu_vm.snapshot(handle); + const auto released_vmid = gpu_vm.snapshot_vmid(7); + ASSERT_TRUE(released_handle); + ASSERT_TRUE(released_vmid); + EXPECT_EQ(released_handle->info().queue_references, 0u); + EXPECT_EQ(released_vmid->info().queue_references, 0u); + EXPECT_EQ(retained_handle->info().queue_references, 1u); + EXPECT_EQ(retained_vmid->info().queue_references, 1u); + EXPECT_TRUE(retained_handle->is_current()); + EXPECT_TRUE(retained_vmid->is_current()); + EXPECT_EQ(gpu_vm.borrow_snapshot(handle), borrowed_handle); + EXPECT_EQ(gpu_vm.borrow_snapshot_vmid(7), borrowed_vmid); +} + +TEST(GpuVmTranslation, AccessBatchAllowsSnapshotMissDuringInvalidation) { + // Isolate the cross-binding lock-order case so a regression becomes a + // bounded failure instead of hanging the test process. + const pid_t child = fork(); + ASSERT_GE(child, 0); + if (child == 0) { + alarm(10); + GpuVm gpu_vm; + auto first_physical = std::make_shared(); + auto second_physical = std::make_shared(); + auto translator = std::make_shared(); + first_physical->system[0] = std::byte{0x11}; + second_physical->system[0] = std::byte{0x22}; + const AddressSpaceHandle first = gpu_vm.register_translated(7, translator, first_physical); + const AddressSpaceHandle second = + gpu_vm.register_unrouted_address_space(8, translator, second_physical); + if (!first || !second) + _exit(1); + + std::atomic invalidation_started{false}; + std::atomic invalidation_complete{false}; + std::thread invalidator; + { + const GpuVmAccessBatchGuard batch; + auto first_access = gpu_vm.snapshot(first); + if (!first_access) + _exit(2); + std::array value{}; + if (first_access->read(0, value) != VmAccessOutcome::Complete || value[0] != std::byte{0x11}) + _exit(3); + + invalidator = std::thread([&] { + invalidation_started.store(true, std::memory_order_release); + const bool invalidated = gpu_vm.invalidate(first); + invalidation_complete.store(invalidated, std::memory_order_release); + }); + while (!invalidation_started.load(std::memory_order_acquire)) + std::this_thread::yield(); + // A miss for another binding must make progress even if invalidation + // has already acquired the registry lock. + auto second_access = gpu_vm.snapshot(second); + if (!second_access) + _exit(5); + value[0] = std::byte{0}; + if (second_access->read(0, value) != VmAccessOutcome::Complete || value[0] != std::byte{0x22}) + _exit(6); + } + + invalidator.join(); + if (!invalidation_complete.load(std::memory_order_acquire)) + _exit(7); + _exit(0); + } + + int status = 0; + ASSERT_EQ(waitpid(child, &status, 0), child); + ASSERT_TRUE(WIFEXITED(status)) << "child status: " << status; + EXPECT_EQ(WEXITSTATUS(status), 0); +} + TEST(GpuVmTranslation, DeviceGartPublishesOnInvalidateAndRejectsItsHandleAfterReset) { GpuVm gpu_vm; auto physical = std::make_shared(); @@ -1383,3 +2012,5 @@ TEST(GpuVmTranslation, LegacyVmidZeroGetsGenerationSafePassthroughBinding) { } // namespace } // namespace rocjitsu::amdgpu + +#undef RJ_GPU_VM_TRANSLATION_TEST_WITH_ASAN diff --git a/emulation/rocjitsu/tests/host_drm_preload_test.cpp b/emulation/rocjitsu/tests/host_drm_preload_test.cpp new file mode 100644 index 000000000000..df95499d0947 --- /dev/null +++ b/emulation/rocjitsu/tests/host_drm_preload_test.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file host_drm_preload_test.cpp +/// @brief Supply a host DRM descriptor identity without requiring a physical GPU. +/// @details Loaded after rocjitsu so its RTLD_NEXT stat calls see this identity. + +#include +#include + +namespace { +thread_local int host_fd = -1; +thread_local dev_t host_device = 0; + +template int finish_stat(int fd, Stat *info, int result) { + if (result == 0 && fd == host_fd) { + info->st_mode = (info->st_mode & ~S_IFMT) | S_IFCHR; + info->st_rdev = host_device; + } + return result; +} +} // namespace + +extern "C" __attribute__((visibility("default"))) void rj_test_host_drm(int fd, dev_t device) { + host_fd = fd; + host_device = device; +} + +extern "C" __attribute__((visibility("default"))) int fstat(int fd, struct stat *info) { + static auto next = reinterpret_cast(dlsym(RTLD_NEXT, "fstat")); + return finish_stat(fd, info, next(fd, info)); +} + +extern "C" __attribute__((visibility("default"))) int fstat64(int fd, struct stat64 *info) { + static auto next = reinterpret_cast(dlsym(RTLD_NEXT, "fstat64")); + return finish_stat(fd, info, next(fd, info)); +} + +extern "C" __attribute__((visibility("default"))) int __fxstat(int version, int fd, + struct stat *info) { + static auto next = reinterpret_cast(dlsym(RTLD_NEXT, "__fxstat")); + return finish_stat(fd, info, next(version, fd, info)); +} + +extern "C" __attribute__((visibility("default"))) int __fxstat64(int version, int fd, + struct stat64 *info) { + static auto next = reinterpret_cast(dlsym(RTLD_NEXT, "__fxstat64")); + return finish_stat(fd, info, next(version, fd, info)); +} diff --git a/emulation/rocjitsu/tests/host_mapping_lock_test.cpp b/emulation/rocjitsu/tests/host_mapping_lock_test.cpp index a5532c868477..59749b3322c0 100644 --- a/emulation/rocjitsu/tests/host_mapping_lock_test.cpp +++ b/emulation/rocjitsu/tests/host_mapping_lock_test.cpp @@ -2,10 +2,14 @@ // SPDX-License-Identifier: MIT #include "rocjitsu/kmd/linux/host_mapping_lock.h" +#include "rocjitsu/vm/amdgpu/gpu_vm.h" #include +#include #include +#include +#include #include #include #include @@ -41,3 +45,58 @@ TEST(HostMappingLockTest, ChildResetsLockHeldByVanishedThread) { // The child's reset must also leave the parent's lock usable. util::ObservableSharedMutex::ExclusiveGuard lock = rocjitsu::host_mapping_lock().lock_exclusive(); } + +TEST(HostMappingLockTest, QuantumKeepsHostMappingLeasesOperationScoped) { + using namespace std::chrono_literals; + + // Isolate the test so an implementation bug that deadlocks a nested access + // becomes a bounded failure rather than hanging the whole suite. + const pid_t child = fork(); + ASSERT_GE(child, 0); + if (child == 0) { + alarm(10); + rocjitsu::reset_host_mapping_lock_after_fork(); + + const rocjitsu::amdgpu::GpuVmAccessBatchGuard outer_batch; + const rocjitsu::amdgpu::GpuVmAccessBatchGuard inner_batch; + std::atomic writer_acquired{false}; + std::thread writer; + { + auto access = rocjitsu::host_mapping_lock().lock_shared(); + writer = std::thread([&] { + auto lock = rocjitsu::host_mapping_lock().lock_exclusive(); + writer_acquired.store(true, std::memory_order_release); + }); + + const auto deadline = std::chrono::steady_clock::now() + 5s; + while (rocjitsu::host_mapping_lock().blocked_writers() == 0) { + if (std::chrono::steady_clock::now() >= deadline) + _exit(1); + std::this_thread::yield(); + } + + if (writer_acquired.load(std::memory_order_acquire)) + _exit(2); + } + writer.join(); + if (!writer_acquired.load(std::memory_order_acquire)) + _exit(3); + + auto exclusive = rocjitsu::host_mapping_lock().lock_exclusive(); + auto ordinary_access = std::async(std::launch::async, [] { + auto access = rocjitsu::host_mapping_lock().lock_shared(); + return true; + }); + if (ordinary_access.wait_for(100ms) != std::future_status::timeout) + _exit(4); + exclusive.unlock(); + if (ordinary_access.wait_for(2s) != std::future_status::ready || !ordinary_access.get()) + _exit(5); + _exit(0); + } + + int status = 0; + ASSERT_EQ(waitpid(child, &status, 0), child); + ASSERT_TRUE(WIFEXITED(status)) << "child status: " << status; + EXPECT_EQ(WEXITSTATUS(status), 0); +} diff --git a/emulation/rocjitsu/tests/instruction_cache_test.cpp b/emulation/rocjitsu/tests/instruction_cache_test.cpp index 34dd183dd059..5d17dbab5491 100644 --- a/emulation/rocjitsu/tests/instruction_cache_test.cpp +++ b/emulation/rocjitsu/tests/instruction_cache_test.cpp @@ -2,7 +2,13 @@ // SPDX-License-Identifier: MIT #include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/machine_insts.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h" +#include "rocjitsu/kmd/linux/kfd_process.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" +#include "rocjitsu/vm/amdgpu/decoded_instruction_cache.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" #include "rocjitsu/vm/amdgpu/instruction_cache.h" #include "rocjitsu/vm/amdgpu/l2_cache.h" @@ -13,10 +19,13 @@ #include #include #include +#include #include #include #include +#include #include +#include #include namespace { @@ -52,6 +61,11 @@ class ExecutableAddressSpace final : public amdgpu::AddressSpaceTranslator, void fill(uint8_t value) { std::ranges::fill(bytes_, static_cast(value)); } + void write_program(std::span words) { + ASSERT_LE(words.size_bytes(), bytes_.size()); + std::memcpy(bytes_.data(), words.data(), words.size_bytes()); + } + amdgpu::VmTranslationResult translate(uint64_t address, std::size_t size, amdgpu::VmAccessKind access) const override { if (access != amdgpu::VmAccessKind::Read && access != amdgpu::VmAccessKind::Execute) @@ -130,6 +144,34 @@ TEST(InstructionCacheTest, FetchMatchesBackingMemoryAtEveryAlignedOffset) { } } +TEST(InstructionCacheTest, RepeatedMaintenanceAdvancesEpochAndInvalidatesRefills) { + GpuMemory memory("memory"); + InstructionCache cache; + uint64_t epoch = cache.epoch(); + const std::vector first = fill_code(memory, InstructionCache::kCacheBytes, 0x31); + for (unsigned cycle = 0; cycle != 3; ++cycle) { + for (unsigned repeat = 0; repeat != 4; ++repeat) { + cache.invalidate_all(); + EXPECT_EQ(cache.epoch(), ++epoch); + uint32_t word = 0; + EXPECT_FALSE(cache.peek_word(kCodeBase, 0, word)); + } + for (uint32_t offset = 0; offset != InstructionCache::kCacheBytes; + offset += InstructionCache::kLineSize) { + const auto bytes = fetch_at(cache, memory, kCodeBase + offset); + EXPECT_TRUE(std::ranges::equal(bytes, std::span(first).subspan(offset, bytes.size()))); + } + } + const std::vector second = fill_code(memory, InstructionCache::kCacheBytes, 0x72); + cache.invalidate_all(); + EXPECT_EQ(cache.epoch(), ++epoch); + for (uint32_t offset = 0; offset != InstructionCache::kCacheBytes; + offset += InstructionCache::kLineSize) { + const auto bytes = fetch_at(cache, memory, kCodeBase + offset); + EXPECT_TRUE(std::ranges::equal(bytes, std::span(second).subspan(offset, bytes.size()))); + } +} + // The straddling offsets are the interesting ones: assert they are actually // exercised above, so the loop cannot silently stop covering them. TEST(InstructionCacheTest, FetchWindowStraddlesALineBoundary) { @@ -222,6 +264,27 @@ TEST(InstructionCacheTest, LinesDoNotAliasAcrossVmids) { EXPECT_EQ(again1, got1) << "the vmid 1 fetch did not cache its line"; } +TEST(InstructionCacheTest, RepeatedMaintenanceInvalidatesTranslatedRefills) { + GpuVm gpu_vm; + InstructionCache cache; + auto backing = std::make_shared(0x11); + const auto handle = gpu_vm.register_translated(7, backing, backing); + ASSERT_TRUE(handle); + const auto access = gpu_vm.snapshot(handle); + ASSERT_TRUE(access); + auto cached = fetch_at(cache, *access); + for (uint8_t value : {0x22, 0x33, 0x44}) { + backing->fill(value); + EXPECT_EQ(fetch_at(cache, *access), cached); + const uint64_t epoch = cache.epoch(); + cache.invalidate_all(); + cache.invalidate_all(); + EXPECT_EQ(cache.epoch(), epoch + 2); + cached = fetch_at(cache, *access); + EXPECT_TRUE(std::ranges::all_of(cached, [value](uint8_t byte) { return byte == value; })); + } +} + TEST(InstructionCacheTest, TranslatedLinesDoNotAliasAcrossRootReplacement) { GpuVm gpu_vm; InstructionCache icache; @@ -352,6 +415,689 @@ class CuFixture { std::unique_ptr cu_; }; +class CountingDecoder : public rocjitsu::Decoder { +public: + explicit CountingDecoder(rj_code_arch_t arch = ROCJITSU_CODE_ARCH_CDNA4) + : decoder_(rocjitsu::Decoder::create(arch)) {} + + rocjitsu::DecodeResult decode(const rj_code_binary_inst_t *words, + const rocjitsu::DecodeErrorEmitter &emit_error) override { + ++calls; + return decoder_->decode(words, emit_error); + } + size_t max_instruction_words() const override { return decoder_->max_instruction_words(); } + + unsigned calls = 0; + +private: + std::unique_ptr decoder_; +}; + +TEST(DecodedInstructionCacheCuTest, ConstructionAndDecoderReplacementPreserveForeignPool) { + rocjitsu::Instruction::ScopedHeapAllocation restore_allocator; + CountingDecoder foreign_decoder; + foreign_decoder.enable_pool(); + const auto alloc_fn = rocjitsu::Instruction::alloc_fn_; + const auto dealloc_fn = rocjitsu::Instruction::dealloc_fn_; + const auto pool = rocjitsu::Instruction::alloc_pool_; + ASSERT_NE(pool, nullptr); + const auto expect_foreign_pool = [&] { + EXPECT_EQ(rocjitsu::Instruction::alloc_fn_, alloc_fn); + EXPECT_EQ(rocjitsu::Instruction::dealloc_fn_, dealloc_fn); + EXPECT_EQ(rocjitsu::Instruction::alloc_pool_, pool); + }; + { + CuFixture fixture("decoded_foreign_pool"); + expect_foreign_pool(); + fixture.cu()->replace_decoder_for_test(std::make_unique()); + expect_foreign_pool(); + fixture.write_program(std::array{s_mov_b32_s0_imm(17), kSEndpgm}); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 17u); + expect_foreign_pool(); + fixture.cu()->step(); + expect_foreign_pool(); + } + expect_foreign_pool(); +} + +TEST(DecodedInstructionCacheCuTest, LoopDecodesOnceAndReadsCurrentRegisters) { + CuFixture fixture("decoded_loop"); + auto decoder = std::make_unique(); + auto *counter = decoder.get(); + fixture.cu()->replace_decoder_for_test(std::move(decoder)); + // s_mov_b32 s0, s1; s_branch -2. The branch revisits both decoded objects. + fixture.write_program(std::array{0xBE800001u, 0xBF82FFFEu}); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + for (uint32_t value = 1; value <= 10; ++value) { + fixture.cu()->write_sgpr(wf->sgpr_alloc().base + 1, value * 19); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), value * 19); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase); + } + EXPECT_EQ(counter->calls, 2u); +} + +TEST(DecodedInstructionCacheCuTest, RewrittenLiteralAndDebuggerBypassUseFreshDecodes) { + CuFixture fixture("decoded_literal"); + auto decoder = std::make_unique(); + auto *counter = decoder.get(); + fixture.cu()->replace_decoder_for_test(std::move(decoder)); + fixture.write_program(std::array{0xBE8000FFu, 17u, kSEndpgm}); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 17u); + + fixture.memory().write32(kCodeBase + 4, 29u); + wf->pc = kCodeBase; + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 17u); + EXPECT_EQ(counter->calls, 1u); + + fixture.cu()->instruction_cache().invalidate_all(); + wf->pc = kCodeBase; + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 29u); + EXPECT_EQ(counter->calls, 2u); + + fixture.cu()->set_debug_active(true); + fixture.memory().write32(kCodeBase + 4, 43u); + wf->pc = kCodeBase; + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 43u); + fixture.cu()->set_debug_active(false); + wf->pc = kCodeBase; + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 43u); + EXPECT_EQ(counter->calls, 4u); +} + +TEST(DecodedInstructionCacheCuTest, VectorReadsCurrentExecAcrossDispatches) { + CuFixture fixture("decoded_vector", 2); + auto decoder = std::make_unique(); + auto *counter = decoder.get(); + fixture.cu()->replace_decoder_for_test(std::move(decoder)); + // v_mov_b32_e32 v0, v1; s_branch -2. + fixture.write_program(std::array{0x7E000301u, 0xBF82FFFEu}); + for (unsigned dispatch = 1; dispatch <= 3; ++dispatch) { + auto *wf = fixture.launch(dispatch, dispatch); + ASSERT_NE(wf, nullptr); + const auto base = wf->vgpr_alloc().base; + for (uint64_t mask : {~uint64_t{0}, uint64_t{0}, uint64_t{0x3333333333333333}}) { + wf->set_exec(mask); + for (unsigned lane = 0; lane < wf->wf_size(); ++lane) { + fixture.cu()->write_vgpr(base, lane, 0xDEAD0000u + lane); + fixture.cu()->write_vgpr(base + 1, lane, dispatch * 1000 + lane); + } + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + for (unsigned lane = 0; lane < wf->wf_size(); ++lane) + EXPECT_EQ(fixture.cu()->read_vgpr(base, lane), + ((mask >> lane) & 1) ? dispatch * 1000 + lane : 0xDEAD0000u + lane); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase); + } + wf->halt(); + } + EXPECT_EQ(counter->calls, 2u); +} + +TEST(DecodedInstructionCacheCuTest, DppDelegatesReadCurrentRegistersAndRestoreAcrossWaves) { + CuFixture fixture("decoded_dpp", 2); + auto *wf0 = fixture.launch(1, 0); + auto *wf1 = fixture.launch(1, 1); + ASSERT_NE(wf0, nullptr); + ASSERT_NE(wf1, nullptr); + ASSERT_NE(wf0->vgpr_alloc().base, wf1->vgpr_alloc().base); + CountingDecoder decoder; + amdgpu::DecodedInstructionCache cache; + // v_mov_b32 v0, v1 with DPP quad permutation [1, 0, 3, 2]. + rocjitsu::cdna4::Vop1VopDppMachineInst raw{}; + raw.src0 = amdgpu::SRC_DPP; + raw.op = rocjitsu::cdna4::kVMovB32Vop1; + raw.encoding = rocjitsu::cdna4::encoding::kVop1 >> 2; + raw.vdst = 0; + raw.vsrc0 = 1; + raw.dpp_ctrl = 0xB1; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + std::array words{}; + std::memcpy(words.data(), &raw, sizeof(raw)); + const rocjitsu::Instruction *identity = nullptr; + for (unsigned repeat = 1; repeat <= 4; ++repeat) + for (auto *wf : {wf0, wf1}) + for (uint64_t mask : {~uint64_t{0}, uint64_t{0}, uint64_t{0x3333333333333333}}) { + const auto base = wf->vgpr_alloc().base; + const auto value = repeat * 1000 + base; + wf->set_exec(mask); + for (unsigned lane = 0; lane < wf->wf_size(); ++lane) { + fixture.cu()->write_vgpr(base, lane, 0xDEAD0000u + lane); + fixture.cu()->write_vgpr(base + 1, lane, value + lane); + } + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + if (!identity) + identity = decoded.value().get(); + EXPECT_EQ(decoded.value().get(), identity); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + ASSERT_TRUE(fixture.cu()->execute_instruction(decoded.value().get(), *wf).succeeded()); + EXPECT_EQ(decoded.value()->src_operand(0)->delegate(), nullptr); + for (unsigned lane = 0; lane < wf->wf_size(); ++lane) + EXPECT_EQ(fixture.cu()->read_vgpr(base, lane), + ((mask >> lane) & 1) ? value + (lane ^ 1u) : 0xDEAD0000u + lane); + } + EXPECT_EQ(decoder.calls, 1u); +} + +TEST(DecodedInstructionCacheTest, InFlightOwnershipPreventsReuse) { + CountingDecoder decoder; + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + std::unique_ptr in_flight; + { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + in_flight = std::move(decoded.value()); + } + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_NE(decoded.value().get(), in_flight.get()); + EXPECT_EQ(decoder.calls, 2u); +} + +TEST(DecodedInstructionCacheTest, FailedExecutionAndExceptionsDoNotRetainInstructions) { + CountingDecoder decoder; + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + returned.discard(); + } + EXPECT_EQ(decoder.calls, 1u); + try { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + throw std::runtime_error("execution callback failed"); + } catch (const std::runtime_error &) { + } + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_EQ(decoder.calls, 3u); +} + +TEST(DecodedInstructionCacheTest, BorrowedVopdEncodingIsNotRetainedAcrossIssues) { + for (const auto arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) { + SCOPED_TRACE(arch); + CountingDecoder decoder(arch); + amdgpu::DecodedInstructionCache cache; + // v_dual_cndmask_b32 v2, v1, v2 :: v_dual_mov_b32 v1, 0. + std::array words{0xCA500501u, 0x02000080u}; + { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + ASSERT_EQ(decoded.value()->raw_encoding(), words.data()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + } + // Use the same bytes from a different fetch buffer on the next issue. + auto next_words = words; + auto decoded = cache.decode(decoder, kCodeBase, 7, next_words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_EQ(decoded.value()->raw_encoding(), next_words.data()); + EXPECT_EQ(decoder.calls, 2u); + } +} + +TEST(DecodedInstructionCacheTest, AddressSpacesCollisionsAndClearCauseMisses) { + CountingDecoder decoder; + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + auto issue = [&](uint64_t pc, uint32_t vmid) { + auto decoded = cache.decode(decoder, pc, vmid, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), pc, vmid, words); + }; + issue(kCodeBase, 7); + issue(kCodeBase, 7); + EXPECT_EQ(decoder.calls, 1u); + issue(kCodeBase, 8); + issue(kCodeBase, 7); + EXPECT_EQ(decoder.calls, 3u); + issue(kCodeBase + 4 * amdgpu::DecodedInstructionCache::kNumEntries, 7); + issue(kCodeBase, 7); + EXPECT_EQ(decoder.calls, 5u); + cache.clear(); + issue(kCodeBase, 7); + EXPECT_EQ(decoder.calls, 6u); +} + +TEST(DecodedInstructionCacheTest, DynamicStateIsNotReusedAndPoolLifetimeIsIndependent) { + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + { + CountingDecoder decoder; + decoder.enable_pool(); + { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + decoded.value()->set_data(std::make_unique()); + } + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_EQ(decoder.calls, 2u); + EXPECT_EQ(decoded.value()->data(), nullptr); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + } + // The decoder and its active allocator pool are gone before the cached + // instruction is destroyed. Cache entries must have independent storage. + cache.clear(); +} + +TEST(DecodedInstructionCacheTest, UncachedInstructionsAreHeapBackedWithAnActivePool) { + rocjitsu::Instruction::ScopedHeapAllocation restore_allocator; + CountingDecoder decoder; + decoder.enable_pool(); + const auto pool = static_cast(rocjitsu::Instruction::alloc_pool_); + ASSERT_NE(pool, nullptr); + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + for (unsigned mode = 0; mode < 4; ++mode) { + SCOPED_TRACE(mode); + std::unique_ptr in_flight; + { + const bool enabled = mode != 2; + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}, enabled); + ASSERT_FALSE(decoded.failed()); + EXPECT_FALSE(pool->owns(decoded.value().get())); + EXPECT_EQ(rocjitsu::Instruction::alloc_pool_, pool); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words, enabled); + if (mode == 0) + returned.discard(); + else if (mode == 1) + decoded.value()->set_data(std::make_unique()); + else if (mode == 3) + in_flight = std::move(decoded.value()); + } + // ArenaAlloc's non-owned-pointer fallback is the supported heap deletion + // path even when an unrelated decoder's pool is ambient on this thread. + in_flight.reset(); + EXPECT_EQ(rocjitsu::Instruction::alloc_pool_, pool); + } + EXPECT_EQ(decoder.calls, 4u); +} + +TEST(DecodedInstructionCacheTest, EntriesOutliveMultipleWorkerPools) { + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + const rocjitsu::Instruction *identity = nullptr; + for (unsigned worker = 0; worker < 4; ++worker) { + std::thread thread([&] { + CountingDecoder decoder; + decoder.enable_pool(); + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_FALSE(static_cast(rocjitsu::Instruction::alloc_pool_) + ->owns(decoded.value().get())); + if (!identity) + identity = decoded.value().get(); + EXPECT_EQ(decoded.value().get(), identity); + EXPECT_EQ(decoder.calls, worker == 0 ? 1u : 0u); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + }); + thread.join(); + } + cache.clear(); +} + +// A warm instruction-cache line must not hide revocation of the VM snapshot +// retained by the CU. Exercise both explicit handles and the VMID fallback. +TEST(InstructionCacheCuTest, VmInvalidationRefreshesAnAlreadyCachedInstruction) { + for (const bool explicit_handle : {false, true}) { + SCOPED_TRACE(explicit_handle); + GpuVm gpu_vm; + CuFixture fixture("vm_invalidation_cu"); + auto backing = std::make_shared(0); + backing->write_program(std::array{kSNop, s_mov_b32_s0_imm(1), kSEndpgm}); + const auto handle = gpu_vm.register_translated(7, backing, backing); + ASSERT_TRUE(handle); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + wf->set_process_id(7); + if (explicit_handle) + wf->set_address_space(handle); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + + backing->write_program(std::array{kSNop, s_mov_b32_s0_imm(2), kSEndpgm}); + ASSERT_TRUE(gpu_vm.invalidate(handle)); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 2u); + } +} + +TEST(InstructionCacheCuTest, RootReplacementRefreshesAnAlreadyCachedInstruction) { + GpuVm gpu_vm; + CuFixture fixture("vm_replacement_cu"); + auto first = std::make_shared(0); + first->write_program(std::array{kSNop, s_mov_b32_s0_imm(1), kSEndpgm}); + const auto handle = gpu_vm.register_translated(7, first, first); + ASSERT_TRUE(handle); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + wf->set_process_id(7); + wf->set_address_space(handle); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + + auto replacement = std::make_shared(0); + replacement->write_program(std::array{kSNop, s_mov_b32_s0_imm(2), kSEndpgm}); + ASSERT_TRUE(gpu_vm.replace_translated(handle, replacement, replacement)); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 2u); +} + +TEST(InstructionCacheCuTest, ReusedVmidCannotReviveAStaleExplicitHandle) { + GpuVm gpu_vm; + CuFixture fixture("vm_stale_handle_cu"); + auto backing = std::make_shared(0); + backing->write_program(std::array{kSNop, s_mov_b32_s0_imm(1), kSEndpgm}); + const auto handle = gpu_vm.register_translated(7, backing, backing); + ASSERT_TRUE(handle); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + wf->set_process_id(7); + wf->set_address_space(handle); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + + ASSERT_TRUE(gpu_vm.unregister_address_space(handle)); + const auto replacement = gpu_vm.register_translated(7, backing, backing); + ASSERT_TRUE(replacement); + EXPECT_NE(handle, replacement); + fixture.cu()->step(); + EXPECT_TRUE(wf->is_halted()); + EXPECT_FALSE(fixture.cu()->has_active_wfs()) + << "the stale address space must abort the dispatch before executing cached code"; +} + +TEST(InstructionCacheCuTest, ChangingAddressSpaceSelectsItsOwnInstruction) { + for (const bool explicit_handle : {false, true}) { + SCOPED_TRACE(explicit_handle); + GpuVm gpu_vm; + CuFixture fixture("vm_switch_cu"); + auto first = std::make_shared(0); + first->write_program(std::array{kSNop, s_mov_b32_s0_imm(1), kSEndpgm}); + const auto first_handle = gpu_vm.register_translated(7, first, first); + auto second = std::make_shared(0); + second->write_program(std::array{kSNop, s_mov_b32_s0_imm(2), kSEndpgm}); + const auto second_handle = gpu_vm.register_translated(8, second, second); + ASSERT_TRUE(first_handle); + ASSERT_TRUE(second_handle); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + wf->set_process_id(7); + if (explicit_handle) + wf->set_address_space(first_handle); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + + wf->set_process_id(8); + if (explicit_handle) + wf->set_address_space(second_handle); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 2u); + } +} + +TEST(InstructionCacheCuTest, SwitchingVmServicesDropsTheRetainedAccess) { + GpuVm first_vm; + GpuVm second_vm; + CuFixture fixture("vm_service_switch_cu"); + auto first = std::make_shared(0); + first->write_program(std::array{kSNop, s_mov_b32_s0_imm(1), kSEndpgm}); + const auto first_handle = first_vm.register_translated(7, first, first); + auto second = std::make_shared(0); + second->write_program(std::array{kSNop, s_mov_b32_s0_imm(2), kSEndpgm}); + const auto second_handle = second_vm.register_translated(7, second, second); + ASSERT_TRUE(first_handle); + ASSERT_EQ(first_handle, second_handle) << "exercise matching keys in different VM services"; + fixture.cu()->set_gpu_vm(&first_vm); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + wf->set_process_id(7); + wf->set_address_space(first_handle); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + + fixture.cu()->set_gpu_vm(&second_vm); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 2u); +} + +struct CountedReporter { + std::shared_ptr process; + unsigned *copies; + + CountedReporter(std::shared_ptr owner, unsigned &count) + : process(std::move(owner)), copies(&count) {} + CountedReporter(const CountedReporter &other) : process(other.process), copies(other.copies) { + ++*copies; + } + void operator()(uint64_t, amdgpu::VmAccessKind) const {} +}; + +// Capture a real process as the KFD binding does. Releasing the registry binding +// must not leave that process owned by an idle CU. +TEST(InstructionCacheCuTest, CompletedWaveDoesNotRetainItsVmOwner) { + GpuVm gpu_vm; + CuFixture fixture("vm_completed_lifetime_cu"); + auto backing = std::make_shared(0); + backing->write_program(std::array{kSNop, kSEndpgm}); + auto owner = std::make_shared(7); + const std::weak_ptr lifetime = owner; + const auto handle = gpu_vm.register_address_space( + 7, backing, backing, [owner](uint64_t, amdgpu::VmAccessKind) { (void)owner; }); + ASSERT_TRUE(handle); + owner.reset(); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + wf->set_dispatch_id(1); + wf->set_process_id(7); + wf->set_address_space(handle); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + fixture.cu()->step(); + ASSERT_TRUE(wf->is_halted()); + ASSERT_FALSE(fixture.cu()->has_active_wfs()); + + ASSERT_TRUE(gpu_vm.unregister_address_space(handle)); + EXPECT_TRUE(lifetime.expired()) << "an idle CU retained the completed process binding"; +} + +TEST(InstructionCacheCuTest, AbortedWaveDoesNotRetainItsVmOwner) { + GpuVm gpu_vm; + CuFixture fixture("vm_aborted_lifetime_cu"); + auto backing = std::make_shared(0); + backing->write_program(std::array{kSNop, kSNop, kSEndpgm}); + auto owner = std::make_shared(7); + const std::weak_ptr lifetime = owner; + const auto handle = gpu_vm.register_address_space( + 7, backing, backing, [owner](uint64_t, amdgpu::VmAccessKind) { (void)owner; }); + ASSERT_TRUE(handle); + owner.reset(); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + wf->set_dispatch_id(1); + wf->set_process_id(7); + wf->set_address_space(handle); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + fixture.cu()->abort_dispatch(1); + ASSERT_TRUE(wf->is_halted()); + + ASSERT_TRUE(gpu_vm.unregister_address_space(handle)); + EXPECT_TRUE(lifetime.expired()) << "an idle CU retained the cancelled process binding"; +} + +TEST(InstructionCacheCuTest, ReentrantFaultCancellationKeepsAccessUntilIssueReturns) { + GpuVm gpu_vm; + CuFixture fixture("vm_reentrant_lifetime_cu"); + auto backing = std::make_shared(0); + auto owner = std::make_shared(7); + const std::weak_ptr lifetime = owner; + unsigned faults = 0; + const auto handle = gpu_vm.register_address_space( + 7, backing, backing, [owner, &fixture, &lifetime, &faults](uint64_t, amdgpu::VmAccessKind) { + (void)owner; + ++faults; + fixture.cu()->abort_dispatch(1); + // The registry and in-flight access share the generation's callback. + // Cancellation must leave its owner alive while the callback executes. + EXPECT_FALSE(lifetime.expired()); + }); + ASSERT_TRUE(handle); + owner.reset(); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *wf = fixture.launch(1, 0, kCodeBase + InstructionCache::kLineSize * 4); + ASSERT_NE(wf, nullptr); + wf->set_dispatch_id(1); + wf->set_process_id(7); + wf->set_address_space(handle); + fixture.cu()->step(); + EXPECT_GE(faults, 1u); + EXPECT_TRUE(wf->is_halted()); + + ASSERT_TRUE(gpu_vm.unregister_address_space(handle)); + EXPECT_TRUE(lifetime.expired()) << "fault handling left the binding retained after issue"; +} + +TEST(InstructionCacheCuTest, ExceptionalQuantumReleasesTheProcessBeforeCancellation) { + GpuVm gpu_vm; + CuFixture fixture("vm_exception_lifetime_cu"); + auto backing = std::make_shared(0); + auto owner = std::make_shared(7); + const std::weak_ptr lifetime = owner; + const auto handle = + gpu_vm.register_address_space(7, backing, backing, [owner](uint64_t, amdgpu::VmAccessKind) { + (void)owner; + throw std::runtime_error("fault callback failed"); + }); + ASSERT_TRUE(handle); + owner.reset(); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *wf = fixture.launch(1, 0, kCodeBase + InstructionCache::kLineSize * 4); + ASSERT_NE(wf, nullptr); + wf->set_dispatch_id(1); + wf->set_process_id(7); + wf->set_address_space(handle); + EXPECT_THROW(fixture.cu()->run_quantum(), std::runtime_error); + ASSERT_TRUE(fixture.cu()->has_active_wfs()); + ASSERT_TRUE(gpu_vm.unregister_address_space(handle)); + EXPECT_TRUE(lifetime.expired()) << "exception unwinding must release the quantum snapshot"; + fixture.cu()->abort_dispatch(1); + EXPECT_TRUE(lifetime.expired()); +} + +TEST(InstructionCacheCuTest, ActiveWaveSharesReporterAcrossQuantaAndDirectSteps) { + GpuVm gpu_vm; + CuFixture fixture("vm_quantum_lifetime_cu"); + auto backing = std::make_shared(0); + backing->write_program(std::array{kSNop, kSNop, kSEndpgm}); + auto process = std::make_shared(7); + const std::weak_ptr lifetime = process; + unsigned copies = 0; + const auto handle = + gpu_vm.register_address_space(7, backing, backing, CountedReporter(process, copies)); + ASSERT_TRUE(handle); + process.reset(); + fixture.cu()->set_gpu_vm(&gpu_vm); + fixture.cu()->set_functional_quantum(1); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + wf->set_dispatch_id(1); + wf->set_process_id(7); + wf->set_address_space(handle); + copies = 0; + fixture.cu()->run_quantum(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + EXPECT_EQ(copies, 0u); + fixture.cu()->run_quantum(); + ASSERT_EQ(wf->pc, kCodeBase + 8); + EXPECT_EQ(copies, 0u) << "new quantum snapshots must share the fault reporter"; + fixture.cu()->step(); + EXPECT_TRUE(wf->is_halted()); + EXPECT_EQ(copies, 0u) << "direct stepping must share the fault reporter"; + + ASSERT_TRUE(gpu_vm.unregister_address_space(handle)); + EXPECT_TRUE(lifetime.expired()) << "the final wave retained its process after retiring"; +} + +TEST(InstructionCacheCuTest, EndingOneWaveKeepsTheOtherWavesVmAccess) { + GpuVm gpu_vm; + CuFixture fixture("vm_multiwave_lifetime_cu", /*wf_slots=*/2); + auto backing = std::make_shared(0); + backing->write_program(std::array{kSNop, kSNop, kSEndpgm}); + auto process = std::make_shared(7); + const std::weak_ptr lifetime = process; + unsigned copies = 0; + const auto handle = + gpu_vm.register_address_space(7, backing, backing, CountedReporter(process, copies)); + ASSERT_TRUE(handle); + process.reset(); + fixture.cu()->set_gpu_vm(&gpu_vm); + auto *first = fixture.launch(1, 0, kCodeBase + 8); + auto *second = fixture.launch(1, 1); + ASSERT_NE(first, nullptr); + ASSERT_NE(second, nullptr); + for (auto *wf : {first, second}) { + wf->set_dispatch_id(1); + wf->set_process_id(7); + wf->set_address_space(handle); + } + copies = 0; + fixture.cu()->step(); + ASSERT_TRUE(first->is_halted()); + ASSERT_FALSE(second->is_halted()); + ASSERT_EQ(second->pc, kCodeBase + 4); + EXPECT_EQ(copies, 0u) << "both waves must share the fault reporter"; + EXPECT_FALSE(lifetime.expired()); + fixture.cu()->step(); + ASSERT_EQ(second->pc, kCodeBase + 8); + EXPECT_EQ(copies, 0u); + fixture.cu()->step(); + EXPECT_TRUE(second->is_halted()); + EXPECT_FALSE(fixture.cu()->has_active_wfs()); + EXPECT_EQ(copies, 0u); + ASSERT_TRUE(gpu_vm.unregister_address_space(handle)); + EXPECT_TRUE(lifetime.expired()); +} + // Self-modifying code: the rewritten instruction only becomes visible to the // fetcher when the wave retires s_icache_inv. This runs the generated // execute_s_icache_inv_sopp body, which is the only thing standing between the diff --git a/emulation/rocjitsu/tests/instruction_execution_harness_test.cpp b/emulation/rocjitsu/tests/instruction_execution_harness_test.cpp index 64c00f0324f6..aca81c57b8ee 100644 --- a/emulation/rocjitsu/tests/instruction_execution_harness_test.cpp +++ b/emulation/rocjitsu/tests/instruction_execution_harness_test.cpp @@ -396,7 +396,6 @@ inline bool should_skip_inst(std::string_view mn) { return mnemonic_has_any_prefix(mn, SKIP_PREFIXES); } -constexpr std::array EXPECTED_CDNA_UNIMPLEMENTED = {"s_setvskip"}; constexpr std::array EXPECTED_RDNA1_UNIMPLEMENTED = { "s_subvector_loop_begin", "s_subvector_loop_end", "s_get_waveid_in_workgroup"}; constexpr std::array EXPECTED_RDNA2_UNIMPLEMENTED = {"s_subvector_loop_begin", @@ -409,8 +408,8 @@ struct HarnessExpectation { }; constexpr HarnessExpectation HARNESS_EXPECTATIONS[] = { - {"cdna1", EXPECTED_CDNA_UNIMPLEMENTED}, {"cdna2", EXPECTED_CDNA_UNIMPLEMENTED}, - {"cdna3", EXPECTED_CDNA_UNIMPLEMENTED}, {"cdna4", EXPECTED_CDNA_UNIMPLEMENTED}, + {"cdna1", EXPECTED_NO_UNIMPLEMENTED}, {"cdna2", EXPECTED_NO_UNIMPLEMENTED}, + {"cdna3", EXPECTED_NO_UNIMPLEMENTED}, {"cdna4", EXPECTED_NO_UNIMPLEMENTED}, {"rdna1", EXPECTED_RDNA1_UNIMPLEMENTED}, {"rdna2", EXPECTED_RDNA2_UNIMPLEMENTED}, {"rdna3", EXPECTED_NO_UNIMPLEMENTED}, {"rdna3_5", EXPECTED_NO_UNIMPLEMENTED}, {"rdna4", EXPECTED_NO_UNIMPLEMENTED}, {"gfx1250", EXPECTED_NO_UNIMPLEMENTED}, @@ -1224,7 +1223,9 @@ void run_scalar_cvt_preserves_scc(rj_code_arch_t arch, std::string_view arch_nam wf->halt(); } -TEST(ScalarSccTest, ScalarCvtPreservesScc) { +// Keep these raw opcode witnesses independent of the generated encodings used +// by ScalarCvtPreservesScc, with a distinct name so each body runs only once. +TEST(ScalarSccTest, ScalarCvtPreservesSccFromRawEncodings) { const uint32_t one_f32 = std::bit_cast(1.0f); const uint32_t one_f16 = util::f32_to_f16(1.0f); const uint32_t qnan_f32 = 0x7FC00000u; @@ -2533,9 +2534,8 @@ TEST(NewerOmodExecutionTest, F32AndF64FinalizeExactResultsInScalarAndSimdPaths) EXPECT_EQ(cu->read_vgpr(vb + 9, lane), 0u); } - // Pin OMOD finalization independently from CLAMP. Positive and negative - // minimum-normal values become subnormals under /2, and both those - // flushed results and an input -0 must be canonicalized to +0. + // Pin OMOD finalization independently from CLAMP. FP32 underflow caused + // by halving a normal result preserves its sign; an input -0 becomes +0. wf->set_exec(0x7u); constexpr std::array kF32Inputs{ std::bit_cast(std::numeric_limits::min()), @@ -2553,7 +2553,7 @@ TEST(NewerOmodExecutionTest, F32AndF64FinalizeExactResultsInScalarAndSimdPaths) ASSERT_NE(unclamped_f32, nullptr); EXPECT_TRUE(cu->execute_instruction(unclamped_f32.get(), *wf).succeeded()); for (std::size_t lane = 0; lane < kF32Inputs.size(); ++lane) - EXPECT_EQ(cu->read_vgpr(vb + 2, lane), 0u); + EXPECT_EQ(cu->read_vgpr(vb + 2, lane), lane == 1 ? 0x80000000u : 0u); constexpr std::array kF64Inputs{ kMinNormalF64, @@ -2719,6 +2719,130 @@ TEST(Cdna4Permlane16SwapExecutionTest, Wave64SwapsAllFourGroups) { wf->halt(); } +class PermlaneSwapWriteRecorder final : public ExecutionPlugin { +public: + PermlaneSwapWriteRecorder() : ExecutionPlugin("permlane_swap_writes") {} + + void onAmdgpuWriteVgprLanes(const amdgpu::Wavefront *, uint32_t reg, uint64_t lanes, + uint8_t) override { + writes.emplace_back(reg, lanes); + } + + std::vector> writes; +}; + +TEST(PermlaneSwapExecutionTest, RespectsExecForBothResults) { + struct SwapCase { + rj_code_arch_t arch; + const char *name; + uint32_t stride; + uint32_t e32_opcode; + uint32_t e64_opcode; + }; + const SwapCase cases[] = { + {ROCJITSU_CODE_ARCH_CDNA4, "cdna4", 16, cdna4::kVPermlane16SwapB32Vop1, + cdna4::kVPermlane16SwapB32Vop3}, + {ROCJITSU_CODE_ARCH_CDNA4, "cdna4", 32, cdna4::kVPermlane32SwapB32Vop1, + cdna4::kVPermlane32SwapB32Vop3}, + {ROCJITSU_CODE_ARCH_CDNA5, "gfx1250", 16, cdna5::kVPermlane16SwapB32Vop1, + cdna5::kVPermlane16SwapB32Vop3}, + }; + for (const auto &test_case : cases) { + SCOPED_TRACE(test_case.name); + const uint32_t stride = test_case.stride; + amdgpu::GpuMemory gpu_mem("permlane_swap_mem"); + amdgpu::L2Cache l2("permlane_swap_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = test_case.arch; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = test_case.arch == ROCJITSU_CODE_ARCH_CDNA5 ? 128 : 106; + cfg.vgprs_per_wf = 256; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create(test_case.name, cfg, &gpu_mem, &l2); + auto decoder = Decoder::create(test_case.arch); + ASSERT_NE(cu, nullptr); + ASSERT_NE(decoder, nullptr); + auto plugins = std::make_shared(PluginSinkConfig{}); + auto recorder = std::make_unique(); + auto *observed = recorder.get(); + plugins->add(std::move(recorder)); + cu->set_plugin_group(plugins); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + ASSERT_NE(wf, nullptr); + const uint32_t wave_size = wf->wf_size(); + ASSERT_EQ(wave_size, test_case.arch == ROCJITSU_CODE_ARCH_CDNA5 ? 32u : 64u); + const uint64_t full_exec = wave_size == 64 ? ~uint64_t{0} : (uint64_t{1} << wave_size) - 1; + const uint32_t base = wf->vgpr_alloc().base; + + // CDNA4 ISA 6.2.2 masks VALU outputs with EXEC. Both operands are outputs + // here, even when the two register selectors alias (ISA 12.8). + for (bool same_register : {false, true}) { + const uint32_t src_reg = same_register ? 0u : 1u; + // -1 is e32; e64 also covers both FI/BOUND_CTRL bits. These modifiers + // cannot permit an inactive destination write. + for (int modifiers : {-1, 0, 1, 2, 3}) { + const auto opsel = static_cast(std::max(modifiers, 0)); + auto words = + test_case.arch == ROCJITSU_CODE_ARCH_CDNA5 + ? encode_vop3(test_case.e64_opcode, 0, vgpr_src(src_reg), 0, 0, 0, opsel) + : encode_cdna_vop3(test_case.e64_opcode, 0, vgpr_src(src_reg), 0, 0, opsel); + if (modifiers < 0) { + const auto src0 = static_cast(vgpr_src(src_reg)); + words[0] = test_case.arch == ROCJITSU_CODE_ARCH_CDNA5 + ? cdna5::build_vop1(test_case.e32_opcode, {.src0 = src0, .vdst = 0})[0] + : cdna4::build_vop1(test_case.e32_opcode, {.src0 = src0, .vdst = 0})[0]; + } + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + for (uint64_t exec : + {full_exec, uint64_t{0}, uint64_t{0x3333333333333333}, uint64_t{1}, uint64_t{1} << 16, + uint64_t{1} << (wave_size / 2), uint64_t{1} << (wave_size - 1)}) { + exec &= full_exec; + SCOPED_TRACE(::testing::Message() << "stride=" << stride << " same=" << same_register + << " modifiers=" << modifiers << " exec=" << exec); + wf->set_exec(exec); + for (uint32_t lane = 0; lane < wave_size; ++lane) { + cu->write_vgpr(base, lane, 0xD0000000u | lane); + cu->write_vgpr(base + 1, lane, 0x50000000u | lane); + } + observed->writes.clear(); + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + uint64_t writes[2] = {}; + for (const auto &[reg, lanes] : observed->writes) { + ASSERT_GE(reg, base); + ASSERT_LE(reg, base + 1); + writes[reg - base] |= lanes; + EXPECT_EQ(lanes & ~exec, 0u); + } + for (uint32_t lane = 0; lane < wave_size; ++lane) { + const bool active = (exec & (uint64_t{1} << lane)) != 0; + const bool peer_active = (exec & (uint64_t{1} << (lane ^ stride))) != 0; + // Active destinations with inactive sources involve separate + // FI/BOUND_CTRL semantics. This regression asserts their inactive + // peers stay unchanged, without prescribing those source rules. + if (active && !peer_active) + continue; + const bool upper = (lane & stride) != 0; + const uint32_t expected_dst = + active && (same_register || upper) + ? (same_register ? 0xD0000000u : 0x50000000u) | (lane ^ stride) + : 0xD0000000u | lane; + const uint32_t expected_src = active && !same_register && !upper + ? 0xD0000000u | (lane ^ stride) + : 0x50000000u | lane; + EXPECT_EQ(cu->read_vgpr(base, lane), expected_dst) << "dst lane=" << lane; + EXPECT_EQ(cu->read_vgpr(base + 1, lane), expected_src) << "src lane=" << lane; + EXPECT_EQ(bool(writes[0] & (uint64_t{1} << lane)), active && (same_register || upper)); + EXPECT_EQ(bool(writes[1] & (uint64_t{1} << lane)), active && !same_register && !upper); + } + } + } + } + if (!wf->is_halted()) + wf->halt(); + } +} + uint32_t gfx1250_fp8_ab_lane(uint32_t row_or_col, uint32_t k) { return row_or_col + 16u * ((k >> 4) & 1u); } @@ -7274,6 +7398,9 @@ TEST(Gfx1250AtomicReturnTest, PackedHalfAddPreservesComponentsAcrossMemoryForms) ASSERT_NE(inst, nullptr); cu.execute_and_route(inst.release(), *wf); uint32_t expected = c.expected; + // LDS selects the incoming NaN when both operands are NaNs. + if (form == Form::Ds && c.old_value == (bf16 ? 0x7fc27fc0u : 0x7e027e00u)) + expected = bf16 ? 0x7fc47fc0u : 0x7e047e00u; if (form == Form::Ds && ((c.input_denorm && !(denorm & 1u)) || (c.output_denorm && !(denorm & 2u)))) expected = c.flushed_value; @@ -7834,6 +7961,100 @@ TEST(HwregHelperTest, Gfx1250UsesManualHwregIdsInsteadOfLegacyAliases) { wf->halt(); } +TEST(HwregTest, Gfx1250GetregReadsWgpId) { + amdgpu::GpuMemory gpu_mem("hwreg_gfx1250_wgp_id_mem"); + amdgpu::L2Cache l2("hwreg_gfx1250_wgp_id_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 2; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 256; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2); + ASSERT_NE(cu, nullptr); + cu->set_shader_engine_location(1, 13, 8); + + auto decoder = Decoder::create(cfg.arch); + ASSERT_NE(decoder, nullptr); + // The instruction emitted by tl.extra.hip.smid(). + constexpr uint16_t kWgpId = encode_hwreg(23, 10, 4); + const auto words = encode_sopk(cdna5::kSGetregB32Sopk, 4, kWgpId); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + + for (uint32_t slot = 0; slot < cfg.num_wf_slots; ++slot) { + auto *wf = cu->dispatch_wf(0, slot, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + ASSERT_NE(wf, nullptr); + EXPECT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 4), 5u); + uint32_t value = 0; + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, kWgpId, value), amdgpu::HwregAccessResult::Success); + EXPECT_EQ(value, 5u); + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(23, 11, 2), value), + amdgpu::HwregAccessResult::Success); + EXPECT_EQ(value, 2u); + EXPECT_EQ(amdgpu::write_hwreg_field(*wf, kWgpId, 0), amdgpu::HwregAccessResult::ReadOnly); + // The remaining identity fields still have no modeled backing. + for (uint16_t hwreg : {encode_hwreg(23), encode_hwreg(23, 9, 2), encode_hwreg(23, 13, 2), + encode_hwreg(23, 16, 1)}) { + value = 0xFFFFFFFFu; + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, hwreg, value), + amdgpu::HwregAccessResult::Unsupported); + EXPECT_EQ(value, 0u); + } + } + for (uint32_t slot = 0; slot < cfg.num_wf_slots; ++slot) + cu->wf(slot)->halt(); +} + +TEST(HwregHelperTest, Gfx1250WgpIdRequiresRepresentableShaderArrayWidth) { + amdgpu::GpuMemory gpu_mem("hwreg_gfx1250_wgp_id_width_mem"); + amdgpu::L2Cache l2("hwreg_gfx1250_wgp_id_width_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 256; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2); + ASSERT_NE(cu, nullptr); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + ASSERT_NE(wf, nullptr); + + constexpr uint32_t kWgpId = encode_hwreg(23, 10, 4); + for (uint32_t cu_index : {15u, 31u}) { + SCOPED_TRACE(cu_index); + cu->set_shader_engine_location(0, cu_index, 16); + uint32_t value = 0; + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, kWgpId, value), amdgpu::HwregAccessResult::Success); + EXPECT_EQ(value, 15u); + } + + cu->set_shader_engine_location(1, 13); + EXPECT_EQ(cu->shader_engine_id(), 1u); + EXPECT_EQ(cu->scratch_scoreboard_base(), 13u * cu->scratch_slots_per_cu()); + EXPECT_EQ(cu->cus_per_shader_array(), 0u); + uint32_t value = 0xFFFFFFFFu; + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, kWgpId, value), amdgpu::HwregAccessResult::Unsupported); + EXPECT_EQ(value, 0u); + + for (uint32_t width : {0u, 17u, 32u, std::numeric_limits::max()}) { + SCOPED_TRACE(width); + for (uint32_t cu_index : {0u, 16u}) { + SCOPED_TRACE(cu_index); + cu->set_shader_engine_location(0, cu_index, width); + for (uint16_t hwreg : {kWgpId, encode_hwreg(23, 11, 2)}) { + SCOPED_TRACE(hwreg); + uint32_t value = 0xFFFFFFFFu; + EXPECT_EQ(amdgpu::read_hwreg_field(*wf, hwreg, value), + amdgpu::HwregAccessResult::Unsupported); + EXPECT_EQ(value, 0u); + } + } + } + wf->halt(); +} + TEST(HwregHelperTest, Gfx1250ReadsModeledIbStsCounters) { amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_ib_sts_mem"); amdgpu::L2Cache l2("hwreg_helper_gfx1250_ib_sts_l2"); diff --git a/emulation/rocjitsu/tests/interposer_dup_test.cpp b/emulation/rocjitsu/tests/interposer_dup_test.cpp index 29a7cfc1b933..3bee071de0e1 100644 --- a/emulation/rocjitsu/tests/interposer_dup_test.cpp +++ b/emulation/rocjitsu/tests/interposer_dup_test.cpp @@ -39,9 +39,11 @@ RJ_DIAGNOSTIC_POP #include #include #include +#include #include #include #include +#include #include #include #include @@ -600,6 +602,7 @@ TEST(InterposerDupTest, DrmCloseReuseRaceNeverMisclassifiesDuplicate) { constexpr int kIterations = 200; for (int iteration = 0; iteration < kIterations; ++iteration) { + SCOPED_TRACE(iteration); int drm = open_drm_render(); ASSERT_GE(drm, 0); drm_syncobj_create create{}; @@ -609,6 +612,7 @@ TEST(InterposerDupTest, DrmCloseReuseRaceNeverMisclassifiesDuplicate) { int replacement = make_sized_memfd(0x1000); ASSERT_GE(replacement, 0); + struct stat replacement_stat {}; ASSERT_EQ(syscall(SYS_fstat, replacement, &replacement_stat), 0); @@ -617,7 +621,7 @@ TEST(InterposerDupTest, DrmCloseReuseRaceNeverMisclassifiesDuplicate) { // runtime teardown may itself open and close temporary descriptors; letting it // overlap the other worker's deliberate reuse of the lowest free fd adds an // unrelated fd-number ABA race. This barrier is after the syscalls, so it does - // not order dup() against close()/dup2(). + // not order dup() against close()/dup(). std::barrier workers_finished(2); std::atomic duplicated{-1}; std::atomic duplicate_errno{0}; @@ -643,7 +647,8 @@ TEST(InterposerDupTest, DrmCloseReuseRaceNeverMisclassifiesDuplicate) { int saved_reuse_errno; do { errno = 0; - reused = dup2(replacement, drm); + // Let the kernel select a free fd without overwriting runtime descriptors. + reused = dup(replacement); saved_reuse_errno = errno; } while (reused < 0 && (saved_reuse_errno == EBUSY || saved_reuse_errno == EINTR)); close_result = closed; @@ -658,7 +663,7 @@ TEST(InterposerDupTest, DrmCloseReuseRaceNeverMisclassifiesDuplicate) { const int duplicate = duplicated.load(); ASSERT_EQ(close_result.load(), 0) << "iteration=" << iteration << " close_errno=" << close_errno.load(); - ASSERT_EQ(reuse_result.load(), drm) + ASSERT_GE(reuse_result.load(), 0) << "iteration=" << iteration << " reuse_errno=" << reuse_errno.load() << " duplicated=" << duplicate << " duplicate_errno=" << duplicate_errno.load(); if (duplicate < 0) { @@ -666,7 +671,8 @@ TEST(InterposerDupTest, DrmCloseReuseRaceNeverMisclassifiesDuplicate) { } struct stat reused_stat {}; - ASSERT_EQ(syscall(SYS_fstat, drm, &reused_stat), 0) << "iteration=" << iteration; + ASSERT_EQ(syscall(SYS_fstat, reuse_result.load(), &reused_stat), 0) + << "iteration=" << iteration; EXPECT_EQ(reused_stat.st_dev, replacement_stat.st_dev); EXPECT_EQ(reused_stat.st_ino, replacement_stat.st_ino); @@ -675,22 +681,19 @@ TEST(InterposerDupTest, DrmCloseReuseRaceNeverMisclassifiesDuplicate) { ASSERT_EQ(syscall(SYS_fstat, duplicate, &duplicate_stat), 0); drm_syncobj_destroy destroy{}; destroy.handle = create.handle; - if (duplicate_stat.st_ino == drm_stat.st_ino) { - EXPECT_EQ(ioctl(duplicate, DRM_IOCTL_SYNCOBJ_DESTROY, &destroy), 0); + if (duplicate_stat.st_dev == drm_stat.st_dev && duplicate_stat.st_ino == drm_stat.st_ino) { + EXPECT_EQ(ioctl(duplicated.load(), DRM_IOCTL_SYNCOBJ_DESTROY, &destroy), 0); } else { - EXPECT_EQ(duplicate_stat.st_ino, replacement_stat.st_ino); - EXPECT_EQ(ioctl(duplicate, DRM_IOCTL_SYNCOBJ_DESTROY, &destroy), -1); + // The old number may now name the replacement or another thread's + // temporary file. Neither may inherit the old DRM namespace. + EXPECT_EQ(ioctl(duplicated.load(), DRM_IOCTL_SYNCOBJ_DESTROY, &destroy), -1); EXPECT_EQ(errno, ENOTTY); } - EXPECT_EQ(close(duplicate), 0) << "iteration=" << iteration; + EXPECT_NE(duplicated.load(), reuse_result.load()); + EXPECT_EQ(close(duplicated.load()), 0); } - // dup() can legally reuse drm's numeric fd if it acquired the source file - // before close() freed that number. In that case duplicate and drm name the - // same post-dup2 descriptor and must be closed only once. - if (duplicate != drm) { - EXPECT_EQ(close(drm), 0) << "iteration=" << iteration; - } + EXPECT_EQ(close(reuse_result.load()), 0); EXPECT_EQ(close(replacement), 0); } EXPECT_EQ(close(kfd), 0); @@ -1058,8 +1061,726 @@ class InterposerPm4Test : public ::testing::Test { wait.timeout_nsec = deadline; return ioctl(drm_, DRM_IOCTL_SYNCOBJ_WAIT, &wait); } + + int query_submission(uint64_t sequence, uint64_t timeout, uint64_t *busy, + uint32_t engine = AMDGPU_HW_IP_COMPUTE, uint32_t ring = 0) { + drm_amdgpu_wait_cs wait{}; + wait.in.handle = sequence; + wait.in.timeout = timeout; + wait.in.ip_type = engine; + wait.in.ring = ring; + wait.in.ctx_id = context_; + const int rc = ioctl(drm_, DRM_IOCTL_AMDGPU_WAIT_CS, &wait); + *busy = wait.out.status; + return rc; + } }; +TEST_F(InterposerPm4Test, SubmissionFenceQueryPollsAndWaitsForCompletion) { + uint64_t busy = 99; + ASSERT_EQ(query_submission(0, 0, &busy), 0); + EXPECT_EQ(busy, 0u); + ASSERT_EQ(query_submission(UINT64_MAX, 0, &busy), 0); + EXPECT_EQ(busy, 0u); + memory_[0] = 0xffff1000; + uint64_t sequence = 0; + ASSERT_EQ(submit(1, true, &sequence), 0); + EXPECT_EQ(sequence, 1u); + ASSERT_EQ(query_submission(sequence, 0, &busy), 0); + EXPECT_EQ(busy, 1u); + ASSERT_EQ( + query_submission(UINT64_MAX, monotonic_deadline_after(std::chrono::milliseconds(10)), &busy), + 0); + EXPECT_EQ(busy, 1u); + int signal_rc = -1; + std::thread release([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + drm_syncobj_array signal{}; + signal.handles = reinterpret_cast(&input_); + signal.count_handles = 1; + signal_rc = ioctl(drm_, DRM_IOCTL_SYNCOBJ_SIGNAL, &signal); + }); + const int rc = + query_submission(sequence, monotonic_deadline_after(std::chrono::seconds(5)), &busy); + release.join(); + EXPECT_EQ(signal_rc, 0); + EXPECT_EQ(rc, 0); + EXPECT_EQ(busy, 0u); + EXPECT_EQ(query_submission(sequence, UINT64_MAX, &busy), 0); + EXPECT_EQ(busy, 0u); +} + +TEST_F(InterposerPm4Test, SubmissionFenceSequencesAreLocalToContextAndEngine) { + memory_[0] = 0xffff1000; + uint64_t pending = 0, graphics = 0, other = 0, busy = 99; + ASSERT_EQ(submit(1, true, &pending), 0); + ASSERT_EQ(submit(1, false, &graphics, AMDGPU_HW_IP_GFX), 0); + EXPECT_EQ(pending, 1u); + EXPECT_EQ(graphics, 1u); + ASSERT_EQ(query_submission(graphics, monotonic_deadline_after(std::chrono::seconds(5)), &busy, + AMDGPU_HW_IP_GFX), + 0); + EXPECT_EQ(busy, 0u); + ASSERT_EQ(query_submission(pending, 0, &busy), 0); + EXPECT_EQ(busy, 1u); + const uint32_t original = context_; + drm_amdgpu_ctx ctx{}; + ctx.in.op = AMDGPU_CTX_OP_ALLOC_CTX; + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_CTX, &ctx), 0); + context_ = ctx.out.alloc.ctx_id; + ASSERT_EQ(submit(1, false, &other), 0); + EXPECT_EQ(other, 1u); + ASSERT_EQ(query_submission(other, monotonic_deadline_after(std::chrono::seconds(5)), &busy), 0); + EXPECT_EQ(busy, 0u); + context_ = original; + ASSERT_EQ(query_submission(pending, 0, &busy), 0); + EXPECT_EQ(busy, 1u); + drm_syncobj_array signal{}; + signal.handles = reinterpret_cast(&input_); + signal.count_handles = 1; + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_SYNCOBJ_SIGNAL, &signal), 0); + EXPECT_EQ(query_submission(pending, monotonic_deadline_after(std::chrono::seconds(5)), &busy), 0); +} + +TEST_F(InterposerPm4Test, SubmissionFenceQueryRejectsInvalidIdentityAndReportsFailure) { + uint64_t busy = 99; + EXPECT_EQ(query_submission(1, 0, &busy), -1); + EXPECT_EQ(errno, EINVAL); + EXPECT_EQ(query_submission(0, 0, &busy, AMDGPU_HW_IP_DMA), -1); + EXPECT_EQ(errno, EINVAL); + EXPECT_EQ(query_submission(0, 0, &busy, AMDGPU_HW_IP_COMPUTE, 4), -1); + EXPECT_EQ(errno, EINVAL); + const uint32_t original = context_; + context_ = UINT32_MAX; + EXPECT_EQ(query_submission(0, 0, &busy), -1); + EXPECT_EQ(errno, EINVAL); + context_ = original; + memory_[0] = 0xc000ff00; + memory_[1] = 0; + uint64_t sequence = 0; + ASSERT_EQ(submit(2, false, &sequence), 0); + EXPECT_EQ(query_submission(sequence, monotonic_deadline_after(std::chrono::seconds(5)), &busy), + -1); + EXPECT_EQ(errno, EIO); + EXPECT_EQ(query_submission(UINT64_MAX, 0, &busy), -1); + EXPECT_EQ(errno, EIO); +} + +TEST_F(InterposerPm4Test, SubmissionFenceQueryRetiresOldHistory) { + memory_[0] = 0xffff1000; + uint64_t sequence = 0, busy = 99; + for (uint64_t i = 1; i <= 80; ++i) { + ASSERT_EQ(submit(1, false, &sequence), 0); + EXPECT_EQ(sequence, i); + ASSERT_EQ(query_submission(sequence, monotonic_deadline_after(std::chrono::seconds(5)), &busy), + 0); + EXPECT_EQ(busy, 0u); + } + EXPECT_EQ(query_submission(1, 0, &busy), 0); + EXPECT_EQ(busy, 0u); + EXPECT_EQ(query_submission(sequence + 1, 0, &busy), -1); + EXPECT_EQ(errno, EINVAL); +} + +TEST_F(InterposerPm4Test, TransferWaitsForSubmissionWithoutWaitingForCompletion) { + drm_syncobj_create target{}; + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_SYNCOBJ_CREATE, &target), 0); + drm_syncobj_transfer transfer{}; + transfer.src_handle = output_; + transfer.dst_handle = target.handle; + EXPECT_EQ(ioctl(drm_, DRM_IOCTL_SYNCOBJ_TRANSFER, &transfer), -1); + EXPECT_EQ(errno, EINVAL); + transfer.flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL; + EXPECT_EQ(ioctl(drm_, DRM_IOCTL_SYNCOBJ_TRANSFER, &transfer), -1); + EXPECT_EQ(errno, EINVAL); + transfer.flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT; + std::atomic started{false}; + int transfer_rc = -1; + std::thread worker([&] { + started.store(true, std::memory_order_release); + transfer_rc = ioctl(drm_, DRM_IOCTL_SYNCOBJ_TRANSFER, &transfer); + }); + while (!started.load(std::memory_order_acquire)) + std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + memory_[0] = 0xffff1000; + uint64_t sequence = 0; + const int submit_rc = submit(1, true, &sequence); + worker.join(); + ASSERT_EQ(submit_rc, 0); + ASSERT_EQ(transfer_rc, 0); + drm_syncobj_wait wait{}; + wait.handles = reinterpret_cast(&target.handle); + wait.count_handles = 1; + EXPECT_EQ(ioctl(drm_, DRM_IOCTL_SYNCOBJ_WAIT, &wait), -1); + EXPECT_EQ(errno, ETIME); + drm_syncobj_array signal{}; + signal.handles = reinterpret_cast(&input_); + signal.count_handles = 1; + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_SYNCOBJ_SIGNAL, &signal), 0); + wait.timeout_nsec = monotonic_deadline_after(std::chrono::seconds(5)); + EXPECT_EQ(ioctl(drm_, DRM_IOCTL_SYNCOBJ_WAIT, &wait), 0); + drm_syncobj_destroy destroy{}; + destroy.handle = target.handle; + EXPECT_EQ(ioctl(drm_, DRM_IOCTL_SYNCOBJ_DESTROY, &destroy), 0); +} + +TEST_F(InterposerPm4Test, GraphicsRegistersRetainAllPacketForms) { + std::vector packets; + const auto emit = [&](uint32_t opcode, std::initializer_list words) { + packets.push_back(0xc0000000 | ((words.size() - 1) << 16) | (opcode << 8)); + packets.insert(packets.end(), words.begin(), words.end()); + }; + emit(0x69, {0x300, 0x11111111, 0x22222222}); + emit(0xb8, {0x302, 0x33333333, 0x303, 0x44444444}); + emit(0xb9, {4, 0x03050304, 0x55555555, 0x66666666, 0x03070306, 0x77777777, 0x88888888}); + emit(0x79, {0x440, 0x99999999}); + emit(0x7a, {0x10000242, 4}); + emit(0xbe, {0x441, 0xaaaaaaaa, 0x442, 0xbbbbbbbb}); + emit(0x9b, {0x30000087, 0xcccccccc}); + const std::array registers{0xa300, 0xa301, 0xa302, 0xa303, 0xa304, 0xa305, 0xa306, + 0xa307, 0xc440, 0xc242, 0xc441, 0xc442, 0x2c87}; + for (uint32_t i = 0; i < registers.size(); ++i) + emit(0x40, + {5u << 8, registers[i], 0, uint32_t(kAddress + 6144 + i * 4), uint32_t(kAddress >> 32)}); + std::memcpy(memory_, packets.data(), packets.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packets.size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + const std::array expected{ + 0x11111111, 0x22222222, 0x33333333, 0x44444444, 0x55555555, 0x66666666, 0x77777777, + 0x88888888, 0x99999999, 4, 0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc}; + for (uint32_t i = 0; i < expected.size(); ++i) + EXPECT_EQ(memory_[1536 + i], expected[i]) << "register " << std::hex << registers[i]; +} + +TEST_F(InterposerPm4Test, ContextMaskedUpdatesAndTranslationPrefetchPreserveOtherState) { + const uint32_t packet[] = {0xc0026900, + 0x300, + 0xa5a5a5a5, + 0x12345678, + 0xc0025100, + 0x300, + 0x00ff00ff, + 0xffff0000, + 0xc0025100, + 0x301, + 0, + 0xffffffff, + 0xc0035d00, + 0x4000000f, + uint32_t(kAddress + 4096), + uint32_t(kAddress >> 32), + 1, + 0xc0044000, + (5u << 8) | (1u << 16), + 0xa300, + 0, + uint32_t(kAddress + 6144), + uint32_t(kAddress >> 32)}; + memory_[1024] = 0xabcdef12; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence, AMDGPU_HW_IP_GFX), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + EXPECT_EQ(memory_[1536], 0xa5ffa500); + EXPECT_EQ(memory_[1537], 0x12345678); + EXPECT_EQ(memory_[1024], 0xabcdef12); +} + +TEST_F(InterposerPm4Test, DispatchInterleaveShadowIsQualifiedForGfx12) { + drm_amdgpu_info_device device{}; + drm_amdgpu_info info{}; + info.query = AMDGPU_INFO_DEV_INFO; + info.return_pointer = reinterpret_cast(&device); + info.return_size = sizeof(device); + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_INFO, &info), 0); + const uint32_t packet[] = {0xc0019b00, + 0x2000022f, + 0x40, + 0xc0044000, + 5u << 8, + 0x2e2f, + 0, + uint32_t(kAddress + 6144), + uint32_t(kAddress >> 32)}; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence, AMDGPU_HW_IP_GFX), 0); + const int rc = wait_output(monotonic_deadline_after(std::chrono::seconds(5))); + if (device.family == AMDGPU_FAMILY_GC_12_0_0) { + EXPECT_EQ(rc, 0); + EXPECT_EQ(memory_[1536], 0x40); + } else { + EXPECT_EQ(rc, -1); + EXPECT_EQ(errno, EIO); + } +} + +TEST_F(InterposerPm4Test, ComputeTranslationPrefetchIgnoresPfpSelector) { + const uint32_t packet[] = { + 0xc0035d00, 0x40000004, uint32_t(kAddress), uint32_t(kAddress >> 32), 1, + 0xc0033700, 5u << 8, uint32_t(kAddress + 6144), uint32_t(kAddress >> 32), 0x12345678}; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence), 0); + EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(2))), 0); + EXPECT_EQ(memory_[1536], 0x12345678u); +} + +TEST_F(InterposerPm4Test, StreamoutQuerySamplesAllFourMemoryCounterPairs) { + drm_amdgpu_info_device device{}; + ASSERT_TRUE(query_drm_device_info(drm_, &device)); + const uint64_t source = kAddress + 5120, destination = kAddress + 6144; + std::vector packets; + for (uint32_t stream = 0; stream < 4; ++stream) { + // Distinct 64-bit counters, including a counter with its high bit already set. + memory_[1284 + 4 * stream] = 17 + stream; + memory_[1285 + 4 * stream] = 3 + stream; + memory_[1286 + 4 * stream] = 31 + stream; + memory_[1287 + 4 * stream] = 0x80000005 + stream; + const uint64_t target = destination + 16 * stream; + packets.insert(packets.end(), {0xc004c300, uint32_t(source), uint32_t(source >> 32), stream, + uint32_t(target), uint32_t(target >> 32)}); + } + std::fill_n(memory_ + 1536, 18, 0xdeadbeef); + std::memcpy(memory_, packets.data(), packets.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packets.size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + const int rc = wait_output(monotonic_deadline_after(std::chrono::seconds(5))); + if (device.family != AMDGPU_FAMILY_GC_12_0_0) { + EXPECT_EQ(rc, -1); + EXPECT_EQ(errno, EIO); + EXPECT_EQ(memory_[1536], 0xdeadbeef); + return; + } + ASSERT_EQ(rc, 0); + for (uint32_t stream = 0; stream < 4; ++stream) { + EXPECT_EQ(memory_[1536 + 4 * stream], 17 + stream); + EXPECT_EQ(memory_[1537 + 4 * stream], 0x80000003 + stream); + EXPECT_EQ(memory_[1538 + 4 * stream], 31 + stream); + EXPECT_EQ(memory_[1539 + 4 * stream], 0x80000005 + stream); + EXPECT_EQ(memory_[1285 + 4 * stream], 3 + stream); + } + EXPECT_EQ(memory_[1552], 0xdeadbeef); +} + +TEST_F(InterposerPm4Test, StreamoutQueryEventMarksZeroCounterSamplesValid) { + drm_amdgpu_info_device device{}; + ASSERT_TRUE(query_drm_device_info(drm_, &device)); + std::vector packets; + for (uint32_t stream = 0; stream < 4; ++stream) { + const uint64_t target = kAddress + 6144 + 16 * stream; + packets.insert(packets.end(), {0xc0024600, 15 | ((stream + 8) << 8), uint32_t(target), + uint32_t(target >> 32)}); + } + std::fill_n(memory_ + 1536, 18, 0xdeadbeef); + std::memcpy(memory_, packets.data(), packets.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packets.size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + const int rc = wait_output(monotonic_deadline_after(std::chrono::seconds(5))); + if (device.family != AMDGPU_FAMILY_GC_11_0_0 && device.family != AMDGPU_FAMILY_GC_11_5_0) { + EXPECT_EQ(rc, -1); + EXPECT_EQ(errno, EIO); + EXPECT_EQ(memory_[1536], 0xdeadbeef); + return; + } + ASSERT_EQ(rc, 0); + for (uint32_t i = 0; i < 8; ++i) { + EXPECT_EQ(memory_[1536 + 2 * i], 0u); + EXPECT_EQ(memory_[1537 + 2 * i], 0x80000000); + } + EXPECT_EQ(memory_[1552], 0xdeadbeef); +} + +TEST_F(InterposerPm4Test, InterleavedDispatchExecutesDirectAndIndirectShaders) { + drm_amdgpu_info_device device{}; + drm_amdgpu_info info{}; + info.query = AMDGPU_INFO_DEV_INFO; + info.return_pointer = reinterpret_cast(&device); + info.return_size = sizeof(device); + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_INFO, &info), 0); + // GFX12: scale the local ID, store a constant, wait for the store, end. + const uint32_t shader[] = {0x30000082, 0x7e0202ff, 0x12345678, 0xee068000, + 0x00800000, 0, 0xbfc10000, 0xbfb00000}; + std::memcpy(memory_ + 1152, shader, sizeof(shader)); + const uint64_t code = kAddress + 4608, data = kAddress + 6144, arguments = kAddress + 5120; + memory_[1280] = memory_[1281] = memory_[1282] = 1; + for (bool indirect : {false, true}) { + std::fill_n(memory_ + 1536, 4, 0xabcdef01u); + std::vector packet{0xc0037600, + 0x207, + 2, + 1, + 1, + 0xc0027600, + 0x20c, + uint32_t(code >> 8), + uint32_t(code >> 40), + 0xc0017600, + 0x213, + 2u << 1, + 0xc0027600, + 0x240, + uint32_t(data), + uint32_t(data >> 32)}; + if (indirect) + packet.insert(packet.end(), {0xc0021100, 1, uint32_t(arguments), uint32_t(arguments >> 32), + 0xc001a802, 0, 0x48005}); + else + packet.insert(packet.end(), {0xc003a702, 1, 1, 1, 0x48005}); + std::memcpy(memory_, packet.data(), packet.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packet.size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + const int rc = wait_output(monotonic_deadline_after(std::chrono::seconds(5))); + if (device.family != AMDGPU_FAMILY_GC_12_0_0) { + EXPECT_EQ(rc, -1); + EXPECT_EQ(errno, EIO); + EXPECT_EQ(memory_[1536], 0xabcdef01u); + return; + } + ASSERT_EQ(rc, 0); + EXPECT_EQ(memory_[1536], 0x12345678u); + EXPECT_EQ(memory_[1537], 0x12345678u); + EXPECT_EQ(memory_[1538], 0xabcdef01u); + } +} + +TEST_F(InterposerPm4Test, ContextRegisterLoadReadsMemoryAndPreservesAdjacentRegisters) { + const uint32_t packet[] = {0xc0046900, + 0x300, + 11, + 22, + 33, + 44, + 0xc0039f00, + uint32_t(kAddress + 4096), + uint32_t(kAddress >> 32), + 0x301, + 2, + 0xc0044000, + 5u << 8, + 0xa300, + 0, + uint32_t(kAddress + 6144), + uint32_t(kAddress >> 32), + 0xc0044000, + 5u << 8, + 0xa301, + 0, + uint32_t(kAddress + 6148), + uint32_t(kAddress >> 32), + 0xc0044000, + 5u << 8, + 0xa302, + 0, + uint32_t(kAddress + 6152), + uint32_t(kAddress >> 32), + 0xc0044000, + 5u << 8, + 0xa303, + 0, + uint32_t(kAddress + 6156), + uint32_t(kAddress >> 32)}; + memory_[1024] = 55; + memory_[1025] = 66; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence, AMDGPU_HW_IP_GFX), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + EXPECT_EQ(memory_[1536], 11); + EXPECT_EQ(memory_[1537], 55); + EXPECT_EQ(memory_[1538], 66); + EXPECT_EQ(memory_[1539], 44); +} + +TEST_F(InterposerPm4Test, ContextRegisterLoadRejectsRangeOverflow) { + const uint32_t packet[] = {0xc0039f00, uint32_t(kAddress + 4096), uint32_t(kAddress >> 32), + 0x1fff, 2}; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence, AMDGPU_HW_IP_GFX), 0); + EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), -1); + EXPECT_EQ(errno, EIO); +} + +TEST_F(InterposerPm4Test, ContextRegisterLoadsRejectComputeEngine) { + for (uint32_t opcode : {0x61u, 0x9fu}) { + SCOPED_TRACE(opcode); + // A rejected packet faults the queue, so each form gets a fresh context. + drm_amdgpu_ctx ctx{}; + ctx.in.op = AMDGPU_CTX_OP_ALLOC_CTX; + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_CTX, &ctx), 0); + context_ = ctx.out.alloc.ctx_id; + const uint32_t packet[] = {0xc0030000 | (opcode << 8), + uint32_t(kAddress + 4096), + uint32_t(kAddress >> 32), + 0x300, + 1, + 0xc0033700, + 5u << 8, + uint32_t(kAddress + 6144), + uint32_t(kAddress >> 32), + 0x12345678}; + memory_[1536] = 0xabcdef12; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence), 0); + EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), -1); + EXPECT_EQ(errno, EIO); + EXPECT_EQ(memory_[1536], 0xabcdef12u); + } +} + +TEST_F(InterposerPm4Test, ClearStatePushPopRestoresContextAcrossSubmissions) { + const uint32_t first[] = {0xc0016900, 0x300, 11, 0xc0017600, 0x240, 55, 0xc0017900, 0x100, + 66, 0xc0001200, 1, 0xc0016900, 0x300, 22, 0xc0017600, 0x240, + 77, 0xc0017900, 0x100, 88}; + std::memcpy(memory_, first, sizeof(first)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(first), false, &sequence, AMDGPU_HW_IP_GFX), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + std::vector second{0xc0001200, 2}; + uint64_t destination = kAddress + 6144; + for (uint32_t reg : {0xa300u, 0x2e40u, 0xc100u}) { + second.insert(second.end(), {0xc0044000, 5u << 8, reg, 0, uint32_t(destination), + uint32_t(destination >> 32)}); + destination += 4; + } + std::memcpy(memory_, second.data(), second.size() * 4); + ASSERT_EQ(submit(second.size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + EXPECT_EQ(memory_[1536], 11u); + EXPECT_EQ(memory_[1537], 77u); + EXPECT_EQ(memory_[1538], 88u); +} + +TEST_F(InterposerPm4Test, ClearStateResetUsesRdna3Defaults) { + drm_amdgpu_info_device device{}; + drm_amdgpu_info info{}; + info.query = AMDGPU_INFO_DEV_INFO; + info.return_pointer = reinterpret_cast(&device); + info.return_size = sizeof(device); + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_INFO, &info), 0); + if (device.family == AMDGPU_FAMILY_GC_12_0_0) + GTEST_SKIP() << "GFX12 CLEAR_STATE has only push and pop modes"; + for (uint32_t command : {0u, 3u}) { + SCOPED_TRACE(command); + std::vector packets; + const auto emit = [&](uint32_t opcode, std::initializer_list words) { + packets.push_back(0xc0000000 | ((words.size() - 1) << 16) | (opcode << 8)); + packets.insert(packets.end(), words.begin(), words.end()); + }; + // A zero default, three nonzero defaults, and a gap outside the shadow ranges. + const uint32_t registers[] = {0, 0xd, 0x81, 0xb5, 0x22}; + for (uint32_t reg : registers) + emit(0x69, {reg, 99}); + if (command == 0) + emit(0x12, {1}); + emit(0x12, {command}); + uint32_t destination = uint32_t(kAddress + 6144); + const auto read = [&](uint32_t reg) { + emit(0x40, {5u << 8, 0xa000 + reg, 0, destination, uint32_t(kAddress >> 32)}); + destination += 4; + }; + for (uint32_t reg : registers) + read(reg); + emit(0x12, {2}); + for (uint32_t reg : registers) + read(reg); + std::memcpy(memory_, packets.data(), packets.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packets.size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + const uint32_t expected[] = {0, 0x40004000, 0x80000000, 0x3f800000, 99}; + for (uint32_t i = 0; i < std::size(registers); ++i) { + EXPECT_EQ(memory_[1536 + i], expected[i]); + EXPECT_EQ(memory_[1536 + std::size(registers) + i], 99u); + } + } +} + +TEST_F(InterposerPm4Test, InvalidClearStateFailsBeforeFollowingWrite) { + drm_amdgpu_info_device device{}; + drm_amdgpu_info info{}; + info.query = AMDGPU_INFO_DEV_INFO; + info.return_pointer = reinterpret_cast(&device); + info.return_size = sizeof(device); + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_INFO, &info), 0); + std::vector> invalid{ + {0xc0001200, 4}, {0xc0001200, 0x10}, {0xc0001200, 2}, {0xc0001200, 1, 0xc0001200, 1}}; + if (device.family == AMDGPU_FAMILY_GC_12_0_0) { + invalid.push_back({0xc0001200, 0}); + invalid.push_back({0xc0001200, 3}); + } + for (auto packets : invalid) { + drm_amdgpu_ctx ctx{}; + ctx.in.op = AMDGPU_CTX_OP_ALLOC_CTX; + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_CTX, &ctx), 0); + context_ = ctx.out.alloc.ctx_id; + packets.insert(packets.end(), {0xc0033700, 5u << 8, uint32_t(kAddress + 6144), + uint32_t(kAddress >> 32), 0x12345678}); + memory_[1536] = 0xabcdef12; + std::memcpy(memory_, packets.data(), packets.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packets.size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), -1); + EXPECT_EQ(errno, EIO); + EXPECT_EQ(memory_[1536], 0xabcdef12u); + } +} + +TEST_F(InterposerPm4Test, ShadowRegisterLoadsUseSparseMemoryOffsets) { + std::vector packets; + const auto emit = [&](uint32_t opcode, std::initializer_list words) { + packets.push_back(0xc0000000 | ((words.size() - 1) << 16) | (opcode << 8)); + packets.insert(packets.end(), words.begin(), words.end()); + }; + // Different apertures share the same sparse shadow layout. Reversing the + // two load ranges distinguishes register offsets from a packed source. + const std::array loads{0x5e, 0x5f, 0x61}; + const std::array sets{0x79, 0x76, 0x69}; + const std::array bases{0xc000, 0x2c00, 0xa000}; + std::fill(memory_ + 1152, memory_ + 1168, 0xdeadbeef); + memory_[1152 + 5] = 55; + memory_[1152 + 6] = 66; + memory_[1152 + 9] = 99; + for (size_t i = 0; i < loads.size(); ++i) { + emit(sets[i], {4, 11, 22, 33, 44, 88, 77}); + emit(loads[i], {uint32_t(kAddress + 4608), uint32_t(kAddress >> 32), 9, 1, 5, 2}); + for (uint32_t reg = 4; reg <= 9; ++reg) + emit(0x40, {5u << 8, bases[i] + reg, 0, uint32_t(kAddress + 6144 + (i * 6 + reg - 4) * 4), + uint32_t(kAddress >> 32)}); + } + std::memcpy(memory_, packets.data(), packets.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packets.size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + const std::array expected{11, 55, 66, 44, 88, 99}; + for (size_t i = 0; i < 18; ++i) + EXPECT_EQ(memory_[1536 + i], expected[i % 6]) << "result " << i; +} + +TEST_F(InterposerPm4Test, MemoryAtomicsApplyIntegerOperationsAtBothWidths) { + struct Case { + uint32_t operation; + uint64_t initial, source, compare, expected; + }; + const std::array cases{Case{7, 3, 7, 0, 7}, + Case{8, 3, 7, 3, 7}, + Case{8, 3, 7, 1, 3}, + Case{15, UINT64_MAX, 2, 0, 1}, + Case{16, 0, 1, 0, UINT64_MAX}, + Case{17, UINT64_MAX, 1, 0, UINT64_MAX}, + Case{18, UINT64_MAX, 1, 0, 1}, + Case{19, UINT64_MAX, 1, 0, 1}, + Case{20, UINT64_MAX, 1, 0, UINT64_MAX}, + Case{21, 0x5a, 0x3c, 0, 0x18}, + Case{22, 0x5a, 0x3c, 0, 0x7e}, + Case{23, 0x5a, 0x3c, 0, 0x66}, + Case{24, 6, 7, 0, 7}, + Case{24, 7, 7, 0, 0}, + Case{24, 9, 7, 0, 0}, + Case{25, 6, 7, 0, 5}, + Case{25, 7, 7, 0, 6}, + Case{25, 9, 7, 0, 7}, + Case{25, 0, 7, 0, 7}}; + std::vector packets; + std::vector expected; + for (uint32_t flags : {0u, 0x20u, 0x40u, 0x60u}) { + for (const auto &test : cases) { + const size_t index = expected.size(); + const uint64_t target = kAddress + 4608 + index * 16; + const bool wide = flags & 0x20; + memory_[1152 + index * 4] = uint32_t(test.initial); + memory_[1153 + index * 4] = wide ? uint32_t(test.initial >> 32) : 0xdeadbeef; + memory_[1154 + index * 4] = 0xa5a5a5a5; + memory_[1155 + index * 4] = 0x5a5a5a5a; + // Use a nonzero ignored cache policy; both return encodings mutate memory. + packets.insert(packets.end(), + {0xc0071e00, test.operation | flags | (1u << 25), uint32_t(target), + uint32_t(target >> 32), uint32_t(test.source), uint32_t(test.source >> 32), + uint32_t(test.compare), uint32_t(test.compare >> 32), 0}); + expected.push_back(wide ? test.expected : (0xdeadbeef00000000ull | uint32_t(test.expected))); + } + } + ASSERT_LT(packets.size(), 1024u); + std::memcpy(memory_, packets.data(), packets.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packets.size(), false, &sequence), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + for (size_t i = 0; i < expected.size(); ++i) { + const uint64_t result = + uint64_t{memory_[1152 + i * 4]} | (uint64_t{memory_[1153 + i * 4]} << 32); + EXPECT_EQ(result, expected[i]) << "atomic case " << i; + EXPECT_EQ(memory_[1154 + i * 4], 0xa5a5a5a5); + EXPECT_EQ(memory_[1155 + i * 4], 0x5a5a5a5a); + } +} + +TEST_F(InterposerPm4Test, InvalidMemoryPacketsFailFenceWithoutWritingMemory) { + const uint32_t low = uint32_t(kAddress + 6144), high = uint32_t(kAddress >> 32); + const std::vector> packets{ + {0xc0071e00, 15 | (1u << 8), low, high, 1, 0, 0, 0, 0}, // Loop command. + {0xc0071e00, 15 | (1u << 7), low, high, 1, 0, 0, 0, 0}, // Reserved control. + {0xc0071e00, 15, low, high, 1, 0, 0, 0, 1u << 13}, // Reserved interval. + {0xc0071e00, 9, low, high, 1, 0, 0, 0, 0}, // Unimplemented TC operation. + {0xc0071e00, 15, low + 1, high, 1, 0, 0, 0, 0}, // Unaligned 32-bit. + {0xc0071e00, 47, low + 4, high, 1, 0, 0, 0, 0}, // Unaligned 64-bit. + {0xc0071e00, 15, low + 0x100000, high, 1, 0, 0, 0, 0}, // Unmapped. + {0xc0061e00, 15, low, high, 1, 0, 0, 0}, // Short atomic. + {0xc0025e00, low, high, 1}, // Unpaired load range. + {0xc0035e00, low + 1, high, 0, 1}, // Unaligned shadow address. + {0xc0035e00, low, high | 0x10000, 0, 1}, // Reserved address bits. + {0xc0035e00, low, high, 0, 0}, // Empty range. + {0xc0035e00, low, high, 0x3fff, 2}, // UCONFIG overflow. + {0xc0035f00, low, high, 0x3ff, 2}, // SH overflow. + {0xc0036100, low, high, 0x1fff, 2}, // CONTEXT overflow. + {0xc0025100, 0x2000, 0xffffffff, 1}, // RMW outside context aperture. + {0xc0015100, 0, 0xffffffff}, // Short RMW packet. + {0xc0035d00, 0x10, uint32_t(kAddress), high, 1}, // Reserved prefetch control. + {0xc0035d00, 7, low, high, 1}, // Unaligned prefetch address. + {0xc0035d00, 7, uint32_t(kAddress), high, 0x4000}, // Reserved page count. + {0xc0019b00, 0x20000240, 1}, // Interleave index on another register. + {0xc0029b00, 0x2000022f, 1, 2}, // Interleave index over multiple registers. + {0xc0024600, 0x180f, low, high}, // Reserved streamout query control. + {0xc0024600, 0x80f, low + 4, high}, // Unaligned query destination. + {0xc0024600, 0x80f, low + 0x100000, high}, // Unmapped query destination. + {0xc004c300, low, high, 4, low + 32, high}, // Reserved stream selection. + {0xc004c300, low + 4, high, 0, low + 32, high}, // Unaligned control buffer. + {0xc004c300, low, high, 0, low + 36, high}, // Unaligned query destination. + {0xc004c300, low + 0x100000, high, 0, low, high}, // Unmapped control buffer. + {0xc004c300, low, high, 0, low + 0x100000, high}, // Unmapped query destination. + {0xc003c300, low, high, 0, low + 32}, // Missing destination high word. + {0xc0035e00, low + 0x100000, high, 0, 1}}; // Unmapped load. + for (size_t i = 0; i < packets.size(); ++i) { + SCOPED_TRACE(i); + // A fault retires the queue. Each malformed packet gets a fresh context. + drm_amdgpu_ctx ctx{}; + ctx.in.op = AMDGPU_CTX_OP_ALLOC_CTX; + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_CTX, &ctx), 0); + context_ = ctx.out.alloc.ctx_id; + memory_[1536] = 0xabcdef12; + memory_[1537] = 0x34567890; + std::memcpy(memory_, packets[i].data(), packets[i].size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packets[i].size(), false, &sequence, AMDGPU_HW_IP_GFX), 0); + EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), -1); + EXPECT_EQ(errno, EIO); + EXPECT_EQ(memory_[1536], 0xabcdef12); + EXPECT_EQ(memory_[1537], 0x34567890); + } +} + +TEST_F(InterposerPm4Test, MalformedPackedGraphicsRegistersFailFence) { + const uint32_t packet[] = {0xc003b900, 4, 0x03010300, 0x11111111, 0x22222222}; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence, AMDGPU_HW_IP_GFX), 0); + EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), -1); + EXPECT_EQ(errno, EIO); +} + TEST_F(InterposerPm4Test, DependencyDefersMemoryWriteAndCompletion) { // WRITE_DATA to the second page, then a header-only NOP. const uint32_t packet[] = {0xc0033700, @@ -1102,8 +1823,19 @@ TEST_F(InterposerPm4Test, FaultedQueueRejectsNewSubmission) { ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), -1); ASSERT_EQ(errno, EIO); memory_[0] = 0xffff1000; - EXPECT_EQ(submit(1, false, &sequence), -1); + // Rejected submissions must not retire the last accepted, failed fence. + for (unsigned attempt = 0; attempt < 65; ++attempt) { + uint64_t rejected_sequence = 0; + ASSERT_EQ(submit(1, false, &rejected_sequence), -1); + ASSERT_EQ(errno, EIO); + } + uint64_t busy = 0; + EXPECT_EQ(query_submission(sequence, 0, &busy), -1); + EXPECT_EQ(errno, EIO); + EXPECT_EQ(query_submission(UINT64_MAX, 0, &busy), -1); EXPECT_EQ(errno, EIO); + EXPECT_EQ(query_submission(sequence + 1, 0, &busy), -1); + EXPECT_EQ(errno, EINVAL); EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), -1); EXPECT_EQ(errno, EIO); } @@ -1192,13 +1924,21 @@ TEST_F(InterposerPm4Test, InsufficientScratchFailsFence) { } TEST_F(InterposerPm4Test, GraphicsQueueAcceptsBufferOwnershipFlushEvents) { - const uint32_t packet[] = {0xc0004600, 44, 0xc0004600, 46}; + const uint32_t packet[] = {0xc0004600, 44, 0xc0004600, 46, 0xc0004600, 38, 0xc0004600, 49}; std::memcpy(memory_, packet, sizeof(packet)); uint64_t sequence = 0; ASSERT_EQ(submit(std::size(packet), false, &sequence, AMDGPU_HW_IP_GFX), 0); EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(2))), 0); } +TEST_F(InterposerPm4Test, ComputeQueueAcceptsPipelineStatisticsControls) { + const uint32_t packet[] = {0xc0004600, 23, 0xc0004600, 24, 0xc0004600, 25, 0xc0004600, 26}; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence), 0); + EXPECT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(2))), 0); +} + TEST_F(InterposerPm4Test, MergedSyncFilesRetainPendingFenceAcrossDupAndImport) { memory_[0] = 0xffff1000; uint64_t sequence = 0; @@ -1436,6 +2176,56 @@ TEST_F(InterposerPm4Test, RepeatedTimelineTransfersVisitSharedDependenciesOnce) } } +TEST_F(InterposerPm4Test, ThreadDimensionsMaskPartialGroupsForDirectAndIndirectDispatch) { + drm_amdgpu_info_device device{}; + drm_amdgpu_info info{}; + info.query = AMDGPU_INFO_DEV_INFO; + info.return_pointer = reinterpret_cast(&device); + info.return_size = sizeof(device); + ASSERT_EQ(ioctl(drm_, DRM_IOCTL_AMDGPU_INFO, &info), 0); + // Every active invocation atomically increments one shared counter. + const uint32_t gfx11[] = {0x7e000280, 0x7e020281, 0xdcd60000, 0x00000100, + 0xbf8903f7, 0xbc7c0000, 0xbfb00000}; + const uint32_t gfx12[] = {0x7e000280, 0x7e020281, 0xee0d4000, 0x00800000, + 0, 0xbfc10000, 0xbfb00000}; + const bool is_gfx12 = device.family == AMDGPU_FAMILY_GC_12_0_0; + std::memcpy(memory_ + 1152, is_gfx12 ? gfx12 : gfx11, sizeof(gfx11)); + const uint64_t code = kAddress + 4608, data = kAddress + 6144, arguments = kAddress + 5120; + for (bool indirect : {false, true}) + for (uint32_t x : {0u, 1u, 64u, 70u}) { + SCOPED_TRACE(testing::Message() << "indirect=" << indirect << " x=" << x); + memory_[1536] = 10; + memory_[1280] = x; + memory_[1281] = memory_[1282] = 3; + std::vector packet{0xc0037600, + 0x207, + 64, + 2, + 2, + 0xc0027600, + 0x20c, + uint32_t(code >> 8), + uint32_t(code >> 40), + 0xc0017600, + 0x213, + 2u << 1, + 0xc0027600, + 0x240, + uint32_t(data), + uint32_t(data >> 32)}; + if (indirect) + packet.insert(packet.end(), + {0xc0021600, uint32_t(arguments), uint32_t(arguments >> 32), 0x8025}); + else + packet.insert(packet.end(), {0xc0031500, x, 3, 3, 0x8025}); + std::memcpy(memory_, packet.data(), packet.size() * 4); + uint64_t sequence = 0; + ASSERT_EQ(submit(packet.size(), false, &sequence), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + EXPECT_EQ(memory_[1536], 10 + x * 9); + } +} + TEST_F(InterposerPm4Test, GridProductOverflowFailsFence) { const uint64_t code = kAddress + 4096; const uint32_t packet[] = {0xc0037600, @@ -1660,6 +2450,64 @@ TEST_F(InterposerPm4Test, NestedIndirectBufferReturns) { EXPECT_EQ(memory_[1024], 0x12345678u); } +TEST(InterposerDrmTest, PrimaryNodePathHasCharacterDeviceMetadata) { + int kfd = open_kfd(); + ASSERT_GE(kfd, 0); + for (const char *symbol : {"stat", "lstat"}) { + SCOPED_TRACE(symbol); + auto function = + reinterpret_cast(dlsym(RTLD_DEFAULT, symbol)); + ASSERT_NE(function, nullptr); + struct stat node {}; + ASSERT_EQ(function("/dev/dri/card0", &node), 0); + EXPECT_TRUE(S_ISCHR(node.st_mode)); + EXPECT_EQ(node.st_rdev, makedev(226, 0)); + EXPECT_EQ(function("/dev/dri/card999999", &node), -1); + EXPECT_EQ(errno, ENOENT); + } + for (const char *symbol : {"stat64", "lstat64"}) { + SCOPED_TRACE(symbol); + auto function = + reinterpret_cast(dlsym(RTLD_DEFAULT, symbol)); + ASSERT_NE(function, nullptr); + struct stat64 node {}; + ASSERT_EQ(function("/dev/dri/card0", &node), 0); + EXPECT_TRUE(S_ISCHR(node.st_mode)); + EXPECT_EQ(node.st_rdev, makedev(226, 0)); + } + close(kfd); +} + +TEST(InterposerDrmTest, CanonicalSysfsPathsRetainSimulatedDevice) { + int kfd = open_kfd(); + ASSERT_GE(kfd, 0); + const char *path = "/sys/dev/char/226:0/device/uevent"; + const auto read_contents = [](const char *name) { + std::ifstream file(name); + return std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); + }; + const auto expected = read_contents(path); + ASSERT_FALSE(expected.empty()); + char *allocated = realpath(path, nullptr); + ASSERT_NE(allocated, nullptr); + EXPECT_EQ(read_contents(allocated), expected); + free(allocated); + + char resolved[PATH_MAX]; + ASSERT_EQ(realpath(path, resolved), resolved); + EXPECT_EQ(read_contents(resolved), expected); + auto checked = reinterpret_cast( + dlsym(RTLD_DEFAULT, "__realpath_chk")); + ASSERT_NE(checked, nullptr); + ASSERT_EQ(checked(path, resolved, sizeof(resolved)), resolved); + EXPECT_EQ(read_contents(resolved), expected); + ASSERT_EQ(checked("/", resolved, sizeof(resolved)), resolved); + EXPECT_STREQ(resolved, "/"); + EXPECT_EQ(realpath("/sys/dev/char/226:999999/device/uevent", resolved), nullptr); + EXPECT_EQ(errno, ENOENT); + close(kfd); +} + TEST(InterposerDrmTest, RenderNodePathMetadataMatchesDescriptor) { int kfd = open_kfd(); ASSERT_GE(kfd, 0); @@ -1720,6 +2568,76 @@ TEST(InterposerDrmTest, RenderNodePathMetadataMatchesDescriptor) { EXPECT_EQ(close(kfd), 0); } +TEST(InterposerDrmTest, BorrowedDisplayDescriptorUsesSimulatedIdentity) { + auto report_host = + reinterpret_cast(dlsym(RTLD_DEFAULT, "rj_test_host_drm")); + ASSERT_NE(report_host, nullptr) << "host DRM identity preload is required"; + int imported = make_sized_memfd(4096); + ASSERT_GE(imported, 0); + // Discover through the borrowed descriptor before any GPU open or enumeration. + // This descriptor bypasses open() just like one received from X11. Neither + // host minor is required to exist in the configured simulated topology. + dev_t presentation_device = 0; + for (dev_t host : {makedev(226, 128), makedev(226, 197)}) { + SCOPED_TRACE(minor(host)); + report_host(imported, host); + struct stat info {}; + ASSERT_EQ(fstat(imported, &info), 0); + ASSERT_TRUE(S_ISCHR(info.st_mode)); + if (presentation_device == 0) + presentation_device = info.st_rdev; + EXPECT_EQ(info.st_rdev, presentation_device); + const std::string sys = "/sys/dev/char/226:" + std::to_string(minor(info.st_rdev)); + struct stat node {}; + EXPECT_EQ(stat((sys + "/device/drm").c_str(), &node), 0); + DIR *dir = opendir("/dev/dri"); + ASSERT_NE(dir, nullptr); + bool matched = false; + while (const dirent *entry = readdir(dir)) { + const std::string path = std::string("/dev/dri/") + entry->d_name; + if (stat(path.c_str(), &node) == 0 && S_ISCHR(node.st_mode) && node.st_rdev == info.st_rdev) + matched = true; + } + EXPECT_EQ(closedir(dir), 0); + EXPECT_TRUE(matched) << "libdrm must find the fd in the enumerated DRM nodes"; + + struct stat64 large {}; + ASSERT_EQ(fstat64(imported, &large), 0); + EXPECT_EQ(large.st_rdev, info.st_rdev); + auto fxstat = + reinterpret_cast(dlsym(RTLD_DEFAULT, "__fxstat")); + auto fxstat64 = + reinterpret_cast(dlsym(RTLD_DEFAULT, "__fxstat64")); + ASSERT_NE(fxstat, nullptr); + ASSERT_NE(fxstat64, nullptr); + ASSERT_EQ(fxstat(1, imported, &node), 0); + EXPECT_EQ(node.st_rdev, info.st_rdev); + ASSERT_EQ(fxstat64(1, imported, &large), 0); + EXPECT_EQ(large.st_rdev, info.st_rdev); + + // An aliased display descriptor is metadata-only, not a physical execution + // endpoint. Passing this ioctl through would produce ENOTTY on our memfd. + drm_version version{}; + EXPECT_EQ(ioctl(imported, DRM_IOCTL_VERSION, &version), -1); + EXPECT_EQ(errno, ENODEV); + } + // Primary DRM nodes and unrelated character devices keep their host identity. + for (dev_t host : {makedev(226, 0), makedev(1, 3)}) { + report_host(imported, host); + struct stat info {}; + ASSERT_EQ(fstat(imported, &info), 0); + EXPECT_EQ(info.st_rdev, host); + drm_version version{}; + EXPECT_EQ(ioctl(imported, DRM_IOCTL_VERSION, &version), -1); + EXPECT_EQ(errno, ENOTTY); + } + report_host(-1, 0); + struct stat info {}; + ASSERT_EQ(fstat(imported, &info), 0); + EXPECT_TRUE(S_ISREG(info.st_mode)); + EXPECT_EQ(close(imported), 0); +} + TEST(InterposerSyncobjTest, VmTimelineWaitObservesSynchronousMapAndUnmap) { int kfd = open_kfd(); ASSERT_GE(kfd, 0); @@ -2371,6 +3289,71 @@ TEST(InterposerSyncobjTest, TimelineWaitReturnsEintrForSignal) { EXPECT_EQ(close(kfd), 0); } +TEST(InterposerSyncobjTest, TimelineWaitAllowsThreadExitFromSignal) { + int drm = open_drm_render(); + ASSERT_GE(drm, 0); + drm_syncobj_create create{}; + ASSERT_EQ(ioctl(drm, DRM_IOCTL_SYNCOBJ_CREATE, &create), 0); + uint64_t point = 1; + drm_syncobj_timeline_wait wait{}; + wait.handles = reinterpret_cast(&create.handle); + wait.points = reinterpret_cast(&point); + wait.count_handles = 1; + wait.flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT; + wait.timeout_nsec = monotonic_deadline_after(std::chrono::seconds(5)); + + // Model Wine terminating a thread while a C driver is blocked in ioctl. Avoid + // inheriting the C++ libc header's noexcept declaration in the caller, too. + auto call = reinterpret_cast(dlsym(RTLD_DEFAULT, "ioctl")); + ASSERT_NE(call, nullptr); + struct sigaction action { + }, old_action{}; + action.sa_handler = +[](int) { pthread_exit(nullptr); }; + sigemptyset(&action.sa_mask); + ASSERT_EQ(sigaction(SIGUSR1, &action, &old_action), 0); + std::atomic tid{0}; + bool unwound = false, returned = false; + std::thread waiter([&] { + struct Cleanup { + bool &unwound; + ~Cleanup() { unwound = true; } + } cleanup{unwound}; + tid.store(syscall(SYS_gettid), std::memory_order_release); + (void)call(drm, DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT, &wait); + returned = true; + }); + // Wait for the actual futex sleep so a signal during setup cannot make this + // pass without exercising forced unwinding through the interposer. + bool blocked = false; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); + while (std::chrono::steady_clock::now() < deadline) { + if (pid_t id = tid.load(std::memory_order_acquire)) { + std::ifstream state("/proc/self/task/" + std::to_string(id) + "/syscall"); + long number; + unsigned long address, operation; + if (state >> number >> std::hex >> address >> operation && number == SYS_futex && + operation == FUTEX_WAIT_BITSET_PRIVATE) { + blocked = true; + break; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + if (blocked) { + EXPECT_EQ(pthread_kill(waiter.native_handle(), SIGUSR1), 0); + } + waiter.join(); + EXPECT_TRUE(blocked); + EXPECT_TRUE(unwound); + EXPECT_FALSE(returned); + EXPECT_EQ(sigaction(SIGUSR1, &old_action, nullptr), 0); + drm_syncobj_destroy destroy{}; + destroy.handle = create.handle; + EXPECT_EQ(ioctl(drm, DRM_IOCTL_SYNCOBJ_DESTROY, &destroy), 0); + // Closing the file also checks that unwinding released its ioctl reservation. + EXPECT_EQ(close(drm), 0); +} + namespace { void check_fresh_fork_backends(bool concurrent_mappings) { @@ -3315,6 +4298,116 @@ TEST(InterposerGemTest, GemNamespaceIsPrivateButSharedByDuplicates) { EXPECT_EQ(close(kfd), 0); } +TEST(InterposerGemTest, SparseReservationsPreserveRangesAndFileOwnership) { + const int drm = open_drm_render(); + ASSERT_GE(drm, 0); + const int alias = dup(drm); + const int foreign = open_drm_render(); + ASSERT_GE(alias, 0); + ASSERT_GE(foreign, 0); + constexpr uint64_t base = 0x2000000000ull; + const auto request = DRM_AMDGPU_GEM_VA_request(); + drm_amdgpu_gem_va sparse{}; + sparse.operation = AMDGPU_VA_OP_MAP; + sparse.flags = AMDGPU_VM_PAGE_PRT; + sparse.va_address = base; + sparse.map_size = 1ull << 36; // A VA reservation must not allocate resident storage. + ASSERT_EQ(ioctl(drm, request, &sparse), 0); + EXPECT_EQ(ioctl(alias, request, &sparse), -1); + EXPECT_EQ(errno, EINVAL); + sparse.operation = AMDGPU_VA_OP_REPLACE; + EXPECT_EQ(ioctl(foreign, request, &sparse), -1); + EXPECT_EQ(errno, EINVAL); + + // Clear a page in the middle; neither remaining tail becomes available. + drm_amdgpu_gem_va clear{}; + clear.operation = AMDGPU_VA_OP_CLEAR; + clear.va_address = base + 4096; + clear.map_size = 1; + EXPECT_EQ(ioctl(alias, request, &clear), -1); + EXPECT_EQ(errno, EINVAL); + clear.va_address += 1; + clear.map_size = 4096; + EXPECT_EQ(ioctl(alias, request, &clear), -1); + EXPECT_EQ(errno, EINVAL); + // Rejected updates must preserve the original reservation as one exact range. + sparse.operation = AMDGPU_VA_OP_UNMAP; + ASSERT_EQ(ioctl(alias, request, &sparse), 0); + sparse.operation = AMDGPU_VA_OP_MAP; + ASSERT_EQ(ioctl(alias, request, &sparse), 0); + clear.va_address = base + 4096; + ASSERT_EQ(ioctl(alias, request, &clear), 0); + sparse.operation = AMDGPU_VA_OP_MAP; + sparse.map_size = 4096; + for (uint64_t offset : {0ull, 8192ull}) { + sparse.va_address = base + offset; + EXPECT_EQ(ioctl(drm, request, &sparse), -1); + EXPECT_EQ(errno, EINVAL); + } + sparse.va_address = base + 4096; + ASSERT_EQ(ioctl(drm, request, &sparse), 0); + sparse.operation = AMDGPU_VA_OP_UNMAP; + ASSERT_EQ(ioctl(alias, request, &sparse), 0); + sparse.operation = AMDGPU_VA_OP_MAP; + sparse.map_size = 0; + EXPECT_EQ(ioctl(drm, request, &sparse), -1); + EXPECT_EQ(errno, EINVAL); + sparse.map_size = 4096; + sparse.va_address = UINT64_MAX - 4095; + EXPECT_EQ(ioctl(drm, request, &sparse), -1); + EXPECT_EQ(errno, EINVAL); + + // Reservations survive duplicate close, but not the last close of their file. + ASSERT_EQ(close(drm), 0); + sparse.va_address = base; + EXPECT_EQ(ioctl(foreign, request, &sparse), -1); + ASSERT_EQ(close(alias), 0); + ASSERT_EQ(ioctl(foreign, request, &sparse), 0); + EXPECT_EQ(close(foreign), 0); +} + +TEST_F(InterposerPm4Test, SparseReplacementPreservesResidentTail) { + constexpr uint64_t base = kAddress + 0x100000; + const auto request = DRM_AMDGPU_GEM_VA_request(); + drm_amdgpu_gem_va update{}; + update.operation = AMDGPU_VA_OP_MAP; + update.flags = AMDGPU_VM_PAGE_PRT; + update.va_address = base; + update.map_size = 4 * 4096; + ASSERT_EQ(ioctl(drm_, request, &update), 0); + update.operation = AMDGPU_VA_OP_REPLACE; + update.flags = AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE; + update.handle = bo_; + update.map_size = 1; + EXPECT_EQ(ioctl(drm_, request, &update), -1); + EXPECT_EQ(errno, EINVAL); + update.map_size = 4096; + update.offset_in_bo = 1; + EXPECT_EQ(ioctl(drm_, request, &update), -1); + EXPECT_EQ(errno, EINVAL); + update.offset_in_bo = 0; + update.map_size = 8192; + ASSERT_EQ(ioctl(drm_, request, &update), 0); + update.flags = AMDGPU_VM_PAGE_PRT; + update.handle = 0; + update.map_size = 4096; + ASSERT_EQ(ioctl(drm_, request, &update), 0); + + // Unbinding the first page must preserve the second page's BO offset and PTE. + constexpr uint64_t address = base + 6144; + const uint32_t packet[] = {0xc0033700, 5u << 8, uint32_t(address), uint32_t(address >> 32), + 0x12345678}; + std::memcpy(memory_, packet, sizeof(packet)); + uint64_t sequence = 0; + ASSERT_EQ(submit(std::size(packet), false, &sequence), 0); + ASSERT_EQ(wait_output(monotonic_deadline_after(std::chrono::seconds(5))), 0); + EXPECT_EQ(memory_[1536], 0x12345678u); + update.operation = AMDGPU_VA_OP_CLEAR; + update.flags = 0; + update.map_size = 4 * 4096; + EXPECT_EQ(ioctl(drm_, request, &update), 0); +} + TEST(InterposerGemTest, IndependentDrmFilesRejectOverlappingMappings) { int kfd = open_kfd(); ASSERT_GE(kfd, 0); @@ -3580,9 +4673,9 @@ TEST(InterposerGemTest, MapSucceedsAfterExportFdClosed) { } // AMDGPU_VA_OP_REPLACE at a VA that overlaps an existing mapping of a DIFFERENT size -// must evict the old mapping (by its own extent), not silently install overlapping -// PTEs. After a REPLACE, the old owner's stale range must be gone: its handle's UNMAP -// of the original range must fail, and no double-unmap can occur. +// must split the old mapping, not silently install overlapping PTEs. After a REPLACE, the old +// owner's stale range must be gone: its handle's UNMAP of the original range must fail, and no +// double-unmap can occur. TEST(InterposerGemTest, ReplaceEvictsOverlappingDifferentSizeRange) { int kfd = open_kfd(); ASSERT_GE(kfd, 0); @@ -3634,6 +4727,11 @@ TEST(InterposerGemTest, ReplaceEvictsOverlappingDifferentSizeRange) { EXPECT_EQ(ioctl(drm, gem_va, &unmap_a), -1) << "A's overlapping range must have been evicted by B's REPLACE"; + // Only the replaced prefix is gone; the rest of A stays mapped. + unmap_a.va_address = va + kSmallBo; + unmap_a.map_size = kBigBo - kSmallBo; + EXPECT_EQ(ioctl(drm, gem_va, &unmap_a), 0); + // B's new range is live and its UNMAP succeeds exactly once. drm_amdgpu_gem_va unmap_b{}; unmap_b.handle = handle_b; diff --git a/emulation/rocjitsu/tests/kfd_events_test.cpp b/emulation/rocjitsu/tests/kfd_events_test.cpp new file mode 100644 index 000000000000..143e2391fac0 --- /dev/null +++ b/emulation/rocjitsu/tests/kfd_events_test.cpp @@ -0,0 +1,317 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "rocjitsu/kmd/linux/events.h" + +#include "rocjitsu/base/rj_compiler.h" +RJ_DIAGNOSTIC_PUSH +RJ_DIAGNOSTIC_IGNORE_PEDANTIC +#include "linux/uapi/kfd_ioctl.h" +RJ_DIAGNOSTIC_POP + +#include + +#include +#include +#include +#include +#include + +namespace { + +class KfdEventsTest : public ::testing::Test { +protected: + rocjitsu::EventState events; + + uint32_t create(bool auto_reset = true, uint32_t type = KFD_IOC_EVENT_SIGNAL) { + kfd_ioctl_create_event_args args{}; + args.event_type = type; + args.auto_reset = auto_reset; + EXPECT_EQ(events.create_event(&args, 1), 0); + return args.event_id; + } + + void signal(uint32_t id, int source = 0) { + if (source == 0) { + kfd_ioctl_set_event_args args{}; + args.event_id = id; + EXPECT_EQ(events.set_event(&args), 0); + } else { + events.signal_interrupt(source == 1 ? id : 0); + } + } + + void reset(uint32_t id) { + kfd_ioctl_reset_event_args args{}; + args.event_id = id; + EXPECT_EQ(events.reset_event(&args), 0); + } + + bool registered(uint32_t id, size_t count = 1) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (std::chrono::steady_clock::now() < deadline) { + if (events.waiter_count(id) == count) + return true; + std::this_thread::yield(); + } + return false; + } + + struct Wait { + rocjitsu::EventState &events; + std::vector data; + kfd_ioctl_wait_events_args args{}; + int rc = -1; + std::atomic done{false}; + std::thread thread; + + Wait(rocjitsu::EventState &events, std::initializer_list ids, uint64_t age = 0, + bool all = true, bool poll = false) + : events(events), data(ids.size()) { + size_t i = 0; + for (uint32_t id : ids) { + data[i].event_id = id; + data[i++].signal_event_data.last_event_age = age; + } + args.events_ptr = reinterpret_cast(data.data()); + args.num_events = data.size(); + args.wait_for_all = all; + args.timeout = poll ? 0 : 5000; + if (poll) + rc = events.wait_events(&args); + else + thread = std::thread([this] { + rc = this->events.wait_events(&args); + done.store(true); + }); + } + + ~Wait() { + if (thread.joinable()) { + events.begin_wait_cancel(); + thread.join(); + } + } + + void complete() { + if (thread.joinable()) + thread.join(); + EXPECT_EQ(rc, 0); + EXPECT_EQ(args.wait_result, KFD_IOC_WAIT_RESULT_COMPLETE); + } + + bool finished() { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!done.load() && std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + return done.load(); + } + }; + + void expect_unsignaled(uint32_t id) { + Wait poll(events, {id}, 0, true, true); + EXPECT_EQ(poll.rc, 0); + EXPECT_EQ(poll.args.wait_result, KFD_IOC_WAIT_RESULT_TIMEOUT); + } +}; + +class KfdEventOrderingTest : public KfdEventsTest, + public ::testing::WithParamInterface> { +}; + +TEST_P(KfdEventOrderingTest, AutoResetSignalReachesWaiter) { + const auto [source, age, signal_first] = GetParam(); + const auto id = create(); + if (signal_first) + signal(id, source); + Wait wait(events, {id}, age); + if (!signal_first) { + ASSERT_TRUE(registered(id)); + signal(id, source); + } + wait.complete(); + expect_unsignaled(id); +} + +INSTANTIATE_TEST_SUITE_P(SetInterruptBroadcast, KfdEventOrderingTest, + ::testing::Combine(::testing::Values(0, 1, 2), + ::testing::Values(uint64_t{0}, uint64_t{1}), + ::testing::Bool())); + +TEST_F(KfdEventsTest, AutoResetSignalReachesEveryRegisteredWaiter) { + const auto id = create(); + Wait legacy(events, {id}); + Wait aged(events, {id}, 1); + Wait second_legacy(events, {id}); + ASSERT_TRUE(registered(id, 3)); + signal(id); + legacy.complete(); + aged.complete(); + second_legacy.complete(); + expect_unsignaled(id); +} + +TEST_F(KfdEventsTest, WaitAllRemembersSignalsWithoutExposingThemToNewWaiters) { + const auto first = create(); + const auto second = create(); + Wait wait(events, {first, second}); + ASSERT_TRUE(registered(first)); + ASSERT_TRUE(registered(second)); + signal(first); + expect_unsignaled(first); + reset(first); + signal(second); + wait.complete(); + expect_unsignaled(first); + expect_unsignaled(second); +} + +TEST_F(KfdEventsTest, WaitAllConsumesInitiallySignaledAutoResetEvent) { + const auto first = create(); + const auto second = create(); + signal(first); + Wait wait(events, {first, second}); + ASSERT_TRUE(registered(second)); + expect_unsignaled(first); + signal(second); + wait.complete(); +} + +TEST_F(KfdEventsTest, ManualResetDoesNotRevokeRegisteredWaiterActivation) { + const auto first = create(false); + const auto second = create(false); + Wait wait(events, {first, second}); + ASSERT_TRUE(registered(first)); + signal(first); + reset(first); + expect_unsignaled(first); + signal(second); + wait.complete(); + Wait poll(events, {second}, 0, true, true); + poll.complete(); + reset(second); + expect_unsignaled(second); +} + +TEST_F(KfdEventsTest, CompletingWaitAllDoesNotConsumeALaterPendingSignal) { + const auto first = create(); + const auto second = create(); + signal(first); + Wait wait(events, {first, second}); + ASSERT_TRUE(registered(second)); + signal(first); + signal(second); + wait.complete(); + Wait later(events, {first}, 0, true, true); + later.complete(); + expect_unsignaled(first); +} + +TEST_F(KfdEventsTest, TimeoutConsumesPartialAutoResetActivation) { + const auto first = create(); + const auto second = create(); + signal(first); + Wait wait(events, {first, second}, 0, true, true); + EXPECT_EQ(wait.args.wait_result, KFD_IOC_WAIT_RESULT_TIMEOUT); + expect_unsignaled(first); + EXPECT_EQ(events.waiter_count(first), 0u); + EXPECT_EQ(events.waiter_count(second), 0u); +} + +TEST_F(KfdEventsTest, AutoResetMemoryFaultReachesAllParkedWaiters) { + const auto id = create(true, KFD_IOC_EVENT_MEMORY); + Wait first(events, {id}); + Wait second(events, {id}); + ASSERT_TRUE(registered(id, 2)); + ASSERT_TRUE(events.signal_memory_fault({.va = 0x1234000, .gpu_id = 7})); + first.complete(); + second.complete(); + EXPECT_EQ(first.data[0].memory_exception_data.va, 0x1234000u); + EXPECT_EQ(second.data[0].memory_exception_data.gpu_id, 7u); + expect_unsignaled(id); +} + +TEST_F(KfdEventsTest, WaitAnyCompletesWhenOnlyOneEventSignals) { + const auto first = create(); + const auto second = create(); + Wait wait(events, {first, second}, 0, false); + ASSERT_TRUE(registered(second)); + signal(second); + wait.complete(); + expect_unsignaled(first); + expect_unsignaled(second); +} + +TEST_F(KfdEventsTest, LegacyWaitKeepsAgeDisabled) { + const auto id = create(); + signal(id); + Wait wait(events, {id}, 0, true, true); + wait.complete(); + EXPECT_EQ(wait.data[0].signal_event_data.last_event_age, 0u); +} + +TEST_F(KfdEventsTest, AgeBasedWaitObservesSignalConsumedByLegacyWaiter) { + const auto id = create(); + signal(id); + Wait legacy(events, {id}, 0, true, true); + legacy.complete(); + Wait aged(events, {id}, 1, true, true); + aged.complete(); + EXPECT_EQ(aged.data[0].signal_event_data.last_event_age, 2u); + Wait unchanged(events, {id}, 2, true, true); + EXPECT_EQ(unchanged.args.wait_result, KFD_IOC_WAIT_RESULT_TIMEOUT); +} + +TEST_F(KfdEventsTest, TimedOutWaitAllPreservesAgesForRetry) { + const auto first = create(); + const auto second = create(); + signal(first); + Wait wait(events, {first, second}, 1, true, true); + EXPECT_EQ(wait.args.wait_result, KFD_IOC_WAIT_RESULT_TIMEOUT); + EXPECT_EQ(wait.data[0].signal_event_data.last_event_age, 1u); + EXPECT_EQ(wait.data[1].signal_event_data.last_event_age, 1u); + signal(second); + EXPECT_EQ(events.wait_events(&wait.args), 0); + EXPECT_EQ(wait.args.wait_result, KFD_IOC_WAIT_RESULT_COMPLETE); + EXPECT_EQ(wait.data[0].signal_event_data.last_event_age, 2u); + EXPECT_EQ(wait.data[1].signal_event_data.last_event_age, 2u); +} + +TEST_F(KfdEventsTest, TimedOutWaitAllDoesNotCopyMemoryFault) { + const auto fault = create(true, KFD_IOC_EVENT_MEMORY); + const auto signal = create(); + ASSERT_TRUE(events.signal_memory_fault({.va = 0x1234000, .gpu_id = 7})); + Wait wait(events, {fault, signal}, 0, true, true); + EXPECT_EQ(wait.args.wait_result, KFD_IOC_WAIT_RESULT_TIMEOUT); + EXPECT_EQ(wait.data[0].memory_exception_data.va, 0u); + EXPECT_EQ(wait.data[0].memory_exception_data.gpu_id, 0u); +} + +TEST_F(KfdEventsTest, DestroyedEventFailsRegisteredWait) { + const auto id = create(); + Wait wait(events, {id}); + ASSERT_TRUE(registered(id)); + kfd_ioctl_destroy_event_args destroy{}; + destroy.event_id = id; + EXPECT_EQ(events.destroy_event(&destroy), 0); + wait.thread.join(); + EXPECT_EQ(wait.rc, 0); + EXPECT_EQ(wait.args.wait_result, KFD_IOC_WAIT_RESULT_FAIL); +} + +TEST_F(KfdEventsTest, DestroyingInitiallySatisfiedEventWakesWaitAll) { + const auto first = create(); + const auto second = create(); + signal(first); + Wait wait(events, {first, second}); + ASSERT_TRUE(registered(second)); + kfd_ioctl_destroy_event_args destroy{}; + destroy.event_id = first; + EXPECT_EQ(events.destroy_event(&destroy), 0); + ASSERT_TRUE(wait.finished()) << "destruction must wake wait-all before its timeout"; + wait.thread.join(); + EXPECT_EQ(wait.rc, 0); + EXPECT_EQ(wait.args.wait_result, KFD_IOC_WAIT_RESULT_FAIL); +} + +} // namespace diff --git a/emulation/rocjitsu/tests/kfd_ioctl_test.cpp b/emulation/rocjitsu/tests/kfd_ioctl_test.cpp index eea530479193..906bdf2b7448 100644 --- a/emulation/rocjitsu/tests/kfd_ioctl_test.cpp +++ b/emulation/rocjitsu/tests/kfd_ioctl_test.cpp @@ -1,6 +1,8 @@ // Copyright (c) 2026 Advanced Micro Devices, Inc. // SPDX-License-Identifier: MIT +#include "aql_queue.h" + #include "rocjitsu/code/builders/instruction_builder.h" #include "rocjitsu/config/config_loader.h" #include "rocjitsu/kmd/linux/cwsr.h" @@ -58,6 +60,7 @@ RJ_DIAGNOSTIC_POP #include #include #include +#include #include #include #include @@ -658,6 +661,53 @@ class KfdIoctlTest : public ::testing::Test { std::vector debug_fds_; }; +TEST_F(KfdIoctlTest, CloseReleasesProcessAfterCompletedOrAbortedInstructionExecution) { + for (const bool abort : {false, true}) { + SCOPED_TRACE(abort ? "aborted dispatch" : "completed quantum"); + if (abort) { + ASSERT_GE(driver_->open(), 0); + } + + constexpr uint64_t kKernelAddress = 0x600000000ULL; + alignas(4096) std::array code{}; + const std::array instructions = {0xBF800000u, 0xBF800000u, + 0xBF810000u}; // s_nop; s_nop; s_endpgm + std::memcpy(code.data(), instructions.data(), sizeof(instructions)); + const uint32_t pid = driver_->local_process_id(); + auto process = driver_->find_process(pid); + ASSERT_NE(process, nullptr); + const std::weak_ptr lifetime = process; + process->map_pages(kKernelAddress, code.data(), code.size()); + + rocjitsu::amdgpu::ComputeUnitCore *cu = nullptr; + soc_->for_each_cp([&](rocjitsu::amdgpu::CommandProcessor *cp) { + if (cu == nullptr && !cp->compute_units().empty()) + cu = cp->compute_units().front(); + }); + ASSERT_NE(cu, nullptr); + auto *wave = cu->dispatch_wf(/*wg_id=*/0, kKernelAddress, /*sgprs=*/16, /*vgprs=*/4); + ASSERT_NE(wave, nullptr); + wave->set_process_id(pid); + wave->set_dispatch_id(7); + if (abort) { + cu->step(); + ASSERT_EQ(wave->pc, kKernelAddress + sizeof(uint32_t)); + ASSERT_FALSE(wave->is_halted()); + cu->abort_dispatch(7); + } else { + cu->set_functional_quantum(4); + EXPECT_EQ(cu->run_quantum().iterations, 2u); + } + ASSERT_TRUE(wave->is_halted()); + process.reset(); + ASSERT_FALSE(lifetime.expired()); + + ASSERT_EQ(driver_->close(), 0); + EXPECT_TRUE(lifetime.expired()) + << "the idle CU retained the closed KFD process while its VM stayed alive"; + } +} + // Same driver surface, brought up on a part whose CWSR layout is not modelled. // Deriving does not inherit KfdIoctlTest's cases: TEST_F registers against the // fixture it names, so only the cases written against this one run here. @@ -1111,10 +1161,389 @@ TEST_F(KfdIoctlTest, CreateQueueDoesNotReplicateSdmaQueue) { EXPECT_EQ(soc_->sdma_queue_scheduler().active_queues(), 0u); } -// KFD exposes PM4 and AQL as distinct compute queue formats. Rocjitsu supports -// general KFD compute dispatch only through AQL; its PM4 processor intentionally -// remains limited to the supported compute-queue packet subset. -TEST_F(KfdIoctlTest, CreateQueueRejectsUnsupportedPm4ComputeAndUnknownTypes) { +struct KfdNativePm4Target { + const char *name; + const char *config; + uint32_t gpu_id; +}; + +class KfdNativePm4Test : public KfdIoctlTest, + public ::testing::WithParamInterface { +protected: + void SetUp() override { SetUpWithConfig(std::string(CONFIG_DIR) + "/" + GetParam().config); } +}; + +INSTANTIATE_TEST_SUITE_P( + KfdTargets, KfdNativePm4Test, + ::testing::Values(KfdNativePm4Target{"gfx1100", "gfx1100_w7900.json", kRdna3GpuId}, + KfdNativePm4Target{"gfx1201", "gfx1201_r9700.json", 8716}, + KfdNativePm4Target{"gfx1151", "gfx1151.json", 5510}), + [](const ::testing::TestParamInfo &info) { return info.param.name; }); + +class KfdAqlPm4Test : public KfdIoctlTest, + public ::testing::WithParamInterface { +protected: + void SetUp() override { SetUpWithConfig(std::string(CONFIG_DIR) + "/" + GetParam().config); } +}; + +INSTANTIATE_TEST_SUITE_P( + KfdTargets, KfdAqlPm4Test, + ::testing::Values(KfdNativePm4Target{"gfx942", "gfx942_cdna3_kmd.json", 50148}, + KfdNativePm4Target{"gfx950", "gfx950_mi355x_kmd.json", kGpuId}), + [](const ::testing::TestParamInfo &info) { return info.param.name; }); + +TEST_P(KfdAqlPm4Test, ExecutesEveryXcdAndRetainsBlockedIndirectBuffer) { + ASSERT_EQ(soc_->num_xcds(), 8u); + alignas(4096) std::array ring{}; + alignas(4096) std::array data{}; + alignas(4096) std::array commands{}; + const uint64_t data_va = reinterpret_cast(data.data()); + const uint64_t ib_va = reinterpret_cast(commands.data()); + auto process = driver_->find_process(driver_->local_process_id()); + ASSERT_NE(process, nullptr); + process->map_pages(data_va, data.data(), sizeof(data)); + process->map_pages(ib_va, commands.data(), sizeof(commands)); + size_t n = 0; + const auto emit = [&](std::initializer_list words) { + for (const uint32_t word : words) + commands[n++] = word; + }; + for (uint32_t xcc = 0; xcc < 8; ++xcc) { + // Only the selected XCD writes its slot; all eight must execute this IB. + const uint64_t address = data_va + (8 + xcc) * 8; + emit({0xc0002300, (1u << (24 + xcc)) | 5}); + emit({0xc0033700, 5u << 8, uint32_t(address), uint32_t(address >> 32), xcc + 1}); + } + // An atomic before the wait catches accidental command replay on each retry. + const uint64_t counter = data_va + 16 * 8; + emit({0xc0071e00, 15, uint32_t(counter), uint32_t(counter >> 32), 1, 0, 0, 0, 0}); + const uint64_t gate = data_va + 17 * 8; + emit({0xc0053c00, (1u << 4) | 3, uint32_t(gate), uint32_t(gate >> 32), 1, UINT32_MAX, 4}); + data[25] = 1; // Signal value at handle + 8. + ring[0] = rocjitsu::test::make_pm4_ib_packet(ib_va, n, data_va + 24 * 8); + ring[0].header |= 1 << HSA_PACKET_HEADER_BARRIER; + const uint64_t successor = ib_va + n * 4; + const size_t start = n; + const uint64_t output = data_va + 18 * 8; + emit({0xc0033700, 5u << 8, uint32_t(output), uint32_t(output >> 32), 99}); + ring[1] = rocjitsu::test::make_pm4_ib_packet(successor, n - start); + + kfd_ioctl_create_queue_args queue{}; + queue.gpu_id = GetParam().gpu_id; + queue.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; + queue.ring_base_address = reinterpret_cast(ring.data()); + queue.ring_size = sizeof(ring); + queue.read_pointer_address = data_va; + queue.write_pointer_address = data_va + 8; + queue.queue_percentage = 100; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &queue), 0); + void *doorbell = driver_->mmap(nullptr, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, + static_cast(queue.doorbell_offset)); + ASSERT_NE(doorbell, MAP_FAILED); + std::atomic_ref(data[1]).store(2, std::memory_order_release); + std::atomic_ref(*static_cast(doorbell)).store(1, std::memory_order_release); + for (unsigned i = 0; i < 2000 && data[16] != 8; ++i) { + (void)engine_->step(); + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + for (uint32_t xcc = 0; xcc < 8; ++xcc) + EXPECT_EQ(data[8 + xcc], xcc + 1); + ASSERT_EQ(data[16], 8u); + for (unsigned i = 0; i < 20; ++i) + (void)engine_->step(); + EXPECT_EQ(data[16], 8u); + EXPECT_EQ(data[18], 0u); + EXPECT_EQ(data[25], 1u); + std::atomic_ref(data[17]).store(1, std::memory_order_release); + for (unsigned i = 0; i < 2000 && (data[18] != 99 || data[25] != 0); ++i) + (void)engine_->step(); + EXPECT_EQ(data[18], 99u); + EXPECT_EQ(data[25], 0u); + EXPECT_EQ(data[16], 8u); + kfd_ioctl_destroy_queue_args destroy{}; + destroy.queue_id = queue.queue_id; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); +} + +TEST_P(KfdNativePm4Test, HostDoorbellExecutesWrappedPacketHdpFlushAndIndirectBuffer) { + alignas(4096) std::array ring{}; + alignas(4096) std::array pointers{}; + alignas(4096) std::array indirect{}; + const uint64_t output = reinterpret_cast(&pointers[2]); + const uint64_t ib = reinterpret_cast(indirect.data()); + auto process = driver_->find_process(driver_->local_process_id()); + ASSERT_NE(process, nullptr); + process->map_pages(output & ~uint64_t(4095), pointers.data(), sizeof(pointers)); + process->map_pages(ib, indirect.data(), sizeof(indirect)); + // A fresh native queue starts at zero regardless of the writeback contents. + // Only the low 32 bits are written by PM4 consumer-pointer publication. + pointers[0] = 0xfeed1234deadbeefull; + auto &read_pointer = *reinterpret_cast(&pointers[0]); + std::fill_n(ring.begin(), 1022, 0x80000000u); // Type-2 NOPs advance to the wrap. + // WRITE_DATA straddles the ring boundary, followed by an indirect buffer. + const std::array packets{ + 0xc0033700, 5u << 8, uint32_t(output), uint32_t(output >> 32), 0x12345678, + 0xc0023f00, uint32_t(ib), uint32_t(ib >> 32), (1u << 23) | 12u}; + // The qualified NBIO/NBIF profiles share this HDP request/completion pair. + indirect[0] = 0xc0053c00; + indirect[1] = (1u << 6) | 5u; + indirect[2] = 0xe26; + indirect[3] = 0xe27; + indirect[4] = indirect[5] = UINT32_MAX; + indirect[6] = 4; + indirect[7] = 0xc0033700; + indirect[8] = 5u << 8; + indirect[9] = uint32_t(output + 4); + indirect[10] = uint32_t((output + 4) >> 32); + indirect[11] = 0xabcdef01; + kfd_ioctl_create_queue_args queue{}; + queue.gpu_id = GetParam().gpu_id; + queue.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; + queue.ring_base_address = reinterpret_cast(ring.data()); + queue.ring_size = sizeof(ring); + queue.read_pointer_address = reinterpret_cast(&pointers[0]); + queue.write_pointer_address = reinterpret_cast(&pointers[1]); + queue.queue_percentage = 100; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &queue), 0); + // Map after queue creation, as tinygrad does for its first queue. + void *doorbell = driver_->mmap(nullptr, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, + static_cast(queue.doorbell_offset)); + ASSERT_NE(doorbell, MAP_FAILED); + std::atomic_ref(*static_cast(doorbell)) + .store(1022, std::memory_order_release); + for (unsigned i = 0; + i < 2000 && std::atomic_ref(read_pointer).load(std::memory_order_acquire) != 1022; + ++i) { + (void)engine_->step(); + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + ASSERT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), 1022u); + for (size_t i = 0; i < packets.size(); ++i) + ring[(1022 + i) % ring.size()] = packets[i]; + constexpr uint64_t producer = 1031; + std::atomic_ref(pointers[1]).store(producer, std::memory_order_release); + std::atomic_ref(*static_cast(doorbell)) + .store(producer, std::memory_order_release); + for (unsigned i = 0; + i < 2000 && std::atomic_ref(read_pointer).load(std::memory_order_acquire) != + producer % ring.size(); + ++i) { + (void)engine_->step(); + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + EXPECT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), + producer % ring.size()); + EXPECT_EQ(pointers[0] >> 32, 0xfeed1234u); + EXPECT_EQ(pointers[2], 0xabcdef0112345678ull); + kfd_ioctl_destroy_queue_args destroy{}; + destroy.queue_id = queue.queue_id; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); + EXPECT_EQ(soc_->queue_registry().active_queues(), 0u); + for (uint32_t i = 0; i < soc_->num_xcds(); ++i) + EXPECT_FALSE(soc_->xcd(i)->command_processor()->doorbell_monitor_running_for_test()); +} + +TEST_P(KfdNativePm4Test, NativePm4PauseResumeRejectsInvalidRingAndTerminalFault) { + alignas(4096) std::array ring{}; + alignas(4096) std::array pointers{}; + const uint64_t output = reinterpret_cast(&pointers[2]); + auto process = driver_->find_process(driver_->local_process_id()); + ASSERT_NE(process, nullptr); + process->map_pages(reinterpret_cast(pointers.data()), pointers.data(), + sizeof(pointers)); + const std::array packet{0xc0033700, 5u << 8, uint32_t(output), + uint32_t(output >> 32), 0x12345678}; + std::copy(packet.begin(), packet.end(), ring.begin()); + kfd_ioctl_create_queue_args queue{}; + queue.gpu_id = GetParam().gpu_id; + queue.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; + queue.ring_base_address = reinterpret_cast(ring.data()); + queue.ring_size = sizeof(ring); + queue.read_pointer_address = reinterpret_cast(&pointers[0]); + queue.write_pointer_address = reinterpret_cast(&pointers[1]); + queue.queue_percentage = 100; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &queue), 0); + void *doorbell = driver_->mmap(nullptr, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, + static_cast(queue.doorbell_offset)); + ASSERT_NE(doorbell, MAP_FAILED); + auto &read_pointer = *reinterpret_cast(&pointers[0]); + const auto pump = [&] { + for (unsigned i = 0; i < 200; ++i) { + (void)engine_->step(); + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + }; + kfd_ioctl_update_queue_args update{}; + update.queue_id = queue.queue_id; + update.ring_base_address = queue.ring_base_address; + update.ring_size = queue.ring_size; + update.queue_percentage = 0; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), 0); + std::atomic_ref(*static_cast(doorbell)).store(5, std::memory_order_release); + pump(); + EXPECT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), 0u); + EXPECT_EQ(pointers[2], 0u); + update.queue_percentage = 100; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), 0); + pump(); + EXPECT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), 5u); + EXPECT_EQ(pointers[2], 0x12345678u); + + // KFD also disables a queue with a null ring address and size, preserving + // its cursor so restoring the original ring resumes already-published work. + update.ring_base_address = 0; + update.ring_size = 0; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), 0); + std::copy(packet.begin(), packet.end(), ring.begin() + 5); + ring[9] = 0x87654321; + std::atomic_ref(*static_cast(doorbell)) + .store(10, std::memory_order_release); + pump(); + EXPECT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), 5u); + EXPECT_EQ(pointers[2], 0x12345678u); + update.ring_base_address = queue.ring_base_address; + update.ring_size = queue.ring_size; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), 0); + pump(); + EXPECT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), 10u); + EXPECT_EQ(pointers[2], 0x87654321u); + + update.ring_base_address = queue.ring_base_address + 1; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), -EINVAL); + update.ring_base_address = queue.ring_base_address; + update.ring_size = 3; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), -EINVAL); + update.ring_size = queue.ring_size; + // Invalid updates must leave the old ring live at its previous cursor. + std::copy(packet.begin(), packet.end(), ring.begin() + 10); + ring[14] = 0xabcdef01; + std::atomic_ref(*static_cast(doorbell)) + .store(15, std::memory_order_release); + pump(); + EXPECT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), 15u); + EXPECT_EQ(pointers[2], 0xabcdef01u); + + ring[15] = 0; // Invalid type: the queue enters terminal failure. + std::atomic_ref(*static_cast(doorbell)) + .store(16, std::memory_order_release); + pump(); + EXPECT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), 15u); + update.queue_percentage = 0; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), -EINVAL); + update.queue_percentage = 100; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), -EINVAL); + kfd_ioctl_destroy_queue_args destroy{}; + destroy.queue_id = queue.queue_id; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); + EXPECT_EQ(soc_->queue_registry().active_queues(), 0u); +} + +TEST_P(KfdNativePm4Test, NativePm4GracefulCloseWaitsForBatchCompletion) { + using rocjitsu::amdgpu::QueueCloseStatus; + alignas(4096) std::array ring{}; + alignas(4096) std::array pointers{}; + const uint64_t output = reinterpret_cast(&pointers[2]); + const uint64_t wait_address = reinterpret_cast(&pointers[3]); + auto process = driver_->find_process(driver_->local_process_id()); + ASSERT_NE(process, nullptr); + process->map_pages(reinterpret_cast(pointers.data()), pointers.data(), + sizeof(pointers)); + const std::array packets{0xc0033700, + 5u << 8, + uint32_t(output), + uint32_t(output >> 32), + 0x12345678, + 0xc0053c00, + (1u << 4) | 3u, + uint32_t(wait_address), + uint32_t(wait_address >> 32), + 1, + UINT32_MAX, + 4}; + std::copy(packets.begin(), packets.end(), ring.begin()); + kfd_ioctl_create_queue_args queue{}; + queue.gpu_id = GetParam().gpu_id; + queue.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; + queue.ring_base_address = reinterpret_cast(ring.data()); + queue.ring_size = sizeof(ring); + queue.read_pointer_address = reinterpret_cast(&pointers[0]); + queue.write_pointer_address = reinterpret_cast(&pointers[1]); + queue.queue_percentage = 100; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &queue), 0); + const auto handle = process->queue_doorbell_map_.at(queue.queue_id).queue_handle; + void *doorbell = driver_->mmap(nullptr, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, + static_cast(queue.doorbell_offset)); + ASSERT_NE(doorbell, MAP_FAILED); + std::atomic_ref(*static_cast(doorbell)) + .store(packets.size(), std::memory_order_release); + for (unsigned i = 0; i < 2000 && pointers[2] != 0x12345678u; ++i) { + (void)engine_->step(); + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + ASSERT_EQ(pointers[2], 0x12345678u); + EXPECT_EQ(soc_->queue_registry().unregister_queue(handle).status, QueueCloseStatus::Busy); + EXPECT_TRUE(soc_->queue_registry().contains(handle)); + pointers[3] = 1; + auto &read_pointer = *reinterpret_cast(&pointers[0]); + for (unsigned i = 0; + i < 2000 && + std::atomic_ref(read_pointer).load(std::memory_order_acquire) != packets.size(); + ++i) { + (void)engine_->step(); + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + ASSERT_EQ(std::atomic_ref(read_pointer).load(std::memory_order_acquire), + packets.size()); + EXPECT_EQ(soc_->queue_registry().unregister_queue(handle).status, QueueCloseStatus::Closed); + EXPECT_FALSE(soc_->queue_registry().contains(handle)); + EXPECT_FALSE(soc_->xcd(0)->command_processor()->doorbell_monitor_running_for_test()); + kfd_ioctl_destroy_queue_args destroy{}; + destroy.queue_id = queue.queue_id; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); +} + +TEST_F(KfdIoctlTest, SmallAllocationsMapTheWholeTailPage) { + using rocjitsu::amdgpu::VmAccessOutcome; + alignas(4096) std::array host{}; + for (const uint32_t kind : {KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, KFD_IOC_ALLOC_MEM_FLAGS_GTT}) { + kfd_ioctl_alloc_memory_of_gpu_args allocation{}; + allocation.size = 12; + allocation.gpu_id = kGpuId; + allocation.flags = kind | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE; + if (kind == KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { + allocation.va_addr = reinterpret_cast(host.data()); + allocation.mmap_offset = allocation.va_addr; + } + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &allocation), 0); + EXPECT_EQ(allocation.size, 12u) << "Preserve the ioctl's requested size"; + void *mapping = host.data(); + if (kind == KFD_IOC_ALLOC_MEM_FLAGS_GTT) { + mapping = driver_->mmap(nullptr, 12, PROT_READ | PROT_WRITE, MAP_SHARED, + static_cast(allocation.mmap_offset)); + ASSERT_NE(mapping, MAP_FAILED); + } + static_cast(mapping)[4095] = std::byte{0x5a}; + const auto access = soc_->gpu_vm().snapshot_vmid(driver_->local_process_id()); + ASSERT_TRUE(access); + std::byte last{}; + EXPECT_EQ(access->read(allocation.va_addr + 4095, std::span(&last, 1)), + VmAccessOutcome::Complete); + EXPECT_EQ(last, std::byte{0x5a}); + kfd_ioctl_free_memory_of_gpu_args free{}; + free.handle = allocation.handle; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free), 0); + } + for (uint64_t size : {uint64_t(0), UINT64_MAX}) { + kfd_ioctl_alloc_memory_of_gpu_args invalid{}; + invalid.gpu_id = kGpuId; + invalid.flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT; + invalid.size = size; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &invalid), -EINVAL); + } +} + +// Both compute formats share queue ownership; a native PM4 queue stays on one XCD. +TEST_F(KfdIoctlTest, CreateQueueAcceptsPm4ComputeAndRejectsUnknownTypes) { const uint32_t num_xcds = soc_->num_xcds(); ASSERT_GT(num_xcds, 1u); @@ -1137,7 +1566,50 @@ TEST_F(KfdIoctlTest, CreateQueueRejectsUnsupportedPm4ComputeAndUnknownTypes) { pm4.read_pointer_address = reinterpret_cast(&ptrs[0]); pm4.write_pointer_address = reinterpret_cast(&ptrs[1]); pm4.queue_percentage = 100; - EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &pm4), -ENOTSUP); + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &pm4), 0); + EXPECT_EQ(registered_on_all_xcds(), 1u); + kfd_ioctl_destroy_queue_args destroy{}; + destroy.queue_id = pm4.queue_id; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); + EXPECT_EQ(registered_on_all_xcds(), 0u); + + // PM4 target placement is initial queue configuration, separate from decoding. + ASSERT_GT(num_xcds, 2u); + pm4.queue_percentage = 100 | (2u << 8); + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &pm4), 0); + EXPECT_EQ(soc_->xcd(2)->command_processor()->registered_queue_count_for_test(), 1u); + EXPECT_EQ(soc_->xcd(0)->command_processor()->registered_queue_count_for_test(), 0u); + destroy.queue_id = pm4.queue_id; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); + EXPECT_EQ(registered_on_all_xcds(), 0u); + pm4.queue_percentage = 100 | (num_xcds << 8); + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &pm4), -EINVAL); + + pm4.queue_percentage = (2u << 8) | 101; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &pm4), -EINVAL); + pm4.queue_percentage = 2u << 8; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &pm4), 0); + auto *owner = soc_->xcd(2)->command_processor(); + EXPECT_TRUE(owner->queue_runtime_suspended_for_test(pm4.queue_id, driver_->local_process_id())); + kfd_ioctl_update_queue_args update{}; + update.queue_id = pm4.queue_id; + update.ring_base_address = pm4.ring_base_address; + update.ring_size = pm4.ring_size; + update.queue_percentage = (2u << 8) | 100; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), 0); + EXPECT_FALSE(owner->queue_runtime_suspended_for_test(pm4.queue_id, driver_->local_process_id())); + update.queue_percentage = 2u << 8; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), 0); + EXPECT_TRUE(owner->queue_runtime_suspended_for_test(pm4.queue_id, driver_->local_process_id())); + update.queue_percentage = (2u << 8) | 101; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), -EINVAL); + update.queue_percentage = (num_xcds << 8) | 100; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), -EINVAL); + update.queue_percentage = (1u << 8) | 100; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_UPDATE_QUEUE, &update), -EOPNOTSUPP); + EXPECT_TRUE(owner->queue_runtime_suspended_for_test(pm4.queue_id, driver_->local_process_id())); + destroy.queue_id = pm4.queue_id; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); EXPECT_EQ(registered_on_all_xcds(), 0u); kfd_ioctl_create_queue_args unknown{}; @@ -1152,6 +1624,35 @@ TEST_F(KfdIoctlTest, CreateQueueRejectsUnsupportedPm4ComputeAndUnknownTypes) { EXPECT_EQ(registered_on_all_xcds(), 0u); } +TEST_P(KfdNativePm4Test, QueueAdmissionIsIndependentOfDiscoveryMetadata) { + alignas(4096) std::array ring{}; + alignas(8) std::array pointers{}; + kfd_ioctl_create_queue_args queue{}; + queue.gpu_id = GetParam().gpu_id; + queue.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; + queue.ring_base_address = reinterpret_cast(ring.data()); + queue.ring_size = sizeof(ring); + queue.read_pointer_address = reinterpret_cast(&pointers[0]); + queue.write_pointer_address = reinterpret_cast(&pointers[1]); + queue.queue_percentage = 100; + + auto device = loaded_.device; + device.ip_versions.clear(); + driver_->setup_topology(device, 1); + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &queue), 0); + kfd_ioctl_destroy_queue_args destroy{}; + destroy.queue_id = queue.queue_id; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); + + device.ip_versions = loaded_.device.ip_versions; + device.gfx_target_version = 110002; // Discovery does not govern queue admission. + driver_->setup_topology(device, 1); + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &queue), 0); + destroy.queue_id = queue.queue_id; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); + EXPECT_EQ(soc_->queue_registry().active_queues(), 0u); +} + // Replication makes every CP hold a host-accessible queue, so "does this CP have // a KFD queue" stops answering "does this CP have a queue of its own". Two // queues land on different XCDs, and destroying the first leaves the second's @@ -1570,6 +2071,7 @@ TEST_F(KfdIoctlCdna5Test, RuntimeTrapInterruptSignalsQueueExceptionFromM0) { kfd_ioctl_create_queue_args create{}; create.gpu_id = kCdna5GpuId; create.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; + create.queue_percentage = 100; create.ring_base_address = reinterpret_cast(ring.data()); create.ring_size = static_cast(ring.size()); create.read_pointer_address = reinterpret_cast(&read_pointer); @@ -2032,6 +2534,105 @@ TEST_F(KfdIoctlCdna5Test, DebuggerResumeRejectsPendingRuntimeExceptionPublicatio }); } +TEST_F(KfdIoctlCdna5Test, DebuggerForwardsQueueExceptionsWithoutRuntimeAcknowledgment) { + constexpr uint64_t kExceptionStatusAddress = 0x600900000ULL; + constexpr uint32_t kExceptionEventId = 84; + constexpr uint64_t kWaveAbort = KFD_EC_MASK(EC_QUEUE_WAVE_ABORT); + constexpr uint64_t kMathError = KFD_EC_MASK(EC_QUEUE_WAVE_MATH_ERROR); + using Page = std::array; + alignas(rocjitsu::amdgpu::kAqlPacketBytes) Page ring{}; + alignas(4096) Page cwsr{}; + Page exception_page{}; + auto process = driver_->find_process(driver_->local_process_id()); + ASSERT_NE(process, nullptr); + process->map_pages(kExceptionStatusAddress, exception_page.data(), exception_page.size()); + std::memcpy(cwsr.data() + 6 * sizeof(uint32_t), &kExceptionStatusAddress, + sizeof(kExceptionStatusAddress)); + std::memcpy(cwsr.data() + 6 * sizeof(uint32_t) + sizeof(uint64_t), &kExceptionEventId, + sizeof(kExceptionEventId)); + + kfd_ioctl_runtime_enable_args runtime{}; + runtime.mode_mask = KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_RUNTIME_ENABLE, &runtime), 0); + kfd_ioctl_dbg_trap_args enable{}; + enable.pid = static_cast(getpid()); + enable.op = KFD_IOC_DBG_TRAP_ENABLE; + enable.enable.dbg_fd = make_debug_fd(); + enable.enable.exception_mask = kWaveAbort | kMathError; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DBG_TRAP, &enable), 0); + + alignas(4096) amd_queue_v2_t queue_descriptor{}; + process->map_pages(reinterpret_cast(&queue_descriptor), &queue_descriptor, + sizeof(queue_descriptor)); + kfd_ioctl_create_queue_args create{}; + create.gpu_id = kCdna5GpuId; + create.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; + create.ring_base_address = reinterpret_cast(ring.data()); + create.ring_size = static_cast(ring.size()); + create.read_pointer_address = reinterpret_cast(&queue_descriptor.read_dispatch_id); + create.write_pointer_address = reinterpret_cast(&queue_descriptor.write_dispatch_id); + create.ctx_save_restore_address = reinterpret_cast(cwsr.data()); + create.ctx_save_restore_size = static_cast(cwsr.size()); + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_CREATE_QUEUE, &create), 0); + + // An immediate deadline makes the old acknowledgment requirement fail + // deterministically without a wall-clock assertion or a one-second wait. + soc_->for_each_cp([](rocjitsu::amdgpu::CommandProcessor *cp) { + cp->set_runtime_exception_ack_timeout_for_testing(std::chrono::milliseconds(0)); + }); + std::vector delivered; + driver_->set_interrupt_callback_for_testing([&](uint32_t process_id, uint32_t event_id) { + if (event_id != kExceptionEventId) + return; + EXPECT_EQ(process_id, driver_->local_process_id()); + delivered.push_back(vm_read64(kExceptionStatusAddress, process_id)); + // ROCr clears the signal and removes its async handler after the first + // fatal exception. Later waves still forward their stops to this queue. + if (delivered.size() == 1) + (void)vm_write64(kExceptionStatusAddress, 0, process_id); + }); + struct ObserverGuard { + rocjitsu::SimulatedKfd *driver; + ~ObserverGuard() { driver->set_interrupt_callback_for_testing(nullptr); } + } observer_guard{driver_}; + + kfd_ioctl_dbg_trap_args forward{}; + forward.pid = static_cast(getpid()); + forward.op = KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT; + forward.send_runtime_event.gpu_id = kCdna5GpuId; + forward.send_runtime_event.queue_id = create.queue_id; + forward.send_runtime_event.exception_mask = kWaveAbort; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_DBG_TRAP, &forward), 0); + EXPECT_EQ(vm_read64(kExceptionStatusAddress, driver_->local_process_id()), 0u); + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_DBG_TRAP, &forward), 0); + EXPECT_EQ(vm_read64(kExceptionStatusAddress, driver_->local_process_id()), kWaveAbort); + forward.send_runtime_event.exception_mask = kMathError; + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_DBG_TRAP, &forward), 0); + EXPECT_EQ(delivered, (std::vector{kWaveAbort, kWaveAbort, kWaveAbort | kMathError})); + soc_->for_each_cp([&](rocjitsu::amdgpu::CommandProcessor *cp) { + EXPECT_TRUE( + cp->queue_exception_suspended_for_test(create.queue_id, driver_->local_process_id())); + }); + + // Runtime-owned delivery still requires acknowledgment before releasing its + // retained debugger fallback. The debugger ioctl's contract is different. + EXPECT_FALSE(driver_->signal_runtime_queue_exception_for_testing( + kCdna5GpuId, create.queue_id, driver_->local_process_id(), kWaveAbort)); + const size_t delivered_before_unmap = delivered.size(); + process->unmap_pages(kExceptionStatusAddress, exception_page.size()); + EXPECT_LT(driver_->ioctl(AMDKFD_IOC_DBG_TRAP, &forward), 0) + << "a live queue with inaccessible exception storage must not report publication success"; + EXPECT_EQ(delivered.size(), delivered_before_unmap); + + // KFD accepts a late forwarding request for a queue already destroyed by + // the runtime; there is no live queue to signal or storage to touch. + kfd_ioctl_destroy_queue_args destroy{}; + destroy.queue_id = create.queue_id; + ASSERT_EQ(driver_->ioctl(AMDKFD_IOC_DESTROY_QUEUE, &destroy), 0); + EXPECT_EQ(driver_->ioctl(AMDKFD_IOC_DBG_TRAP, &forward), 0); + EXPECT_EQ(delivered.size(), delivered_before_unmap); +} + TEST_F(KfdIoctlCdna5Test, RuntimeExceptionFanoutCanDrainPendingCuNotification) { GTEST_FLAG_SET(death_test_style, "threadsafe"); using Page = std::array; @@ -4654,8 +5255,7 @@ TEST(RemoteDriverPtracerGrantTest, RefusesAListenerThatIsNotTheLaunchedDaemon) { EXPECT_EQ(*driver.ptracer_verdict(), rocjitsu::PtracerGrantVerdict::RefusedPeerMismatch); } -// Attach mode: no launcher named a daemon, so there is nothing to compare the -// peer against and the grant is withheld rather than given to whoever answered. +// Attach mode trusts the same-user socket peer without a launcher-provided PID. TEST(RemoteDriverPtracerGrantTest, TrustsTheSocketWhenNoLauncherNamedADaemon) { // Empty rather than unset: both mean "no launcher named one", and the empty // spelling is also what an inherited-but-cleared environment looks like. @@ -4668,10 +5268,15 @@ TEST(RemoteDriverPtracerGrantTest, TrustsTheSocketWhenNoLauncherNamedADaemon) { }); rocjitsu::RemoteDriver driver(sv[0]); - ASSERT_GE(driver.open(), 0); + testing::internal::CaptureStderr(); + const int fd = driver.open(); + const std::string stderr_output = testing::internal::GetCapturedStderr(); + ASSERT_GE(fd, 0); ASSERT_TRUE(driver.ptracer_verdict().has_value()); EXPECT_EQ(*driver.ptracer_verdict(), rocjitsu::PtracerGrantVerdict::GrantUnverifiedPeer); + EXPECT_EQ(stderr_output.find("[rj warn] daemon: authorizing pid"), std::string::npos) + << stderr_output; } // Having named a daemon outranks trusting the socket, so a peer failing the PID @@ -7276,7 +7881,7 @@ TEST_F(KfdIoctlCdna5Test, ScratchGrowthPreservesSpillsFromOverlappingDispatches) alignas(64) std::array, 2> pointers{}; std::array registrations{}; for (uint32_t i = 0; i < cps.size(); ++i) { - amdgpu::AqlQueueConfig queue{}; + amdgpu::ComputeQueueConfig queue{}; queue.address_space = process->gpu(0).address_space; queue.process_id = process_id; queue.queue_id = i + 1; @@ -7715,6 +8320,7 @@ TEST_F(KfdIoctlCdna5Test, DbgTrapHandlerExceptionReportsExactMaskBeforeExplicitC kfd_ioctl_create_queue_args create{}; create.gpu_id = kCdna5GpuId; create.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; + create.queue_percentage = 100; create.ring_base_address = reinterpret_cast(ring.data()); create.ring_size = static_cast(ring.size()); create.read_pointer_address = reinterpret_cast(&read_pointer); diff --git a/emulation/rocjitsu/tests/l2_cache_test.cpp b/emulation/rocjitsu/tests/l2_cache_test.cpp index 66cae10a42b1..ae1a94357766 100644 --- a/emulation/rocjitsu/tests/l2_cache_test.cpp +++ b/emulation/rocjitsu/tests/l2_cache_test.cpp @@ -89,7 +89,8 @@ using rocjitsu::amdgpu::RequestMtypeResolver; class IdentityAddressSpace final : public rocjitsu::amdgpu::AddressSpaceTranslator { public: - explicit IdentityAddressSpace(uint64_t size) : size_(size) {} + explicit IdentityAddressSpace(uint64_t size, Mtype mtype = Mtype::RW) + : size_(size), mtype_(mtype) {} rocjitsu::amdgpu::VmTranslationResult translate(uint64_t address, std::size_t size, rocjitsu::amdgpu::VmAccessKind) const override { @@ -100,12 +101,13 @@ class IdentityAddressSpace final : public rocjitsu::amdgpu::AddressSpaceTranslat .translation = {.domain = rocjitsu::amdgpu::VmMemoryDomain::System, .address = address, .contiguous_bytes = size_ - address, - .mtype = Mtype::RW, + .mtype = mtype_, .permissions = {.readable = true, .writable = true, .executable = true}}}; } private: uint64_t size_; + Mtype mtype_; }; class ConfigurablePhysicalMemory final : public rocjitsu::amdgpu::PhysicalMemoryAccess { @@ -1259,6 +1261,7 @@ class L1CacheMtypeTest : public testing::Test { rocjitsu::amdgpu::LegacyGpuVmAdapter legacy_vm_{gpu_vm_, &memory_}; rocjitsu::amdgpu::AddressSpaceHandle address_space_; L2Cache l2_{"l2"}; + rocjitsu::amdgpu::VmMtypeCache mtype_cache_; }; class LegacySubPageCacheTest : public testing::Test { @@ -1287,6 +1290,832 @@ class LegacySubPageCacheTest : public testing::Test { L2Cache l2_{"l2"}; }; +class ScalarRamBatchTest : public testing::Test, public rocjitsu::amdgpu::MemoryFaultReporter { +protected: + using Owner = rocjitsu::amdgpu::LegacyHostExtentOwner; + using Outcome = rocjitsu::amdgpu::VmAccessOutcome; + static constexpr uint32_t kVmid = 29; + static constexpr uint64_t kBase = 0x800000; + static constexpr uint32_t kSentinel = 0xdeadbeef; + + void SetUp() override { + for (size_t i = 0; i < backing_.size(); ++i) + backing_[i] = static_cast(i * 17 + 3); + address_space_ = adapter_.register_address_space( + kVmid, {.page_table = &process_.page_table_, + .page_table_mutex = &process_.page_table_mutex_, + .page_table_generation = process_.page_table_generation(), + .request_mutex = process_.page_table_request_mutex(), + .mutation_epoch = process_.page_table_mutation_epoch(), + .page_table_cache_state = process_.page_table_cache_state(), + .fault_reporter = this}); + ASSERT_TRUE(address_space_); + l2_.set_backing_memory(&memory_); + l2_.set_gpu_vm(&vm_); + l1_.set_gpu_vm(&vm_); + } + + void map(Owner owner = Owner::DriverSealedRam, Mtype mtype = Mtype::UC, size_t size = 8192) { + process_.map_pages(kBase, backing_.data(), size, mtype, owner); + } + + void report_memory_fault(uint32_t vmid, uint64_t address, + rocjitsu::amdgpu::MemoryFaultCause) override { + EXPECT_EQ(vmid, kVmid); + faults_.push_back(address); + } + + void expect_words(const std::array &words, uint32_t count, size_t offset = 0) { + EXPECT_EQ(std::memcmp(words.data(), backing_.data() + offset, count * sizeof(uint32_t)), 0); + for (uint32_t i = count; i < words.size(); ++i) + EXPECT_EQ(words[i], kSentinel); + } + + std::array backing_{}; + rocjitsu::KfdProcess process_{kVmid}; + GpuMemory memory_{"scalar_ram"}; + rocjitsu::amdgpu::GpuVm vm_; + rocjitsu::amdgpu::LegacyGpuVmAdapter adapter_{vm_, &memory_}; + rocjitsu::amdgpu::AddressSpaceHandle address_space_; + L2Cache l2_{"scalar_ram_l2"}; + L1ScalarCache l1_{&l2_}; + std::vector faults_; +}; + +class VectorRamBatchTest : public ScalarRamBatchTest { +protected: + using Store = rocjitsu::amdgpu::VmRamDwordStore; + void SetUp() override { + ScalarRamBatchTest::SetUp(); + vector_.set_gpu_vm(&vm_); + for (size_t i = 0; i < values_.size(); ++i) + values_[i] = 0x10203000u + static_cast(i); + } + void reset_bytes() { + vector_.invalidate_all(); + l2_.invalidate_all(); + backing_.fill(0x5a); + faults_.clear(); + } + std::array stores() const { + return {{{kBase + 4, reinterpret_cast(&values_[0])}, + {kBase + 12, reinterpret_cast(&values_[1])}, + {kBase + 4, reinterpret_cast(&values_[2])}}}; + } + L1VectorCache vector_{&l2_}; + std::array values_{}; +}; + +TEST_F(VectorRamBatchTest, SwizzledMasksDuplicatesAndBoundariesMatchOrdinaryStores) { + map(Owner::DriverSealedRam, Mtype::RW); + for (const Mtype policy : {Mtype::UC, Mtype::RW, Mtype::CC, Mtype::NT}) { + for (const uint32_t unit : {4u, 16u}) { + for (const uint32_t offset : {0u, 120u, 4080u}) { + for (const bool partial : {false, true}) { + SCOPED_TRACE(static_cast(policy)); + SCOPED_TRACE(unit); + SCOPED_TRACE(offset); + SCOPED_TRACE(partial); + std::array addresses; + for (size_t lane = 0; lane < addresses.size(); ++lane) + addresses[lane] = kBase + offset + lane * unit; + addresses[3] = addresses[2]; + const uint64_t lanes = partial ? 0x7fff00ffff00ffffull : UINT64_MAX; + const std::array masks{lanes, lanes & 0x5555555555555555ull, + lanes & 0x3333333333333333ull, lanes}; + std::span element_masks; + if (partial) + element_masks = masks; + std::array ordinary; + std::array ordinary_read{}; + uint64_t ordinary_transactions = 0; + for (const bool allow : {false, true}) { + reset_bytes(); + const uint64_t before = l2_.backing_write_transactions(); + ASSERT_EQ(vector_.store(addresses.data(), lanes, 4, 4, + reinterpret_cast(values_.data()), policy, + policy == Mtype::NT, 64, kVmid, unit * 32, offset % unit, + element_masks, unit, allow), + Outcome::Complete); + std::array read{}; + ASSERT_EQ(vector_.load(addresses.data(), lanes, 4, 4, read.data(), policy, + policy == Mtype::NT, false, 64, kVmid, unit * 32, offset % unit, + element_masks, unit), + Outcome::Complete); + if (!allow) { + ordinary = backing_; + ordinary_read = read; + ordinary_transactions = l2_.backing_write_transactions() - before; + } else { + EXPECT_EQ(backing_, ordinary); + EXPECT_EQ(read, ordinary_read); + EXPECT_EQ(l2_.backing_write_transactions() - before, ordinary_transactions); + } + EXPECT_TRUE(faults_.empty()); + } + } + } + } + } +} + +TEST_F(VectorRamBatchTest, DirectBatchKeepsDuplicateOrderCountersAndAliasVisibility) { + for (const Mtype policy : {Mtype::UC, Mtype::CC, Mtype::RW}) { + map(Owner::DriverSealedRam, Mtype::RW); + process_.map_pages(kBase + 8192, backing_.data(), backing_.size(), Mtype::RW, + Owner::DriverSealedRam); + reset_bytes(); + uint32_t alias_before = 0; + ASSERT_EQ( + l2_.read(kBase + 8192 + 4, reinterpret_cast(&alias_before), 4, Mtype::RW, kVmid), + Outcome::Complete); + ASSERT_EQ(l2_.write(kBase, reinterpret_cast(&values_[3]), 4, policy, kVmid), + Outcome::Complete); + const uint64_t transactions = l2_.backing_write_transactions(); + const uint64_t writes = l2_.write_count(); + ASSERT_TRUE(l2_.try_write_private_dwords(stores(), policy, policy, kVmid)); + EXPECT_EQ(l2_.backing_write_transactions() - transactions, 3u); + EXPECT_EQ(l2_.write_count() - writes, policy == Mtype::UC ? 0u : 3u); + uint32_t value = 0; + std::memcpy(&value, backing_.data() + 4, 4); + EXPECT_EQ(value, values_[2]); + ASSERT_EQ(l2_.read(kBase + 4, reinterpret_cast(&value), 4, policy, kVmid), + Outcome::Complete); + EXPECT_EQ(value, values_[2]); + ASSERT_EQ(l2_.read(kBase + 8192 + 4, reinterpret_cast(&value), 4, Mtype::RW, kVmid), + Outcome::Complete); + EXPECT_EQ(value, alias_before); + ASSERT_EQ(l2_.read(kBase + 8192 + 4, reinterpret_cast(&value), 4, Mtype::UC, kVmid), + Outcome::Complete); + EXPECT_EQ(value, values_[2]); + } +} + +TEST_F(VectorRamBatchTest, OwnerPolicySourceAliasAndDirtyRefusalsHaveNoEffects) { + auto access = vm_.snapshot(address_space_); + ASSERT_TRUE(access); + for (const Owner owner : {Owner::Application, Owner::Driver, Owner::DriverSealedRam}) { + map(owner, Mtype::UC); + reset_bytes(); + const auto before = backing_; + EXPECT_EQ(access->try_write_private_dwords(stores(), Mtype::RW, Mtype::UC), + owner == Owner::DriverSealedRam); + if (owner != Owner::DriverSealedRam) { + EXPECT_EQ(backing_, before); + } + } + map(Owner::DriverSealedRam, Mtype::RW); + reset_bytes(); + const auto before = backing_; + EXPECT_FALSE(access->try_write_private_dwords(stores(), Mtype::RW, Mtype::UC)); + auto overlapping_sources = stores(); + overlapping_sources[0].source = backing_.data() + 4; + EXPECT_FALSE(access->try_write_private_dwords(overlapping_sources, Mtype::RW, Mtype::RW)); + EXPECT_EQ(backing_, before); + std::array dirty; + dirty.fill(0xa5); + ASSERT_EQ(l2_.writeback_line(kBase, dirty.data(), Mtype::RW, kVmid), Outcome::Complete); + const uint64_t transactions = l2_.backing_write_transactions(); + EXPECT_FALSE(l2_.try_write_private_dwords(stores(), Mtype::RW, Mtype::RW, kVmid)); + EXPECT_FALSE(l2_.try_write_private_dwords(stores(), Mtype::UC, Mtype::UC, kVmid)); + EXPECT_EQ(l2_.backing_write_transactions(), transactions); + EXPECT_EQ(backing_, before); + ASSERT_EQ(l2_.flush_line(kBase, kVmid), Outcome::Complete); + EXPECT_TRUE(std::ranges::equal(dirty, std::span(backing_).first(dirty.size()))); +} + +TEST_F(VectorRamBatchTest, FirstRequestKeepsDirtyPrefixAndLaterFlushContents) { + map(Owner::DriverSealedRam, Mtype::RW); + std::array addresses; + for (size_t lane = 0; lane < addresses.size(); ++lane) + addresses[lane] = kBase + lane * 4; + std::array dirty; + dirty.fill(0xa5); + for (const Mtype policy : {Mtype::UC, Mtype::RW}) { + std::array ordinary_before_flush, ordinary_after_flush; + uint64_t ordinary_transactions = 0; + for (const bool allow : {false, true}) { + reset_bytes(); + ASSERT_EQ(l2_.writeback_line(kBase, dirty.data(), Mtype::RW, kVmid), Outcome::Complete); + const uint64_t before = l2_.backing_write_transactions(); + ASSERT_EQ(vector_.store(addresses.data(), 255, 4, 1, + reinterpret_cast(values_.data()), policy, false, 8, + kVmid, 32, 0, {}, 4, allow), + Outcome::Complete); + if (!allow) + ordinary_before_flush = backing_; + else + EXPECT_EQ(backing_, ordinary_before_flush); + ASSERT_EQ(l2_.flush_all(), Outcome::Complete); + if (!allow) { + ordinary_after_flush = backing_; + ordinary_transactions = l2_.backing_write_transactions() - before; + } else { + EXPECT_EQ(backing_, ordinary_after_flush); + EXPECT_EQ(l2_.backing_write_transactions() - before, ordinary_transactions); + } + EXPECT_EQ(std::memcmp(backing_.data(), values_.data(), 8 * sizeof(uint32_t)), 0); + EXPECT_TRUE(std::ranges::all_of(std::span(backing_).subspan(32, 96), + [](uint8_t value) { return value == 0xa5; })); + EXPECT_TRUE(faults_.empty()); + } + } +} + +TEST_F(VectorRamBatchTest, HintsNeverRefreshOrAuthorizeChangedMappings) { + auto access = vm_.snapshot(address_space_); + ASSERT_TRUE(access); + rocjitsu::amdgpu::VmMtypeCache cache; + for (const Mtype policy : {Mtype::UC, Mtype::RW}) { + map(Owner::DriverSealedRam, policy); + ASSERT_EQ(access->query_mtype(kBase, cache), policy); + ASSERT_EQ(access->cached_private_ram_mtype(kBase, cache), policy); + map(Owner::Application, policy); + EXPECT_FALSE(access->cached_private_ram_mtype(kBase, cache)); + const auto before = backing_; + EXPECT_FALSE(access->try_write_private_dwords(stores(), Mtype::RW, policy)); + EXPECT_EQ(backing_, before); + ASSERT_EQ(access->query_mtype(kBase, cache), policy); + EXPECT_FALSE(access->cached_private_ram_mtype(kBase, cache)); + } + map(Owner::DriverSealedRam, Mtype::UC); + ASSERT_EQ(access->query_mtype(kBase, cache), Mtype::UC); + auto physical = std::make_shared(256); + ASSERT_TRUE(adapter_.unregister_address_space(address_space_)); + address_space_ = vm_.register_address_space(kVmid, std::make_shared(256), + physical, {}, true); + ASSERT_TRUE(address_space_); + EXPECT_FALSE(access->try_write_private_dwords(stores(), Mtype::RW, Mtype::UC)); + auto replacement = vm_.snapshot(address_space_); + ASSERT_TRUE(replacement); + EXPECT_FALSE(replacement->cached_private_ram_mtype(kBase, cache)); + EXPECT_FALSE(replacement->try_write_private_dwords(stores(), Mtype::RW, Mtype::UC)); + EXPECT_EQ(physical->write_calls, 0u); +} + +TEST_F(VectorRamBatchTest, StrictSubextentFailuresRetainExactWrittenPrefix) { + for (const uint32_t kind : {0u, 1u, 2u}) { + for (const bool allow : {false, true}) { + map(Owner::DriverSealedRam, Mtype::UC, kind == 0 ? 10 : 8192); + if (kind == 1) + process_.page_table_[kBase >> rocjitsu::KfdProcess::kPageShift].host_extents = { + {backing_.data(), 6, 0, Owner::DriverSealedRam}, + {backing_.data() + 6, 58, 6, Owner::DriverSealedRam}}; + if (kind == 2) + process_.page_table_[kBase >> rocjitsu::KfdProcess::kPageShift].host_extents = { + {backing_.data(), 64, 0, Owner::DriverSealedRam}, + {backing_.data() + 8, 8, 8, Owner::DriverSealedRam}}; + reset_bytes(); + std::array addresses; + for (size_t i = 0; i < addresses.size(); ++i) + addresses[i] = kBase + i * 4; + const uint64_t before = l2_.backing_write_transactions(); + EXPECT_EQ(vector_.store(addresses.data(), 255, 4, 1, + reinterpret_cast(values_.data()), Mtype::UC, false, + 8, kVmid, 32, 0, {}, 4, allow), + Outcome::Faulted); + const size_t prefix = kind == 1 ? 4 : 8; + EXPECT_EQ(std::memcmp(backing_.data(), values_.data(), prefix), 0); + EXPECT_TRUE(std::ranges::all_of(std::span(backing_).subspan(prefix), + [](uint8_t value) { return value == 0x5a; })); + EXPECT_EQ(faults_, (std::vector{kBase + prefix})); + EXPECT_EQ(l2_.backing_write_transactions() - before, prefix / 4 + 1); + } + } +} + +TEST_F(VectorRamBatchTest, AdmissionPinsPageTableUntilMappingGuardAllowsCommit) { + map(); + auto access = vm_.snapshot(address_space_); + ASSERT_TRUE(access); + const auto before = backing_; + const auto request_mutex = process_.page_table_request_mutex(); + std::barrier start(2); + bool copied = false; + std::thread worker([&] { + start.arrive_and_wait(); + copied = access->try_write_private_dwords(stores(), Mtype::RW, Mtype::UC); + }); + auto mapping = rocjitsu::host_mapping_lock().lock_exclusive(); + start.arrive_and_wait(); + bool pinned = false; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (std::chrono::steady_clock::now() < deadline) { + if (!process_.page_table_mutex_.try_lock()) { + pinned = true; + break; + } + process_.page_table_mutex_.unlock(); + std::this_thread::yield(); + } + const bool writable = request_mutex->try_lock(); + if (writable) + request_mutex->unlock(); + const bool unchanged = backing_ == before; + mapping.unlock(); + worker.join(); + EXPECT_TRUE(pinned); + EXPECT_FALSE(writable); + EXPECT_TRUE(unchanged); + EXPECT_TRUE(copied); + EXPECT_TRUE(faults_.empty()); +} + +TEST(VectorRamBatch, CacheInspectionDoesNotChangeReplacementOrder) { + simdojo::Cache<7, 1, 2> cache; + cache.allocate(0); + cache.allocate(128); + ASSERT_NE(cache.peek(0), nullptr); + EXPECT_EQ(cache.peek(256), nullptr); + simdojo::CacheTag evicted; + cache.allocate(256, 0, &evicted); + EXPECT_EQ(evicted.tag, 0u); + EXPECT_EQ(cache.peek(0), nullptr); + EXPECT_NE(cache.peek(128), nullptr); +} + +TEST(VectorRamBatch, UnknownBackendAddsNoOptionalQueriesOrRepeatedStoreEffects) { + using namespace rocjitsu::amdgpu; + class ObservedPolicy final : public AddressSpaceTranslator { + public: + VmTranslationResult translate(uint64_t address, size_t size, + VmAccessKind access) const override { + ++queries; + return inner.translate(address, size, access); + } + bool try_write_private_dwords(PhysicalMemoryAccess &, std::span, Mtype, + Mtype) const override { + ++optional_attempts; + return false; + } + mutable uint32_t queries = 0, optional_attempts = 0; + IdentityAddressSpace inner{256, Mtype::UC}; + }; + class ObservedMemory final : public PhysicalMemoryAccess { + public: + VmAccessOutcome read(VmMemoryDomain, uint64_t, std::span) override { + ADD_FAILURE() << "UC store must not read backing"; + return VmAccessOutcome::Malformed; + } + VmAccessOutcome write(VmMemoryDomain, uint64_t address, + std::span bytes) override { + stores.emplace_back(address, bytes.size()); + if (stores.size() == 5 && result != VmAccessOutcome::Complete) + return result; + std::copy(bytes.begin(), bytes.end(), backing.begin() + address); + return VmAccessOutcome::Complete; + } + VmAccessOutcome result = VmAccessOutcome::Complete; + std::array backing{}; + std::vector> stores; + }; + for (const auto outcome : + {VmAccessOutcome::Complete, VmAccessOutcome::Faulted, VmAccessOutcome::Unavailable}) { + uint32_t ordinary_queries = 0; + for (const bool allow : {false, true}) { + GpuVm vm; + auto policy = std::make_shared(); + auto physical = std::make_shared(); + physical->result = outcome; + const auto handle = vm.register_address_space(7, policy, physical, {}, true); + ASSERT_TRUE(handle); + GpuMemory memory("store_custom"); + L2Cache l2("store_custom_l2"); + l2.set_backing_memory(&memory); + l2.set_gpu_vm(&vm); + L1VectorCache l1(&l2); + l1.set_gpu_vm(&vm); + std::array addresses; + std::array values; + for (uint32_t lane = 0; lane < 8; ++lane) { + addresses[lane] = 64 + lane * 4; + values[lane] = 0x76540000u + lane; + } + EXPECT_EQ(l1.store(addresses.data(), 255, 4, 1, + reinterpret_cast(values.data()), Mtype::UC, false, 8, 7, + 32, 0, {}, 4, allow), + outcome); + const size_t complete = outcome == VmAccessOutcome::Complete ? 8 : 4; + const size_t attempts = outcome == VmAccessOutcome::Complete ? 8 : 5; + ASSERT_EQ(physical->stores.size(), attempts); + for (size_t index = 0; index < attempts; ++index) + EXPECT_EQ(physical->stores[index], (std::pair{64 + index * 4, 4})); + EXPECT_EQ(std::memcmp(physical->backing.data() + 64, values.data(), complete * 4), 0); + EXPECT_TRUE(std::ranges::all_of(std::span(physical->backing).subspan(64 + complete * 4), + [](std::byte value) { return value == std::byte{0}; })); + EXPECT_EQ(l2.backing_write_transactions(), attempts); + EXPECT_EQ(policy->optional_attempts, 0u); + if (!allow) + ordinary_queries = policy->queries; + else + EXPECT_EQ(policy->queries, ordinary_queries); + } + } +} + +TEST(VectorRamBatch, RawRegistryReplacementRetainsOnlyOriginalRequestOwner) { + using namespace rocjitsu::amdgpu; + GpuMemory memory("store_owner"); + GpuVm vm; + LegacyGpuVmAdapter adapter(vm, &memory); + std::array backing{}; + LegacyPageTable table; + table[8] = {backing.data(), Mtype::UC, backing.size(), 0}; + table[8].host_extents.front().owner = LegacyHostExtentOwner::DriverSealedRam; + util::DistributedSharedMutex page_mutex; + const auto destroyed = std::make_shared(false); + auto request = std::shared_ptr(new util::DistributedSharedMutex, + [destroyed](auto *mutex) { + *destroyed = true; + delete mutex; + }); + const auto handle = adapter.register_address_space( + 7, {.page_table = &table, .page_table_mutex = &page_mutex, .request_mutex = request}); + ASSERT_TRUE(handle); + auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + std::array words; + words.fill(std::byte{0x5a}); + ASSERT_TRUE(access->try_read_uncached_ram(8 * GpuMemory::PAGE_SIZE, words)); + EXPECT_EQ(words, (std::array{})); + adapter.address_space(7)->register_process(7, &table, &page_mutex, nullptr, + std::make_shared()); + request.reset(); + EXPECT_FALSE(*destroyed); + const uint32_t word = 0x12345678; + const std::array stores{ + {{8 * GpuMemory::PAGE_SIZE, reinterpret_cast(&word)}, + {8 * GpuMemory::PAGE_SIZE + 4, reinterpret_cast(&word)}}}; + EXPECT_FALSE(access->try_write_private_dwords(stores, Mtype::UC, Mtype::UC)); + words.fill(std::byte{0x5a}); + EXPECT_FALSE(access->try_read_uncached_ram(8 * GpuMemory::PAGE_SIZE, words)); + EXPECT_TRUE(std::ranges::all_of(words, [](std::byte value) { return value == std::byte{0x5a}; })); + EXPECT_FALSE(*destroyed); + EXPECT_TRUE(std::ranges::all_of(backing, [](uint8_t value) { return value == 0; })); + ASSERT_TRUE(adapter.unregister_address_space(handle)); + access.reset(); + EXPECT_TRUE(*destroyed); +} + +TEST_F(ScalarRamBatchTest, SealedUncachedWidthsCopyOnceWithoutPopulatingCache) { + map(); + for (const uint32_t width : {1u, 4u, 8u, 16u}) { + for (uint32_t repeat = 0; repeat < 2; ++repeat) { + backing_[0] ^= 0x5a; + std::array words; + words.fill(kSentinel); + const uint64_t before = l2_.backing_read_transactions(); + ASSERT_EQ(l1_.load(kBase, width, words.data(), kVmid, true), Outcome::Complete); + expect_words(words, width); + EXPECT_EQ(l2_.backing_read_transactions() - before, 1u); + } + } + EXPECT_TRUE(faults_.empty()); +} + +TEST_F(ScalarRamBatchTest, PrivateCopyUsesPageBoundsAndDeclinesGuestAliases) { + map(); + auto access = vm_.snapshot(address_space_); + ASSERT_TRUE(access); + std::array copied{}; + for (std::size_t size : {128u, 4096u}) { + EXPECT_TRUE(access->try_read_uncached_ram(kBase, std::span(copied).first(size))); + EXPECT_EQ(std::memcmp(copied.data(), backing_.data(), size), 0); + } + const auto original = backing_; + EXPECT_FALSE( + access->try_read_uncached_ram(kBase, std::as_writable_bytes(std::span(backing_).first(128)))); + EXPECT_EQ(backing_, original); + process_.unmap_pages(kBase, backing_.size()); + map(Owner::DriverSealedRam, Mtype::UC, 128); + copied.fill(std::byte{0x5a}); + EXPECT_FALSE(access->try_read_uncached_ram(kBase, std::span(copied).first(256))); + EXPECT_TRUE( + std::ranges::all_of(copied, [](std::byte value) { return value == std::byte{0x5a}; })); +} + +TEST_F(ScalarRamBatchTest, DisabledBatchAndUnprovedOwnersKeepDwordTransactions) { + for (const Owner owner : {Owner::Application, Owner::Driver, Owner::DriverSealedRam}) { + map(owner); + for (const bool allow : {false, true}) { + std::array words; + words.fill(kSentinel); + const uint64_t before = l2_.backing_read_transactions(); + ASSERT_EQ(l1_.load(kBase, 8, words.data(), kVmid, allow), Outcome::Complete); + expect_words(words, 8); + EXPECT_EQ(l2_.backing_read_transactions() - before, + allow && owner == Owner::DriverSealedRam ? 1u : 8u); + } + } +} + +TEST_F(ScalarRamBatchTest, RefusesAbsentMappingAndClippedExtentBeforeOriginalFault) { + map(Owner::DriverSealedRam, Mtype::UC, 22); + for (const bool allow : {false, true}) { + faults_.clear(); + std::array words; + words.fill(kSentinel); + const uint64_t before = l2_.backing_read_transactions(); + EXPECT_EQ(l1_.load(kBase, 8, words.data(), kVmid, allow), Outcome::Faulted); + expect_words(words, 5); + EXPECT_EQ(faults_, (std::vector{kBase + 20})); + EXPECT_EQ(l2_.backing_read_transactions() - before, 6u); + } + auto access = vm_.snapshot(address_space_); + ASSERT_TRUE(access); + std::array bytes; + bytes.fill(std::byte{0xcc}); + faults_.clear(); + EXPECT_FALSE(access->try_read_uncached_ram(kBase, bytes)); + EXPECT_FALSE(access->try_read_uncached_ram(kBase + GpuMemory::PAGE_SIZE, bytes)); + EXPECT_TRUE(std::ranges::all_of(bytes, [](std::byte b) { return b == std::byte{0xcc}; })); + EXPECT_TRUE(faults_.empty()); +} + +TEST_F(ScalarRamBatchTest, AlignmentLineAndPageCrossingsKeepOriginalChunks) { + map(); + for (const uint32_t offset : {1u, 60u, static_cast(GpuMemory::PAGE_SIZE - 4)}) { + uint64_t original_reads = 0; + for (const bool allow : {false, true}) { + std::array words; + words.fill(kSentinel); + const uint64_t before = l2_.backing_read_transactions(); + ASSERT_EQ(l1_.load(kBase + offset, 8, words.data(), kVmid, allow), Outcome::Complete); + expect_words(words, 8, offset); + if (!allow) + original_reads = l2_.backing_read_transactions() - before; + else + EXPECT_EQ(l2_.backing_read_transactions() - before, original_reads); + } + } +} + +TEST_F(ScalarRamBatchTest, AdjacentSealedExtentsRetainDwordFaultBoundaries) { + for (const uint32_t split : {2u, 6u}) { + map(); + // Populate before first access rather than using map_pages, which merges + // contiguous extents. The translation interface also accepts split PTEs. + process_.page_table_[kBase >> rocjitsu::KfdProcess::kPageShift].host_extents = { + {backing_.data(), split, 0, Owner::DriverSealedRam}, + {backing_.data() + split, 64 - split, split, Owner::DriverSealedRam}}; + for (const bool allow : {false, true}) { + faults_.clear(); + std::array words; + words.fill(kSentinel); + const uint64_t reads = l2_.backing_read_transactions(); + EXPECT_EQ(l1_.load(kBase, 8, words.data(), kVmid, allow), Outcome::Faulted); + expect_words(words, split / 4); + EXPECT_EQ(faults_, (std::vector{kBase + (split / 4) * 4})); + EXPECT_EQ(l2_.backing_read_transactions() - reads, split / 4 + 1); + } + } +} + +TEST_F(ScalarRamBatchTest, OverlappingSealedExtentsKeepSuccessfulDwordPrefix) { + map(); + process_.page_table_[kBase >> rocjitsu::KfdProcess::kPageShift].host_extents = { + {backing_.data(), 32, 0, Owner::DriverSealedRam}, + {backing_.data() + 8, 8, 8, Owner::DriverSealedRam}}; + for (const bool allow : {false, true}) { + faults_.clear(); + std::array words; + words.fill(kSentinel); + const uint64_t reads = l2_.backing_read_transactions(); + EXPECT_EQ(l1_.load(kBase, 8, words.data(), kVmid, allow), Outcome::Faulted); + expect_words(words, 2); + EXPECT_EQ(faults_, (std::vector{kBase + 8})); + EXPECT_EQ(l2_.backing_read_transactions() - reads, 3u); + } +} + +TEST_F(ScalarRamBatchTest, ResidentCleanAndDirtyLinesKeepFlushBeforeRead) { + map(); + std::array cached{}; + ASSERT_EQ(l2_.read(kBase, cached.data(), cached.size(), Mtype::RW, kVmid), Outcome::Complete); + for (const bool dirty : {false, true}) { + if (dirty) { + cached.fill(0xa5); + ASSERT_EQ(l2_.writeback_line(kBase, cached.data(), Mtype::RW, kVmid), Outcome::Complete); + } + std::array words; + words.fill(kSentinel); + const uint64_t reads = l2_.backing_read_transactions(); + const uint64_t writes = l2_.backing_write_transactions(); + ASSERT_EQ(l1_.load(kBase, 8, words.data(), kVmid, true), Outcome::Complete); + expect_words(words, 8); + EXPECT_EQ(l2_.backing_read_transactions() - reads, 8u); + EXPECT_EQ(l2_.backing_write_transactions() - writes, dirty ? 1u : 0u); + if (dirty) { + EXPECT_EQ(words[0], 0xa5a5a5a5u); + } + } +} + +TEST_F(ScalarRamBatchTest, CacheablePolicyAndRetiredSnapshotsDeclinePrivateCopy) { + map(Owner::DriverSealedRam, Mtype::RW); + auto access = vm_.snapshot(address_space_); + ASSERT_TRUE(access); + std::array bytes; + bytes.fill(std::byte{0xcc}); + EXPECT_FALSE(access->try_read_uncached_ram(kBase, bytes)); + std::array words; + words.fill(kSentinel); + ASSERT_EQ(l1_.load(kBase, 8, words.data(), kVmid, true), Outcome::Complete); + expect_words(words, 8); + const uint64_t reads = l2_.backing_read_transactions(); + ASSERT_EQ(l1_.load(kBase, 8, words.data(), kVmid, true), Outcome::Complete); + EXPECT_EQ(l2_.backing_read_transactions(), reads); + EXPECT_TRUE(vm_.unregister_address_space(address_space_)); + EXPECT_FALSE(access->try_read_uncached_ram(kBase, bytes)); + EXPECT_TRUE(std::ranges::all_of(bytes, [](std::byte b) { return b == std::byte{0xcc}; })); + EXPECT_TRUE(faults_.empty()); +} + +TEST_F(ScalarRamBatchTest, OwnershipHintsFollowMutationEpochWithinQuantum) { + map(Owner::Application); + auto access = vm_.snapshot(address_space_); + ASSERT_TRUE(access); + rocjitsu::amdgpu::VmMtypeCache cache; + rocjitsu::amdgpu::GpuVmAccessBatchGuard quantum; + for (const Owner owner : {Owner::Application, Owner::DriverSealedRam, Owner::Driver, + Owner::DriverSealedRam, Owner::Application}) { + map(owner); + ASSERT_EQ(access->query_mtype(kBase, cache), Mtype::UC); + EXPECT_EQ(access->cached_private_uc_hint(kBase, cache), owner == Owner::DriverSealedRam); + std::array words; + words.fill(kSentinel); + const uint64_t reads = l2_.backing_read_transactions(); + ASSERT_EQ(l1_.load(kBase, 8, words.data(), kVmid, true), Outcome::Complete); + expect_words(words, 8); + EXPECT_EQ(l2_.backing_read_transactions() - reads, owner == Owner::DriverSealedRam ? 1u : 8u); + } + EXPECT_TRUE(faults_.empty()); +} + +TEST_F(ScalarRamBatchTest, StalePositiveHintNeverAuthorizesChangedOwnerOrPolicy) { + map(); + auto access = vm_.snapshot(address_space_); + ASSERT_TRUE(access); + rocjitsu::amdgpu::VmMtypeCache cache; + for (const bool change_owner : {true, false}) { + map(); + ASSERT_EQ(access->query_mtype(kBase, cache), Mtype::UC); + ASSERT_TRUE(access->cached_private_uc_hint(kBase, cache)); + map(change_owner ? Owner::Application : Owner::DriverSealedRam, + change_owner ? Mtype::UC : Mtype::RW); + // Deliberately retain the positive hint across the mutation. The copy + // helper must prove current policy independently before any guest effect. + std::array words; + words.fill(kSentinel); + const uint64_t reads = l2_.backing_read_transactions(); + EXPECT_FALSE(l2_.try_read_scalar_ram(kBase, words.data(), 8, kVmid)); + expect_words(words, 0); + EXPECT_EQ(l2_.backing_read_transactions(), reads); + EXPECT_TRUE(faults_.empty()); + EXPECT_FALSE(access->cached_private_uc_hint(kBase, cache)); + EXPECT_EQ(access->query_mtype(kBase, cache), change_owner ? Mtype::UC : Mtype::RW); + EXPECT_FALSE(access->cached_private_uc_hint(kBase, cache)); + } +} + +TEST_F(ScalarRamBatchTest, BindingReplacementDiscardsCachedOwnershipHint) { + map(); + auto old_access = vm_.snapshot(address_space_); + ASSERT_TRUE(old_access); + rocjitsu::amdgpu::VmMtypeCache cache; + ASSERT_EQ(old_access->query_mtype(kBase, cache), Mtype::UC); + ASSERT_TRUE(old_access->cached_private_uc_hint(kBase, cache)); + rocjitsu::amdgpu::GpuVmAccessBatchGuard quantum; + std::array words{}; + ASSERT_EQ(l1_.load(kBase, 8, words.data(), kVmid, true), Outcome::Complete); + ASSERT_TRUE(vm_.unregister_address_space(address_space_)); + auto translator = std::make_shared(kBase + 8192, Mtype::UC); + auto physical = std::make_shared(kBase + 8192); + auto replacement = vm_.register_address_space(kVmid, translator, physical, {}, true); + ASSERT_TRUE(replacement); + auto new_access = vm_.snapshot(replacement); + ASSERT_TRUE(new_access); + EXPECT_FALSE(new_access->cached_private_uc_hint(kBase, cache)); + EXPECT_FALSE(old_access->cached_private_uc_hint(kBase, cache)); + EXPECT_EQ(new_access->query_mtype(kBase, cache), Mtype::UC); + EXPECT_FALSE(new_access->cached_private_uc_hint(kBase, cache)); + EXPECT_FALSE(old_access->query_mtype(kBase, cache)); + EXPECT_FALSE(old_access->cached_private_uc_hint(kBase, cache)); + const uint64_t reads = l2_.backing_read_transactions(); + ASSERT_EQ(l1_.load(kBase, 8, words.data(), kVmid, true), Outcome::Complete); + EXPECT_EQ(l2_.backing_read_transactions() - reads, 8u); + EXPECT_EQ(physical->read_calls, 8u); + EXPECT_TRUE(faults_.empty()); +} + +TEST(ScalarRamBatch, UnknownEligibilityAddsNoPolicyOrBackingObservations) { + using namespace rocjitsu::amdgpu; + class CountingPolicy final : public AddressSpaceTranslator { + public: + VmTranslationResult translate(uint64_t address, size_t size, + VmAccessKind access) const override { + ++translations; + return inner_.translate(address, size, access); + } + bool try_read_uncached_ram(PhysicalMemoryAccess &, uint64_t, + std::span) const override { + ++optional_attempts; + return false; + } + mutable uint64_t translations = 0; + mutable uint64_t optional_attempts = 0; + + private: + IdentityAddressSpace inner_{256, Mtype::UC}; + }; + uint64_t original_translations = 0; + for (const bool allow : {false, true}) { + GpuVm vm; + auto translator = std::make_shared(); + auto physical = std::make_shared(256); + const auto handle = vm.register_address_space(7, translator, physical, {}, true); + ASSERT_TRUE(handle); + auto access = vm.snapshot(handle); + ASSERT_TRUE(access); + VmMtypeCache empty_cache; + EXPECT_FALSE(access->cached_private_uc_hint(64, empty_cache)); + EXPECT_FALSE(access->cached_private_uc_hint(64, empty_cache)); + EXPECT_EQ(translator->translations, 0u); + EXPECT_EQ(physical->read_calls, 0u); + GpuMemory memory("unknown_hint"); + L2Cache l2("unknown_hint_l2"); + l2.set_backing_memory(&memory); + l2.set_gpu_vm(&vm); + L1ScalarCache l1(&l2); + l1.set_gpu_vm(&vm); + GpuVmAccessBatchGuard quantum; + std::array words{}; + ASSERT_EQ(l1.load(64, words.size(), words.data(), 7, allow), VmAccessOutcome::Complete); + EXPECT_EQ(physical->read_calls, 8u); + EXPECT_EQ(translator->optional_attempts, 0u); + if (!allow) + original_translations = translator->translations; + else + EXPECT_EQ(translator->translations, original_translations); + } +} + +TEST(ScalarRamBatch, CustomTransportKeepsEachReadEffectAndIncompletePrefix) { + using Outcome = rocjitsu::amdgpu::VmAccessOutcome; + class ObservedMemory final : public rocjitsu::amdgpu::PhysicalMemoryAccess { + public: + Outcome read(rocjitsu::amdgpu::VmMemoryDomain, uint64_t address, + std::span bytes) override { + reads.emplace_back(address, bytes.size()); + if (reads.size() == 5 && incomplete != Outcome::Complete) + return incomplete; + const uint32_t value = 0x12340000 + static_cast(reads.size()); + EXPECT_EQ(bytes.size(), sizeof(value)); + if (bytes.size() != sizeof(value)) + return Outcome::Malformed; + std::memcpy(bytes.data(), &value, bytes.size()); + return Outcome::Complete; + } + Outcome write(rocjitsu::amdgpu::VmMemoryDomain, uint64_t, std::span) override { + ADD_FAILURE() << "scalar read unexpectedly wrote backing"; + return Outcome::Malformed; + } + bool try_read_contiguous(rocjitsu::amdgpu::VmMemoryDomain, uint64_t, + std::span) override { + ADD_FAILURE() << "generic optional transport must not be used as private RAM proof"; + return false; + } + Outcome incomplete = Outcome::Complete; + std::vector> reads; + }; + for (const auto outcome : {Outcome::Complete, Outcome::Faulted, Outcome::Unavailable}) { + for (const bool allow : {false, true}) { + rocjitsu::amdgpu::GpuVm vm; + auto translator = std::make_shared(256, Mtype::UC); + auto physical = std::make_shared(); + physical->incomplete = outcome; + ASSERT_TRUE(vm.register_address_space(7, translator, physical, {}, true)); + GpuMemory memory("observed"); + L2Cache l2("observed_l2"); + l2.set_backing_memory(&memory); + l2.set_gpu_vm(&vm); + L1ScalarCache l1(&l2); + l1.set_gpu_vm(&vm); + std::array words; + words.fill(0xdeadbeef); + EXPECT_EQ(l1.load(64, words.size(), words.data(), 7, allow), outcome); + const size_t completed = outcome == Outcome::Complete ? 8 : 4; + const size_t attempts = outcome == Outcome::Complete ? 8 : 5; + ASSERT_EQ(physical->reads.size(), attempts); + for (size_t i = 0; i < attempts; ++i) + EXPECT_EQ(physical->reads[i], (std::pair{64 + i * 4, 4})); + for (size_t i = 0; i < words.size(); ++i) + EXPECT_EQ(words[i], i < completed ? 0x12340001u + i : 0xdeadbeefu); + EXPECT_EQ(l2.backing_read_transactions(), attempts); + } + } +} + TEST_F(LegacySubPageCacheTest, VectorAccessCachesOnlyAccessibleBytesOfIncompleteLine) { constexpr uint64_t kAddress = kBase + 8; constexpr size_t kAccessSize = kBackingSize - 8; @@ -1430,7 +2259,7 @@ TEST_F(L1CacheMtypeTest, PageMtypeMutationRefreshesLiveResolver) { l1.load(kAddr, 1, &result, kVmid); ASSERT_EQ(result, kFirst); - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kAddr), Mtype::RW); process_.set_page_mtype(kBase, GpuMemory::PAGE_SIZE, Mtype::CC); write_words(kFirstReplacement, kSecond); @@ -1457,7 +2286,7 @@ TEST_F(L1CacheMtypeTest, VmidRebindingRefreshesNewResolverAndUsesNewPolicy) { replacement_process.map_pages(kBase, replacement_page.data(), replacement_page.size(), Mtype::UC); { - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kAddr), Mtype::RW); } ASSERT_TRUE(legacy_vm_.unregister_vmid(kVmid)); @@ -1466,7 +2295,7 @@ TEST_F(L1CacheMtypeTest, VmidRebindingRefreshesNewResolverAndUsesNewPolicy) { kVmid, &replacement_process.page_table_, &replacement_process.page_table_mutex_, replacement_process.page_table_generation(), replacement_process.page_table_request_mutex()); ASSERT_TRUE(address_space_); - RequestMtypeResolver replacement_request(&gpu_vm_, kVmid); + RequestMtypeResolver replacement_request(&gpu_vm_, kVmid, mtype_cache_); EXPECT_EQ(replacement_request.at(kAddr + 1), Mtype::UC); l1.load(kAddr, 1, &result, kVmid); @@ -1478,20 +2307,20 @@ TEST_F(L1CacheMtypeTest, VmidRebindingRefreshesNewResolverAndUsesNewPolicy) { TEST_F(L1CacheMtypeTest, VmidUnregistrationRevokesLiveResolverSnapshot) { map_pages(Mtype::UC, Mtype::UC); - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kAddr), Mtype::UC); ASSERT_TRUE(legacy_vm_.unregister_vmid(kVmid)); address_space_ = {}; EXPECT_EQ(request.at(kAddr + 1), Mtype::RW); EXPECT_FALSE(gpu_vm_.snapshot_vmid(kVmid)); - RequestMtypeResolver new_request(&gpu_vm_, kVmid); + RequestMtypeResolver new_request(&gpu_vm_, kVmid, mtype_cache_); EXPECT_EQ(new_request.at(kAddr + 1), Mtype::RW); } TEST_F(L1CacheMtypeTest, SamePageHitDoesNotTakePageTableLocks) { map_pages(Mtype::UC, Mtype::RW); - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kBase), Mtype::UC); std::unique_lock request_lock(*process_.page_table_request_mutex()); std::unique_lock table_lock(process_.page_table_mutex_); @@ -1503,9 +2332,53 @@ TEST_F(L1CacheMtypeTest, SamePageHitDoesNotTakePageTableLocks) { EXPECT_EQ(hit.get(), Mtype::UC); } +TEST_F(L1CacheMtypeTest, NewRequestsReusePolicyWithoutPageTableLocks) { + write_words(kFirst, kSecond); + map_pages(Mtype::RW, Mtype::RW); + for (bool vector : {false, true}) { + SCOPED_TRACE(vector ? "vector" : "scalar"); + L1ScalarCache scalar_cache(&l2_); + L1VectorCache vector_cache(&l2_); + scalar_cache.set_gpu_vm(&gpu_vm_); + vector_cache.set_gpu_vm(&gpu_vm_); + uint32_t observed = 0; + auto load = [&] { + if (vector) + return vector_cache.load(&kAddr, /*lane_mask=*/1, sizeof(observed), /*num_elems=*/1, + reinterpret_cast(&observed), Mtype::RW, + /*non_temporal=*/false, /*request_l1_bypass=*/false, + /*wf_size=*/1, kVmid); + return scalar_cache.load(kAddr, 1, &observed, kVmid); + }; + ASSERT_EQ(load(), rocjitsu::amdgpu::VmAccessOutcome::Complete); + ASSERT_EQ(observed, kFirst); + observed = 0; + + std::unique_lock request_lock(*process_.page_table_request_mutex()); + std::unique_lock table_lock(process_.page_table_mutex_); + auto hit = std::async(std::launch::async, load); + const auto status = hit.wait_for(std::chrono::seconds(2)); + table_lock.unlock(); + request_lock.unlock(); + EXPECT_EQ(status, std::future_status::ready); + EXPECT_EQ(hit.get(), rocjitsu::amdgpu::VmAccessOutcome::Complete); + EXPECT_EQ(observed, kFirst); + } +} + +TEST_F(L1CacheMtypeTest, CachedPagePolicyDoesNotRetainPreviousInstructionPolicy) { + map_pages(Mtype::RW, Mtype::RW); + EXPECT_EQ(RequestMtypeResolver(&gpu_vm_, kVmid, mtype_cache_, Mtype::NT).at(kBase), Mtype::NT); + EXPECT_EQ(RequestMtypeResolver(&gpu_vm_, kVmid, mtype_cache_, Mtype::RW).at(kBase), Mtype::RW); + EXPECT_EQ(RequestMtypeResolver(&gpu_vm_, kVmid, mtype_cache_).at(kBase), Mtype::RW); + + process_.set_page_mtype(kBase, GpuMemory::PAGE_SIZE, Mtype::UC); + EXPECT_EQ(RequestMtypeResolver(&gpu_vm_, kVmid, mtype_cache_, Mtype::NT).at(kBase), Mtype::UC); +} + TEST_F(L1CacheMtypeTest, InstructionPolicyIsPreservedAcrossHitsAndMutation) { map_pages(Mtype::RW, Mtype::RW); - RequestMtypeResolver request(&gpu_vm_, kVmid, Mtype::NT); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_, Mtype::NT); ASSERT_EQ(request.at(kBase), Mtype::NT); EXPECT_EQ(request.at(kBase + 4), Mtype::NT); process_.set_page_mtype(kBase, GpuMemory::PAGE_SIZE, Mtype::UC); @@ -1516,7 +2389,7 @@ TEST_F(L1CacheMtypeTest, InstructionPolicyIsPreservedAcrossHitsAndMutation) { TEST_F(L1CacheMtypeTest, LiveResolverObservesUnmapAndRemapOnSamePage) { map_pages(Mtype::UC, Mtype::RW); - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kAddr), Mtype::UC); EXPECT_EQ(request.at(kAddr + 1), Mtype::UC); @@ -1536,7 +2409,7 @@ TEST_F(L1CacheMtypeTest, LiveResolverSurvivesRegisteredProcessDestruction) { .request_mutex = process->page_table_request_mutex(), .mutation_epoch = process->page_table_mutation_epoch()}); ASSERT_TRUE(address_space_); - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kAddr), Mtype::UC); ASSERT_TRUE(legacy_vm_.unregister_vmid(kVmid)); address_space_ = {}; @@ -1546,7 +2419,7 @@ TEST_F(L1CacheMtypeTest, LiveResolverSurvivesRegisteredProcessDestruction) { map_pages(Mtype::CC, Mtype::RW); // A retired snapshot must not adopt a new owner of the same numeric VMID. EXPECT_EQ(request.at(kAddr), Mtype::RW); - RequestMtypeResolver replacement(&gpu_vm_, kVmid); + RequestMtypeResolver replacement(&gpu_vm_, kVmid, mtype_cache_); EXPECT_EQ(replacement.at(kAddr), Mtype::CC); } @@ -1560,7 +2433,7 @@ TEST_F(L1CacheMtypeTest, LiveResolverWithoutMutationTokenStillRefreshes) { .request_mutex = process_.page_table_request_mutex(), .mutation_epoch = {}}); ASSERT_TRUE(address_space_); - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kAddr), Mtype::UC); process_.set_page_mtype(kBase, GpuMemory::PAGE_SIZE, Mtype::CC); EXPECT_EQ(request.at(kAddr + 1), Mtype::CC); @@ -1576,7 +2449,7 @@ TEST_F(L1CacheMtypeTest, LiveResolverWithoutGenerationDoesNotCacheMutationToken) .request_mutex = process_.page_table_request_mutex(), .mutation_epoch = process_.page_table_mutation_epoch()}); ASSERT_TRUE(address_space_); - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kAddr), Mtype::UC); { // Legacy registrations may omit generation publication; they must keep @@ -1599,7 +2472,7 @@ TEST_F(L1CacheMtypeTest, MutationTokenRefreshesPolicyWithoutGenerationPublicatio .request_mutex = process_.page_table_request_mutex(), .mutation_epoch = epoch}); ASSERT_TRUE(address_space_); - RequestMtypeResolver request(&gpu_vm_, kVmid); + RequestMtypeResolver request(&gpu_vm_, kVmid, mtype_cache_); ASSERT_EQ(request.at(kAddr), Mtype::UC); { // Model a mutation that invalidates snapshots and changes a PTE, then @@ -2026,6 +2899,67 @@ TEST(L2CacheThreadingTest, CrossL2AtomicRmwSameAddressIsSerialized) { EXPECT_EQ(memory.read32(kTarget), kThreads * kIterations); } +TEST(L2CacheThreadingTest, BatchedAndIndividualAtomicsShareCoherence) { + GpuMemory memory("memory"); + auto coherence = std::make_shared(); + L2Cache l2a("l2a", coherence), l2b("l2b", coherence); + l2a.set_backing_memory(&memory); + l2b.set_backing_memory(&memory); + constexpr uint64_t kTarget = 0x280000; + constexpr uint32_t kBatches = 64, kLanes = 32; + memory.write32(kTarget, 0); + const auto increment = [](uint8_t *target, uint32_t offset) { + uint32_t value; + std::memcpy(&value, target + offset, sizeof(value)); + ++value; + std::memcpy(target + offset, &value, sizeof(value)); + }; + std::barrier start(2); + std::atomic failures = 0; + std::thread batched([&] { + start.arrive_and_wait(); + for (uint32_t batch = 0; batch < kBatches; ++batch) { + auto boundary = coherence->acquire_atomic_boundary(); + for (uint32_t lane = 0; lane < kLanes; ++lane) + if (l2a.atomic_rmw(boundary, kTarget, sizeof(uint32_t), increment, 0, std::nullopt) != + rocjitsu::amdgpu::VmAccessOutcome::Complete) + ++failures; + } + }); + start.arrive_and_wait(); + for (uint32_t i = 0; i < kBatches * kLanes; ++i) + if (l2b.atomic_rmw(kTarget, sizeof(uint32_t), increment) != + rocjitsu::amdgpu::VmAccessOutcome::Complete) + ++failures; + batched.join(); + EXPECT_EQ(failures.load(), 0u); + EXPECT_EQ(memory.read32(kTarget), 2 * kBatches * kLanes); +} + +TEST(L2CacheTest, AtomicBatchRejectsWrongDomainAndMovedBoundary) { + GpuMemory memory("memory"); + L2Cache l2("l2"); + l2.set_backing_memory(&memory); + auto other = std::make_shared(); + bool called = false; + auto mutation = [&](uint8_t *, uint32_t) { called = true; }; + { + auto wrong_boundary = other->acquire_atomic_boundary(); + EXPECT_EQ(l2.atomic_rmw(wrong_boundary, 0x1000, sizeof(uint32_t), mutation, 0, std::nullopt), + rocjitsu::amdgpu::VmAccessOutcome::Malformed); + } + { + auto boundary = l2.coherence_domain()->acquire_atomic_boundary(); + auto moved = std::move(boundary); + EXPECT_EQ(l2.atomic_rmw(boundary, 0x1000, sizeof(uint32_t), mutation, 0, std::nullopt), + rocjitsu::amdgpu::VmAccessOutcome::Malformed); + EXPECT_FALSE(called); + EXPECT_EQ(l2.atomic_rmw(moved, 0x1000, sizeof(uint32_t), mutation, 0, std::nullopt), + rocjitsu::amdgpu::VmAccessOutcome::Complete); + EXPECT_TRUE(called); + } +} + TEST(L2CacheThreadingTest, CrossL2AtomicRmwAliasedVasIsSerialized) { GpuMemory memory("memory"); auto coherence = std::make_shared(); @@ -2852,4 +3786,114 @@ TEST(GpuMemoryTest, BlockAccessHandlesPageBoundaries) { EXPECT_EQ(output, input); } +TEST(L2DiagnosticCounters, JoinedTotalsCoverDistributedAndSharedLines) { + using namespace rocjitsu::amdgpu; + class StatelessMemory final : public PhysicalMemoryAccess { + public: + VmAccessOutcome read(VmMemoryDomain, uint64_t, std::span bytes) override { + std::ranges::fill(bytes, std::byte{0}); + return VmAccessOutcome::Complete; + } + VmAccessOutcome write(VmMemoryDomain, uint64_t, std::span) override { + return VmAccessOutcome::Complete; + } + }; + constexpr uint32_t kThreads = 8, kOperations = 64; + for (const bool distributed : {false, true}) { + GpuVm vm; + auto translator = std::make_shared(1u << 20); + auto physical = std::make_shared(); + ASSERT_TRUE(vm.register_address_space(7, translator, physical, {}, true)); + GpuMemory memory("diagnostic_memory"); + L2Cache l2("diagnostic_l2"); + l2.set_backing_memory(&memory); + l2.set_gpu_vm(&vm); + std::barrier start(kThreads); + std::atomic succeeded{true}; + std::vector workers; + for (uint32_t thread = 0; thread < kThreads; ++thread) { + workers.emplace_back([&, thread] { + const uint32_t value = thread; + start.arrive_and_wait(); + for (uint32_t i = 0; i < kOperations; ++i) { + const uint64_t address = distributed ? (thread + i * kThreads) * L2Cache::LINE_SIZE : 0; + for (const auto policy : {Mtype::RW, Mtype::UC}) { + if (l2.write(address, reinterpret_cast(&value), sizeof(value), policy, + 7) != VmAccessOutcome::Complete) + succeeded.store(false, std::memory_order_relaxed); + } + } + }); + } + for (auto &worker : workers) + worker.join(); + ASSERT_TRUE(succeeded.load(std::memory_order_relaxed)); + EXPECT_EQ(l2.write_count(), kThreads * kOperations); + EXPECT_EQ(l2.backing_write_transactions(), 2u * kThreads * kOperations); + EXPECT_EQ(l2.backing_read_transactions(), 0u); + } +} + +TEST(L2DiagnosticCounters, ReentrantQueriesKeepAttemptAndSuccessBoundaries) { + using namespace rocjitsu::amdgpu; + class ObservedMemory final : public PhysicalMemoryAccess { + public: + void observe() { + snapshots.push_back({cache->backing_read_transactions(), cache->backing_write_transactions(), + cache->write_count()}); + } + VmAccessOutcome read(VmMemoryDomain, uint64_t, std::span bytes) override { + observe(); + std::ranges::fill(bytes, std::byte{0}); + return VmAccessOutcome::Complete; + } + VmAccessOutcome write(VmMemoryDomain, uint64_t, std::span) override { + observe(); + return write_outcome; + } + VmAccessOutcome atomic_modify(VmMemoryDomain, uint64_t, uint32_t width, + const AtomicMutation &mutation) override { + observe(); + std::array bytes{}; + if (width != bytes.size()) + return VmAccessOutcome::Malformed; + mutation(bytes); + return VmAccessOutcome::Complete; + } + L2Cache *cache = nullptr; + VmAccessOutcome write_outcome = VmAccessOutcome::Complete; + std::vector> snapshots; + }; + GpuVm vm; + auto translator = std::make_shared(1u << 20); + auto physical = std::make_shared(); + ASSERT_TRUE(vm.register_address_space(7, translator, physical, {}, true)); + GpuMemory memory("diagnostic_observed_memory"); + L2Cache l2("diagnostic_observed_l2"); + physical->cache = &l2; + l2.set_backing_memory(&memory); + l2.set_gpu_vm(&vm); + const uint32_t value = 0x12345678; + const auto *bytes = reinterpret_cast(&value); + ASSERT_EQ(l2.write(0x68000, bytes, sizeof(value), Mtype::RW, 7), VmAccessOutcome::Complete); + physical->write_outcome = VmAccessOutcome::Faulted; + EXPECT_EQ(l2.write(0xe0000, bytes, sizeof(value), Mtype::RW, 7), VmAccessOutcome::Faulted); + physical->write_outcome = VmAccessOutcome::Complete; + ASSERT_EQ(l2.write(0x48000, bytes, sizeof(value), Mtype::UC, 7), VmAccessOutcome::Complete); + uint32_t loaded = 1; + ASSERT_EQ(l2.read(0x50000, reinterpret_cast(&loaded), sizeof(loaded), Mtype::UC, 7), + VmAccessOutcome::Complete); + ASSERT_EQ(l2.atomic_rmw( + 0x58000, sizeof(value), [](uint8_t *, uint32_t) {}, 7), + VmAccessOutcome::Complete); + const std::vector> expected{ + {0, 1, 0}, {0, 2, 1}, {0, 3, 1}, {1, 3, 1}, {2, 4, 1}}; + EXPECT_EQ(physical->snapshots, expected); + ASSERT_EQ(l2.flush_all(), VmAccessOutcome::Complete); + l2.set_coherence_domain(std::make_shared()); + EXPECT_EQ(l2.backing_read_transactions(), 2u); + EXPECT_EQ(l2.backing_write_transactions(), 4u); + EXPECT_EQ(l2.write_count(), 1u); +} + } // namespace diff --git a/emulation/rocjitsu/tests/legacy_gpu_memory_fixture.h b/emulation/rocjitsu/tests/legacy_gpu_memory_fixture.h index c76ef65ccef0..2f8fd28b33c8 100644 --- a/emulation/rocjitsu/tests/legacy_gpu_memory_fixture.h +++ b/emulation/rocjitsu/tests/legacy_gpu_memory_fixture.h @@ -67,20 +67,25 @@ class LegacyGpuMemoryFixture : public amdgpu::GpuMemory { using GpuMemory::read32; using GpuMemory::write32; - void register_process(uint32_t vmid, amdgpu::LegacyPageTable *page_table, - util::DistributedSharedMutex *page_table_mutex, - const uint64_t *page_table_generation = nullptr, - std::shared_ptr request_mutex = {}) { + void + register_process(uint32_t vmid, amdgpu::LegacyPageTable *page_table, + util::DistributedSharedMutex *page_table_mutex, + const uint64_t *page_table_generation = nullptr, + std::shared_ptr request_mutex = {}, + std::shared_ptr> mutation_epoch = {}, + std::shared_ptr page_table_cache_state = {}) { unregister_process(vmid); - const amdgpu::AddressSpaceHandle handle = - legacy_vm_.register_address_space(vmid, {.page_table = page_table, - .page_table_mutex = page_table_mutex, - .page_table_generation = page_table_generation, - .request_mutex = std::move(request_mutex), - .client_pid = client_pids_[vmid], - .client_mem_fd = -1, - .passthrough = passthrough_, - .fault_reporter = fault_reporter_}); + const amdgpu::AddressSpaceHandle handle = legacy_vm_.register_address_space( + vmid, {.page_table = page_table, + .page_table_mutex = page_table_mutex, + .page_table_generation = page_table_generation, + .request_mutex = std::move(request_mutex), + .mutation_epoch = std::move(mutation_epoch), + .page_table_cache_state = std::move(page_table_cache_state), + .client_pid = client_pids_[vmid], + .client_mem_fd = -1, + .passthrough = passthrough_, + .fault_reporter = fault_reporter_}); if (handle) { handles_[vmid] = handle; if (amdgpu::LegacyAddressSpace *address_space = legacy_vm_.address_space(vmid)) diff --git a/emulation/rocjitsu/tests/legacy_page_table_cache_test.cpp b/emulation/rocjitsu/tests/legacy_page_table_cache_test.cpp new file mode 100644 index 000000000000..34e903e0f7fd --- /dev/null +++ b/emulation/rocjitsu/tests/legacy_page_table_cache_test.cpp @@ -0,0 +1,196 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "rocjitsu/kmd/linux/kfd_process.h" +#include "rocjitsu/vm/amdgpu/legacy_address_space.h" +#include "rocjitsu/vm/amdgpu/legacy_page_table.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace rocjitsu::amdgpu { +namespace { + +TEST(LegacyPageTableCacheStateTest, RegistrationRequiresMutationSerialization) { + GpuMemory memory("page_cache_registration"); + LegacyAddressSpace space(memory); + LegacyPageTable pages; + util::DistributedSharedMutex page_mutex; + auto state = std::make_shared(); + EXPECT_THROW(space.register_process(7, &pages, &page_mutex, nullptr, {}, state), + std::invalid_argument); + auto request_mutex = std::make_shared(); + EXPECT_NO_THROW(space.register_process(7, &pages, &page_mutex, nullptr, request_mutex, state)); +} + +TEST(LegacyPageTableCacheStateTest, InvalidatesOldCopiesAndAdmitsPublishedGeneration) { + LegacyPageTableCacheState state; + const uint64_t old_generation = state.generation(); + ASSERT_TRUE(state.try_acquire(old_generation)); + state.release(); + state.invalidate_and_wait(); + EXPECT_FALSE(state.try_acquire(old_generation)); + ASSERT_TRUE(state.try_acquire(state.generation())); + state.release(); +} + +TEST(LegacyPageTableCacheStateTest, NoOpMutationRestoresLockFreeHits) { + EXPECT_EXIT( + ([] { + alarm(10); + GpuMemory memory("page_cache_noop"); + LegacyAddressSpace space(memory); + KfdProcess process(7); + constexpr uint64_t kAddress = 0x40000000; + std::array page{}; + page[0] = 0x37; + process.map_pages(kAddress, page.data(), page.size(), Mtype::RW); + space.register_process(7, &process.page_table_, &process.page_table_mutex_, + process.page_table_generation(), process.page_table_request_mutex(), + process.page_table_cache_state()); + if (space.read8(kAddress, 7) != page[0]) + _exit(1); + const uint64_t generation = *process.page_table_generation(); + process.set_page_mtype(kAddress, page.size(), Mtype::RW); + if (*process.page_table_generation() != generation || space.read8(kAddress, 7) != page[0]) + _exit(2); + + // The lookup after the no-op must restore admission. Holding + // the table lock on another thread distinguishes a copied-PTE + // hit from a fallback that still takes the reader locks. + std::atomic locked{false}; + std::atomic read_done{false}; + std::thread locker([&] { + std::unique_lock lock(process.page_table_mutex_); + locked.store(true); + while (!read_done.load()) + std::this_thread::yield(); + }); + while (!locked.load()) + std::this_thread::yield(); + if (space.read8(kAddress, 7) != page[0]) + _exit(3); + read_done.store(true); + locker.join(); + alarm(0); + _exit(0); + }()), + ::testing::ExitedWithCode(0), ""); +} + +TEST(LegacyPageTableCacheStateTest, InvalidationDrainsEveryNestedReader) { + EXPECT_EXIT(([] { + alarm(10); + constexpr size_t kReaders = 32; + LegacyPageTableCacheState state; + const uint64_t generation = state.generation(); + std::atomic entered{0}; + std::atomic partially_released{0}; + std::atomic phase{0}; + std::atomic writer_done{false}; + std::array, kReaders> release_last{}; + std::array, kReaders> left{}; + std::vector readers; + for (size_t index = 0; index < kReaders; ++index) { + readers.emplace_back([&, index] { + if (!state.try_acquire(generation) || !state.try_acquire(generation)) + _exit(1); + entered.fetch_add(1); + while (phase.load() == 0) + std::this_thread::yield(); + state.release(); + partially_released.fetch_add(1); + while (!release_last[index].load()) + std::this_thread::yield(); + state.release(); + left[index].store(true); + }); + } + while (entered.load() != kReaders) + std::this_thread::yield(); + std::thread writer([&] { + state.invalidate_and_wait(); + writer_done.store(true); + }); + while (state.generation() == generation) + std::this_thread::yield(); + if (state.try_acquire(generation) || writer_done.load()) + _exit(2); + phase.store(1); + while (partially_released.load() != kReaders) + std::this_thread::yield(); + // Release readers individually: a writer must drain every admission, + // including collisions and the second admission on each thread. + for (size_t index = 0; index < kReaders; ++index) { + if (writer_done.load()) + _exit(3); + release_last[index].store(true); + while (!left[index].load()) + std::this_thread::yield(); + } + for (auto &reader : readers) + reader.join(); + writer.join(); + if (!writer_done.load() || state.try_acquire(generation)) + _exit(4); + alarm(0); + _exit(0); + }()), + ::testing::ExitedWithCode(0), ""); +} + +TEST(LegacyPageTableCacheStateTest, ConcurrentAdmissionsObserveOnlyPublishedCopies) { + EXPECT_EXIT(([] { + alarm(10); + LegacyPageTableCacheState state; + std::atomic published{state.generation()}; + uint64_t payload = published.load(); + std::atomic stop{false}; + std::atomic started{0}; + std::atomic reads{0}; + std::vector readers; + for (size_t index = 0; index < 16; ++index) { + readers.emplace_back([&] { + started.fetch_add(1); + while (!stop.load()) { + const uint64_t copy_generation = published.load(); + if (!state.try_acquire(copy_generation)) + continue; + if (payload != copy_generation) + _exit(1); + reads.fetch_add(1, std::memory_order_relaxed); + state.release(); + } + }); + } + while (started.load() != readers.size() || reads.load() == 0) + std::this_thread::yield(); + for (size_t mutation = 0; mutation < 2000; ++mutation) { + state.invalidate_and_wait(); + // Real PTE copies are published under the mutation-side locks only + // after the old admissions drain. This publication models that + // boundary, independently of the cache-state generation itself. + payload = state.generation(); + published.store(payload); + std::this_thread::yield(); + } + stop.store(true); + for (auto &reader : readers) + reader.join(); + state.invalidate_and_wait(); + if (state.try_acquire(published.load())) + _exit(2); + alarm(0); + _exit(0); + }()), + ::testing::ExitedWithCode(0), ""); +} + +} // namespace +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/tests/matrix_coexecution_test.cpp b/emulation/rocjitsu/tests/matrix_coexecution_test.cpp index 372544ea480a..32128623933b 100644 --- a/emulation/rocjitsu/tests/matrix_coexecution_test.cpp +++ b/emulation/rocjitsu/tests/matrix_coexecution_test.cpp @@ -30,7 +30,9 @@ #include #include #include +#include #include +#include #include namespace { @@ -523,6 +525,99 @@ TEST(MatrixCoexecutionTest, SharedPoolHandlesConcurrentIssuersAndZeroCapacity) { } } +TEST(MatrixCoexecutionTest, SharedPoolScalesPreferredStartToCapacity) { + struct Case { + uint32_t hash; + std::size_t capacity; + unsigned expected; + }; + // Include bucket boundaries: masking an unscaled hash would concentrate + // small-pool issuers on slot zero instead of reaching these four preferences. + for (const Case c : {Case{0, 0, 0}, + {0xffffffffu, 1, 0}, + {0, 4, 0}, + {0x3fffffffu, 4, 0}, + {0x40000000u, 4, 1}, + {0x7fffffffu, 4, 1}, + {0x80000000u, 4, 2}, + {0xffffffffu, 4, 3}, + {0xffffffffu, 8, 7}, + {0xffffffffu, 63, 62}, + {0xffffffffu, 64, 63}, + {0x80000000u, 65, 32}, + {0xffffffffu, 65, 0}, + {0x40000000u, 128, 32}, + {0x80000000u, 128, 0}, + {0xffffffffu, 128, 63}}) { + SCOPED_TRACE(c.capacity); + SCOPED_TRACE(c.hash); + EXPECT_EQ(mc::preferred_helper_start(c.hash, c.capacity), c.expected); + } +} + +TEST(MatrixCoexecutionTest, SharedPoolIssuersReusePreferredHelpers) { + mc::SharedPool pool(4); + Instruction record("record", [](Instruction &, void *opaque) { + *static_cast(opaque) = std::this_thread::get_id(); + }); + auto helpers = [&](unsigned width, unsigned batches) { + std::set used; + for (unsigned batch = 0; batch != batches; ++batch) { + std::array ids; + std::array tickets{}; + for (unsigned i = 0; i != width; ++i) { + tickets[i] = pool.submit(record, &ids[i]); + EXPECT_TRUE(tickets[i]); + } + for (unsigned i = 0; i != width; ++i) + if (tickets[i]) { + EXPECT_FALSE(pool.finish(tickets[i])); + used.insert(ids[i]); + } + } + return used; + }; + EXPECT_EQ(helpers(4, 1).size(), 4u); + EXPECT_EQ(helpers(1, 16).size(), 1u); + // While the preferred helper's ticket is held, the same next helper takes the job. + EXPECT_EQ(helpers(2, 8).size(), 2u); + std::thread([&] { EXPECT_EQ(helpers(1, 16).size(), 1u); }).join(); +} + +TEST(MatrixCoexecutionTest, SharedPoolClaimsAndReclaimsCapacityAcrossBitmapWords) { + Instruction record("record", [](Instruction &, void *opaque) { + *static_cast(opaque) = std::this_thread::get_id(); + }); + for (unsigned capacity : {0u, 1u, 4u, 64u, 65u, 128u}) { + SCOPED_TRACE(capacity); + mc::SharedPool pool(capacity); + for (unsigned batch = 0; batch != 3; ++batch) { + std::vector ids(capacity); + std::vector tickets(capacity); + for (unsigned i = 0; i != capacity; ++i) { + tickets[i] = pool.submit(record, &ids[i]); + EXPECT_TRUE(tickets[i]); + } + // Claims remain held until finish(), even if a helper has completed. + EXPECT_FALSE(pool.available()); + std::thread::id extra_id; + const auto extra = pool.submit(record, &extra_id); + EXPECT_FALSE(extra); + if (extra) { + EXPECT_FALSE(pool.finish(extra)); + } + std::set used; + for (unsigned i = 0; i != capacity; ++i) + if (tickets[i]) { + EXPECT_FALSE(pool.finish(tickets[i])); + used.insert(ids[i]); + } + EXPECT_EQ(used.size(), capacity); + EXPECT_EQ(pool.available(), capacity != 0); + } + } +} + class QueueRetirementProbe final : public DynamicInstState { public: QueueRetirementProbe(std::vector &retired, unsigned id) diff --git a/emulation/rocjitsu/tests/memory_wait_scoreboard_test.cpp b/emulation/rocjitsu/tests/memory_wait_scoreboard_test.cpp index e2b192e937c0..c33cb2ed8813 100644 --- a/emulation/rocjitsu/tests/memory_wait_scoreboard_test.cpp +++ b/emulation/rocjitsu/tests/memory_wait_scoreboard_test.cpp @@ -1051,7 +1051,7 @@ TEST(MemoryWaitExecutionTest, ScalarMissingWaitWarnsWithoutChangingTheResult) { } } -TEST(MemoryWaitExecutionTest, FlatLanesHaveSeparateDependenciesAndKeepBothQueueEntries) { +TEST(MemoryWaitExecutionTest, FlatLanesHaveSeparateDependenciesAndCounterParticipation) { for (auto arch : {ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) { for (uint64_t shared_lanes : {0u, 0b0101u, 0b1111u}) { @@ -1084,8 +1084,8 @@ TEST(MemoryWaitExecutionTest, FlatLanesHaveSeparateDependenciesAndKeepBothQueueE inst.set_data(std::move(data)); cu->track_memory_wait(inst, *wf, shared_lanes); auto &state = *wf->memory_wait_scoreboard(); - EXPECT_EQ(state.outstanding(WaitCounterKind::Load), 1u); - EXPECT_EQ(state.outstanding(WaitCounterKind::Ds), 1u); + EXPECT_EQ(state.outstanding(WaitCounterKind::Load), shared_lanes != 0b1111 ? 1u : 0u); + EXPECT_EQ(state.outstanding(WaitCounterKind::Ds), shared_lanes != 0 ? 1u : 0u); std::vector hazards; state.bind(0x200, &hazards, [](void *p, const auto &hazard) { static_cast(p)->push_back(hazard); @@ -1099,7 +1099,7 @@ TEST(MemoryWaitExecutionTest, FlatLanesHaveSeparateDependenciesAndKeepBothQueueE state.wait(waited, 0); state.access(result, ready_lanes, 0xf, false); EXPECT_TRUE(hazards.empty()); - EXPECT_EQ(state.outstanding(pending), 1u); + EXPECT_EQ(state.outstanding(pending), pending_lanes ? 1u : 0u); state.access(result, pending_lanes, 0xf, false); ASSERT_EQ(hazards.size(), pending_lanes ? 1u : 0u); if (pending_lanes) { diff --git a/emulation/rocjitsu/tests/packet_processor_contract_test.cpp b/emulation/rocjitsu/tests/packet_processor_contract_test.cpp new file mode 100644 index 000000000000..c2c0453fd4f9 --- /dev/null +++ b/emulation/rocjitsu/tests/packet_processor_contract_test.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "rocjitsu/vm/amdgpu/packet_processor.h" + +#include + +namespace rocjitsu::amdgpu { +namespace { + +TEST(PacketProcessorContractTest, AcceptsLegalCommonResults) { + EXPECT_TRUE(valid_packet_process_result({.status = PacketProcessStatus::Complete, + .retirement = PacketRetirement::Retire, + .retirement_bytes = 4}, + 4, 4)); + EXPECT_TRUE(valid_packet_process_result( + {.status = PacketProcessStatus::NeedInput, .required_bytes = 12}, 4, 4)); + EXPECT_TRUE(valid_packet_process_result( + {.status = PacketProcessStatus::Blocked, .retirement_bytes = 12}, 12, 4)); + EXPECT_TRUE(valid_packet_process_result({.status = PacketProcessStatus::Faulted, + .retirement = PacketRetirement::Retire, + .retirement_bytes = 12}, + 12, 4)); + EXPECT_TRUE(valid_packet_process_result({.status = PacketProcessStatus::Complete, + .retirement = PacketRetirement::Retire, + .retirement_bytes = 28}, + 20, 4)); +} + +TEST(PacketProcessorContractTest, RejectsContradictoryCommonResults) { + EXPECT_FALSE(valid_packet_process_result( + {.status = PacketProcessStatus::Complete, .retirement_bytes = 4}, 4, 4)); + EXPECT_FALSE(valid_packet_process_result({.status = PacketProcessStatus::NeedInput, + .retirement = PacketRetirement::Retire, + .retirement_bytes = 4, + .required_bytes = 8}, + 4, 4)); + EXPECT_FALSE(valid_packet_process_result( + {.status = PacketProcessStatus::NeedInput, .required_bytes = 4}, 4, 4)); + EXPECT_FALSE(valid_packet_process_result( + {.status = PacketProcessStatus::Malformed, .retirement = PacketRetirement::Retire}, 4, 4)); + EXPECT_FALSE(valid_packet_process_result( + {.status = PacketProcessStatus::Blocked, .required_bytes = 8}, 4, 4)); +} + +} // namespace +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/tests/partitioning_test.cpp b/emulation/rocjitsu/tests/partitioning_test.cpp index 070dd12b4562..64db85c250f9 100644 --- a/emulation/rocjitsu/tests/partitioning_test.cpp +++ b/emulation/rocjitsu/tests/partitioning_test.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include namespace { @@ -38,6 +39,8 @@ const std::string CONFIG_PATH = std::string(CONFIG_DIR) + "/gfx950_mi355x.json"; const std::string CONFIG_KMD_PATH = std::string(CONFIG_DIR) + "/gfx950_mi355x_kmd.json"; const std::string CONFIG_2GPU_PATH = std::string(CONFIG_DIR) + "/gfx950_mi355x_kmd_2gpu.json"; const std::string CONFIG_1XCD_PATH = std::string(CONFIG_DIR) + "/gfx1100_w7900.json"; +const std::string CONFIG_CDNA5_PATH = std::string(CONFIG_DIR) + "/gfx1250_mi455x.json"; +const std::string CONFIG_RDNA4_PATH = std::string(CONFIG_DIR) + "/gfx1201_r9700.json"; // Most shipped configs omit num_threads so they pick up the default; the 2-GPU // KMD config pins it. Replace the pin when there is one, otherwise insert an @@ -186,11 +189,11 @@ TEST(CpuDispatchBudgetTest, JsonDistinguishesOmittedZeroAndExplicitSchemaDefault } using Choice = config::ExecutionThreadChoice; -constexpr Choice kCdna4Choices[] = {{1, 1, 0}, {1, 2, 0}, {1, 4, 0}, {2, 7, 0}, - {8, 9, 0}, {8, 17, 0}, {8, 25, 0}, {8, 25, 2}, - {8, 25, 4}, {8, 25, 8}, {8, 25, 16}}; +constexpr Choice kAdditiveHelperChoices[] = {{1, 1, 0}, {1, 2, 0}, {1, 4, 0}, {2, 7, 0}, + {8, 9, 0}, {8, 17, 0}, {8, 25, 0}, {8, 25, 2}, + {8, 25, 4}, {8, 25, 8}, {8, 25, 16}}; -TEST(ExecutionThreadBudgetTest, SelectsLargestFittingGranuleAndCapsAutomaticBudget) { +TEST(ExecutionThreadBudgetTest, SelectsLargestFittingGranule) { struct Case { uint32_t budget, engines, dispatch, helpers; }; @@ -206,23 +209,25 @@ TEST(ExecutionThreadBudgetTest, SelectsLargestFittingGranuleAndCapsAutomaticBudg for (bool explicit_budget : {false, true}) { auto plan = config::resolve_execution_threads({.budget = explicit_budget ? c.budget : 0}, explicit_budget ? 128 : c.budget, 8, capacities, - true, kCdna4Choices); + true, kAdditiveHelperChoices); EXPECT_EQ(plan.engines, c.engines); EXPECT_EQ(plan.dispatch, (std::vector{c.dispatch})); EXPECT_EQ(plan.helpers, c.helpers); } } - auto unknown = config::resolve_execution_threads({}, 0, 8, capacities, true, kCdna4Choices); + auto unknown = + config::resolve_execution_threads({}, 0, 8, capacities, true, kAdditiveHelperChoices); EXPECT_EQ(unknown.engines, 1u); EXPECT_EQ(unknown.dispatch, (std::vector{1})); EXPECT_EQ(unknown.helpers, 0u); const Choice larger[] = {{8, 25, 0}, {8, 33, 8}}; auto automatic = config::resolve_execution_threads({}, 64, 8, capacities, true, larger); - EXPECT_EQ(automatic.dispatch, (std::vector{25})); + EXPECT_EQ(automatic.dispatch, (std::vector{33})); + EXPECT_EQ(automatic.helpers, 8u); auto explicit_plan = - config::resolve_execution_threads({.budget = 48}, 64, 8, capacities, true, larger); - EXPECT_EQ(explicit_plan.dispatch, (std::vector{33})); - EXPECT_EQ(explicit_plan.helpers, 8u); + config::resolve_execution_threads({.budget = 40}, 64, 8, capacities, true, larger); + EXPECT_EQ(explicit_plan.dispatch, (std::vector{25})); + EXPECT_EQ(explicit_plan.helpers, 0u); } TEST(ExecutionThreadBudgetTest, ConfigCanStopAtAFewThreads) { @@ -251,12 +256,12 @@ TEST(ExecutionThreadBudgetTest, AccountsForEveryGpuAndTopologyClamps) { TEST(ExecutionThreadBudgetTest, ExplicitOverridesAndSafeFallback) { const std::array capacities{64u}; auto plan = config::resolve_execution_threads({.engines = 8, .dispatch = 33, .helpers = 8}, 4, 8, - capacities, true, kCdna4Choices); + capacities, true, kAdditiveHelperChoices); EXPECT_EQ(plan.engines, 8u); EXPECT_EQ(plan.dispatch, (std::vector{33})); EXPECT_EQ(plan.helpers, 8u); plan = config::resolve_execution_threads({.dispatch = 1, .helpers = 0}, 32, 8, capacities, true, - kCdna4Choices); + kAdditiveHelperChoices); EXPECT_EQ(plan.engines, 8u); EXPECT_EQ(plan.dispatch, (std::vector{1})); EXPECT_EQ(plan.helpers, 0u); @@ -264,7 +269,8 @@ TEST(ExecutionThreadBudgetTest, ExplicitOverridesAndSafeFallback) { EXPECT_EQ(plan.engines, 1u); EXPECT_EQ(plan.dispatch, (std::vector{1})); EXPECT_EQ(plan.helpers, 0u); - plan = config::resolve_execution_threads({}, 4, 8, capacities, true, kCdna4Choices, true); + plan = + config::resolve_execution_threads({}, 4, 8, capacities, true, kAdditiveHelperChoices, true); EXPECT_EQ(plan.engines, 4u); EXPECT_EQ(plan.dispatch, (std::vector{1})); EXPECT_EQ(plan.helpers, 0u); @@ -287,8 +293,8 @@ TEST(ExecutionThreadBudgetTest, AutomaticPlansStayWithinAffinityAndCapacity) { for (uint32_t xcds : {1u, 2u, 3u, 4u, 7u, 8u, 16u, 32u}) for (bool async : {false, true}) { const std::array capacities{2u, 36u}; - auto plan = - config::resolve_execution_threads({}, host, xcds, capacities, async, kCdna4Choices); + auto plan = config::resolve_execution_threads({}, host, xcds, capacities, async, + kAdditiveHelperChoices); uint64_t used = plan.engines + plan.helpers; for (size_t i = 0; i < capacities.size(); ++i) { EXPECT_GE(plan.dispatch[i], 1u); @@ -296,20 +302,22 @@ TEST(ExecutionThreadBudgetTest, AutomaticPlansStayWithinAffinityAndCapacity) { used += plan.dispatch[i] - 1; } EXPECT_LE(used, std::max(host, 1u)); - EXPECT_LE(used - plan.helpers, 32u); EXPECT_LE(plan.engines, xcds); } } TEST(ExecutionThreadBudgetTest, MetadataMatchesBuiltTopology) { - for (const auto &path : {CONFIG_PATH, CONFIG_2GPU_PATH, CONFIG_1XCD_PATH}) { + for (const auto &path : + {CONFIG_PATH, CONFIG_2GPU_PATH, CONFIG_1XCD_PATH, CONFIG_CDNA5_PATH, CONFIG_RDNA4_PATH}) { auto settings = config::load_execution_thread_settings(path, rocjitsu::kEmbeddedSchema); - auto loaded = config::load_config(path, rocjitsu::kEmbeddedSchema, 32); - auto plan = settings.resolve(32); + auto loaded = config::load_config(path, rocjitsu::kEmbeddedSchema, 48); + auto plan = settings.resolve(48); EXPECT_EQ(plan.engines, loaded.execution_threads.engines); EXPECT_EQ(plan.dispatch, loaded.execution_threads.dispatch); EXPECT_EQ(plan.helpers, loaded.execution_threads.helpers); EXPECT_EQ(settings.xcds, loaded.build_result.num_xcds * (1 + loaded.extra_gpu_builds.size())); + loaded.apply_cpu_dispatch_threads(); + EXPECT_EQ(loaded.soc()->dispatch_threads(), plan.dispatch.front()); } } @@ -379,9 +387,9 @@ TEST(ExecutionThreadBudgetTest, ShippedServerChoicesAndExplicitSerial) { json.insert(json.find('{') + 1, R"("cpu_thread_budget":32,)"); auto loaded = config::load_config_from_string(json, rocjitsu::kEmbeddedSchema, 64); EXPECT_EQ(loaded.execution_threads.engines, 8u); - EXPECT_EQ(loaded.execution_threads.dispatch, (std::vector{25})); - EXPECT_EQ(loaded.execution_threads.helpers, 0u); - EXPECT_EQ(loaded.async_resources->helpers(), 0u); + EXPECT_EQ(loaded.execution_threads.dispatch, (std::vector{17})); + EXPECT_EQ(loaded.execution_threads.helpers, 8u); + EXPECT_EQ(loaded.async_resources->helpers(), 8u); json.insert(json.find('{') + 1, R"("cpu_dispatch_threads":1,"async_helper_threads":0,)"); auto serial = config::load_config_from_string(json, rocjitsu::kEmbeddedSchema, 64); EXPECT_EQ(serial.cpu_dispatch_threads, 1u); @@ -395,10 +403,11 @@ TEST(ExecutionThreadBudgetTest, ShippedServerChoicesAndExplicitSerial) { TEST(ExecutionThreadBudgetTest, LaunchBudgetReplanesOnlyTheConfigItRewrites) { std::string json = config_json_with_num_threads(CONFIG_PATH, 0); json.insert(json.find('{') + 1, R"("cpu_thread_budget":32,)"); - const std::string relaunched = config::json_with_cpu_thread_budget(json, 4); + FailureOr relaunched = config::json_with_cpu_thread_budget(json, 4); + ASSERT_TRUE(relaunched.succeeded()); const config::LoadedConfig loaded = - config::load_config_from_string(relaunched, rocjitsu::kEmbeddedSchema, 64); + config::load_config_from_string(relaunched.value(), rocjitsu::kEmbeddedSchema, 64); EXPECT_EQ(loaded.cpu_thread_budget, 4u); EXPECT_EQ(loaded.execution_threads.engines, 2u); EXPECT_EQ(loaded.execution_threads.dispatch, (std::vector{3})); @@ -408,7 +417,8 @@ TEST(ExecutionThreadBudgetTest, LaunchBudgetReplanesOnlyTheConfigItRewrites) { config::load_config_from_string(json, rocjitsu::kEmbeddedSchema, 64); EXPECT_EQ(untouched.cpu_thread_budget, 32u); EXPECT_EQ(untouched.execution_threads.engines, 8u); - EXPECT_EQ(untouched.execution_threads.dispatch, (std::vector{25})); + EXPECT_EQ(untouched.execution_threads.dispatch, (std::vector{17})); + EXPECT_EQ(untouched.execution_threads.helpers, 8u); } TEST(ExecutionThreadBudgetTest, PresetsKeepSiblingTablesConsistent) { @@ -467,7 +477,7 @@ TEST(ExecutionThreadBudgetTest, OlderCdnaPresetsUseDispatchWorkers) { settings.request.budget = 32; auto parallel = settings.resolve(64); EXPECT_EQ(parallel.engines, settings.xcds); - EXPECT_EQ(parallel.dispatch, (std::vector{33 - parallel.engines})); + EXPECT_EQ(parallel.dispatch, (std::vector{25 - parallel.engines})); } } @@ -475,11 +485,11 @@ TEST(ExecutionThreadBudgetTest, DesktopTablesStopAtTheMeasuredCeiling) { for (const auto *name : {"gfx1100_w7900", "gfx1151", "gfx1201_r9700"}) { auto settings = config::load_execution_thread_settings( std::string(CONFIG_DIR) + "/" + name + ".json", rocjitsu::kEmbeddedSchema); - for (uint32_t budget : {32u, 64u}) { + for (uint32_t budget : {24u, 32u, 64u}) { settings.request.budget = budget; auto plan = settings.resolve(128); EXPECT_EQ(plan.engines, 1u); - EXPECT_EQ(plan.dispatch, (std::vector{32})); + EXPECT_EQ(plan.dispatch, (std::vector{24})); } } } @@ -489,11 +499,13 @@ TEST(ExecutionThreadBudgetTest, DisablingHelpersKeepsSynchronousServerGranules) uint32_t budget, engines, dispatch; }; for (const auto *name : {"gfx950_mi355x.json", "gfx1250_mi455x.json"}) { + const bool cdna5 = std::string_view(name) == "gfx1250_mi455x.json"; auto settings = config::load_execution_thread_settings(std::string(CONFIG_DIR) + "/" + name, rocjitsu::kEmbeddedSchema); settings.request.helpers = 0; - for (Case c : {Case{1, 1, 1}, Case{2, 1, 2}, Case{8, 2, 7}, Case{16, 8, 9}, Case{24, 8, 17}, - Case{32, 8, 25}}) { + for (Case c : {Case{1, 1, 1}, Case{2, 1, 2}, Case{4, cdna5 ? 1u : 2u, cdna5 ? 4u : 3u}, + Case{8, 2, 7}, Case{16, 8, 9}, Case{24, 8, 17}, Case{32, 8, cdna5 ? 25u : 17u}, + Case{40, 8, cdna5 ? 33u : 17u}, Case{48, 8, cdna5 ? 33u : 17u}}) { SCOPED_TRACE(name); SCOPED_TRACE(c.budget); settings.request.budget = c.budget; @@ -507,33 +519,32 @@ TEST(ExecutionThreadBudgetTest, DisablingHelpersKeepsSynchronousServerGranules) TEST(ExecutionThreadBudgetTest, ServerPresetsAddHelpersWithoutDisplacingCpuWorkers) { for (const char *name : {"gfx950_mi355x.json", "gfx950_mi355x_kmd.json", "gfx1250_mi455x.json"}) { + SCOPED_TRACE(name); + const bool cdna5 = std::string_view(name) == "gfx1250_mi455x.json"; + const uint32_t max_cpu = cdna5 ? 40 : 24; auto settings = config::load_execution_thread_settings(std::string(CONFIG_DIR) + "/" + name, rocjitsu::kEmbeddedSchema); for (uint32_t affinity = 1; affinity <= 128; ++affinity) { const auto plan = settings.resolve(affinity); ASSERT_EQ(plan.dispatch.size(), 1u); const auto cpu = plan.engines + plan.dispatch[0] - 1; - EXPECT_LE(cpu, std::min(affinity, 32u)); - EXPECT_LE(cpu + plan.helpers, std::min(affinity, 48u)); - const unsigned expected = affinity >= 48 ? 16 - : affinity >= 40 ? 8 - : affinity >= 36 ? 4 - : affinity >= 34 ? 2 - : 0; - EXPECT_EQ(plan.helpers, expected) << name << " affinity=" << affinity; - if (affinity >= 32) { + EXPECT_LE(cpu, std::min(affinity, max_cpu)); + EXPECT_LE(cpu + plan.helpers, std::min(affinity, max_cpu + 8)); + EXPECT_EQ(plan.helpers, affinity >= max_cpu + 8 ? 8u : 0u); + if (affinity >= 24) { EXPECT_EQ(plan.engines, 8u); - EXPECT_EQ(plan.dispatch[0], 25u); + const uint32_t expected_cpu = cdna5 && affinity >= 32 ? (affinity >= 40 ? 40 : 32) : 24; + EXPECT_EQ(cpu, expected_cpu); } } - settings.request.budget = 32; + settings.request.budget = max_cpu; EXPECT_EQ(settings.resolve(128).helpers, 0u); - settings.request.budget = 48; - EXPECT_EQ(settings.resolve(128).helpers, 16u); + settings.request.budget = max_cpu + 8; + EXPECT_EQ(settings.resolve(128).helpers, 8u); settings.request.helpers = 0; const auto disabled = settings.resolve(128); EXPECT_EQ(disabled.engines, 8u); - EXPECT_EQ(disabled.dispatch, (std::vector{25})); + EXPECT_EQ(disabled.dispatch, (std::vector{max_cpu - 7})); EXPECT_EQ(disabled.helpers, 0u); } } diff --git a/emulation/rocjitsu/tests/patch/smem_builder_test.cpp b/emulation/rocjitsu/tests/patch/smem_builder_test.cpp new file mode 100644 index 000000000000..eafacc66ffee --- /dev/null +++ b/emulation/rocjitsu/tests/patch/smem_builder_test.cpp @@ -0,0 +1,268 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file smem_builder_test.cpp +/// @brief Encoding contract for the scalar-memory builders, across all ten +/// AMDGPU targets. +/// +/// The load is pinned by decoding it back through the same generated decoder the +/// simulator uses, not against a hand-computed word. A literal expected word +/// would restate the fields this builder has to get right (the halved SBASE, and +/// the two shapes of "immediate offset, no SGPR offset") rather than check them. +/// +/// Decoding cannot check RDNA's SOFFSET, because the model builds its offset +/// operand from the immediate alone and ignores that field. It is asserted +/// against the raw encoded word instead, per generation. + +#include "rocjitsu/code/builders/smem_builders.h" + +#include "rocjitsu/code/builders/spill_builders.h" +#include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" +#include "rocjitsu/isa/operand.h" +#include "rocjitsu/isa/register_set.h" + +#include "../decode_test_util.h" + +#include + +#include +#include +#include +#include +#include + +namespace rocjitsu { +namespace { + +// How a target spells "immediate offset, no SGPR offset". CDNA1-4 clear +// SOFFSET_EN and set IMM; RDNA has neither bit and must name NULL in SOFFSET. +enum class SmemOffsetShape { CdnaEnableBits, RdnaSoffsetNull }; + +struct SmemArch { + rj_code_arch_t arch; + const char *name; ///< For failure messages; the enum prints as an int. + const char *load_mnemonic; + uint32_t max_byte_offset; + SmemOffsetShape shape; + std::optional soffset_null; ///< Set only for RdnaSoffsetNull. + const char *wait_mnemonic; +}; + +// All ten AMDGPU targets. The per-target constants are spelled out rather than +// derived from the builder, so a generated table changing underneath us fails +// here instead of the test silently agreeing with the new value. +constexpr std::array kArchs{{ + {ROCJITSU_CODE_ARCH_CDNA1, "cdna1", "s_load_dwordx2", 0x0FFFFFu, + SmemOffsetShape::CdnaEnableBits, std::nullopt, "s_waitcnt"}, + {ROCJITSU_CODE_ARCH_CDNA2, "cdna2", "s_load_dwordx2", 0x0FFFFFu, + SmemOffsetShape::CdnaEnableBits, std::nullopt, "s_waitcnt"}, + {ROCJITSU_CODE_ARCH_CDNA3, "cdna3", "s_load_dwordx2", 0x0FFFFFu, + SmemOffsetShape::CdnaEnableBits, std::nullopt, "s_waitcnt"}, + {ROCJITSU_CODE_ARCH_CDNA4, "cdna4", "s_load_dwordx2", 0x0FFFFFu, + SmemOffsetShape::CdnaEnableBits, std::nullopt, "s_waitcnt"}, + // NULL is 125 on RDNA1/2 and 124 from RDNA3 on. A dispatch that hardcoded + // one value would encode a real SGPR as the offset on the other half. + {ROCJITSU_CODE_ARCH_RDNA1, "rdna1", "s_load_dwordx2", 0x0FFFFFu, + SmemOffsetShape::RdnaSoffsetNull, 125, "s_waitcnt"}, + {ROCJITSU_CODE_ARCH_RDNA2, "rdna2", "s_load_dwordx2", 0x0FFFFFu, + SmemOffsetShape::RdnaSoffsetNull, 125, "s_waitcnt"}, + {ROCJITSU_CODE_ARCH_RDNA3, "rdna3", "s_load_b64", 0x0FFFFFu, SmemOffsetShape::RdnaSoffsetNull, + 124, "s_waitcnt"}, + {ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5", "s_load_b64", 0x0FFFFFu, + SmemOffsetShape::RdnaSoffsetNull, 124, "s_waitcnt"}, + // GFX12 widens the immediate to 24 bits and splits the wait counters. + {ROCJITSU_CODE_ARCH_RDNA4, "rdna4", "s_load_b64", 0x7FFFFFu, SmemOffsetShape::RdnaSoffsetNull, + 124, "s_wait_kmcnt"}, + {ROCJITSU_CODE_ARCH_CDNA5, "cdna5", "s_load_b64", 0x7FFFFFu, SmemOffsetShape::RdnaSoffsetNull, + 124, "s_wait_kmcnt"}, +}}; + +constexpr std::array kNonAmdGpuArchs{ROCJITSU_CODE_ARCH_RV32I, + ROCJITSU_CODE_ARCH_RV64I}; + +// Decode the instruction starting at `words[0]`, or null if it did not decode. +// Two words are always passed so an 8-byte encoding has its second word. +std::unique_ptr decode_one(const std::array &words, rj_code_arch_t arch) { + auto decoder = Decoder::create(arch); + if (!decoder) + return nullptr; + return std::unique_ptr(decode_valid(*decoder, words.data())); +} + +// SOFFSET occupies bits [63:57], the second word's high 7 bits, on every +// generation whose encoding carries it. +constexpr uint8_t encoded_soffset(const std::array &words) { + return static_cast((words[1] >> 25) & 0x7Fu); +} + +// The builder's job is the three fields the encoding does not take literally: +// the destination pair, the address pair (encoded halved), and the immediate. +// Decoding recovers all three on every target. +TEST(SmemBuilder, LoadDwordx2RoundTripsThroughTheDecoderOnEveryTarget) { + constexpr uint16_t kSdst = 4; + constexpr uint16_t kSbase = 6; + constexpr uint32_t kOffset = 0x1010; + + for (const SmemArch &a : kArchs) { + const auto inst = decode_one(build_s_load_dwordx2(kSdst, kSbase, kOffset, a.arch), a.arch); + ASSERT_NE(inst, nullptr) << a.name; + EXPECT_EQ(inst->mnemonic(), std::string_view(a.load_mnemonic)) << a.name; + + ASSERT_GE(inst->num_dst_operands(), 1) << a.name; + const auto dst_ref = inst->dst_operand(0)->to_register_ref(); + ASSERT_TRUE(dst_ref.has_value()) << a.name; + EXPECT_EQ(dst_ref->cls, RegClass::SGPR) << a.name; + EXPECT_EQ(dst_ref->index, kSdst) << a.name; + EXPECT_EQ(dst_ref->width, 2) << a.name; + + // SBASE is encoded halved and the decoder doubles it back, so an unhalved + // builder would name s12 here. + ASSERT_GE(inst->num_src_operands(), 1) << a.name; + const auto base_ref = inst->src_operand(0)->to_register_ref(); + ASSERT_TRUE(base_ref.has_value()) << a.name; + EXPECT_EQ(base_ref->cls, RegClass::SGPR) << a.name; + EXPECT_EQ(base_ref->index, kSbase) << a.name; + EXPECT_EQ(base_ref->width, 2) << a.name; + + // The immediate. Without this the builder can drop its only variable + // argument and every other assertion here still holds. On CDNA it also + // covers the IMM bit: with IMM clear the same 21 bits decode as an SGPR + // selector taking `offset & 0x7F` (shared/addr_calc_scalar.h reads it that + // way on the address path, not just in disassembly), so the value would come + // back as 0x10 rather than 0x1010. + ASSERT_GE(inst->num_src_operands(), 2) << a.name; + const Operand *offset = inst->src_operand(1); + ASSERT_NE(offset, nullptr) << a.name; + EXPECT_EQ(offset->encoding_value(), static_cast(kOffset)) << a.name; + EXPECT_FALSE(offset->to_register_ref().has_value()) + << a.name << ": immediate offset decoded as a register"; + } +} + +// RDNA carries no SOFFSET_EN, so an unset SOFFSET field names s0 and the load +// silently adds whatever s0 holds. The decoder cannot catch this, since it builds +// the offset operand from the immediate alone. Assert the encoded word instead, +// and the generation's own NULL code, which moved between RDNA2 and RDNA3. +TEST(SmemBuilder, RdnaNamesItsOwnNullCodeInSoffset) { + bool saw_125 = false; + bool saw_124 = false; + for (const SmemArch &a : kArchs) { + if (a.shape != SmemOffsetShape::RdnaSoffsetNull) + continue; + const auto words = build_s_load_dwordx2(/*sdst=*/4, /*sbase=*/6, /*byte_offset=*/0x40, a.arch); + ASSERT_TRUE(a.soffset_null.has_value()) << a.name; + EXPECT_EQ(encoded_soffset(words), *a.soffset_null) << a.name; + EXPECT_NE(encoded_soffset(words), 0) << a.name << ": SOFFSET 0 would name s0"; + saw_125 |= *a.soffset_null == 125; + saw_124 |= *a.soffset_null == 124; + } + // Both codes are actually exercised, so a shared constant cannot pass this. + EXPECT_TRUE(saw_125); + EXPECT_TRUE(saw_124); +} + +// Both operands name a 64-bit pair, so an odd base has no encoding. It must be +// refused rather than silently truncated to the even pair below it, which would +// load through, or into, the wrong register. +TEST(SmemBuilder, OddPairBaseIsRejectedOnEveryTarget) { + for (const SmemArch &a : kArchs) { + EXPECT_THROW((void)build_s_load_dwordx2(/*sdst=*/4, /*sbase=*/7, /*byte_offset=*/0, a.arch), + util::InvalidInst) + << a.name << ": odd sbase"; + EXPECT_THROW((void)build_s_load_dwordx2(/*sdst=*/5, /*sbase=*/6, /*byte_offset=*/0, a.arch), + util::InvalidInst) + << a.name << ": odd sdst"; + } +} + +// The encoded fields are narrower than the uint16_t parameters and the generated +// set_field masks rather than rejects, so an out-of-range index would encode a +// different register: sbase 128 halves to 64, which a 6-bit SBASE truncates to 0, +// silently loading through s[0:1]. +TEST(SmemBuilder, OutOfRangeRegisterIndexIsRejectedOnEveryTarget) { + for (const SmemArch &a : kArchs) { + EXPECT_NO_THROW( + (void)build_s_load_dwordx2(kMaxSmemSdata - 1, kMaxSmemSbase, /*byte_offset=*/0, a.arch)) + << a.name; + EXPECT_THROW( + (void)build_s_load_dwordx2(/*sdst=*/4, kMaxSmemSbase + 2, /*byte_offset=*/0, a.arch), + util::InvalidInst) + << a.name << ": sbase past SBASE"; + EXPECT_THROW( + (void)build_s_load_dwordx2(kMaxSmemSdata + 1, /*sbase=*/6, /*byte_offset=*/0, a.arch), + util::InvalidInst) + << a.name << ": sdst past SDATA"; + } +} + +// Two immediate widths, both read as signed, so the bound is the signed-positive +// maximum: one bit below the field width. Pinning the exact values keeps a later +// "the field is 21 bits wide" simplification from doubling the accepted range. +TEST(SmemBuilder, OffsetIsBoundedByTheArchSignedImmediateRange) { + for (const SmemArch &a : kArchs) { + EXPECT_EQ(max_smem_byte_offset(a.arch), a.max_byte_offset) << a.name; + const uint32_t aligned_max = a.max_byte_offset & ~0x3u; + EXPECT_NO_THROW((void)build_s_load_dwordx2(/*sdst=*/4, /*sbase=*/0, aligned_max, a.arch)) + << a.name; + EXPECT_THROW((void)build_s_load_dwordx2(/*sdst=*/4, /*sbase=*/0, a.max_byte_offset + 1, a.arch), + util::InvalidInst) + << a.name; + + // The offset an unsigned reading of the field would have admitted is + // refused. Nothing in simulation distinguishes these, since the decoder + // zero-extends, so this bound is the only guard against an offset that runs + // backwards on hardware. + const uint32_t unsigned_max = ((a.max_byte_offset << 1) | 1u) & ~0x3u; + EXPECT_THROW((void)build_s_load_dwordx2(/*sdst=*/4, /*sbase=*/0, unsigned_max, a.arch), + util::InvalidInst) + << a.name; + } + + // GFX12's field is wider, so an offset legal there is not legal earlier. + EXPECT_THROW((void)build_s_load_dwordx2(4, 0, 0x100000u, ROCJITSU_CODE_ARCH_CDNA4), + util::InvalidInst); + EXPECT_NO_THROW((void)build_s_load_dwordx2(4, 0, 0x100000u, ROCJITSU_CODE_ARCH_RDNA4)); +} + +// Scalar loads drop the low two address bits, so offset 1 would load from the +// base itself. +TEST(SmemBuilder, UnalignedOffsetIsRejectedOnEveryTarget) { + for (const SmemArch &a : kArchs) { + EXPECT_THROW((void)build_s_load_dwordx2(/*sdst=*/4, /*sbase=*/0, /*byte_offset=*/1, a.arch), + util::InvalidInst) + << a.name; + } +} + +// The scalar-load wait is its own counter. On GFX12 it must be KMCNT: the +// LOADCNT wait in spill_builders.h orders VMEM and would leave a scalar load in +// flight. CDNA5 has no s_waitcnt opcode at all. +TEST(SmemBuilder, ScalarLoadWaitSelectsTheScalarCounter) { + for (const SmemArch &a : kArchs) { + const std::array wait{build_wait_scalar_loads_complete(a.arch), 0}; + const auto inst = decode_one(wait, a.arch); + ASSERT_NE(inst, nullptr) << a.name; + EXPECT_EQ(inst->mnemonic(), std::string_view(a.wait_mnemonic)) << a.name; + } + + // Through RDNA3.5 the monolithic wait covers both counters; GFX12 splits them, + // and only there is the distinction observable in the encoding. + EXPECT_EQ(build_wait_scalar_loads_complete(ROCJITSU_CODE_ARCH_CDNA3), + build_wait_loads_complete(ROCJITSU_CODE_ARCH_CDNA3)); + EXPECT_NE(build_wait_scalar_loads_complete(ROCJITSU_CODE_ARCH_RDNA4), + build_wait_loads_complete(ROCJITSU_CODE_ARCH_RDNA4)); +} + +TEST(SmemBuilder, NonAmdGpuArchsThrow) { + for (const rj_code_arch_t arch : kNonAmdGpuArchs) { + EXPECT_THROW((void)build_s_load_dwordx2(/*sdst=*/4, /*sbase=*/0, /*byte_offset=*/0, arch), + util::UnimplementedInst); + EXPECT_THROW((void)build_wait_scalar_loads_complete(arch), util::UnimplementedInst); + EXPECT_THROW((void)max_smem_byte_offset(arch), util::UnimplementedInst); + } +} + +} // namespace +} // namespace rocjitsu diff --git a/emulation/rocjitsu/tests/pci/gpu_pci_device_test.cpp b/emulation/rocjitsu/tests/pci/gpu_pci_device_test.cpp index 97c56224b5c8..041a958bdac6 100644 --- a/emulation/rocjitsu/tests/pci/gpu_pci_device_test.cpp +++ b/emulation/rocjitsu/tests/pci/gpu_pci_device_test.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: MIT #include "rocjitsu/kmd/linux/legacy_gpu_vm.h" -#include "rocjitsu/vm/amdgpu/aql/aql_queue_binding_factory.h" #include "rocjitsu/vm/amdgpu/command_processor.h" +#include "rocjitsu/vm/amdgpu/compute_queue_binding_factory.h" #include "rocjitsu/vm/amdgpu/pci/bar_access_trace.h" #include "rocjitsu/vm/amdgpu/pci/gpu_pci_device.h" #include "rocjitsu/vm/amdgpu/pci/gpu_pci_device_spec.h" @@ -2474,7 +2474,7 @@ TEST_F(GpuDeviceMes, LostConnectionReleasesOnlyPciQueuesAndAllowsPasidReuse) { .consumer_pointer_address = 0x2000, .producer_pointer_address = 0x3000}, .doorbell = {}, - .binding_factory = rocjitsu::amdgpu::make_aql_queue_binding_factory(legacy_cp), + .binding_factory = rocjitsu::amdgpu::make_compute_queue_binding_factory(legacy_cp), .type = rocjitsu::amdgpu::QueueType::Compute, .packet_format = rocjitsu::amdgpu::QueuePacketFormat::Aql, }); diff --git a/emulation/rocjitsu/tests/perfsim_fake_backend.cpp b/emulation/rocjitsu/tests/perfsim_fake_backend.cpp index d569e9c81dac..b31d8303369b 100644 --- a/emulation/rocjitsu/tests/perfsim_fake_backend.cpp +++ b/emulation/rocjitsu/tests/perfsim_fake_backend.cpp @@ -542,7 +542,7 @@ ffm_observer_plugin_get_api(uint32_t host_api_version) { : mode_is("too_old_version") ? 7 : mode_is("bad_version") ? 14 : 13; - api.name = "rocjitsu-perfsim-fake"; + api.name = mode_is("gpucsim_name") ? "GPUCompilerSim" : "rocjitsu-perfsim-fake"; api.on_init = on_init; api.on_dispatch_begin = on_dispatch_begin; api.on_dispatch_end = on_dispatch_end; diff --git a/emulation/rocjitsu/tests/perfsim_plugin_test.cpp b/emulation/rocjitsu/tests/perfsim_plugin_test.cpp index b8d99d8820cc..8756f37fcaca 100644 --- a/emulation/rocjitsu/tests/perfsim_plugin_test.cpp +++ b/emulation/rocjitsu/tests/perfsim_plugin_test.cpp @@ -219,6 +219,16 @@ std::string plugin_config_with_staging_budget(uint64_t max_staged_bytes) { ",\"max_staged_bytes\":" + std::to_string(max_staged_bytes) + "}"; } +std::string plugin_config_with_dispatch_name(std::string_view dispatch_name) { + return std::string{"{\"library_path\":"} + json_string(PERFSIM_FAKE_BACKEND_PATH) + + ",\"dispatch_name\":" + json_string(dispatch_name) + "}"; +} + +std::string plugin_config_with_observed_workgroup_cap(uint64_t max_observed_workgroups) { + return std::string{"{\"library_path\":"} + json_string(PERFSIM_FAKE_BACKEND_PATH) + + ",\"max_observed_workgroups\":" + std::to_string(max_observed_workgroups) + "}"; +} + std::string read_file(const std::string &path) { std::ifstream input(path); return {std::istreambuf_iterator(input), std::istreambuf_iterator()}; @@ -352,6 +362,372 @@ TEST(PerfsimPluginConfigTest, RejectsInvalidStagingBudgets) { } } +TEST(PerfsimPluginConfigTest, RejectsInvalidDispatchNames) { + const std::string prefix = std::string{"{\"library_path\":"} + + json_string(PERFSIM_FAKE_BACKEND_PATH) + ",\"dispatch_name\":"; + for (std::string_view value : {"\"\"", "7", "true", "[]"}) { + SCOPED_TRACE(value); + const std::string config = prefix + std::string(value) + "}"; + EXPECT_THROW(PerfsimPlugin(config.c_str()), std::invalid_argument); + } +} + +TEST(PerfsimPluginConfigTest, RejectsInvalidObservedWorkgroupCaps) { + const std::string prefix = std::string{"{\"library_path\":"} + + json_string(PERFSIM_FAKE_BACKEND_PATH) + + ",\"max_observed_workgroups\":"; + for (std::string_view value : {"0", "-1", "1.5", "\"1\""}) { + SCOPED_TRACE(value); + const std::string config = prefix + std::string(value) + "}"; + EXPECT_THROW(PerfsimPlugin(config.c_str()), std::invalid_argument); + } +} + +TEST_F(PerfsimPluginTest, GpuCompilerSimV13StagesUntilDispatchIsAccepted) { + WaveFixture fixture; + setenv("ROCJITSU_PERFSIM_FAKE_MODE", "gpucsim_name", 1); + + PerfsimPlugin plugin(plugin_config().c_str()); + EXPECT_TRUE(plugin.requires_serial_hot_hooks()); + plugin.onInit(); + + const KernelDispatchInfo info = dispatch_info(43); + plugin.onAmdgpuDispatchPacketProcessed(info); + plugin.onAmdgpuDispatchExecutionBegin(info.dispatch_id); + Wavefront &wave = fixture.wave(info.dispatch_id, 0, {0, 0, 0}, 0); + plugin.onAmdgpuWavefrontDispatched(wave); + + const std::array end_words{0xBF810000}; + SyntheticInstruction end("s_endpgm", end_words, PROGRAM_TERMINATOR); + plugin.onAmdgpuBeforeExecuteInstruction(0x4000, end, wave); + plugin.onAmdgpuWavefrontHalted(wave); + + const auto partial_trace = lines(read_file(trace_.path())); + EXPECT_EQ(line_with_prefix(partial_trace, "begin 43 "), partial_trace.size()); + EXPECT_EQ(line_with_prefix(partial_trace, "instruction 43 "), partial_trace.size()); + + plugin.onAmdgpuDispatchExecutionEnd(info.dispatch_id); + plugin.onShutdown(); + + const auto trace = lines(read_file(trace_.path())); + EXPECT_NE(line_with_prefix(trace, "begin 43 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "instruction 43 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "end 43 "), trace.size()); +} + +TEST_F(PerfsimPluginTest, ReclaimsFaultAbortedWaveStateBeforePhysicalSlotReuse) { + WaveFixture fixture(1); + testing::internal::CaptureStderr(); + { + PerfsimPlugin plugin(plugin_config().c_str()); + plugin.onInit(); + + constexpr uint32_t AbortedDispatch = 44; + plugin.onAmdgpuDispatchPacketProcessed(dispatch_info(AbortedDispatch)); + plugin.onAmdgpuDispatchExecutionBegin(AbortedDispatch); + Wavefront &aborted = fixture.wave(AbortedDispatch, 0, {0, 0, 0}, 0); + plugin.onAmdgpuWavefrontDispatched(aborted); + const std::array add_words{0x7E000200}; + SyntheticInstruction add("v_add_f32", add_words); + plugin.onAmdgpuBeforeExecuteInstruction(0x4000, add, aborted); + + // Terminal VM-fault recovery resets resident waves without delivering a + // wave-halt callback. The next dispatch must safely reclaim this slot and + // must not retain or dereference the old PerfsimWavefrontState. + fixture.cu->abort_dispatch(AbortedDispatch); + + constexpr uint32_t ReusedDispatch = 45; + plugin.onAmdgpuDispatchPacketProcessed(dispatch_info(ReusedDispatch)); + plugin.onAmdgpuDispatchExecutionBegin(ReusedDispatch); + Wavefront &reused = fixture.wave(ReusedDispatch, 1, {1, 0, 0}, 0); + ASSERT_EQ(&reused, &aborted); + plugin.onAmdgpuWavefrontDispatched(reused); + EXPECT_TRUE(plugin.observes_hot_hooks_for_wavefront(&reused)); + + const std::array end_words{0xBF810000}; + SyntheticInstruction end("s_endpgm", end_words, PROGRAM_TERMINATOR); + plugin.onAmdgpuBeforeExecuteInstruction(0x5000, end, reused); + plugin.onAmdgpuWavefrontHalted(reused); + plugin.onAmdgpuDispatchExecutionEnd(ReusedDispatch); + plugin.onShutdown(); + } + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_NE(diagnostic.find("wavefront slot was reset without a halt callback"), std::string::npos); + + const auto trace = lines(read_file(trace_.path())); + EXPECT_EQ(line_with_prefix(trace, "begin 44 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "instruction 44 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "end 44 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "begin 45 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "instruction 45 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "end 45 "), trace.size()); +} + +TEST_F(PerfsimPluginTest, RetainsRejectedDispatchStateUntilLiveWaveRetires) { + WaveFixture fixture(1); + testing::internal::CaptureStderr(); + { + PerfsimPlugin plugin(plugin_config().c_str()); + plugin.onInit(); + + constexpr uint32_t Dispatch = 50; + plugin.onAmdgpuDispatchPacketProcessed(dispatch_info(Dispatch)); + plugin.onAmdgpuDispatchExecutionBegin(Dispatch); + Wavefront &wave = fixture.wave(Dispatch, 0, {0, 0, 0}, 0); + plugin.onAmdgpuWavefrontDispatched(wave); + + // Execution end normally follows every wave halt. Exercise the malformed + // order explicitly: rejecting the end must not free the dispatch state + // cached by the still-subscribed wave. ASan catches the former use after + // free on this instruction callback. + plugin.onAmdgpuDispatchExecutionEnd(Dispatch); + const std::array add_words{0x7E000200}; + SyntheticInstruction add("v_add_f32", add_words); + plugin.onAmdgpuBeforeExecuteInstruction(0x4000, add, wave); + EXPECT_TRUE(plugin.observes_hot_hooks_for_wavefront(&wave)); + + plugin.onAmdgpuWavefrontHalted(wave); + EXPECT_FALSE(plugin.observes_hot_hooks_for_wavefront(&wave)); + plugin.onShutdown(); + } + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_NE(diagnostic.find("dispatch ended with live wavefronts"), std::string::npos); + + const auto trace = lines(read_file(trace_.path())); + EXPECT_EQ(line_with_prefix(trace, "begin 50 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "instruction 50 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "end 50 "), trace.size()); +} + +TEST_F(PerfsimPluginTest, SelectsExactDispatchNameWithoutRejectingOtherDispatches) { + WaveFixture fixture; + const std::string config = plugin_config_with_dispatch_name("target_kernel"); + testing::internal::CaptureStderr(); + { + PerfsimPlugin plugin(config.c_str()); + plugin.onInit(); + + KernelDispatchInfo setup = dispatch_info(40); + setup.kernel_name = "target_kernel_suffix"; + plugin.onAmdgpuDispatchPacketProcessed(setup); + plugin.onAmdgpuDispatchExecutionBegin(setup.dispatch_id); + Wavefront &setup_wave = fixture.wave(setup.dispatch_id, 0, {0, 0, 0}, 0); + plugin.onAmdgpuWavefrontDispatched(setup_wave); + EXPECT_FALSE(plugin.observes_hot_hooks_for_wavefront(&setup_wave)); + const std::array add_words{0x7E000200}; + SyntheticInstruction add("v_add_f32", add_words); + plugin.onAmdgpuBeforeExecuteInstruction(0x1000, add, setup_wave); + plugin.onAmdgpuWavefrontHalted(setup_wave); + plugin.onAmdgpuDispatchExecutionEnd(setup.dispatch_id); + + KernelDispatchInfo target = dispatch_info(41); + target.kernel_name = "target_kernel"; + plugin.onAmdgpuDispatchPacketProcessed(target); + plugin.onAmdgpuDispatchExecutionBegin(target.dispatch_id); + Wavefront &target_wave = fixture.wave(target.dispatch_id, 1, {1, 0, 0}, 0); + plugin.onAmdgpuWavefrontDispatched(target_wave); + EXPECT_TRUE(plugin.observes_hot_hooks_for_wavefront(&target_wave)); + const std::array end_words{0xBF810000}; + SyntheticInstruction end("s_endpgm", end_words, PROGRAM_TERMINATOR); + plugin.onAmdgpuBeforeExecuteInstruction(0x2000, end, target_wave); + plugin.onAmdgpuWavefrontHalted(target_wave); + plugin.onAmdgpuDispatchExecutionEnd(target.dispatch_id); + plugin.onShutdown(); + } + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_EQ(diagnostic.find("skipped dispatch"), std::string::npos); + + const auto trace = lines(read_file(trace_.path())); + EXPECT_EQ(line_with_prefix(trace, "begin 40 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "instruction 40 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "end 40 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "begin 41 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "instruction 41 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "end 41 "), trace.size()); +} + +TEST_F(PerfsimPluginTest, DiscardsUnselectedFaultAbortedDispatchWithoutEnd) { + WaveFixture fixture(1); + const std::string config = plugin_config_with_dispatch_name("target_kernel"); + testing::internal::CaptureStderr(); + { + PerfsimPlugin plugin(config.c_str()); + plugin.onInit(); + + KernelDispatchInfo filtered = dispatch_info(46); + filtered.kernel_name = "other_kernel"; + plugin.onAmdgpuDispatchPacketProcessed(filtered); + plugin.onAmdgpuDispatchExecutionBegin(filtered.dispatch_id); + Wavefront &filtered_wave = fixture.wave(filtered.dispatch_id, 0, {0, 0, 0}, 0); + plugin.onAmdgpuWavefrontDispatched(filtered_wave); + EXPECT_FALSE(plugin.observes_hot_hooks_for_wavefront(&filtered_wave)); + + // Terminal VM-fault recovery does not deliver an execution-end callback. + // A dispatch excluded by dispatch_name must still be silently discarded. + fixture.cu->abort_dispatch(filtered.dispatch_id); + + KernelDispatchInfo target = dispatch_info(47); + target.kernel_name = "target_kernel"; + plugin.onAmdgpuDispatchPacketProcessed(target); + plugin.onAmdgpuDispatchExecutionBegin(target.dispatch_id); + Wavefront &target_wave = fixture.wave(target.dispatch_id, 1, {1, 0, 0}, 0); + plugin.onAmdgpuWavefrontDispatched(target_wave); + EXPECT_TRUE(plugin.observes_hot_hooks_for_wavefront(&target_wave)); + + const std::array end_words{0xBF810000}; + SyntheticInstruction end("s_endpgm", end_words, PROGRAM_TERMINATOR); + plugin.onAmdgpuBeforeExecuteInstruction(0x2000, end, target_wave); + plugin.onAmdgpuWavefrontHalted(target_wave); + plugin.onAmdgpuDispatchExecutionEnd(target.dispatch_id); + plugin.onShutdown(); + } + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_EQ(diagnostic.find("skipped dispatch 46"), std::string::npos); + EXPECT_EQ(diagnostic.find("dispatch was incomplete at plugin shutdown"), std::string::npos); + EXPECT_EQ(diagnostic.find("matched no dispatches"), std::string::npos); + + const auto trace = lines(read_file(trace_.path())); + EXPECT_EQ(line_with_prefix(trace, "begin 46 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "end 46 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "begin 47 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "end 47 "), trace.size()); +} + +TEST_F(PerfsimPluginTest, ReportsConfiguredDispatchNameWhenEveryDispatchMisses) { + const std::string config = plugin_config_with_dispatch_name("missing_kernel"); + testing::internal::CaptureStderr(); + { + PerfsimPlugin plugin(config.c_str()); + plugin.onInit(); + + for (uint32_t dispatch_id : {48u, 49u}) { + KernelDispatchInfo info = dispatch_info(dispatch_id); + info.kernel_name = dispatch_id == 48 ? "other_kernel" : "missing_kernel_suffix"; + plugin.onAmdgpuDispatchPacketProcessed(info); + plugin.onAmdgpuDispatchExecutionBegin(info.dispatch_id); + plugin.onAmdgpuDispatchExecutionEnd(info.dispatch_id); + } + plugin.onShutdown(); + } + const std::string diagnostic = testing::internal::GetCapturedStderr(); + const std::string expected = + "[rocjitsu:perfsim] configured dispatch_name 'missing_kernel' matched no dispatches"; + const size_t first = diagnostic.find(expected); + ASSERT_NE(first, std::string::npos); + EXPECT_EQ(diagnostic.find(expected, first + expected.size()), std::string::npos); + EXPECT_EQ(diagnostic.find("skipped dispatch"), std::string::npos); + + const auto trace = lines(read_file(trace_.path())); + EXPECT_EQ(line_with_prefix(trace, "begin 48 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "begin 49 "), trace.size()); +} + +TEST_F(PerfsimPluginTest, CapsStagingToFirstDistinctWorkgroupsWithoutBlockingReplay) { + WaveFixture fixture(3); + const std::string config = plugin_config_with_observed_workgroup_cap(1); + testing::internal::CaptureStderr(); + { + ExecutionPluginGroup group(PluginSinkConfig{}); + ASSERT_TRUE(group.add(std::make_unique(config.c_str()))); + group.onInit(); + + KernelDispatchInfo info = dispatch_info(42); + info.wfs_per_workgroup = 2; + group.onAmdgpuDispatchPacketProcessed(info); + group.onAmdgpuDispatchExecutionBegin(info.dispatch_id); + Wavefront &first = fixture.wave(info.dispatch_id, 10, {1000, 0, 0}, 0); + Wavefront &same_workgroup = fixture.wave(info.dispatch_id, 10, {1000, 0, 0}, 1); + // This coordinate collides with {1000,0,0} under FFM's packed cluster ID. + // The observation cap is defined in terms of RocJITsu workgroup identity, + // so the collision must not admit this distinct workgroup. + Wavefront &capped = fixture.wave(info.dispatch_id, 11, {0, 1, 0}, 0); + group.onAmdgpuWavefrontDispatched(first); + group.onAmdgpuWavefrontDispatched(same_workgroup); + group.onAmdgpuWavefrontDispatched(capped); + EXPECT_TRUE(group.observes_memory_routing(first)); + EXPECT_TRUE(group.observes_tensor_dma_memory_access(first)); + EXPECT_TRUE(group.observes_memory_routing(same_workgroup)); + EXPECT_TRUE(group.observes_tensor_dma_memory_access(same_workgroup)); + EXPECT_FALSE(group.observes_memory_routing(capped)); + EXPECT_FALSE(group.observes_tensor_dma_memory_access(capped)); + + // Exercise the VM-facing group entry points for a capped-out wave. Its + // cached subscription excludes both callbacks, and the adapter must not + // turn that intentional omission into a dispatch rejection. + const std::array global_words{0xDC508000}; + SyntheticInstruction global("global_load_b32", global_words, MEMORY_OP); + group.onAmdgpuBeforeExecuteInstruction(0x5000, global, capped); + std::array addresses{}; + addresses[0] = 0x100000; + MemoryAccessObservation memory_access; + memory_access.mnemonic = "global_load_b32"; + memory_access.pc = 0x5000; + memory_access.compute_unit_id = static_cast(capped.cu().id()); + memory_access.dispatch_id = info.dispatch_id; + memory_access.queue_id = capped.queue_id(); + memory_access.workgroup_id = capped.wg_id(); + memory_access.wavefront_id = capped.wf_id(); + memory_access.process_id = capped.process_id(); + memory_access.route = MemoryRoute::GLOBAL; + memory_access.decoded_space = DecodedMemorySpace::GLOBAL; + memory_access.is_load = true; + memory_access.wavefront_size = 32; + memory_access.element_size_bytes = 4; + memory_access.elements_per_lane = 1; + memory_access.active_lane_mask = 1; + memory_access.architectural_exec_lane_mask = 1; + memory_access.valid_lane_mask = 1; + memory_access.request_lane_mask = 1; + memory_access.addresses = addresses; + group.onAmdgpuMemoryAccessRouted(memory_access, global, capped); + + const std::array tensor_words{0xD0710001, 0x7C000000, 0x18140C00}; + SyntheticInstruction tensor("tensor_load_to_lds", tensor_words, MEMORY_OP); + group.onAmdgpuBeforeExecuteInstruction(0x6000, tensor, capped); + const std::array tensor_addresses{0x300000, 0x300004}; + TensorDmaMemoryAccessObservation tensor_access; + tensor_access.mnemonic = "tensor_load_to_lds"; + tensor_access.pc = 0x6000; + tensor_access.compute_unit_id = static_cast(capped.cu().id()); + tensor_access.dispatch_id = info.dispatch_id; + tensor_access.queue_id = capped.queue_id(); + tensor_access.workgroup_id = capped.wg_id(); + tensor_access.wavefront_id = capped.wf_id(); + tensor_access.process_id = capped.process_id(); + tensor_access.element_size_bytes = 4; + tensor_access.tile_dim0 = 2; + tensor_access.tile_dim1 = 1; + tensor_access.data_size = 2; + tensor_access.tensor_dim0_stride = 8; + tensor_access.tensor_dim1_stride = 16; + tensor_access.is_load = true; + tensor_access.addresses = std::span(tensor_addresses); + group.onAmdgpuTensorDmaMemoryAccess(tensor_access, capped); + + const std::array end_words{0xBF810000}; + SyntheticInstruction end("s_endpgm", end_words, PROGRAM_TERMINATOR); + group.onAmdgpuBeforeExecuteInstruction(0x3000, end, first); + group.onAmdgpuBeforeExecuteInstruction(0x4000, end, same_workgroup); + group.onAmdgpuBeforeExecuteInstruction(0x7000, end, capped); + group.onAmdgpuWavefrontHalted(first); + group.onAmdgpuWavefrontHalted(same_workgroup); + group.onAmdgpuWavefrontHalted(capped); + group.onAmdgpuDispatchExecutionEnd(info.dispatch_id); + group.onShutdown(); + } + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_EQ(diagnostic.find("skipped dispatch"), std::string::npos); + + const auto trace = lines(read_file(trace_.path())); + EXPECT_NE(line_with_prefix(trace, "begin 42 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "instruction 42 1000 0 0 0 0 12288 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "instruction 42 1000 0 1 1 0 16384 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "instruction 42 1000 0 0 0 0 28672 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "memory 42 "), trace.size()); + EXPECT_EQ(line_with_prefix(trace, "tdm 42 "), trace.size()); + EXPECT_NE(line_with_prefix(trace, "end 42 "), trace.size()); +} + TEST_F(PerfsimPluginTest, RequestsV13AndForwardsOwnedFieldsInOrder) { WaveFixture fixture; const std::string config = plugin_config(); @@ -396,7 +772,7 @@ TEST_F(PerfsimPluginTest, RequestsV13AndForwardsOwnedFieldsInOrder) { access.valid_lane_mask = 0x9; access.request_lane_mask = 0x9; access.addresses = addresses; - plugin.onAmdgpuMemoryAccessRouted(access); + plugin.onAmdgpuMemoryAccessRouted(access, wave); addresses[0] = 0xDEADBEEF; const std::array tensor_words{0xD0710001, 0x7C000000, 0x18140C00}; @@ -420,7 +796,7 @@ TEST_F(PerfsimPluginTest, RequestsV13AndForwardsOwnedFieldsInOrder) { tensor_access.tensor_dim1_stride = 104; tensor_access.is_load = true; tensor_access.addresses = std::span(tensor_addresses); - plugin.onAmdgpuTensorDmaMemoryAccess(tensor_access); + plugin.onAmdgpuTensorDmaMemoryAccess(tensor_access, wave); tensor_addresses[0] = 0xDEADBEEF; const std::array end_words{0xBF810000}; @@ -1291,6 +1667,7 @@ TEST_F(PerfsimPluginTest, ForwardsNominalWidthsForPartialOobVbufferAccesses) { TEST_F(PerfsimPluginTest, RejectsWholeDispatchWhenFlatAtomicResolvesToScratch) { WaveFixture fixture; + setenv("ROCJITSU_PERFSIM_FAKE_MODE", "gpucsim_name", 1); const std::string config = plugin_config(); testing::internal::CaptureStderr(); { @@ -1351,6 +1728,7 @@ TEST_F(PerfsimPluginTest, RejectsWholeDispatchWhenFlatAtomicResolvesToScratch) { TEST_F(PerfsimPluginTest, ReplaysInterleavedDispatchesAsOneGloballyOrderedEpoch) { WaveFixture fixture; + setenv("ROCJITSU_PERFSIM_FAKE_MODE", "gpucsim_name", 1); const std::string config = plugin_config(); PerfsimPlugin plugin(config.c_str()); plugin.onInit(); @@ -1909,9 +2287,11 @@ TEST_F(PerfsimPluginTest, SequentialInstancesShutdownOnceWithoutUnloadingBackend EXPECT_EQ(line_with_prefix(trace, "unload"), trace.size()); } -TEST_F(PerfsimPluginTest, LoadsAsRocjitsuPluginWithStagingBudgetSchema) { - const std::string config = - std::string{"{\"plugins\":{\"perfsim\":"} + plugin_config_with_staging_budget(4096) + "}}"; +TEST_F(PerfsimPluginTest, LoadsAsRocjitsuPluginWithPublicOptionSchema) { + const std::string adapter_config = std::string{"{\"library_path\":"} + + json_string(PERFSIM_FAKE_BACKEND_PATH) + + ",\"max_staged_bytes\":4096,\"max_observed_workgroups\":1}"; + const std::string config = std::string{"{\"plugins\":{\"perfsim\":"} + adapter_config + "}}"; ExecutionPluginGroup group(PluginSinkConfig{}); testing::internal::CaptureStderr(); const size_t loaded = PluginLoader::load_from_config(config, group, PERFSIM_PLUGIN_DIR); @@ -2000,6 +2380,11 @@ TEST_F(PerfsimPluginTest, RealBackendMatchesDirectFfmForCanonicalStream) { rocjitsu::test::ScopedEnvironmentVariable target{"GPUCSIM_TARGET", "gfx1250"}; rocjitsu::test::ScopedEnvironmentVariable wmma_only{"GPUCSIM_WMMA_ONLY", "0"}; rocjitsu::test::ScopedEnvironmentVariable verbose{"GPUCSIM_VERBOSE", "0"}; + // The current GPUCompilerSim backend keeps dispatch identity in its + // diagnostic report schema. Request that schema explicitly because this + // parity test compares individual dispatches, not the public summary. + rocjitsu::test::ScopedEnvironmentVariable detailed_report{"GPUCSIM_INTERNAL_DETAILED_REPORT", + "1"}; rocjitsu::test::ScopedEnvironmentVariable probes{"GPUCSIM_PROBE", "0"}; rocjitsu::test::ScopedEnvironmentVariable smem_model{"GPUCSIM_SMEM_SQC_MODEL", "0"}; rocjitsu::test::ScopedEnvironmentVariable wait_histogram{"GPUCSIM_WAIT_HISTOGRAM", "0"}; @@ -2181,7 +2566,7 @@ TEST_F(PerfsimPluginTest, RealBackendMatchesDirectFfmForCanonicalStream) { memory_access.valid_lane_mask = 1; memory_access.request_lane_mask = 1; memory_access.addresses = global_addresses; - plugin.onAmdgpuMemoryAccessRouted(memory_access); + plugin.onAmdgpuMemoryAccessRouted(memory_access, wave); SyntheticInstruction tensor("tensor_load_to_lds", kTensorEncoding, MEMORY_OP); plugin.onAmdgpuBeforeExecuteInstruction(kTensorPc, tensor, wave, kTensorFetch); @@ -2202,7 +2587,7 @@ TEST_F(PerfsimPluginTest, RealBackendMatchesDirectFfmForCanonicalStream) { tensor_access.tensor_dim1_stride = 16; tensor_access.is_load = true; tensor_access.addresses = std::span(kTensorAddresses); - plugin.onAmdgpuTensorDmaMemoryAccess(tensor_access); + plugin.onAmdgpuTensorDmaMemoryAccess(tensor_access, wave); SyntheticInstruction wait("s_wait_loadcnt", kWaitEncoding); plugin.onAmdgpuBeforeExecuteInstruction(kWaitPc, wait, wave, kWaitFetch); diff --git a/emulation/rocjitsu/tests/plugin_config_resolver_test.cpp b/emulation/rocjitsu/tests/plugin_config_resolver_test.cpp index ff8a6c3b65db..5d012661bf7a 100644 --- a/emulation/rocjitsu/tests/plugin_config_resolver_test.cpp +++ b/emulation/rocjitsu/tests/plugin_config_resolver_test.cpp @@ -74,6 +74,24 @@ TEST(PluginConfigResolver, RequiredArgProvidedSucceeds) { EXPECT_NE(out.find("\"path\": \"/tmp/log\""), std::string::npos); } +TEST(PluginConfigResolver, OptionalArgMayBeAbsentOrProvided) { + const char *schema = R"({ "label": { "type": "string", "optional": true } })"; + std::string out; + EXPECT_TRUE(resolve(schema, "", out)); + EXPECT_EQ(out.find("\"label\""), std::string::npos); + + EXPECT_TRUE(resolve(schema, R"({"label":"target"})", out)); + EXPECT_NE(out.find("\"label\": \"target\""), std::string::npos); +} + +TEST(PluginConfigResolver, DefaultTakesPrecedenceOverOptionalMarker) { + const char *schema = + R"({ "label": { "type": "string", "default": "fallback", "optional": true } })"; + std::string out; + EXPECT_TRUE(resolve(schema, "", out)); + EXPECT_NE(out.find("\"label\": \"fallback\""), std::string::npos); +} + TEST(PluginConfigResolver, WrongTypeFails) { const char *schema = R"({ "level": { "type": "number", "default": 3 } })"; std::string out; diff --git a/emulation/rocjitsu/tests/pm4_compute_queue_test.cpp b/emulation/rocjitsu/tests/pm4_compute_queue_test.cpp new file mode 100644 index 000000000000..84ecbf2582a8 --- /dev/null +++ b/emulation/rocjitsu/tests/pm4_compute_queue_test.cpp @@ -0,0 +1,1103 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "embedded_schema.h" +#include "rocjitsu/config/config_loader.h" +#include "rocjitsu/vm/amdgpu/command_processor.h" +#include "rocjitsu/vm/amdgpu/compute_queue_binding_factory.h" +#include "rocjitsu/vm/amdgpu/gpu_vm.h" +#include "rocjitsu/vm/soc.h" +#include "simdojo/sim/simulation.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rocjitsu::amdgpu { +namespace { + +class Pm4QueueMemory final : public AddressSpaceTranslator, public PhysicalMemoryAccess { +public: + Pm4QueueMemory() : bytes_(0x4000) {} + + VmTranslationResult translate(uint64_t address, std::size_t size, VmAccessKind) const override { + if (size == 0 || address > bytes_.size() || size > bytes_.size() - address) + return {.outcome = VmAccessOutcome::Faulted, .translation = {}}; + return { + .outcome = VmAccessOutcome::Complete, + .translation = {.domain = VmMemoryDomain::System, + .address = address, + .contiguous_bytes = bytes_.size() - address, + .mtype = Mtype::RW, + .permissions = {.readable = true, .writable = true, .executable = false}}}; + } + + VmAccessOutcome read(VmMemoryDomain, uint64_t address, std::span bytes) override { + if (unavailable_read_ && address == *unavailable_read_) + return VmAccessOutcome::Unavailable; + if (address > bytes_.size() || bytes.size() > bytes_.size() - address) + return VmAccessOutcome::Faulted; + std::ranges::copy_n(bytes_.begin() + static_cast(address), bytes.size(), + bytes.begin()); + return VmAccessOutcome::Complete; + } + + VmAccessOutcome write(VmMemoryDomain, uint64_t address, + std::span bytes) override { + if (address > bytes_.size() || bytes.size() > bytes_.size() - address) + return VmAccessOutcome::Faulted; + std::ranges::copy(bytes, bytes_.begin() + static_cast(address)); + return VmAccessOutcome::Complete; + } + + AtomicLoadResult atomic_load(VmMemoryDomain, uint64_t address, uint32_t width) override { + if (unavailable_read_ && address == *unavailable_read_) + return {.outcome = VmAccessOutcome::Unavailable}; + if ((width != 4 && width != 8) || address % width != 0 || address > bytes_.size() || + width > bytes_.size() - address) { + return {.outcome = VmAccessOutcome::Malformed}; + } + uint64_t value = 0; + std::memcpy(&value, bytes_.data() + address, width); + return {.outcome = VmAccessOutcome::Complete, .value = value}; + } + + VmAccessOutcome atomic_store(VmMemoryDomain, uint64_t address, uint32_t width, + uint64_t value) override { + if (unavailable_store_ && address == *unavailable_store_) + return VmAccessOutcome::Unavailable; + if (faulted_store_ && address == *faulted_store_) + return VmAccessOutcome::Faulted; + if ((width != 4 && width != 8) || address % width != 0 || address > bytes_.size() || + width > bytes_.size() - address) { + return VmAccessOutcome::Malformed; + } + std::memcpy(bytes_.data() + address, &value, width); + return VmAccessOutcome::Complete; + } + + AtomicCompareExchangeResult compare_exchange(VmMemoryDomain, uint64_t, uint32_t, uint64_t, + uint64_t) override { + return {.outcome = VmAccessOutcome::Malformed}; + } + + template void store(uint64_t address, T value) { + std::memcpy(bytes_.data() + address, &value, sizeof(value)); + } + + template T load(uint64_t address) const { + T value{}; + std::memcpy(&value, bytes_.data() + address, sizeof(value)); + return value; + } + + void make_read_unavailable(uint64_t address) { unavailable_read_ = address; } + void make_store_unavailable(uint64_t address) { unavailable_store_ = address; } + void make_store_faulted(uint64_t address) { faulted_store_ = address; } + void make_available() { + unavailable_read_.reset(); + unavailable_store_.reset(); + faulted_store_.reset(); + } + +private: + std::vector bytes_; + std::optional unavailable_read_; + std::optional unavailable_store_; + std::optional faulted_store_; +}; + +class Pm4ComputeQueueTest : public ::testing::Test { +protected: + void SetUp() override { + memory = std::make_shared(); + address_space = gpu_vm.register_translated(1, memory, memory); + ASSERT_TRUE(address_space); + memory->store(kReadPointer, 0); + auto loaded = config::load_config_from_string(R"({"max_ticks":100000,"num_threads":1, + "vm":{"arch":"cdna3"},"topology":{"root":{"name":"soc","type":"soc","children":[ + {"name":"vram","type":"gpu_memory"},{"name":"xcd0","type":"xcd","children":[ + {"name":"l2","type":"l2_cache"},{"name":"cp","type":"command_processor"}, + {"name":"se0","type":"shader_engine","children":[{"name":"cu0","type":"compute_unit"}]}]}]}, + "links":[{"src":"xcd0.cp.req_0","dst":"xcd0.se0.cu0.cpl","latency":1,"weight":2}, + {"src":"xcd0.se0.cu0.req","dst":"xcd0.l2.cpl_0","latency":1,"weight":10}]}})", + kEmbeddedSchema); + cp = loaded.soc()->xcd(0)->command_processor(); + engine = std::make_unique(loaded.engine_config); + engine->topology().set_root(loaded.take_root()); + loaded.wire_links(engine->topology()); + engine->create(); + cp->set_gpu_vm(&gpu_vm, address_space); + } + + uint64_t attach(Pm4PacketCallbacks callbacks, uint64_t ring = kRing, + uint64_t read_pointer = kReadPointer, uint32_t ring_bytes = kRingBytes) { + return cp->register_queue({.address_space = address_space, + .queue_id = next_queue_id++, + .ring_base_va = ring, + .ring_size = ring_bytes, + .read_ptr_va = read_pointer, + .packet_format = QueuePacketFormat::Pm4, + .packet_callbacks = std::move(callbacks)}); + } + + void service() { (void)engine->step(); } + + static constexpr uint64_t kRing = 0x100; + static constexpr uint32_t kRingBytes = 64; + static constexpr uint64_t kReadPointer = 0x200; + + GpuVm gpu_vm; + std::shared_ptr memory; + AddressSpaceHandle address_space; + std::unique_ptr engine; + CommandProcessor *cp = nullptr; + uint32_t next_queue_id = 1; +}; + +TEST_F(Pm4ComputeQueueTest, SubmissionDefersEffectsUntilTheCpServicesTheQueue) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t writes = 0; + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + + EXPECT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + EXPECT_EQ(writes, 0u); + EXPECT_EQ(memory->load(kReadPointer), 0u); + + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 3u); +} + +TEST_F(Pm4ComputeQueueTest, NativeRingResumesAfterPublishingAnInitiallyUnreadyGart) { + constexpr uint64_t aperture = 0x10000; + constexpr uint64_t page_table = 0x1000; + constexpr uint64_t physical_page = 0x2000; + // One readable/writable system page, also marked as a leaf PDE. + constexpr uint64_t page_flags = (uint64_t{1} << 63) | 0x63; + memory->store(page_table, physical_page | page_flags); + memory->store(physical_page + kRing, 0xc0017900); + memory->store(physical_page + kRing + 4, 0x40); + memory->store(physical_page + kRing + 8, 0xdeadbeef); + address_space = gpu_vm.initialize_gart_address_space(); + ASSERT_TRUE(address_space); + ASSERT_FALSE(gpu_vm.lookup(address_space)->ready); + uint32_t writes = 0; + const auto registration = attach({.write_uconfig_register = + [&](uint64_t, uint32_t value) { + EXPECT_EQ(value, 0xdeadbeefu); + ++writes; + return Pm4RegisterWriteStatus::Complete; + }}, + aperture + kRing, aperture + kReadPointer); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + + service(); + EXPECT_EQ(writes, 0u); + EXPECT_EQ(memory->load(physical_page + kReadPointer), 0u); + ASSERT_TRUE(gpu_vm.publish_gart( + {.page_table_base = page_table, .aperture_start = aperture, .aperture_end = aperture + 0xfff}, + memory)); + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(physical_page + kReadPointer), 3u); + service(); + EXPECT_EQ(writes, 1u); +} + +TEST_F(Pm4ComputeQueueTest, SubmittedStreamResumesAfterPublishingAnInitiallyUnreadyGart) { + constexpr uint64_t aperture = 0x10000; + constexpr uint64_t page_table = 0x1000; + constexpr uint64_t physical_page = 0x2000; + constexpr uint64_t page_flags = (uint64_t{1} << 63) | 0x63; + memory->store(page_table, physical_page | page_flags); + memory->store(physical_page + kRing, 0xc0033700); + memory->store(physical_page + kRing + 4, 0x100); + memory->store(physical_page + kRing + 8, aperture + kReadPointer); + memory->store(physical_page + kRing + 12, 0); + memory->store(physical_page + kRing + 16, 0xdeadbeef); + address_space = gpu_vm.initialize_gart_address_space(); + ASSERT_TRUE(address_space); + ASSERT_TRUE( + cp->register_drm_queue({.address_space = address_space, .process_id = 1, .queue_id = 1})); + uint32_t completions = 0; + Pm4Submission submission; + submission.buffers.push_back({.address = aperture + kRing, .dwords = 5}); + submission.complete = [&](bool success) { + EXPECT_TRUE(success); + ++completions; + }; + ASSERT_TRUE(cp->submit_pm4(1, 1, std::move(submission))); + + service(); + EXPECT_EQ(completions, 0u); + EXPECT_EQ(memory->load(physical_page + kReadPointer), 0u); + ASSERT_TRUE(gpu_vm.publish_gart( + {.page_table_base = page_table, .aperture_start = aperture, .aperture_end = aperture + 0xfff}, + memory)); + service(); + EXPECT_EQ(completions, 1u); + EXPECT_EQ(memory->load(physical_page + kReadPointer), 0xdeadbeefu); + service(); + EXPECT_EQ(completions, 1u); +} + +TEST_F(Pm4ComputeQueueTest, BlockedSubmissionsKeepStallRecheckWorkBounded) { + bool ready = false; + uint32_t completed = 0; + for (uint32_t id : {1u, 2u}) { + ASSERT_TRUE(cp->register_drm_queue({.address_space = address_space, .queue_id = id})); + Pm4Submission submission; + submission.ready = [&] { return ready; }; + submission.complete = [&](bool success) { + EXPECT_TRUE(success); + ++completed; + }; + ASSERT_TRUE(cp->submit_pm4(id, 0, std::move(submission))); + } + + // A persistent dependency must back off even when several queues and several + // service sites request a retry. Duplicate retries would multiply at each tick + // and prevent later shader or dependency-completion events from progressing. + while (engine->context(cp->partition_id()).current_tick() < 20000u) + ASSERT_TRUE(engine->step()); + EXPECT_LT(cp->doorbell_handle_count_for_test(), 64u); + EXPECT_EQ(completed, 0u); + + ready = true; + // Readiness alone must be observed by the coalesced timer. + for (uint32_t i = 0; i < 8 && completed != 2; ++i) + service(); + EXPECT_EQ(completed, 2u); +} + +TEST_F(Pm4ComputeQueueTest, ProducerCompletionWakesEarlierConsumerPromptly) { + bool fence = false; + uint32_t consumer_done = 0, producer_done = 0; + ASSERT_TRUE(cp->register_drm_queue({.address_space = address_space, .queue_id = 1})); + ASSERT_TRUE(cp->register_drm_queue({.address_space = address_space, .queue_id = 2})); + Pm4Submission consumer; + consumer.ready = [&] { return fence; }; + consumer.complete = [&](bool success) { + EXPECT_TRUE(success); + ++consumer_done; + }; + ASSERT_TRUE(cp->submit_pm4(1, 0, std::move(consumer))); + auto tick = [&] { return engine->context(cp->partition_id()).current_tick(); }; + while (tick() < 20000) + ASSERT_TRUE(engine->step()); + Pm4Submission producer; + producer.complete = [&](bool success) { + EXPECT_TRUE(success); + fence = true; + ++producer_done; + }; + const auto start = tick(); + ASSERT_TRUE(cp->submit_pm4(2, 0, std::move(producer))); + ASSERT_TRUE(engine->step()); + ASSERT_EQ(producer_done, 1u); + ASSERT_EQ(consumer_done, 0u); + for (unsigned i = 0; i < 8 && consumer_done == 0; ++i) + ASSERT_TRUE(engine->step()); + EXPECT_EQ(consumer_done, 1u); + EXPECT_LE(tick() - start, 1u); + const auto passes = cp->doorbell_handle_count_for_test(); + unsigned stale_steps = 0; + while (engine->step()) + ASSERT_LT(++stale_steps, 20u); + // Completing both submissions may request one final retry, but superseded + // long deadlines must not enter the CP handler after all work is retired. + EXPECT_LE(cp->doorbell_handle_count_for_test() - passes, 1u); +} + +TEST_F(Pm4ComputeQueueTest, BoundsEachQueueServiceTurn) { + constexpr uint64_t kLargeRing = 0x400; + constexpr uint64_t kLargeReadPointer = 0xe00; + constexpr uint32_t kPacketCount = 257; + constexpr uint32_t kLargeRingBytes = 512 * sizeof(uint32_t); + for (uint32_t packet = 0; packet < kPacketCount; ++packet) + memory->store(kLargeRing + packet * sizeof(uint32_t), 0xffff1000); + memory->store(kLargeReadPointer, 0); + + ASSERT_TRUE(cp->register_drm_queue({.address_space = address_space, .queue_id = 99})); + Pm4Submission blocked; + blocked.ready = [] { return false; }; + ASSERT_TRUE(cp->submit_pm4(99, 0, std::move(blocked))); + auto tick = [&] { return engine->context(cp->partition_id()).current_tick(); }; + while (tick() < 20000) + ASSERT_TRUE(engine->step()); + const auto start = tick(); + + const auto registration = attach({}, kLargeRing, kLargeReadPointer, kLargeRingBytes); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, kPacketCount), + QueueSubmissionStatus::Accepted); + + service(); + EXPECT_EQ(memory->load(kLargeReadPointer), 256u); + + service(); + EXPECT_EQ(memory->load(kLargeReadPointer), kPacketCount); + EXPECT_LE(tick() - start, 1u); +} + +TEST_F(Pm4ComputeQueueTest, RetainsItsAddressSpaceUntilDetach) { + const uint64_t registration = attach({}); + ASSERT_NE(registration, 0u); + + EXPECT_FALSE(gpu_vm.unregister_address_space(address_space)); + EXPECT_TRUE(cp->unregister_pm4_queue_registration(registration)); + EXPECT_TRUE(gpu_vm.unregister_address_space(address_space)); +} + +TEST_F(Pm4ComputeQueueTest, CursorPublicationRetryDoesNotReplayThePacketEffect) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t writes = 0; + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + memory->make_store_unavailable(kReadPointer); + + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 0u); + + memory->make_available(); + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 3u); +} + +TEST_F(Pm4ComputeQueueTest, RootReplacementDoesNotMovePendingCursorPublication) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t writes = 0; + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + memory->make_store_unavailable(kReadPointer); + + service(); + EXPECT_EQ(writes, 1u); + + auto replacement = std::make_shared(); + replacement->store(kReadPointer, 99); + ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); + memory->make_available(); + + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 3u); + EXPECT_EQ(replacement->load(kReadPointer), 99u); +} + +TEST_F(Pm4ComputeQueueTest, RootReplacementDoesNotMoveBlockedPacketFetch) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t written_value = 0; + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t value) { + written_value = value; + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + memory->make_read_unavailable(kRing); + + service(); + EXPECT_EQ(written_value, 0u); + + auto replacement = std::make_shared(); + replacement->store(kRing, 0xc0017900); + replacement->store(kRing + 4, 0x40); + replacement->store(kRing + 8, 0x12345678); + replacement->store(kReadPointer, 99); + ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); + memory->make_available(); + + service(); + EXPECT_EQ(written_value, 0xdeadbeefu); + EXPECT_EQ(memory->load(kReadPointer), 3u); + EXPECT_EQ(replacement->load(kReadPointer), 99u); +} + +TEST_F(Pm4ComputeQueueTest, LaterDoorbellUsesFreshSnapshotAfterBlockedTransaction) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0x11111111); + std::vector values; + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t value) { + values.push_back(value); + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + memory->make_store_unavailable(kReadPointer); + + service(); + ASSERT_EQ(values.size(), 1u); + EXPECT_EQ(values[0], 0x11111111u); + + auto replacement = std::make_shared(); + replacement->store(kRing + 12, 0xc0017900); + replacement->store(kRing + 16, 0x40); + replacement->store(kRing + 20, 0x22222222); + replacement->store(kReadPointer, 3); + ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 6), QueueSubmissionStatus::Accepted); + memory->make_available(); + + service(); + EXPECT_EQ(memory->load(kReadPointer), 3u); + service(); + ASSERT_EQ(values.size(), 2u); + EXPECT_EQ(values[1], 0x22222222u); + EXPECT_EQ(replacement->load(kReadPointer), 6u); +} + +TEST_F(Pm4ComputeQueueTest, PublishesEarlierProgressBeforeRetryingALaterFetch) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + memory->store(kRing + 12, 0xffff1000); + uint32_t writes = 0; + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 4), QueueSubmissionStatus::Accepted); + memory->make_read_unavailable(kRing + 12); + + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 3u); + + memory->make_available(); + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 4u); +} + +TEST_F(Pm4ComputeQueueTest, RejectsInvalidRingsAtRegistration) { + EXPECT_EQ(cp->register_pm4_queue({.address_space = address_space, + .ring_base = kRing + 1, + .ring_size_bytes = kRingBytes, + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = std::nullopt, + .packet_callbacks = {}}), + 0u); + EXPECT_EQ(cp->register_pm4_queue({.address_space = address_space, + .ring_base = kRing, + .ring_size_bytes = kRingBytes - 1, + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = std::nullopt, + .packet_callbacks = {}}), + 0u); + EXPECT_EQ(cp->register_pm4_queue({.address_space = address_space, + .ring_base = kRing, + .ring_size_bytes = kRingBytes, + .consumer_pointer_address = kReadPointer + sizeof(uint16_t), + .initial_consumer_cursor = std::nullopt, + .packet_callbacks = {}}), + 0u); +} + +TEST_F(Pm4ComputeQueueTest, NormalizesWrappedNativeProducerBeforeRetainingProgress) { + constexpr uint32_t kRingDwords = 4; + memory->store(kReadPointer, 3); + memory->store(kRing + 3 * sizeof(uint32_t), 0xc0017900); + memory->store(kRing, 0x40); + memory->store(kRing + sizeof(uint32_t), 0xdeadbeef); + + uint32_t writes = 0; + const auto registration = cp->register_pm4_queue({ + .address_space = address_space, + .ring_base = kRing, + .ring_size_bytes = kRingDwords * sizeof(uint32_t), + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = 3, + .packet_callbacks = {.write_uconfig_register = + [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }}, + }); + ASSERT_NE(registration, 0u); + + EXPECT_EQ(cp->notify_pm4_queue_doorbell(registration, 2), QueueSubmissionStatus::Accepted); + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 2u); + service(); + EXPECT_EQ(writes, 1u); +} + +TEST_F(Pm4ComputeQueueTest, BlockedQueuesRetryIndependently) { + constexpr uint64_t kOtherRing = 0x300; + constexpr uint64_t kOtherReadPointer = 0x380; + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 1); + memory->store(kOtherRing, 0xc0017900); + memory->store(kOtherRing + 4, 0x40); + memory->store(kOtherRing + 8, 2); + memory->store(kOtherReadPointer, 0); + + bool first_blocked = true; + bool second_blocked = true; + uint32_t first_attempts = 0; + uint32_t second_attempts = 0; + const auto first = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + ++first_attempts; + return first_blocked ? Pm4RegisterWriteStatus::Blocked : Pm4RegisterWriteStatus::Complete; + }}); + const auto second = attach({.write_uconfig_register = + [&](uint64_t, uint32_t) { + ++second_attempts; + return second_blocked ? Pm4RegisterWriteStatus::Blocked + : Pm4RegisterWriteStatus::Complete; + }}, + kOtherRing, kOtherReadPointer); + ASSERT_NE(first, 0u); + ASSERT_NE(second, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(first, 3), QueueSubmissionStatus::Accepted); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(second, 3), QueueSubmissionStatus::Accepted); + + service(); + EXPECT_GE(first_attempts, 1u); + EXPECT_GE(second_attempts, 1u); + EXPECT_EQ(memory->load(kReadPointer), 0u); + EXPECT_EQ(memory->load(kOtherReadPointer), 0u); + + const uint32_t first_blocked_attempts = first_attempts; + first_blocked = false; + service(); + EXPECT_EQ(first_attempts, first_blocked_attempts + 1); + EXPECT_EQ(memory->load(kReadPointer), 3u); + EXPECT_EQ(memory->load(kOtherReadPointer), 0u); + second_blocked = false; + service(); + EXPECT_EQ(first_attempts, first_blocked_attempts + 1); + EXPECT_EQ(memory->load(kOtherReadPointer), 3u); +} + +TEST_F(Pm4ComputeQueueTest, DetachWaitsForActiveCommandProcessorService) { + using namespace std::chrono_literals; + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + std::promise callback_entered; + std::promise release_callback; + std::shared_future release = release_callback.get_future().share(); + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + callback_entered.set_value(); + release.wait(); + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + + auto service_task = std::async(std::launch::async, [&] { service(); }); + ASSERT_EQ(callback_entered.get_future().wait_for(1s), std::future_status::ready); + auto detach = std::async(std::launch::async, + [&] { return cp->unregister_pm4_queue_registration(registration); }); + + EXPECT_EQ(detach.wait_for(20ms), std::future_status::timeout); + release_callback.set_value(); + service_task.get(); + EXPECT_TRUE(detach.get()); +} + +TEST_F(Pm4ComputeQueueTest, ReconfigureWaitsForActiveService) { + using namespace std::chrono_literals; + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + std::promise callback_entered; + std::promise release_callback; + std::shared_future release = release_callback.get_future().share(); + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + callback_entered.set_value(); + release.wait(); + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + + auto service_task = std::async(std::launch::async, [&] { service(); }); + ASSERT_EQ(callback_entered.get_future().wait_for(1s), std::future_status::ready); + auto reconfigure = std::async(std::launch::async, [&] { + return cp->update_pm4_queue_registration( + registration, + {.ring_base_address = kRing, .ring_size_bytes = kRingBytes, .scheduling_percentage = 100}); + }); + + EXPECT_EQ(reconfigure.wait_for(20ms), std::future_status::timeout); + release_callback.set_value(); + service_task.get(); + EXPECT_EQ(reconfigure.get(), QueueReconfigureStatus::Applied); +} + +TEST_F(Pm4ComputeQueueTest, GracefulDetachWaitsForPendingCursorPublication) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t writes = 0; + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + memory->make_store_unavailable(kReadPointer); + + service(); + EXPECT_EQ(cp->prepare_unregister_pm4_queue_registration(registration), + QueuePrepareCloseStatus::Busy); + EXPECT_EQ(cp->registered_pm4_queue_count_for_test(), 1u); + EXPECT_EQ(writes, 1u); + + memory->make_available(); + service(); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 3u); + EXPECT_EQ(cp->prepare_unregister_pm4_queue_registration(registration), + QueuePrepareCloseStatus::Ready); + EXPECT_EQ(cp->registered_pm4_queue_count_for_test(), 0u); +} + +TEST_F(Pm4ComputeQueueTest, GracefulDetachWaitsForBlockedCommandStream) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t attempts = 0; + const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { + ++attempts; + return Pm4RegisterWriteStatus::Blocked; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + + service(); + EXPECT_EQ(attempts, 1u); + EXPECT_EQ(cp->prepare_unregister_pm4_queue_registration(registration), + QueuePrepareCloseStatus::Busy); + EXPECT_EQ(cp->registered_pm4_queue_count_for_test(), 1u); + EXPECT_TRUE(cp->unregister_pm4_queue_registration(registration)); +} + +TEST_F(Pm4ComputeQueueTest, GracefulDetachReportsFaultedPublicationWithoutRetryingForever) { + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + const auto registration = attach({.write_uconfig_register = [](uint64_t, uint32_t) { + return Pm4RegisterWriteStatus::Complete; + }}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 3), QueueSubmissionStatus::Accepted); + memory->make_store_faulted(kReadPointer); + + service(); + EXPECT_EQ(cp->prepare_unregister_pm4_queue_registration(registration), + QueuePrepareCloseStatus::Faulted); + EXPECT_EQ(cp->registered_pm4_queue_count_for_test(), 1u); + EXPECT_TRUE(cp->unregister_pm4_queue_registration(registration)); + EXPECT_EQ(cp->registered_pm4_queue_count_for_test(), 0u); +} + +TEST_F(Pm4ComputeQueueTest, DrainedQueueCanBeDisabledAndReconfigured) { + constexpr uint64_t kReplacementRing = 0x300; + memory->store(kRing, 0xffff1000); + const auto registration = attach({}); + ASSERT_NE(registration, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(registration, 1), QueueSubmissionStatus::Accepted); + service(); + + EXPECT_EQ( + cp->update_pm4_queue_registration( + registration, {.ring_base_address = 0, .ring_size_bytes = 0, .scheduling_percentage = 0}), + QueueReconfigureStatus::Disabled); + memory->store(kReplacementRing, 0xffff1000); + EXPECT_EQ(cp->update_pm4_queue_registration(registration, {.ring_base_address = kReplacementRing, + .ring_size_bytes = kRingBytes, + .scheduling_percentage = 100}), + QueueReconfigureStatus::Applied); + EXPECT_EQ(cp->notify_pm4_queue_doorbell(registration, 1), QueueSubmissionStatus::Accepted); + service(); + EXPECT_EQ(cp->prepare_unregister_pm4_queue_registration(registration), + QueuePrepareCloseStatus::Ready); +} + +TEST_F(Pm4ComputeQueueTest, InitialCursorRetryKeepsTheOriginalVmSnapshot) { + memory->store(kRing, 0xffff1000); + memory->make_read_unavailable(kReadPointer); + const auto id = attach({}); + ASSERT_NE(id, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(id, 1), QueueSubmissionStatus::Accepted); + service(); + EXPECT_EQ(memory->load(kReadPointer), 0u); + EXPECT_EQ(cp->prepare_unregister_pm4_queue_registration(id), QueuePrepareCloseStatus::Busy); + auto replacement = std::make_shared(); + replacement->store(kReadPointer, 99); + ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); + memory->make_available(); + service(); + EXPECT_EQ(memory->load(kReadPointer), 1u); + EXPECT_EQ(replacement->load(kReadPointer), 99u); +} + +TEST_F(Pm4ComputeQueueTest, InitialCursorRetryIsReportedWithoutAnEngine) { + CommandProcessor standalone("standalone"); + standalone.set_gpu_vm(&gpu_vm); + memory->make_read_unavailable(kReadPointer); + const auto id = standalone.register_pm4_queue({.address_space = address_space, + .ring_base = kRing, + .ring_size_bytes = kRingBytes, + .consumer_pointer_address = kReadPointer}); + ASSERT_NE(id, 0u); + EXPECT_EQ(standalone.notify_pm4_queue_doorbell(id, 1), QueueSubmissionStatus::Retry); +} + +TEST_F(Pm4ComputeQueueTest, ChangedRingWaitsForANewDoorbell) { + constexpr uint64_t replacement_ring = 0x300; + memory->store(kRing, 0xffff1000); + memory->store(replacement_ring, 0xffff1000); + const auto id = attach({}); + ASSERT_NE(id, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(id, 1), QueueSubmissionStatus::Accepted); + service(); + ASSERT_EQ(memory->load(kReadPointer), 1u); + ASSERT_EQ(cp->update_pm4_queue_registration(id, {}), QueueReconfigureStatus::Disabled); + memory->store(kReadPointer, 0); + ASSERT_EQ(cp->update_pm4_queue_registration(id, {.ring_base_address = replacement_ring, + .ring_size_bytes = kRingBytes, + .scheduling_percentage = 100}), + QueueReconfigureStatus::Applied); + service(); + EXPECT_EQ(memory->load(kReadPointer), 0u); + EXPECT_FALSE(cp->queue_faulted_for_test(1, 0)); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(id, 1), QueueSubmissionStatus::Accepted); + service(); + EXPECT_EQ(memory->load(kReadPointer), 1u); +} + +TEST_F(Pm4ComputeQueueTest, SameRingRuntimeResumePreservesSubmittedProgress) { + memory->store(kRing, 0xffff1000); + const auto id = attach({}); + ASSERT_NE(id, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(id, 1), QueueSubmissionStatus::Accepted); + ASSERT_EQ(cp->update_pm4_queue_registration(id, {}), QueueReconfigureStatus::Disabled); + service(); + EXPECT_EQ(memory->load(kReadPointer), 0u); + ASSERT_EQ(cp->update_pm4_queue_registration(id, {.ring_base_address = kRing, + .ring_size_bytes = kRingBytes, + .scheduling_percentage = 100}), + QueueReconfigureStatus::Applied); + service(); + EXPECT_EQ(memory->load(kReadPointer), 1u); +} + +TEST_F(Pm4ComputeQueueTest, DebugResumeServicesTheExistingDoorbell) { + memory->store(kRing, 0xffff1000); + const auto id = attach({}); + ASSERT_NE(id, 0u); + cp->set_queue_debug_suspended(1, 0, true); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(id, 1), QueueSubmissionStatus::Accepted); + service(); + EXPECT_EQ(memory->load(kReadPointer), 0u); + cp->set_queue_debug_suspended(1, 0, false); + service(); + EXPECT_EQ(memory->load(kReadPointer), 1u); +} + +TEST_F(Pm4ComputeQueueTest, EngineNotificationRejectsDisabledInvalidAndFaultedQueues) { + const auto id = attach({}); + ASSERT_NE(id, 0u); + ASSERT_EQ(cp->update_pm4_queue_registration(id, {}), QueueReconfigureStatus::Disabled); + EXPECT_EQ(cp->notify_pm4_queue_doorbell(id, 1), QueueSubmissionStatus::Faulted); + ASSERT_EQ(cp->update_pm4_queue_registration(id, {.ring_base_address = kRing, + .ring_size_bytes = kRingBytes, + .scheduling_percentage = 100}), + QueueReconfigureStatus::Applied); + EXPECT_EQ(cp->notify_pm4_queue_doorbell(id, kRingBytes / 4 + 1), QueueSubmissionStatus::Faulted); + EXPECT_EQ(cp->notify_pm4_queue_doorbell(id, 1), QueueSubmissionStatus::Faulted); + EXPECT_EQ(cp->notify_pm4_queue_doorbell(id + 100, 1), QueueSubmissionStatus::Faulted); + EXPECT_EQ(memory->load(kReadPointer), 0u); +} + +TEST_F(Pm4ComputeQueueTest, NativeRegisterWritesRequireSupportedCallbacks) { + for (const uint32_t opcode : {0x79u, 0xbeu}) { + SCOPED_TRACE(opcode); + memory->store(kRing, 0xc0010000 | opcode << 8); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t writes = 0; + Pm4PacketCallbacks callbacks; + if (opcode == 0xbe) + callbacks.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }; + const auto id = attach(std::move(callbacks)); + ASSERT_NE(id, 0u); + ASSERT_EQ(cp->notify_pm4_queue_doorbell(id, 3), QueueSubmissionStatus::Accepted); + service(); + EXPECT_EQ(cp->notify_pm4_queue_doorbell(id, 3), QueueSubmissionStatus::Faulted); + EXPECT_EQ(writes, 0u); + EXPECT_EQ(memory->load(kReadPointer), 0u); + ASSERT_TRUE(cp->unregister_pm4_queue_registration(id)); + } +} + +TEST(Pm4ComputeQueueStateTest, PendingWorkIncludesStreamsAndShaderDispatches) { + ComputeQueueRecord queue; + EXPECT_FALSE(queue.has_pending_commands()); + queue.commands.submissions.emplace_back(); + EXPECT_TRUE(queue.has_pending_commands()); + queue.commands.submissions.clear(); + queue.dispatches.entries.emplace_back(); + EXPECT_TRUE(queue.has_pending_commands()); + queue.dispatches.entries.clear(); + queue.entries.emplace_back(); + EXPECT_TRUE(queue.has_pending_commands()); + queue.entries.clear(); + EXPECT_FALSE(queue.has_pending_commands()); +} + +TEST(Pm4QueueBindingTest, RejectsUnsupportedVmPollingThroughTheComputeQueueBinding) { + constexpr uint64_t kRing = 0x100; + constexpr uint64_t kReadPointer = 0x200; + constexpr uint64_t kWritePointer = 0x208; + alignas(8) std::array doorbell{}; + auto memory = std::make_shared(); + GpuVm gpu_vm; + const AddressSpaceHandle address_space = gpu_vm.register_translated(1, memory, memory); + ASSERT_TRUE(address_space); + CommandProcessor command_processor("cp"); + command_processor.set_gpu_vm(&gpu_vm); + GpuQueueRegistry registry(gpu_vm); + const std::shared_ptr binding_factory = + make_compute_queue_binding_factory(command_processor, {}); + + for (const QueueDoorbellMode mode : + {QueueDoorbellMode::HostPolled, QueueDoorbellMode::VmPolled}) { + const QueueHandle queue = registry.register_queue({ + .identity = {.address_space = address_space, .process_id = 1, .queue_id = 7}, + .ring = {.base_address = kRing, + .size_bytes = 64, + .consumer_pointer_address = kReadPointer, + .producer_pointer_address = kWritePointer}, + .doorbell = {.mode = mode, .host_base = doorbell.data(), .address = 0x300}, + .binding_factory = binding_factory, + .type = QueueType::Compute, + .packet_format = QueuePacketFormat::Pm4, + }); + if (mode == QueueDoorbellMode::VmPolled) { + EXPECT_FALSE(queue); + EXPECT_EQ(command_processor.registered_pm4_queue_count_for_test(), 0u); + continue; + } + ASSERT_TRUE(queue); + EXPECT_EQ(registry.active_queues(), 1u); + EXPECT_EQ(command_processor.registered_pm4_queue_count_for_test(), 1u); + EXPECT_TRUE(registry.unregister_queue(queue)); + EXPECT_EQ(command_processor.registered_queue_count_for_test(), 0u); + } + + EXPECT_TRUE(gpu_vm.unregister_address_space(address_space)); +} + +TEST(Pm4QueueBindingTest, GracefulRegistryRemovalKeepsHandleUntilPublicationCompletes) { + constexpr uint64_t kRing = 0x100; + constexpr uint64_t kReadPointer = 0x200; + auto memory = std::make_shared(); + GpuVm gpu_vm; + const AddressSpaceHandle address_space = gpu_vm.register_translated(1, memory, memory); + ASSERT_TRUE(address_space); + CommandProcessor command_processor("cp"); + command_processor.set_gpu_vm(&gpu_vm); + GpuQueueRegistry registry(gpu_vm); + uint32_t writes = 0; + const std::shared_ptr binding_factory = make_compute_queue_binding_factory( + command_processor, {.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }}); + const QueueHandle queue = registry.register_queue({ + .identity = {.address_space = address_space, .process_id = 1, .queue_id = 7}, + .ring = {.base_address = kRing, .size_bytes = 64, .consumer_pointer_address = kReadPointer}, + .doorbell = {}, + .binding_factory = binding_factory, + .type = QueueType::Compute, + .packet_format = QueuePacketFormat::Pm4, + }); + ASSERT_TRUE(queue); + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + memory->make_store_unavailable(kReadPointer); + + const QueueSubmissionResult blocked = registry.submit_producer(queue, 3); + ASSERT_TRUE(blocked.found); + EXPECT_EQ(blocked.status, QueueSubmissionStatus::Retry); + EXPECT_EQ(writes, 1u); + EXPECT_FALSE(registry.unregister_queue(queue)); + EXPECT_TRUE(registry.contains(queue)); + + memory->make_available(); + const QueueSubmissionResult completed = registry.submit_producer(queue, 3); + ASSERT_TRUE(completed.found); + EXPECT_EQ(completed.status, QueueSubmissionStatus::Accepted); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 3u); + EXPECT_TRUE(registry.unregister_queue(queue)); + EXPECT_FALSE(registry.contains(queue)); + EXPECT_TRUE(gpu_vm.unregister_address_space(address_space)); +} + +} // namespace +TEST_F(Pm4ComputeQueueTest, ComputeQueueRetainsCursorPublicationWithoutReplayingEffects) { + CommandProcessor cp("compute"); + cp.set_gpu_vm(&gpu_vm, address_space); + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t effects = 0; + const auto id = cp.register_pm4_queue( + {.address_space = address_space, + .ring_base = kRing, + .ring_size_bytes = kRingBytes, + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = std::nullopt, + .packet_callbacks = {.write_uconfig_register = [&](uint64_t, uint32_t) { + ++effects; + return Pm4RegisterWriteStatus::Complete; + }}}); + ASSERT_NE(id, 0u); + memory->make_store_unavailable(kReadPointer); + EXPECT_EQ(cp.notify_pm4_queue_doorbell(id, 3), QueueSubmissionStatus::Retry); + EXPECT_EQ(effects, 1u); + EXPECT_EQ(memory->load(kReadPointer), 0u); + EXPECT_EQ(cp.prepare_unregister_queue_registration(id), QueuePrepareCloseStatus::Busy); + memory->make_available(); + EXPECT_EQ(cp.notify_pm4_queue_doorbell(id, 3), QueueSubmissionStatus::Accepted); + EXPECT_EQ(effects, 1u); + EXPECT_EQ(memory->load(kReadPointer), 3u); + EXPECT_EQ(cp.prepare_unregister_queue_registration(id), QueuePrepareCloseStatus::Ready); +} + +TEST_F(Pm4ComputeQueueTest, ComputeQueueUsesDwordCursorsAcrossRingWrap) { + CommandProcessor cp("compute"); + cp.set_gpu_vm(&gpu_vm, address_space); + memory->store(kReadPointer, 15); + memory->store(kRing + 60, 0xc0017900); + memory->store(kRing, 0x40); + memory->store(kRing + 4, 0x1234); + uint32_t effects = 0; + const auto id = cp.register_pm4_queue( + {.address_space = address_space, + .ring_base = kRing, + .ring_size_bytes = kRingBytes, + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = std::nullopt, + .packet_callbacks = {.write_uconfig_register = [&](uint64_t reg, uint32_t value) { + EXPECT_EQ(reg, 0xc040u); + EXPECT_EQ(value, 0x1234u); + ++effects; + return Pm4RegisterWriteStatus::Complete; + }}}); + ASSERT_NE(id, 0u); + EXPECT_EQ(cp.notify_pm4_queue_doorbell(id, 2), QueueSubmissionStatus::Accepted); + EXPECT_EQ(effects, 1u); + EXPECT_EQ(memory->load(kReadPointer), 2u); + EXPECT_TRUE(cp.unregister_queue_registration(id)); +} + +TEST_F(Pm4ComputeQueueTest, ComputeQueueRetainsItsSnapshotWhilePacketFetchIsBlocked) { + CommandProcessor cp("compute"); + cp.set_gpu_vm(&gpu_vm, address_space); + memory->store(kRing, 0xc0017900); + memory->store(kRing + 4, 0x40); + memory->store(kRing + 8, 0xdeadbeef); + uint32_t written = 0; + const auto id = cp.register_pm4_queue( + {.address_space = address_space, + .ring_base = kRing, + .ring_size_bytes = kRingBytes, + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = std::nullopt, + .packet_callbacks = {.write_uconfig_register = [&](uint64_t, uint32_t value) { + written = value; + return Pm4RegisterWriteStatus::Complete; + }}}); + ASSERT_NE(id, 0u); + memory->make_read_unavailable(kRing); + EXPECT_EQ(cp.notify_pm4_queue_doorbell(id, 3), QueueSubmissionStatus::Retry); + auto replacement = std::make_shared(); + replacement->store(kRing, 0xc0017900); + replacement->store(kRing + 4, 0x40); + replacement->store(kRing + 8, 0x1234); + replacement->store(kReadPointer, 99); + ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); + memory->make_available(); + EXPECT_EQ(cp.notify_pm4_queue_doorbell(id, 3), QueueSubmissionStatus::Accepted); + EXPECT_EQ(written, 0xdeadbeefu); + EXPECT_EQ(memory->load(kReadPointer), 3u); + EXPECT_EQ(replacement->load(kReadPointer), 99u); + EXPECT_TRUE(cp.unregister_queue_registration(id)); +} + +TEST_F(Pm4ComputeQueueTest, ComputeQueueCancellationDropsItsNestedStreamAndVmLease) { + CommandProcessor cp("compute"); + cp.set_gpu_vm(&gpu_vm, address_space); + constexpr uint64_t child = 0x400, gate = 0x700; + const std::array root{0xc0023f00, uint32_t(child), 0, (1u << 23) | 7}; + const std::array wait{0xc0053c00, 0x13, uint32_t(gate), 0, 1, 0xffffffff, 4}; + for (uint32_t i = 0; i < root.size(); ++i) + memory->store(kRing + i * 4, root[i]); + for (uint32_t i = 0; i < wait.size(); ++i) + memory->store(child + i * 4, wait[i]); + const auto id = cp.register_pm4_queue({.address_space = address_space, + .ring_base = kRing, + .ring_size_bytes = kRingBytes, + .consumer_pointer_address = kReadPointer, + .initial_consumer_cursor = std::nullopt, + .packet_callbacks = {}}); + ASSERT_NE(id, 0u); + EXPECT_EQ(cp.notify_pm4_queue_doorbell(id, root.size()), QueueSubmissionStatus::Retry); + EXPECT_FALSE(gpu_vm.unregister_address_space(address_space)); + EXPECT_EQ(cp.prepare_unregister_queue_registration(id), QueuePrepareCloseStatus::Busy); + EXPECT_TRUE(cp.unregister_queue_registration(id)); + EXPECT_EQ(cp.registered_queue_count_for_test(), 0u); + EXPECT_TRUE(gpu_vm.unregister_address_space(address_space)); +} + +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/tests/pm4_packet_processor_test.cpp b/emulation/rocjitsu/tests/pm4_packet_processor_test.cpp index b789a15da952..85b8f79e4425 100644 --- a/emulation/rocjitsu/tests/pm4_packet_processor_test.cpp +++ b/emulation/rocjitsu/tests/pm4_packet_processor_test.cpp @@ -3,153 +3,435 @@ #include "rocjitsu/vm/amdgpu/pm4/pm4_packet_processor.h" +#include "rocjitsu/vm/amdgpu/dispatch_entry.h" + #include +#include #include +#include #include +#include +#include +#include +#include +#include +#include +#include namespace rocjitsu::amdgpu { namespace { -TEST(PacketProcessorContractTest, AcceptsLegalCommonResults) { - EXPECT_TRUE(valid_packet_process_result({.status = PacketProcessStatus::Complete, - .retirement = PacketRetirement::Retire, - .retirement_bytes = 4}, - 4, 4)); - EXPECT_TRUE(valid_packet_process_result( - {.status = PacketProcessStatus::NeedInput, .required_bytes = 12}, 4, 4)); - EXPECT_TRUE(valid_packet_process_result( - {.status = PacketProcessStatus::Blocked, .retirement_bytes = 12}, 12, 4)); - EXPECT_TRUE(valid_packet_process_result({.status = PacketProcessStatus::Faulted, - .retirement = PacketRetirement::Retire, - .retirement_bytes = 12}, - 12, 4)); - EXPECT_TRUE(valid_packet_process_result({.status = PacketProcessStatus::Complete, - .retirement = PacketRetirement::Retire, - .retirement_bytes = 28}, - 20, 4)); -} - -TEST(PacketProcessorContractTest, RejectsContradictoryCommonResults) { - EXPECT_FALSE(valid_packet_process_result( - {.status = PacketProcessStatus::Complete, .retirement_bytes = 4}, 4, 4)); - EXPECT_FALSE(valid_packet_process_result({.status = PacketProcessStatus::NeedInput, - .retirement = PacketRetirement::Retire, - .retirement_bytes = 4, - .required_bytes = 8}, - 4, 4)); - EXPECT_FALSE(valid_packet_process_result( - {.status = PacketProcessStatus::NeedInput, .required_bytes = 4}, 4, 4)); - EXPECT_FALSE(valid_packet_process_result( - {.status = PacketProcessStatus::Malformed, .retirement = PacketRetirement::Retire}, 4, 4)); - EXPECT_FALSE(valid_packet_process_result( - {.status = PacketProcessStatus::Blocked, .required_bytes = 8}, 4, 4)); -} - -TEST(Pm4PacketProcessorTest, RetiresSupportedOneDwordNop) { - Pm4PacketProcessor processor; - const std::array packet = {0xffff1000}; - - const Pm4PacketProcessResult result = processor.process({.available_dwords = packet}); - - EXPECT_EQ(result.packet.status, PacketProcessStatus::Complete); - EXPECT_EQ(result.packet.retirement, PacketRetirement::Retire); - EXPECT_EQ(result.packet.retirement_bytes, sizeof(uint32_t)); -} - -TEST(Pm4PacketProcessorTest, RejectsCountDerivedTwoDwordNop) { - Pm4PacketProcessor processor; - const std::array packet = {0xc0001000, 0}; - - const Pm4PacketProcessResult result = processor.process({.available_dwords = packet}); - - EXPECT_EQ(result.packet.status, PacketProcessStatus::Unsupported); - EXPECT_EQ(result.packet.retirement, PacketRetirement::Hold); - EXPECT_EQ(result.packet.retirement_bytes, 0u); - EXPECT_EQ(result.diagnostic, Pm4PacketDiagnostic::UnsupportedPacket); -} - -TEST(Pm4PacketProcessorTest, RequestsTheCompleteRegisterPacketBeforeApplyingItsEffect) { +class FlatMemory final : public AddressSpaceTranslator, public PhysicalMemoryAccess { +public: + FlatMemory() : bytes_(0x4000) {} + + VmTranslationResult translate(uint64_t address, std::size_t size, VmAccessKind) const override { + if (size == 0 || address > bytes_.size() || size > bytes_.size() - address) + return {.outcome = VmAccessOutcome::Faulted, .translation = {}}; + return { + .outcome = VmAccessOutcome::Complete, + .translation = {.domain = VmMemoryDomain::System, + .address = address, + .contiguous_bytes = bytes_.size() - address, + .mtype = Mtype::RW, + .permissions = {.readable = true, .writable = true, .executable = false}}}; + } + + VmAccessOutcome read(VmMemoryDomain, uint64_t address, std::span bytes) override { + if (unavailable_read_ && address == *unavailable_read_) + return VmAccessOutcome::Unavailable; + if (address > bytes_.size() || bytes.size() > bytes_.size() - address) + return VmAccessOutcome::Faulted; + std::ranges::copy_n(bytes_.begin() + static_cast(address), bytes.size(), + bytes.begin()); + return VmAccessOutcome::Complete; + } + + VmAccessOutcome write(VmMemoryDomain, uint64_t address, + std::span bytes) override { + if (address > bytes_.size() || bytes.size() > bytes_.size() - address) + return VmAccessOutcome::Faulted; + std::ranges::copy(bytes, bytes_.begin() + static_cast(address)); + return VmAccessOutcome::Complete; + } + + AtomicLoadResult atomic_load(VmMemoryDomain, uint64_t address, uint32_t width) override { + if ((width != 4 && width != 8) || address % width != 0 || address > bytes_.size() || + width > bytes_.size() - address) + return {.outcome = VmAccessOutcome::Malformed}; + uint64_t value = 0; + std::memcpy(&value, bytes_.data() + address, width); + return {.outcome = VmAccessOutcome::Complete, .value = value}; + } + + VmAccessOutcome atomic_store(VmMemoryDomain, uint64_t address, uint32_t width, + uint64_t value) override { + if ((width != 4 && width != 8) || address % width != 0 || address > bytes_.size() || + width > bytes_.size() - address) + return VmAccessOutcome::Malformed; + if (store_outcome_once_ && address == store_outcome_once_->first) { + const VmAccessOutcome outcome = store_outcome_once_->second; + store_outcome_once_.reset(); + return outcome; + } + if (unavailable_store_once_ && address == *unavailable_store_once_) { + unavailable_store_once_.reset(); + return VmAccessOutcome::Unavailable; + } + std::memcpy(bytes_.data() + address, &value, width); + return VmAccessOutcome::Complete; + } + + AtomicCompareExchangeResult compare_exchange(VmMemoryDomain, uint64_t, uint32_t, uint64_t, + uint64_t) override { + return {.outcome = VmAccessOutcome::Malformed}; + } + + template void store(uint64_t address, T value) { + std::memcpy(bytes_.data() + address, &value, sizeof(value)); + } + + template T load(uint64_t address) const { + T value{}; + std::memcpy(&value, bytes_.data() + address, sizeof(value)); + return value; + } + + void make_read_unavailable(uint64_t address) { unavailable_read_ = address; } + void make_store_unavailable_once(uint64_t address) { unavailable_store_once_ = address; } + void make_store_faulted_once(uint64_t address) { + store_outcome_once_ = std::pair{address, VmAccessOutcome::Faulted}; + } + +private: + std::vector bytes_; + std::optional unavailable_read_; + std::optional unavailable_store_once_; + std::optional> store_outcome_once_; +}; + +enum class Pm4TestStatus { Ready, Blocked, Faulted }; + +class Pm4PacketProcessorTest : public ::testing::Test { +protected: + static constexpr uint64_t kRing = 0x100; + static constexpr uint64_t kReadPointer = 0x80; + static constexpr uint32_t kOutput = 0x800; + + Pm4PacketProcessorTest() : memory(std::make_shared()) { + address_space = vm.register_translated(1, memory, memory); + context.arch = ROCJITSU_CODE_ARCH_RDNA4; + context.retry = [this] { + queue.command_retry_pending = true; + status = Pm4TestStatus::Blocked; + }; + context.fault_queue = [this] { + queue.faulted = true; + status = Pm4TestStatus::Faulted; + }; + context.wake = [] {}; + context.flush_caches = [] {}; + context.dispatch = [this](const std::array &dimensions) { + dispatches.push_back(dimensions); + queue.dispatches.push_entry({}); + }; + } + + void submit(std::span words, bool ring = false, uint64_t initial_cursor = 0, + uint32_t ring_dwords = 64) { + queue.address_space = address_space; + queue.packet_format = QueuePacketFormat::Pm4; + queue.submission_queue = !ring; + queue.read_ptr_va = kReadPointer; + queue.read_pointer_journal = ConsumerCursorJournal(kReadPointer, initial_cursor, 4); + for (size_t word_index = 0; word_index < words.size(); ++word_index) { + const uint64_t offset = ring ? (initial_cursor + word_index) % ring_dwords : word_index; + memory->store(kRing + offset * 4, words[word_index]); + } + Pm4Submission submission; + submission.buffers.push_back({.address = ring ? initial_cursor * 4 : kRing, + .dwords = static_cast(words.size()), + .ring_base = ring ? kRing : 0, + .ring_bytes = ring ? ring_dwords * 4 : 0}); + submission.complete = [this](bool success) { completed = success; }; + queue.commands.submissions.push_back(std::move(submission)); + } + + Pm4TestStatus service() { + // Model CP resuming the queue on its next scheduling turn. + queue.command_retry_pending = false; + status = Pm4TestStatus::Ready; + process_pm4_packets(queue, &vm, context); + return status; + } + + std::shared_ptr memory; + GpuVm vm; + AddressSpaceHandle address_space; + ComputeQueueRecord queue; + Pm4ExecutionContext context; + std::vector> dispatches; + bool completed = false; + Pm4TestStatus status = Pm4TestStatus::Ready; +}; + +TEST_F(Pm4PacketProcessorTest, ProcessesRegisterWriteAcrossNativeRingWrap) { + const std::array words{0xc0017900u, 0x40u, 0xdeadbeefu, 0xffff1000u}; + submit(words, true, 3, 4); uint32_t writes = 0; - const Pm4PacketCallbacks callbacks{.write_uconfig_register = [&](uint64_t, uint32_t) { + queue.packet_callbacks.write_uconfig_register = [&](uint64_t reg, uint32_t value) { + EXPECT_EQ(reg, 0xc040u); + EXPECT_EQ(value, 0xdeadbeefu); ++writes; return Pm4RegisterWriteStatus::Complete; - }}; - Pm4PacketProcessor processor(callbacks); - const std::array header = {0xc0017900}; + }; + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(writes, 1u); + EXPECT_TRUE(completed); + EXPECT_EQ(queue.read_pointer_journal.cursor(), 7u); + EXPECT_EQ(memory->load(kReadPointer), 3u); +} - const Pm4PacketProcessResult result = processor.process({.available_dwords = header}); +TEST_F(Pm4PacketProcessorTest, MaskedPredExecRetainsSkippedCursorAcrossPublicationRetry) { + context.arch = ROCJITSU_CODE_ARCH_CDNA3; + context.xcc_id = 0; + const std::array words{0xc0002300u, (1u << 25) | 5u, 0xc0033700u, 5u << 8, kOutput, 0u, 99u}; + submit(words, true, 6, 8); + memory->make_store_unavailable_once(kReadPointer); + EXPECT_EQ(service(), Pm4TestStatus::Blocked); + EXPECT_EQ(queue.read_pointer_journal.cursor(), 13u); + EXPECT_EQ(memory->load(kOutput), 0u); + EXPECT_TRUE(queue.read_pointer_journal.publication_pending()); + EXPECT_FALSE(completed); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(memory->load(kReadPointer), 5u); + EXPECT_EQ(memory->load(kOutput), 0u); + EXPECT_TRUE(completed); +} - EXPECT_EQ(result.packet.status, PacketProcessStatus::NeedInput); - EXPECT_EQ(result.packet.retirement, PacketRetirement::Hold); - EXPECT_EQ(result.packet.required_bytes, 3 * sizeof(uint32_t)); - EXPECT_EQ(writes, 0u); +TEST_F(Pm4PacketProcessorTest, CondExecRetainsSkippedCursorAcrossPublicationRetry) { + constexpr uint32_t kPredicate = kOutput + 16; + const std::array words{0xc0032200u, kPredicate, 0u, 0u, 5u, + 0xc0033700u, 5u << 8, kOutput, 0u, 99u}; + memory->store(kPredicate, uint32_t{0}); + submit(words, true, 14, 16); + memory->make_store_unavailable_once(kReadPointer); + EXPECT_EQ(service(), Pm4TestStatus::Blocked); + EXPECT_EQ(queue.read_pointer_journal.cursor(), 24u); + EXPECT_EQ(memory->load(kOutput), 0u); + EXPECT_TRUE(queue.read_pointer_journal.publication_pending()); + EXPECT_FALSE(completed); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(memory->load(kReadPointer), 8u); + EXPECT_EQ(memory->load(kOutput), 0u); + EXPECT_TRUE(completed); } -TEST(Pm4PacketProcessorTest, AppliesSupportedRegisterWriteExactlyOnce) { +TEST_F(Pm4PacketProcessorTest, PublishesCommittedEffectBeforeFetchingAnotherPacket) { + const std::array words{0xc0017900u, 0x40u, 7u, 0xffff1000u}; + submit(words, true); uint32_t writes = 0; - uint64_t register_dword = 0; - uint32_t register_value = 0; - const Pm4PacketCallbacks callbacks{.write_uconfig_register = [&](uint64_t reg, uint32_t value) { + queue.packet_callbacks.write_uconfig_register = [&](uint64_t, uint32_t) { ++writes; - register_dword = reg; - register_value = value; return Pm4RegisterWriteStatus::Complete; - }}; - Pm4PacketProcessor processor(callbacks); - const std::array packet = {0xc0017900, 0x40, 0xdeadbeef}; + }; + memory->make_store_unavailable_once(kReadPointer); + EXPECT_EQ(service(), Pm4TestStatus::Blocked); + EXPECT_EQ(writes, 1u); + EXPECT_TRUE(queue.read_pointer_journal.publication_pending()); + EXPECT_FALSE(completed); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(writes, 1u); + EXPECT_EQ(memory->load(kReadPointer), 4u); + EXPECT_TRUE(completed); +} - const Pm4PacketProcessResult result = processor.process({.available_dwords = packet}); +TEST_F(Pm4PacketProcessorTest, RetainsTerminalPublicationFailureForCpCancellation) { + const std::array words{0xc0017900u, 0x40u, 7u}; + submit(words, true); + uint32_t writes = 0; + queue.packet_callbacks.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }; + memory->make_store_faulted_once(kReadPointer); + EXPECT_EQ(service(), Pm4TestStatus::Faulted); + EXPECT_EQ(writes, 1u); + EXPECT_TRUE(queue.publication_faulted); + EXPECT_TRUE(queue.read_pointer_journal.publication_pending()); + EXPECT_FALSE(completed); +} - EXPECT_EQ(result.packet.status, PacketProcessStatus::Complete); - EXPECT_EQ(result.packet.retirement, PacketRetirement::Retire); - EXPECT_EQ(result.packet.retirement_bytes, packet.size() * sizeof(uint32_t)); +TEST_F(Pm4PacketProcessorTest, RetriesRegisterBackpressureWithoutAdvancingThePacket) { + const std::array words{0xc0017900u, 0x40u, 7u}; + submit(words, true); + bool ready = false; + uint32_t writes = 0; + queue.packet_callbacks.write_uconfig_register = [&](uint64_t, uint32_t) { + if (!ready) + return Pm4RegisterWriteStatus::Blocked; + ++writes; + return Pm4RegisterWriteStatus::Complete; + }; + EXPECT_EQ(service(), Pm4TestStatus::Blocked); + EXPECT_EQ(writes, 0u); + EXPECT_EQ(queue.commands.submissions.front().buffers.front().address, 0u); + ready = true; + EXPECT_EQ(service(), Pm4TestStatus::Ready); EXPECT_EQ(writes, 1u); - EXPECT_EQ(register_dword, 0xc040u); - EXPECT_EQ(register_value, 0xdeadbeefu); + EXPECT_EQ(memory->load(kReadPointer), 3u); } -TEST(Pm4PacketProcessorTest, RejectsUndefinedRegisterOffsetBits) { +TEST_F(Pm4PacketProcessorTest, RetriesUnavailableFetchWithoutApplyingAnEffect) { + const std::array words{0xc0017900u, 0x40u, 7u}; + submit(words, true); uint32_t writes = 0; - Pm4PacketProcessor processor({.write_uconfig_register = [&](uint64_t, uint32_t) { + queue.packet_callbacks.write_uconfig_register = [&](uint64_t, uint32_t) { ++writes; return Pm4RegisterWriteStatus::Complete; - }}); - const std::array packet = {0xc0017900, 0x00010040, 0xdeadbeef}; + }; + memory->make_read_unavailable(kRing + 4); + EXPECT_EQ(service(), Pm4TestStatus::Blocked); + EXPECT_EQ(writes, 0u); + EXPECT_EQ(queue.commands.submissions.front().buffers.front().address, 0u); +} - const Pm4PacketProcessResult result = processor.process({.available_dwords = packet}); +TEST_F(Pm4PacketProcessorTest, RejectsTruncatedPacketBeforeCallingItsHandler) { + const std::array words{0xc0017900u, 0x40u}; + submit(words, true); + uint32_t writes = 0; + queue.packet_callbacks.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }; + EXPECT_EQ(service(), Pm4TestStatus::Faulted); + EXPECT_EQ(writes, 0u); + EXPECT_FALSE(completed); +} - EXPECT_EQ(result.packet.status, PacketProcessStatus::Malformed); - EXPECT_EQ(result.packet.retirement, PacketRetirement::Hold); - EXPECT_EQ(result.diagnostic, Pm4PacketDiagnostic::InvalidRegisterOffset); +TEST_F(Pm4PacketProcessorTest, RetiresSupportedNopEncodings) { + const std::array words{0x80000000u, 0xc0001000u, 0u, 0xffff1000u}; + submit(words, true); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(memory->load(kReadPointer), 4u); + EXPECT_TRUE(completed); +} + +TEST_F(Pm4PacketProcessorTest, YieldsAtTheNativePacketBudgetAndResumesWithoutReplay) { + const std::vector words(300, 0xffff1000u); + submit(words, true, 0, 512); + EXPECT_EQ(service(), Pm4TestStatus::Blocked); + EXPECT_EQ(queue.read_pointer_journal.cursor(), 256u); + EXPECT_FALSE(completed); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(queue.read_pointer_journal.cursor(), 300u); + EXPECT_TRUE(completed); +} + +TEST_F(Pm4PacketProcessorTest, RejectsInvalidRegisterOffsetsBeforeApplyingAnEffect) { + const std::array words{0xc0017900u, 0x10040u, 7u}; + submit(words, true); + uint32_t writes = 0; + queue.packet_callbacks.write_uconfig_register = [&](uint64_t, uint32_t) { + ++writes; + return Pm4RegisterWriteStatus::Complete; + }; + EXPECT_EQ(service(), Pm4TestStatus::Faulted); EXPECT_EQ(writes, 0u); } -TEST(Pm4PacketProcessorTest, RejectsUnsupportedPacketWithoutRetiringIt) { - Pm4PacketProcessor processor; - const std::array packet = {0xc0002000}; +TEST_F(Pm4PacketProcessorTest, ReturnsFromAnIndirectBufferWithPersistentRegisterState) { + const std::array child{0xc0017600u, kPm4ComputeNumThreadY, 8u}; + for (size_t word_index = 0; word_index < child.size(); ++word_index) + memory->store(0x400 + word_index * 4, child[word_index]); + const std::array words{0xc0017600u, kPm4ComputeNumThreadX, 4u, 0xc0023f00u, 0x400u, 0u, 3u, + 0xc0017600u, kPm4ComputeNumThreadZ, 2u}; + submit(words); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(queue.commands.sh_registers[kPm4ComputeNumThreadX], 4u); + EXPECT_EQ(queue.commands.sh_registers[kPm4ComputeNumThreadY], 8u); + EXPECT_EQ(queue.commands.sh_registers[kPm4ComputeNumThreadZ], 2u); + EXPECT_TRUE(completed); +} + +TEST_F(Pm4PacketProcessorTest, WaitBlocksLaterMemoryEffectsUntilItsConditionPasses) { + const std::array words{0xc0053c00u, 0x13u, kOutput, 0u, 1u, ~0u, + 0u, 0xc0033700u, 0x100u, kOutput + 4, 0u, 0xdeadbeefu}; + submit(words); + EXPECT_EQ(service(), Pm4TestStatus::Blocked); + EXPECT_EQ(memory->load(kOutput + 4), 0u); + EXPECT_EQ(queue.commands.submissions.front().buffers.front().address, kRing); + memory->store(kOutput, 1); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(memory->load(kOutput + 4), 0xdeadbeefu); + EXPECT_TRUE(completed); +} - const Pm4PacketProcessResult result = processor.process({.available_dwords = packet}); +TEST_F(Pm4PacketProcessorTest, DispatchPausesInterpretationUntilCpRetiresIt) { + const std::array words{0xc0031500u, 2u, 3u, 4u, 1u, + 0xc0033700u, 0x100u, kOutput, 0u, 0xdeadbeefu}; + submit(words); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + ASSERT_EQ(dispatches.size(), 1u); + EXPECT_EQ(dispatches.front(), (std::array{2, 3, 4, 1})); + EXPECT_EQ(memory->load(kOutput), 0u); + EXPECT_FALSE(completed); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(memory->load(kOutput), 0u); + queue.dispatches.entries.clear(); + EXPECT_EQ(service(), Pm4TestStatus::Ready); + EXPECT_EQ(memory->load(kOutput), 0xdeadbeefu); + EXPECT_TRUE(completed); +} - EXPECT_EQ(result.packet.status, PacketProcessStatus::Unsupported); - EXPECT_EQ(result.packet.retirement, PacketRetirement::Hold); - EXPECT_EQ(result.diagnostic, Pm4PacketDiagnostic::UnsupportedPacket); +TEST_F(Pm4PacketProcessorTest, QueueCancellationKeepsThePacketExceptionScope) { + const std::array words{0xc0017900u, 0x40u, 7u}; + submit(words, true); + queue.packet_callbacks.write_uconfig_register = [](uint64_t, uint32_t) { + return Pm4RegisterWriteStatus::Rejected; + }; + uint32_t cancellations = 0; + context.fault_queue = [&] { + if (++cancellations == 1) + throw std::runtime_error("cancellation callback failed once"); + }; + EXPECT_NO_THROW(service()); + EXPECT_EQ(cancellations, 2u); + EXPECT_FALSE(completed); } -TEST(Pm4PacketProcessorTest, PreservesRetryableRegisterBackpressureWithoutRetirement) { - uint32_t attempts = 0; - const Pm4PacketCallbacks callbacks{.write_uconfig_register = [&](uint64_t, uint32_t) { - ++attempts; - return Pm4RegisterWriteStatus::Blocked; - }}; - Pm4PacketProcessor processor(callbacks); - const std::array packet = {0xc0017900, 0x40, 0xdeadbeef}; +TEST_F(Pm4PacketProcessorTest, AlreadyFailedSubmissionCancellationRemainsOutsideTheTryBlock) { + const std::array words{0xffff1000u}; + submit(words); + queue.commands.submissions.front().failure->failed.store(true); + uint32_t cancellations = 0; + context.fault_queue = [&] { + ++cancellations; + throw std::runtime_error("cancellation callback failed"); + }; + EXPECT_THROW(service(), std::runtime_error); + EXPECT_EQ(cancellations, 1u); +} - const Pm4PacketProcessResult result = processor.process({.available_dwords = packet}); +TEST_F(Pm4PacketProcessorTest, CancellationFromTheCatchBlockIsNotCaughtAgain) { + const std::array words{0xc000ff00u, 0u}; + submit(words); + uint32_t cancellations = 0; + context.fault_queue = [&] { + ++cancellations; + throw std::runtime_error("cancellation callback failed"); + }; + EXPECT_THROW(service(), std::runtime_error); + EXPECT_EQ(cancellations, 1u); +} - EXPECT_EQ(result.packet.status, PacketProcessStatus::Blocked); - EXPECT_EQ(result.packet.retirement, PacketRetirement::Hold); - EXPECT_EQ(attempts, 1u); +TEST(Pm4ProducerCursorTest, NormalizesWrappedPointersAndRejectsOverruns) { + EXPECT_EQ(normalize_pm4_producer_cursor(2, 3, 4), 6u); + EXPECT_EQ(normalize_pm4_producer_cursor(6, 3, 4), 6u); + EXPECT_FALSE(normalize_pm4_producer_cursor(8, 3, 4)); + EXPECT_FALSE(normalize_pm4_producer_cursor(0, 0, 0)); + EXPECT_FALSE(normalize_pm4_producer_cursor(0, UINT64_MAX, 4)); } } // namespace diff --git a/emulation/rocjitsu/tests/pm4_queue_controller_test.cpp b/emulation/rocjitsu/tests/pm4_queue_controller_test.cpp deleted file mode 100644 index 612ac337f3d9..000000000000 --- a/emulation/rocjitsu/tests/pm4_queue_controller_test.cpp +++ /dev/null @@ -1,662 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -#include "rocjitsu/vm/amdgpu/command_processor.h" -#include "rocjitsu/vm/amdgpu/gpu_vm.h" -#include "rocjitsu/vm/amdgpu/pm4/pm4_queue_controller.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace rocjitsu::amdgpu { -namespace { - -class Pm4QueueMemory final : public AddressSpaceTranslator, public PhysicalMemoryAccess { -public: - Pm4QueueMemory() : bytes_(0x1000) {} - - VmTranslationResult translate(uint64_t address, std::size_t size, VmAccessKind) const override { - if (size == 0 || address > bytes_.size() || size > bytes_.size() - address) - return {.outcome = VmAccessOutcome::Faulted, .translation = {}}; - return { - .outcome = VmAccessOutcome::Complete, - .translation = {.domain = VmMemoryDomain::System, - .address = address, - .contiguous_bytes = bytes_.size() - address, - .mtype = Mtype::RW, - .permissions = {.readable = true, .writable = true, .executable = false}}}; - } - - VmAccessOutcome read(VmMemoryDomain, uint64_t address, std::span bytes) override { - if (unavailable_read_ && address == *unavailable_read_) - return VmAccessOutcome::Unavailable; - if (address > bytes_.size() || bytes.size() > bytes_.size() - address) - return VmAccessOutcome::Faulted; - std::ranges::copy_n(bytes_.begin() + static_cast(address), bytes.size(), - bytes.begin()); - return VmAccessOutcome::Complete; - } - - VmAccessOutcome write(VmMemoryDomain, uint64_t address, - std::span bytes) override { - if (address > bytes_.size() || bytes.size() > bytes_.size() - address) - return VmAccessOutcome::Faulted; - std::ranges::copy(bytes, bytes_.begin() + static_cast(address)); - return VmAccessOutcome::Complete; - } - - AtomicLoadResult atomic_load(VmMemoryDomain, uint64_t address, uint32_t width) override { - if ((width != 4 && width != 8) || address % width != 0 || address > bytes_.size() || - width > bytes_.size() - address) { - return {.outcome = VmAccessOutcome::Malformed}; - } - uint64_t value = 0; - std::memcpy(&value, bytes_.data() + address, width); - return {.outcome = VmAccessOutcome::Complete, .value = value}; - } - - VmAccessOutcome atomic_store(VmMemoryDomain, uint64_t address, uint32_t width, - uint64_t value) override { - if (unavailable_store_ && address == *unavailable_store_) - return VmAccessOutcome::Unavailable; - if (faulted_store_ && address == *faulted_store_) - return VmAccessOutcome::Faulted; - if ((width != 4 && width != 8) || address % width != 0 || address > bytes_.size() || - width > bytes_.size() - address) { - return VmAccessOutcome::Malformed; - } - std::memcpy(bytes_.data() + address, &value, width); - return VmAccessOutcome::Complete; - } - - AtomicCompareExchangeResult compare_exchange(VmMemoryDomain, uint64_t, uint32_t, uint64_t, - uint64_t) override { - return {.outcome = VmAccessOutcome::Malformed}; - } - - template void store(uint64_t address, T value) { - std::memcpy(bytes_.data() + address, &value, sizeof(value)); - } - - template T load(uint64_t address) const { - T value{}; - std::memcpy(&value, bytes_.data() + address, sizeof(value)); - return value; - } - - void make_read_unavailable(uint64_t address) { unavailable_read_ = address; } - void make_store_unavailable(uint64_t address) { unavailable_store_ = address; } - void make_store_faulted(uint64_t address) { faulted_store_ = address; } - void make_available() { - unavailable_read_.reset(); - unavailable_store_.reset(); - faulted_store_.reset(); - } - -private: - std::vector bytes_; - std::optional unavailable_read_; - std::optional unavailable_store_; - std::optional faulted_store_; -}; - -class Pm4QueueControllerTest : public ::testing::Test { -protected: - void SetUp() override { - memory = std::make_shared(); - address_space = gpu_vm.register_translated(1, memory, memory); - ASSERT_TRUE(address_space); - memory->store(kReadPointer, 0); - } - - Pm4QueueController::RegistrationId attach(Pm4PacketCallbacks callbacks, uint64_t ring = kRing, - uint64_t read_pointer = kReadPointer, - uint32_t ring_bytes = kRingBytes) { - return controller.attach({.address_space = address_space, - .ring_base = ring, - .ring_size_bytes = ring_bytes, - .consumer_pointer_address = read_pointer, - .initial_consumer_cursor = std::nullopt, - .packet_callbacks = std::move(callbacks)}); - } - - static constexpr uint64_t kRing = 0x100; - static constexpr uint32_t kRingBytes = 64; - static constexpr uint64_t kReadPointer = 0x200; - - GpuVm gpu_vm; - std::shared_ptr memory; - AddressSpaceHandle address_space; - Pm4QueueController controller{gpu_vm}; -}; - -TEST_F(Pm4QueueControllerTest, SubmissionDefersEffectsUntilTheCpServicesTheQueue) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - uint32_t writes = 0; - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - ++writes; - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - - EXPECT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - EXPECT_EQ(writes, 0u); - EXPECT_EQ(memory->load(kReadPointer), 0u); - - EXPECT_FALSE(controller.service()); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 3u); -} - -TEST_F(Pm4QueueControllerTest, BoundsEachQueueServiceTurn) { - constexpr uint64_t kLargeRing = 0x400; - constexpr uint64_t kLargeReadPointer = 0xe00; - constexpr uint32_t kPacketCount = 257; - constexpr uint32_t kLargeRingBytes = 512 * sizeof(uint32_t); - for (uint32_t packet = 0; packet < kPacketCount; ++packet) - memory->store(kLargeRing + packet * sizeof(uint32_t), 0xffff1000); - memory->store(kLargeReadPointer, 0); - - const auto registration = attach({}, kLargeRing, kLargeReadPointer, kLargeRingBytes); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, kPacketCount), QueueSubmissionStatus::Accepted); - - EXPECT_TRUE(controller.service()); - EXPECT_EQ(memory->load(kLargeReadPointer), 256u); - - EXPECT_FALSE(controller.service()); - EXPECT_EQ(memory->load(kLargeReadPointer), kPacketCount); -} - -TEST_F(Pm4QueueControllerTest, RetainsItsAddressSpaceUntilDetach) { - const Pm4QueueController::RegistrationId registration = attach({}); - ASSERT_NE(registration, 0u); - - EXPECT_FALSE(gpu_vm.unregister_address_space(address_space)); - EXPECT_TRUE(controller.detach(registration)); - EXPECT_TRUE(gpu_vm.unregister_address_space(address_space)); -} - -TEST_F(Pm4QueueControllerTest, CursorPublicationRetryDoesNotReplayThePacketEffect) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - uint32_t writes = 0; - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - ++writes; - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - memory->make_store_unavailable(kReadPointer); - - EXPECT_TRUE(controller.service()); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 0u); - - memory->make_available(); - EXPECT_FALSE(controller.service()); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 3u); -} - -TEST_F(Pm4QueueControllerTest, RootReplacementDoesNotMovePendingCursorPublication) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - uint32_t writes = 0; - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - ++writes; - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - memory->make_store_unavailable(kReadPointer); - - EXPECT_TRUE(controller.service()); - EXPECT_EQ(writes, 1u); - - auto replacement = std::make_shared(); - replacement->store(kReadPointer, 99); - ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); - memory->make_available(); - - EXPECT_FALSE(controller.service()); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 3u); - EXPECT_EQ(replacement->load(kReadPointer), 99u); -} - -TEST_F(Pm4QueueControllerTest, RootReplacementDoesNotMoveBlockedPacketFetch) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - uint32_t written_value = 0; - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t value) { - written_value = value; - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - memory->make_read_unavailable(kRing); - - EXPECT_TRUE(controller.service()); - EXPECT_EQ(written_value, 0u); - - auto replacement = std::make_shared(); - replacement->store(kRing, 0xc0017900); - replacement->store(kRing + 4, 0x40); - replacement->store(kRing + 8, 0x12345678); - replacement->store(kReadPointer, 99); - ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); - memory->make_available(); - - EXPECT_FALSE(controller.service()); - EXPECT_EQ(written_value, 0xdeadbeefu); - EXPECT_EQ(memory->load(kReadPointer), 3u); - EXPECT_EQ(replacement->load(kReadPointer), 99u); -} - -TEST_F(Pm4QueueControllerTest, LaterDoorbellUsesFreshSnapshotAfterBlockedTransaction) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0x11111111); - std::vector values; - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t value) { - values.push_back(value); - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - memory->make_store_unavailable(kReadPointer); - - EXPECT_TRUE(controller.service()); - ASSERT_EQ(values.size(), 1u); - EXPECT_EQ(values[0], 0x11111111u); - - auto replacement = std::make_shared(); - replacement->store(kRing + 12, 0xc0017900); - replacement->store(kRing + 16, 0x40); - replacement->store(kRing + 20, 0x22222222); - replacement->store(kReadPointer, 3); - ASSERT_TRUE(gpu_vm.replace_translated(address_space, replacement, replacement)); - ASSERT_EQ(controller.notify(registration, 6), QueueSubmissionStatus::Accepted); - memory->make_available(); - - EXPECT_TRUE(controller.service()); - ASSERT_EQ(values.size(), 1u); - EXPECT_EQ(memory->load(kReadPointer), 3u); - EXPECT_EQ(replacement->load(kReadPointer), 3u); - - EXPECT_FALSE(controller.service()); - ASSERT_EQ(values.size(), 2u); - EXPECT_EQ(values[1], 0x22222222u); - EXPECT_EQ(replacement->load(kReadPointer), 6u); -} - -TEST_F(Pm4QueueControllerTest, PublishesEarlierProgressBeforeRetryingALaterFetch) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - memory->store(kRing + 12, 0xffff1000); - uint32_t writes = 0; - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - ++writes; - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 4), QueueSubmissionStatus::Accepted); - memory->make_read_unavailable(kRing + 12); - - EXPECT_TRUE(controller.service()); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 3u); - - memory->make_available(); - EXPECT_FALSE(controller.service()); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 4u); -} - -TEST_F(Pm4QueueControllerTest, RejectsInvalidRingsAtRegistration) { - EXPECT_EQ(controller.attach({.address_space = address_space, - .ring_base = kRing + 1, - .ring_size_bytes = kRingBytes, - .consumer_pointer_address = kReadPointer, - .initial_consumer_cursor = std::nullopt, - .packet_callbacks = {}}), - 0u); - EXPECT_EQ(controller.attach({.address_space = address_space, - .ring_base = kRing, - .ring_size_bytes = kRingBytes - 1, - .consumer_pointer_address = kReadPointer, - .initial_consumer_cursor = std::nullopt, - .packet_callbacks = {}}), - 0u); - EXPECT_EQ(controller.attach({.address_space = address_space, - .ring_base = kRing, - .ring_size_bytes = kRingBytes, - .consumer_pointer_address = kReadPointer + sizeof(uint32_t), - .initial_consumer_cursor = std::nullopt, - .packet_callbacks = {}}), - 0u); -} - -TEST_F(Pm4QueueControllerTest, NormalizesWrappedNativeProducerBeforeRetainingProgress) { - constexpr uint32_t kRingDwords = 4; - memory->store(kReadPointer, 3); - memory->store(kRing + 3 * sizeof(uint32_t), 0xc0017900); - memory->store(kRing, 0x40); - memory->store(kRing + sizeof(uint32_t), 0xdeadbeef); - - uint32_t writes = 0; - const auto registration = controller.attach({ - .address_space = address_space, - .ring_base = kRing, - .ring_size_bytes = kRingDwords * sizeof(uint32_t), - .consumer_pointer_address = kReadPointer, - .initial_consumer_cursor = 3, - .packet_callbacks = {.write_uconfig_register = - [&](uint64_t, uint32_t) { - ++writes; - return Pm4RegisterWriteStatus::Complete; - }}, - }); - ASSERT_NE(registration, 0u); - - EXPECT_EQ(controller.notify(registration, 2), QueueSubmissionStatus::Accepted); - EXPECT_FALSE(controller.service()); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 2u); - EXPECT_FALSE(controller.service()) << "the consumed wrapped cursor remained spuriously ready"; - EXPECT_EQ(writes, 1u); -} - -TEST_F(Pm4QueueControllerTest, BlockedQueuesRetryIndependently) { - constexpr uint64_t kOtherRing = 0x300; - constexpr uint64_t kOtherReadPointer = 0x380; - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 1); - memory->store(kOtherRing, 0xc0017900); - memory->store(kOtherRing + 4, 0x40); - memory->store(kOtherRing + 8, 2); - memory->store(kOtherReadPointer, 0); - - bool first_blocked = true; - bool second_blocked = true; - uint32_t first_attempts = 0; - uint32_t second_attempts = 0; - const auto first = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - ++first_attempts; - return first_blocked ? Pm4RegisterWriteStatus::Blocked : Pm4RegisterWriteStatus::Complete; - }}); - const auto second = attach({.write_uconfig_register = - [&](uint64_t, uint32_t) { - ++second_attempts; - return second_blocked ? Pm4RegisterWriteStatus::Blocked - : Pm4RegisterWriteStatus::Complete; - }}, - kOtherRing, kOtherReadPointer); - ASSERT_NE(first, 0u); - ASSERT_NE(second, 0u); - ASSERT_EQ(controller.notify(first, 3), QueueSubmissionStatus::Accepted); - ASSERT_EQ(controller.notify(second, 3), QueueSubmissionStatus::Accepted); - - EXPECT_TRUE(controller.service()); - EXPECT_EQ(first_attempts, 1u); - EXPECT_EQ(second_attempts, 1u); - EXPECT_EQ(memory->load(kReadPointer), 0u); - EXPECT_EQ(memory->load(kOtherReadPointer), 0u); - - first_blocked = false; - second_blocked = false; - EXPECT_FALSE(controller.service()); - EXPECT_EQ(first_attempts, 2u); - EXPECT_EQ(second_attempts, 2u); - EXPECT_EQ(memory->load(kReadPointer), 3u); - EXPECT_EQ(memory->load(kOtherReadPointer), 3u); -} - -TEST_F(Pm4QueueControllerTest, DetachWaitsForActiveServiceWithoutHoldingControllerMutex) { - using namespace std::chrono_literals; - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - std::promise callback_entered; - std::promise release_callback; - std::shared_future release = release_callback.get_future().share(); - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - callback_entered.set_value(); - release.wait(); - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - - auto service = std::async(std::launch::async, [&] { return controller.service(); }); - ASSERT_EQ(callback_entered.get_future().wait_for(1s), std::future_status::ready); - auto detach = std::async(std::launch::async, [&] { return controller.detach(registration); }); - - EXPECT_EQ(detach.wait_for(20ms), std::future_status::timeout); - EXPECT_EQ(controller.active_queues(), 1u); - release_callback.set_value(); - EXPECT_FALSE(service.get()); - EXPECT_TRUE(detach.get()); -} - -TEST_F(Pm4QueueControllerTest, ReconfigureWaitsForActiveService) { - using namespace std::chrono_literals; - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - std::promise callback_entered; - std::promise release_callback; - std::shared_future release = release_callback.get_future().share(); - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - callback_entered.set_value(); - release.wait(); - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - - auto service = std::async(std::launch::async, [&] { return controller.service(); }); - ASSERT_EQ(callback_entered.get_future().wait_for(1s), std::future_status::ready); - auto reconfigure = std::async(std::launch::async, [&] { - return controller.update( - registration, - {.ring_base_address = kRing, .ring_size_bytes = kRingBytes, .scheduling_percentage = 100}); - }); - - EXPECT_EQ(reconfigure.wait_for(20ms), std::future_status::timeout); - release_callback.set_value(); - EXPECT_FALSE(service.get()); - EXPECT_EQ(reconfigure.get(), QueueReconfigureStatus::Applied); -} - -TEST_F(Pm4QueueControllerTest, GracefulDetachWaitsForPendingCursorPublication) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - uint32_t writes = 0; - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - ++writes; - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - memory->make_store_unavailable(kReadPointer); - - EXPECT_TRUE(controller.service()); - EXPECT_EQ(controller.prepare_detach(registration), QueuePrepareCloseStatus::Busy); - EXPECT_EQ(controller.active_queues(), 1u); - EXPECT_EQ(writes, 1u); - - memory->make_available(); - EXPECT_FALSE(controller.service()); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 3u); - EXPECT_EQ(controller.prepare_detach(registration), QueuePrepareCloseStatus::Ready); - EXPECT_EQ(controller.active_queues(), 0u); -} - -TEST_F(Pm4QueueControllerTest, GracefulDetachCancelsBlockedPacketBeforeAnyEffect) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - uint32_t attempts = 0; - const auto registration = attach({.write_uconfig_register = [&](uint64_t, uint32_t) { - ++attempts; - return Pm4RegisterWriteStatus::Blocked; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - - EXPECT_TRUE(controller.service()); - EXPECT_EQ(attempts, 1u); - EXPECT_EQ(controller.prepare_detach(registration), QueuePrepareCloseStatus::Ready); - EXPECT_EQ(controller.active_queues(), 0u); -} - -TEST_F(Pm4QueueControllerTest, GracefulDetachReportsFaultedPublicationWithoutRetryingForever) { - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - const auto registration = attach({.write_uconfig_register = [](uint64_t, uint32_t) { - return Pm4RegisterWriteStatus::Complete; - }}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 3), QueueSubmissionStatus::Accepted); - memory->make_store_faulted(kReadPointer); - - EXPECT_FALSE(controller.service()); - EXPECT_EQ(controller.prepare_detach(registration), QueuePrepareCloseStatus::Faulted); - EXPECT_EQ(controller.active_queues(), 1u); - EXPECT_TRUE(controller.detach(registration)); - EXPECT_EQ(controller.active_queues(), 0u); -} - -TEST_F(Pm4QueueControllerTest, TerminalQueueCanBeDisabledAndReconfigured) { - constexpr uint64_t kReplacementRing = 0x300; - memory->store(kRing, 0xffff1000); - const auto registration = attach({}); - ASSERT_NE(registration, 0u); - ASSERT_EQ(controller.notify(registration, 1), QueueSubmissionStatus::Accepted); - EXPECT_FALSE(controller.service()); - - EXPECT_EQ( - controller.update(registration, - {.ring_base_address = 0, .ring_size_bytes = 0, .scheduling_percentage = 0}), - QueueReconfigureStatus::Disabled); - memory->store(kReplacementRing, 0xffff1000); - EXPECT_EQ(controller.update(registration, {.ring_base_address = kReplacementRing, - .ring_size_bytes = kRingBytes, - .scheduling_percentage = 100}), - QueueReconfigureStatus::Applied); - EXPECT_EQ(controller.notify(registration, 1), QueueSubmissionStatus::Accepted); - EXPECT_FALSE(controller.service()); - EXPECT_EQ(controller.prepare_detach(registration), QueuePrepareCloseStatus::Ready); -} - -TEST(Pm4QueueBindingTest, RejectsPollingModesWithoutPollingSupport) { - constexpr uint64_t kRing = 0x100; - constexpr uint64_t kReadPointer = 0x200; - constexpr uint64_t kWritePointer = 0x208; - alignas(8) std::array doorbell{}; - auto memory = std::make_shared(); - GpuVm gpu_vm; - const AddressSpaceHandle address_space = gpu_vm.register_translated(1, memory, memory); - ASSERT_TRUE(address_space); - CommandProcessor command_processor("cp"); - command_processor.set_gpu_vm(&gpu_vm); - GpuQueueRegistry registry(gpu_vm); - const std::shared_ptr binding_factory = - command_processor.make_pm4_queue_binding_factory({}); - - for (const QueueDoorbellMode mode : - {QueueDoorbellMode::HostPolled, QueueDoorbellMode::VmPolled}) { - const QueueHandle queue = registry.register_queue({ - .identity = {.address_space = address_space, .process_id = 1, .queue_id = 7}, - .ring = {.base_address = kRing, - .size_bytes = 64, - .consumer_pointer_address = kReadPointer, - .producer_pointer_address = kWritePointer}, - .doorbell = {.mode = mode, .host_base = doorbell.data(), .address = 0x300}, - .binding_factory = binding_factory, - .type = QueueType::Compute, - .packet_format = QueuePacketFormat::Pm4, - }); - EXPECT_FALSE(queue); - EXPECT_EQ(registry.active_queues(), 0u); - EXPECT_EQ(command_processor.registered_pm4_queue_count_for_test(), 0u); - } - - EXPECT_TRUE(gpu_vm.unregister_address_space(address_space)); -} - -TEST(Pm4QueueBindingTest, GracefulRegistryRemovalKeepsHandleUntilPublicationCompletes) { - constexpr uint64_t kRing = 0x100; - constexpr uint64_t kReadPointer = 0x200; - auto memory = std::make_shared(); - GpuVm gpu_vm; - const AddressSpaceHandle address_space = gpu_vm.register_translated(1, memory, memory); - ASSERT_TRUE(address_space); - CommandProcessor command_processor("cp"); - command_processor.set_gpu_vm(&gpu_vm); - GpuQueueRegistry registry(gpu_vm); - uint32_t writes = 0; - const std::shared_ptr binding_factory = - command_processor.make_pm4_queue_binding_factory( - {.write_uconfig_register = [&](uint64_t, uint32_t) { - ++writes; - return Pm4RegisterWriteStatus::Complete; - }}); - const QueueHandle queue = registry.register_queue({ - .identity = {.address_space = address_space, .process_id = 1, .queue_id = 7}, - .ring = {.base_address = kRing, .size_bytes = 64, .consumer_pointer_address = kReadPointer}, - .doorbell = {}, - .binding_factory = binding_factory, - .type = QueueType::Compute, - .packet_format = QueuePacketFormat::Pm4, - }); - ASSERT_TRUE(queue); - memory->store(kRing, 0xc0017900); - memory->store(kRing + 4, 0x40); - memory->store(kRing + 8, 0xdeadbeef); - memory->make_store_unavailable(kReadPointer); - - const QueueSubmissionResult blocked = registry.submit_producer(queue, 3); - ASSERT_TRUE(blocked.found); - EXPECT_EQ(blocked.status, QueueSubmissionStatus::Retry); - EXPECT_EQ(writes, 1u); - EXPECT_FALSE(registry.unregister_queue(queue)); - EXPECT_TRUE(registry.contains(queue)); - - memory->make_available(); - const QueueSubmissionResult completed = registry.submit_producer(queue, 3); - ASSERT_TRUE(completed.found); - EXPECT_EQ(completed.status, QueueSubmissionStatus::Accepted); - EXPECT_EQ(writes, 1u); - EXPECT_EQ(memory->load(kReadPointer), 3u); - EXPECT_TRUE(registry.unregister_queue(queue)); - EXPECT_FALSE(registry.contains(queue)); - EXPECT_TRUE(gpu_vm.unregister_address_space(address_space)); -} - -} // namespace -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/tests/pm4_ring_consumer_test.cpp b/emulation/rocjitsu/tests/pm4_ring_consumer_test.cpp deleted file mode 100644 index 67a26c96054d..000000000000 --- a/emulation/rocjitsu/tests/pm4_ring_consumer_test.cpp +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -#include "rocjitsu/vm/amdgpu/pm4/pm4_ring_consumer.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace rocjitsu::amdgpu { -namespace { - -class FlatMemory final : public AddressSpaceTranslator, public PhysicalMemoryAccess { -public: - FlatMemory() : bytes_(0x4000) {} - - VmTranslationResult translate(uint64_t address, std::size_t size, VmAccessKind) const override { - if (size == 0 || address > bytes_.size() || size > bytes_.size() - address) - return {.outcome = VmAccessOutcome::Faulted, .translation = {}}; - return { - .outcome = VmAccessOutcome::Complete, - .translation = {.domain = VmMemoryDomain::System, - .address = address, - .contiguous_bytes = bytes_.size() - address, - .mtype = Mtype::RW, - .permissions = {.readable = true, .writable = true, .executable = false}}}; - } - - VmAccessOutcome read(VmMemoryDomain, uint64_t address, std::span bytes) override { - if (unavailable_read_ && address == *unavailable_read_) - return VmAccessOutcome::Unavailable; - if (address > bytes_.size() || bytes.size() > bytes_.size() - address) - return VmAccessOutcome::Faulted; - std::ranges::copy_n(bytes_.begin() + static_cast(address), bytes.size(), - bytes.begin()); - return VmAccessOutcome::Complete; - } - - VmAccessOutcome write(VmMemoryDomain, uint64_t address, - std::span bytes) override { - if (address > bytes_.size() || bytes.size() > bytes_.size() - address) - return VmAccessOutcome::Faulted; - std::ranges::copy(bytes, bytes_.begin() + static_cast(address)); - return VmAccessOutcome::Complete; - } - - AtomicLoadResult atomic_load(VmMemoryDomain, uint64_t address, uint32_t width) override { - if ((width != 4 && width != 8) || address % width != 0 || address > bytes_.size() || - width > bytes_.size() - address) - return {.outcome = VmAccessOutcome::Malformed}; - uint64_t value = 0; - std::memcpy(&value, bytes_.data() + address, width); - return {.outcome = VmAccessOutcome::Complete, .value = value}; - } - - VmAccessOutcome atomic_store(VmMemoryDomain, uint64_t address, uint32_t width, - uint64_t value) override { - if ((width != 4 && width != 8) || address % width != 0 || address > bytes_.size() || - width > bytes_.size() - address) - return VmAccessOutcome::Malformed; - if (store_outcome_once_ && address == store_outcome_once_->first) { - const VmAccessOutcome outcome = store_outcome_once_->second; - store_outcome_once_.reset(); - return outcome; - } - if (unavailable_store_once_ && address == *unavailable_store_once_) { - unavailable_store_once_.reset(); - return VmAccessOutcome::Unavailable; - } - std::memcpy(bytes_.data() + address, &value, width); - return VmAccessOutcome::Complete; - } - - AtomicCompareExchangeResult compare_exchange(VmMemoryDomain, uint64_t, uint32_t, uint64_t, - uint64_t) override { - return {.outcome = VmAccessOutcome::Malformed}; - } - - template void store(uint64_t address, T value) { - std::memcpy(bytes_.data() + address, &value, sizeof(value)); - } - - template T load(uint64_t address) const { - T value{}; - std::memcpy(&value, bytes_.data() + address, sizeof(value)); - return value; - } - - void make_read_unavailable(uint64_t address) { unavailable_read_ = address; } - void make_store_unavailable_once(uint64_t address) { unavailable_store_once_ = address; } - void make_store_faulted_once(uint64_t address) { - store_outcome_once_ = std::pair{address, VmAccessOutcome::Faulted}; - } - -private: - std::vector bytes_; - std::optional unavailable_read_; - std::optional unavailable_store_once_; - std::optional> store_outcome_once_; -}; - -class RingConsumerFixture { -public: - RingConsumerFixture() : memory(std::make_shared()) { - handle = vm.register_translated(1, memory, memory); - access = vm.snapshot(handle); - } - - std::shared_ptr memory; - GpuVm vm; - AddressSpaceHandle handle; - std::optional access; -}; - -TEST(Pm4RingConsumerTest, ProcessesTheBoundedSupportedSubsetAcrossRingWrap) { - RingConsumerFixture fixture; - ASSERT_TRUE(fixture.access); - constexpr uint64_t kRing = 0x100; - constexpr uint64_t kRingDwords = 4; - fixture.memory->store(kRing + 3 * sizeof(uint32_t), 0xc0017900); - fixture.memory->store(kRing + 0 * sizeof(uint32_t), 0x40); - fixture.memory->store(kRing + 1 * sizeof(uint32_t), 0xdeadbeef); - fixture.memory->store(kRing + 2 * sizeof(uint32_t), 0xffff1000); - - uint64_t register_dword = 0; - uint32_t register_value = 0; - Pm4PacketProcessor packet_processor({.write_uconfig_register = [&](uint64_t reg, uint32_t value) { - register_dword = reg; - register_value = value; - return Pm4RegisterWriteStatus::Complete; - }}); - Pm4RingConsumer consumer(packet_processor, fixture.vm, fixture.handle, kRing, - kRingDwords * sizeof(uint32_t), 0x80, 3); - const Pm4RingResult result = consumer.consume(7); - - EXPECT_EQ(result.status, Pm4RingStatus::Complete); - EXPECT_EQ(result.read_pointer, 7u); - EXPECT_EQ(fixture.memory->load(0x80), 3u); - EXPECT_EQ(register_dword, 0xc040u); - EXPECT_EQ(register_value, 0xdeadbeefu); -} - -TEST(Pm4RingConsumerTest, StopsAtThePacketBudgetAndResumesAtTheNextPacket) { - RingConsumerFixture fixture; - ASSERT_TRUE(fixture.access); - constexpr uint64_t kRing = 0x100; - constexpr uint64_t kReadPointer = 0x80; - fixture.memory->store(kRing, 0xffff1000); - fixture.memory->store(kRing + sizeof(uint32_t), 0xffff1000); - - Pm4PacketProcessor packet_processor; - Pm4RingConsumer consumer(packet_processor, fixture.vm, fixture.handle, kRing, - 4 * sizeof(uint32_t), kReadPointer, 0); - - const Pm4RingResult first = consumer.consume(2, 1); - EXPECT_EQ(first.status, Pm4RingStatus::Complete); - EXPECT_EQ(first.read_pointer, 1u); - EXPECT_EQ(fixture.memory->load(kReadPointer), 1u); - - const Pm4RingResult second = consumer.consume(2, 1); - EXPECT_EQ(second.status, Pm4RingStatus::Complete); - EXPECT_EQ(second.read_pointer, 2u); - EXPECT_EQ(fixture.memory->load(kReadPointer), 2u); -} - -TEST(Pm4RingConsumerTest, LeavesTheUnsupportedPacketAtTheReadPointer) { - RingConsumerFixture fixture; - ASSERT_TRUE(fixture.access); - constexpr uint64_t kRing = 0x100; - fixture.memory->store(kRing, 0xc0002000); - - Pm4PacketProcessor packet_processor; - Pm4RingConsumer consumer(packet_processor, fixture.vm, fixture.handle, kRing, - 4 * sizeof(uint32_t), 0x80, 0); - const Pm4RingResult result = consumer.consume(1); - - EXPECT_EQ(result.status, Pm4RingStatus::UnsupportedPacket); - EXPECT_EQ(result.read_pointer, 0u); - EXPECT_EQ(result.packet_header, 0xc0002000u); -} - -TEST(Pm4RingConsumerTest, PreservesUnavailableAsARetryableOutcome) { - RingConsumerFixture fixture; - ASSERT_TRUE(fixture.access); - constexpr uint64_t kRing = 0x100; - fixture.memory->make_read_unavailable(kRing); - - Pm4PacketProcessor packet_processor; - Pm4RingConsumer consumer(packet_processor, fixture.vm, fixture.handle, kRing, - 4 * sizeof(uint32_t), 0x80, 0); - const Pm4RingResult result = consumer.consume(1); - - EXPECT_EQ(result.status, Pm4RingStatus::Blocked); - EXPECT_EQ(result.read_pointer, 0u); -} - -TEST(Pm4RingConsumerTest, RejectsARingWhoseAddressRangeWraps) { - RingConsumerFixture fixture; - ASSERT_TRUE(fixture.access); - - Pm4PacketProcessor packet_processor; - Pm4RingConsumer consumer(packet_processor, fixture.vm, fixture.handle, UINT64_MAX - 7, - 4 * sizeof(uint32_t), 0x80, 0); - const Pm4RingResult result = consumer.consume(1); - - EXPECT_EQ(result.status, Pm4RingStatus::Malformed); - EXPECT_EQ(result.read_pointer, 0u); -} - -TEST(Pm4RingConsumerTest, PreservesTerminalPacketAcrossCursorPublicationRetry) { - RingConsumerFixture fixture; - ASSERT_TRUE(fixture.access); - constexpr uint64_t kRing = 0x100; - constexpr uint64_t kReadPointer = 0x80; - constexpr uint32_t kUnsupportedHeader = 0xc0002000; - fixture.memory->store(kRing, 0xc0017900); - fixture.memory->store(kRing + sizeof(uint32_t), 0x40); - fixture.memory->store(kRing + 2 * sizeof(uint32_t), 0xdeadbeef); - fixture.memory->store(kRing + 3 * sizeof(uint32_t), kUnsupportedHeader); - fixture.memory->make_store_unavailable_once(kReadPointer); - - uint32_t register_writes = 0; - Pm4PacketProcessor packet_processor({ - .write_uconfig_register = - [&](uint64_t, uint32_t) { - ++register_writes; - return Pm4RegisterWriteStatus::Complete; - }, - }); - Pm4RingConsumer consumer(packet_processor, fixture.vm, fixture.handle, kRing, - 8 * sizeof(uint32_t), kReadPointer, 0); - - const Pm4RingResult blocked = consumer.consume(4); - EXPECT_EQ(blocked.status, Pm4RingStatus::Blocked); - EXPECT_EQ(blocked.read_pointer, 3u); - EXPECT_EQ(register_writes, 1u); - - const Pm4RingResult terminal = consumer.consume(4); - EXPECT_EQ(terminal.status, Pm4RingStatus::UnsupportedPacket); - EXPECT_EQ(terminal.read_pointer, 3u); - EXPECT_EQ(terminal.packet_header, kUnsupportedHeader); - EXPECT_EQ(terminal.register_dword, 0u); - EXPECT_EQ(register_writes, 1u); - EXPECT_FALSE(consumer.in_flight()); -} - -TEST(Pm4RingConsumerTest, ReleasesAnUnreadyGartSnapshotBeforeRetry) { - auto memory = std::make_shared(); - GpuVm vm; - const AddressSpaceHandle handle = vm.initialize_gart_address_space(); - ASSERT_TRUE(handle); - constexpr uint64_t kTable = 0x1000; - constexpr uint64_t kPhysicalPage = 0x2000; - constexpr uint64_t kAperture = 0x10000; - constexpr uint64_t kRing = kAperture + 0x100; - constexpr uint64_t kReadPointer = kAperture + 0x80; - constexpr uint64_t kValidSystemReadWrite = (uint64_t{1} << 0) | (uint64_t{1} << 1) | - (uint64_t{1} << 5) | (uint64_t{1} << 6) | - (uint64_t{1} << 63); - - Pm4PacketProcessor packet_processor; - Pm4RingConsumer consumer(packet_processor, vm, handle, kRing, 4 * sizeof(uint32_t), kReadPointer, - 0); - - const Pm4RingResult blocked = consumer.consume(1); - EXPECT_EQ(blocked.status, Pm4RingStatus::Blocked); - EXPECT_FALSE(consumer.in_flight()); - - memory->store(kTable, kPhysicalPage | kValidSystemReadWrite); - memory->store(kPhysicalPage + 0x100, 0xffff1000); - ASSERT_TRUE(vm.publish_gart( - {.page_table_base = kTable, .aperture_start = kAperture, .aperture_end = kAperture + 0xfff}, - memory)); - - const Pm4RingResult complete = consumer.consume(1); - EXPECT_EQ(complete.status, Pm4RingStatus::Complete); - EXPECT_EQ(complete.read_pointer, 1u); - EXPECT_EQ(memory->load(kPhysicalPage + 0x80), 1u); -} - -TEST(Pm4RingConsumerTest, ClearsPublicationFaultAfterASuccessfulRetry) { - RingConsumerFixture fixture; - ASSERT_TRUE(fixture.access); - constexpr uint64_t kRing = 0x100; - constexpr uint64_t kReadPointer = 0x80; - fixture.memory->store(kRing, 0xffff1000); - fixture.memory->store(kRing + sizeof(uint32_t), 0xffff1000); - - Pm4PacketProcessor packet_processor; - Pm4RingConsumer consumer(packet_processor, fixture.vm, fixture.handle, kRing, - 4 * sizeof(uint32_t), kReadPointer, 0); - fixture.memory->make_store_faulted_once(kReadPointer); - - EXPECT_EQ(consumer.consume(1).status, Pm4RingStatus::Faulted); - EXPECT_EQ(consumer.consume(1).status, Pm4RingStatus::Complete); - - fixture.memory->make_store_unavailable_once(kReadPointer); - EXPECT_EQ(consumer.consume(2).status, Pm4RingStatus::Blocked); - EXPECT_EQ(consumer.prepare_close(), QueuePrepareCloseStatus::Busy); -} - -} // namespace -} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/tests/pseudo_scalar_exec_test.cpp b/emulation/rocjitsu/tests/pseudo_scalar_exec_test.cpp index ab7ae7610c63..f3eb42461e97 100644 --- a/emulation/rocjitsu/tests/pseudo_scalar_exec_test.cpp +++ b/emulation/rocjitsu/tests/pseudo_scalar_exec_test.cpp @@ -11,7 +11,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/opcodes.h" #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/test_encodings.h" #include "rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h" -#include "rocjitsu/isa/arch/amdgpu/shared/pseudo_scalar.h" +#include "rocjitsu/isa/arch/amdgpu/shared/transcendental.h" #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" @@ -166,7 +166,7 @@ struct PseudoScalarSpecialCase { uint32_t mode = 0; }; -constexpr std::array kSpecialCases{{ +constexpr std::array kSpecialCases{{ {"literal_f32", "v_s_sqrt_f32", 0, @@ -190,9 +190,24 @@ constexpr std::array kSpecialCases{{ "", {.source_opsel = true}, amdgpu::Wavefront::FP16_OVFL_BIT}, + // Physical gfx1201 scalar instructions match the vector hardware mappings. + {"f32_exp_approximation", "v_s_exp_f32", 0xB6128C18u, 0x3F7FFFE6u, "", {}}, + {"f32_log_approximation", "v_s_log_f32", 0x266D35EFu, 0xC248709Eu, "", {}}, {"f32_input_denorm_flush", "v_s_log_f32", 0x00000001u, 0xFF800000u, "", {}}, - {"f32_input_denorm_allow", "v_s_log_f32", 0x00000001u, f32_bits(-149.0f), "", {}, 1u << 4}, - {"f32_output_denorm_allow", "v_s_exp_f32", f32_bits(-149.0f), 0x00000001u, "", {}, 1u << 5}, + {"f32_log_flushes_allowed_input_denorm", + "v_s_log_f32", + 0x00000001u, + 0xFF800000u, + "", + {}, + 1u << 4}, + {"f32_exp_flushes_allowed_output_denorm", + "v_s_exp_f32", + f32_bits(-149.0f), + 0x00000000u, + "", + {}, + 1u << 5}, {"f32_ignores_f16_output_denorm_mode", "v_s_exp_f32", f32_bits(-149.0f), @@ -214,36 +229,36 @@ constexpr std::array kSpecialCases{{ "", {.source_opsel = true}, 1u << 7}, - {"f32_round_toward_positive", "v_s_exp_f32", f32_bits(0.5f), 0x3FB504F4u, "", {}, 1u}, + {"f32_round_toward_positive", "v_s_exp_f32", f32_bits(0.5f), 0x3FB504F3u, "", {}, 1u}, {"f16_round_toward_positive", "v_s_exp_f16", 0xCAFE3800u, - 0x00003DA9u, + 0x00003DA8u, "", {.source_opsel = true}, 1u << 2}, - {"f32_round_toward_negative", "v_s_sqrt_f32", f32_bits(5.0f), 0x400F1BBCu, "", {}, 2u}, - {"f32_round_toward_zero", "v_s_sqrt_f32", f32_bits(5.0f), 0x400F1BBCu, "", {}, 3u}, + {"f32_round_toward_negative", "v_s_sqrt_f32", f32_bits(5.0f), 0x400F1BBDu, "", {}, 2u}, + {"f32_round_toward_zero", "v_s_sqrt_f32", f32_bits(5.0f), 0x400F1BBDu, "", {}, 3u}, {"f16_round_toward_negative", "v_s_sqrt_f16", 0xCAFE4200u, - 0x00003EEDu, + 0x00003EEEu, "", {.source_opsel = true}, 2u << 2}, {"f16_round_toward_zero", "v_s_sqrt_f16", 0xCAFE4200u, - 0x00003EEDu, + 0x00003EEEu, "", {.source_opsel = true}, 3u << 2}, - {"f32_negative_round_toward_negative", "v_s_rcp_f32", f32_bits(-3.0f), 0xBEAAAAABu, "", {}, 2u}, + {"f32_negative_round_toward_negative", "v_s_rcp_f32", f32_bits(-3.0f), 0xBEAAAAAAu, "", {}, 2u}, {"f32_negative_round_toward_zero", "v_s_rcp_f32", f32_bits(-3.0f), 0xBEAAAAAAu, "", {}, 3u}, {"f16_negative_round_toward_negative", "v_s_rcp_f16", 0xCAFEC200u, - 0x0000B556u, + 0x0000B555u, "", {.source_opsel = true}, 2u << 2}, @@ -257,14 +272,14 @@ constexpr std::array kSpecialCases{{ {"f32_finite_overflow_round_toward_negative", "v_s_exp_f32", f32_bits(1024.0f), - 0x7F7FFFFFu, + 0x7F800000u, "", {}, 2u}, {"f32_finite_overflow_round_toward_zero", "v_s_exp_f32", f32_bits(2000.0f), - 0x7F7FFFFFu, + 0x7F800000u, "", {}, 3u}, @@ -279,7 +294,7 @@ constexpr std::array kSpecialCases{{ {"f32_finite_underflow_round_toward_positive", "v_s_exp_f32", f32_bits(-2000.0f), - 0x00000001u, + 0x00000000u, "", {}, 1u | (1u << 5)}, @@ -320,14 +335,14 @@ constexpr std::array kSpecialCases{{ {"f32_destination_overflow_round_toward_negative", "v_s_exp_f32", f32_bits(128.0f), - 0x7F7FFFFFu, + 0x7F800000u, "", {}, 2u}, {"f32_destination_overflow_round_toward_zero", "v_s_exp_f32", f32_bits(128.0f), - 0x7F7FFFFFu, + 0x7F800000u, "", {}, 3u}, @@ -341,7 +356,7 @@ constexpr std::array kSpecialCases{{ {"f32_destination_underflow_round_toward_positive", "v_s_exp_f32", f32_bits(-150.0f), - 0x00000001u, + 0x00000000u, "", {}, 1u | (1u << 5)}, @@ -375,14 +390,14 @@ constexpr std::array kSpecialCases{{ {"f16_destination_overflow_round_toward_negative", "v_s_exp_f16", 0xCAFE4C00u, - 0x00007BFFu, + 0x00007C00u, "", {.source_opsel = true}, 2u << 2}, {"f16_destination_overflow_round_toward_zero", "v_s_exp_f16", 0xCAFE4C00u, - 0x00007BFFu, + 0x00007C00u, "", {.source_opsel = true}, 3u << 2}, @@ -396,7 +411,7 @@ constexpr std::array kSpecialCases{{ {"f16_destination_underflow_round_toward_positive", "v_s_exp_f16", 0xCAFECE40u, - 0x00000001u, + 0x00000000u, "", {.source_opsel = true}, (1u << 2) | (1u << 7)}, @@ -430,14 +445,14 @@ constexpr std::array kSpecialCases{{ {"f32_rcp_omod_destination_overflow_round_toward_negative", "v_s_rcp_f32", 0x00800000u, - 0x7F7FFFFFu, + 0x7F800000u, "", {.omod = 2}, 2u}, {"f32_rcp_omod_destination_overflow_round_toward_zero", "v_s_rcp_f32", 0x00800000u, - 0x7F7FFFFFu, + 0x7F800000u, "", {.omod = 2}, 3u}, @@ -457,14 +472,14 @@ constexpr std::array kSpecialCases{{ {"f16_rcp_omod_destination_overflow_round_toward_negative", "v_s_rcp_f16", 0xCAFE0400u, - 0x00007BFFu, + 0x00007C00u, "", {.source_opsel = true, .omod = 2}, 2u << 2}, {"f16_rcp_omod_destination_overflow_round_toward_zero", "v_s_rcp_f16", 0xCAFE0400u, - 0x00007BFFu, + 0x00007C00u, "", {.source_opsel = true, .omod = 2}, 3u << 2}, @@ -496,7 +511,7 @@ constexpr std::array kSpecialCases{{ {"f16_finite_overflow_round_toward_zero", "v_s_exp_f16", 0xCAFE7BFFu, - 0x00007BFFu, + 0x00007C00u, "", {.source_opsel = true}, 3u << 2}, @@ -516,7 +531,7 @@ constexpr std::array kSpecialCases{{ {"f16_finite_overflow_round_toward_negative", "v_s_exp_f16", 0xCAFE7BFFu, - 0x00007BFFu, + 0x00007C00u, "", {.source_opsel = true}, 2u << 2}, @@ -530,7 +545,7 @@ constexpr std::array kSpecialCases{{ {"f16_finite_underflow_round_toward_positive", "v_s_exp_f16", 0xCAFEFBFFu, - 0x00000001u, + 0x00000000u, "", {.source_opsel = true}, (1u << 2) | (1u << 7)}, @@ -565,10 +580,16 @@ constexpr std::array kSpecialCases{{ {"f16_divide_by_zero_with_fp16_ovfl", "v_s_rcp_f16", 0xCAFE0000u, - 0x00007C00u, + 0x00007BFFu, "", {.source_opsel = true}, amdgpu::Wavefront::FP16_OVFL_BIT}, + // Raw gfx1201 captures: half rounding/flush precedes OMOD; guest rounding is ignored. + {"f16_exp_round_before_omod", "v_s_exp_f16", 0x4c00u, 0x7c00u, "", {.omod = 3}, 0x30u}, + {"f16_exp_saturate_before_omod", "v_s_exp_f16", 0x4c00u, 0x77ffu, "", {.omod = 3}, 0x800030u}, + {"f16_exp_flush_before_omod", "v_s_exp_f16", 0xcb80u, 0x0000u, "", {.omod = 1}, 0xc0u}, + {"f16_exp_ignores_guest_rounding", "v_s_exp_f16", 0x0001u, 0x3c00u, "", {}, 0xc4u}, + {"f16_log_zero_saturates", "v_s_log_f16", 0x0000u, 0xfbffu, "", {}, 0x8000c0u}, {"f32_omod_flushes_output_denorm", "v_s_exp_f32", f32_bits(-149.0f), @@ -806,15 +827,15 @@ TEST(PseudoScalarHelperTest, HandlesExplicitSpecialCasesWithoutHostInvalidOrDivi Operation::EXP2, std::bit_cast(0xFF800000u), false, false, 0, 3, 0, false); const uint32_t f32_signaling_nan = amdgpu::pseudo_scalar::execute_f32( Operation::SQRT, std::bit_cast(0x7FA00001u), false, false, 0, 3, 0, false); - const uint32_t f16_rcp_zero = amdgpu::pseudo_scalar::execute_f16( + const uint32_t f16_rcp_zero = amdgpu::transcendental::execute_pseudo_f16( Operation::RCP, util::f16_to_f32(0x0000u), false, false, 0, 3, 0, false, true); - const uint32_t f16_sqrt_negative = amdgpu::pseudo_scalar::execute_f16( + const uint32_t f16_sqrt_negative = amdgpu::transcendental::execute_pseudo_f16( Operation::SQRT, util::f16_to_f32(0xBC00u), false, false, 0, 3, 0, false, false); - const uint32_t f16_rsq_negative = amdgpu::pseudo_scalar::execute_f16( + const uint32_t f16_rsq_negative = amdgpu::transcendental::execute_pseudo_f16( Operation::RSQ, util::f16_to_f32(0xBC00u), false, false, 0, 3, 0, false, false); - const uint32_t f16_log_negative = amdgpu::pseudo_scalar::execute_f16( + const uint32_t f16_log_negative = amdgpu::transcendental::execute_pseudo_f16( Operation::LOG2, util::f16_to_f32(0xBC00u), false, false, 0, 3, 0, false, false); - const uint32_t f16_signaling_nan = amdgpu::pseudo_scalar::execute_f16( + const uint32_t f16_signaling_nan = amdgpu::transcendental::execute_pseudo_f16( Operation::SQRT, util::f16_to_f32(0x7D01u), false, false, 0, 3, 0, false, false); const int leaked_exceptions = std::fetestexcept(FE_INVALID | FE_DIVBYZERO); const int restore_result = std::fesetenv(&saved_environment); @@ -831,7 +852,7 @@ TEST(PseudoScalarHelperTest, HandlesExplicitSpecialCasesWithoutHostInvalidOrDivi EXPECT_EQ(f32_exp_negative_infinity, 0x00000000u); EXPECT_TRUE(std::isnan(std::bit_cast(f32_signaling_nan))); EXPECT_NE(f32_signaling_nan & 0x00400000u, 0u); - EXPECT_EQ(f16_rcp_zero, 0x00007C00u); + EXPECT_EQ(f16_rcp_zero, 0x00007BFFu); EXPECT_EQ(f16_sqrt_negative, 0x0000FE00u); EXPECT_EQ(f16_rsq_negative, 0x0000FE00u); EXPECT_EQ(f16_log_negative, 0x0000FE00u); @@ -840,6 +861,126 @@ TEST(PseudoScalarHelperTest, HandlesExplicitSpecialCasesWithoutHostInvalidOrDivi EXPECT_NE(f16_signaling_nan & 0x0200u, 0u); } +TEST(PseudoScalarHelperTest, QuietsSignalingNanWithoutHostExceptions) { + using amdgpu::pseudo_scalar::Operation; + std::fenv_t saved_environment{}; + ASSERT_EQ(std::fegetenv(&saved_environment), 0); + for (Operation operation : + {Operation::EXP2, Operation::LOG2, Operation::RCP, Operation::RSQ, Operation::SQRT}) + for (uint32_t input : {0x7f800001u, 0xffbfffffu}) + for (uint32_t omod = 0; omod < 4; ++omod) + for (bool clamp : {false, true}) { + // Keep the source dynamic so constant folding cannot hide an unsafe + // host floating-point comparison in the NaN classification. + volatile uint32_t source = input; + EXPECT_EQ(std::feclearexcept(FE_ALL_EXCEPT), 0); + const uint32_t result = amdgpu::pseudo_scalar::execute_f32( + operation, std::bit_cast(uint32_t{source}), false, false, 0, 3, omod, clamp); + const int exceptions = std::fetestexcept(FE_INVALID | FE_DIVBYZERO); + EXPECT_EQ(std::fesetenv(&saved_environment), 0); + EXPECT_EQ(exceptions, 0) << "operation=" << static_cast(operation); + EXPECT_EQ(result, clamp ? 0u : (input | 0x00400000u)); + } +} + +TEST(PseudoScalarHelperTest, ReciprocalSquareRootMatchesPhysicalGfx12ModesAndModifiers) { + // Physical scalar and vector RSQ agree in all sixteen FP_ROUND/FP_DENORM modes. + const uint32_t cases[][5] = { + {0x7f7fffffu, 0x1f800000u, 0x20000000u, 0x20800000u, 0x1f000000u}, + {0xff7fffffu, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u}, + {0x00800000u, 0x5f000000u, 0x5f800000u, 0x60000000u, 0x5e800000u}, + {0x00000001u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u}, + {0x3f800000u, 0x3f800000u, 0x40000000u, 0x40800000u, 0x3f000000u}, + {0x40400000u, 0x3f13cd3au, 0x3f93cd3au, 0x4013cd3au, 0x3e93cd3au}, + {0x007fffffu, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u}, + {0x80800000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u}, + {0x3d5389d4u, 0x408ccf83u, 0x410ccf83u, 0x418ccf83u, 0x400ccf83u}, + {0x3dd28815u, 0x40479ca3u, 0x40c79ca3u, 0x41479ca3u, 0x3fc79ca3u}, + {0x4314a8c6u, 0x3da7f88au, 0x3e27f88au, 0x3ea7f88au, 0x3d27f88au}, + {0x496e2c06u, 0x3a84b446u, 0x3b04b446u, 0x3b84b446u, 0x3a04b446u}, + {0x83f75dfcu, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u}, + }; + std::fenv_t saved_environment{}; + ASSERT_EQ(std::fegetenv(&saved_environment), 0); + for (int host_mode : {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO}) { + EXPECT_EQ(std::fesetround(host_mode), 0); + for (const auto &test : cases) + for (uint32_t round_mode = 0; round_mode < 4; ++round_mode) + for (uint32_t denorm_mode = 0; denorm_mode < 4; ++denorm_mode) + for (uint32_t omod = 0; omod < 4; ++omod) + EXPECT_EQ(amdgpu::pseudo_scalar::execute_f32( + amdgpu::pseudo_scalar::Operation::RSQ, std::bit_cast(test[0]), + false, false, round_mode, denorm_mode, omod, false), + test[omod + 1]) + << "host rounding mode=" << host_mode; + } + EXPECT_EQ(std::fesetenv(&saved_environment), 0); +} + +TEST(PseudoScalarHelperTest, HalfReciprocalSquareRootMatchesPhysicalModesAndModifiers) { + // Physical scalar/vector results ignore rounding, including all four OMOD settings. + const uint16_t cases[][5] = { + {0x0401, 0x57ff, 0x5bff, 0x5fff, 0x53ff}, + {0x7c02, 0x7e02, 0x7e02, 0x7e02, 0x7e02}, + {0xbc00, 0xfe00, 0xfe00, 0xfe00, 0xfe00}, + {0x7bff, 0x1c00, 0x2000, 0x2400, 0x1800}, + }; + std::fenv_t saved_environment{}; + ASSERT_EQ(std::fegetenv(&saved_environment), 0); + for (int host_mode : {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO}) { + EXPECT_EQ(std::fesetround(host_mode), 0); + for (uint32_t round_mode = 0; round_mode < 4; ++round_mode) + for (uint32_t denorm_mode = 0; denorm_mode < 4; ++denorm_mode) { + for (const auto &test : cases) + for (uint32_t omod = 0; omod < 4; ++omod) + EXPECT_EQ(amdgpu::transcendental::execute_pseudo_f16( + amdgpu::pseudo_scalar::Operation::RSQ, util::f16_to_f32(test[0]), false, + false, round_mode, denorm_mode, omod, false, false), + test[omod + 1]); + EXPECT_EQ(amdgpu::transcendental::execute_pseudo_f16( + amdgpu::pseudo_scalar::Operation::RSQ, util::f16_to_f32(0x0001), false, false, + round_mode, denorm_mode, 0, false, false), + (denorm_mode & 1u) ? 0x6c00u : 0x7c00u); + } + } + EXPECT_EQ(std::fesetenv(&saved_environment), 0); +} + +TEST(PseudoScalarHelperTest, ReciprocalMatchesPhysicalGfx12ModesAndModifiers) { + // Physical V_S_RCP_F32 captures are identical for all sixteen combinations + // of FP_ROUND and FP_DENORM. OMOD also retains reciprocal's overflow policy. + const uint32_t cases[][5] = { + {0x7F7FFFFFu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0xFF7FFFFFu, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00800000u, 0x7E800000u, 0x7F000000u, 0x7F800000u, 0x7E000000u}, + {0x00000001u, 0x7F800000u, 0x7F800000u, 0x7F800000u, 0x7F800000u}, + {0x3F800000u, 0x3F800000u, 0x40000000u, 0x40800000u, 0x3F000000u}, + {0x40400000u, 0x3EAAAAAAu, 0x3F2AAAAAu, 0x3FAAAAAAu, 0x3E2AAAAAu}, + {0x007FFFFFu, 0x7F800000u, 0x7F800000u, 0x7F800000u, 0x7F800000u}, + {0x80800000u, 0xFE800000u, 0xFF000000u, 0xFF800000u, 0xFE000000u}, + }; + std::fenv_t saved_environment{}; + ASSERT_EQ(std::fegetenv(&saved_environment), 0); + for (int host_mode : {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO}) { + EXPECT_EQ(std::fesetround(host_mode), 0); + for (const auto &test : cases) + for (uint32_t round_mode = 0; round_mode < 4; ++round_mode) + for (uint32_t denorm_mode = 0; denorm_mode < 4; ++denorm_mode) + for (uint32_t omod = 0; omod < 4; ++omod) + EXPECT_EQ(amdgpu::pseudo_scalar::execute_f32( + amdgpu::pseudo_scalar::Operation::RCP, std::bit_cast(test[0]), + false, false, round_mode, denorm_mode, omod, false), + test[omod + 1]) + << "host rounding mode=" << host_mode; + for (uint32_t sign : {0u, 0x80000000u}) + EXPECT_EQ(amdgpu::pseudo_scalar::execute_f32(amdgpu::pseudo_scalar::Operation::RCP, + std::bit_cast(0x00800000u | sign), false, + false, 3, 3, 2, true), + sign ? 0u : 0x3F800000u); + } + EXPECT_EQ(std::fesetenv(&saved_environment), 0); +} + TEST(PseudoScalarHelperTest, PreservesAndFlushesSignedDenormals) { using amdgpu::pseudo_scalar::Operation; @@ -850,28 +991,29 @@ TEST(PseudoScalarHelperTest, PreservesAndFlushesSignedDenormals) { 0x80000000u); const uint32_t f32_allowed_negative_input = amdgpu::pseudo_scalar::execute_f32( Operation::SQRT, f32_negative_minimum, false, false, 0, 1, 0, false); - EXPECT_TRUE(std::isnan(std::bit_cast(f32_allowed_negative_input))); + // F32 SQRT flushes input denormals independently of MODE. + EXPECT_EQ(f32_allowed_negative_input, 0x80000000u); EXPECT_EQ(amdgpu::pseudo_scalar::execute_f32(Operation::RCP, f32_negative_maximum, false, false, 0, 2, 0, false), - 0x80200000u); + 0x80000000u); EXPECT_EQ(amdgpu::pseudo_scalar::execute_f32(Operation::RCP, f32_negative_maximum, false, false, 0, 0, 0, false), 0x80000000u); const float f16_negative_minimum = util::f16_to_f32(0x8001u); const float f16_negative_maximum = util::f16_to_f32(0xFBFFu); - EXPECT_EQ(amdgpu::pseudo_scalar::execute_f16(Operation::SQRT, f16_negative_minimum, false, false, - 0, 0, 0, false, false), + EXPECT_EQ(amdgpu::transcendental::execute_pseudo_f16(Operation::SQRT, f16_negative_minimum, false, + false, 0, 0, 0, false, false), 0x00008000u); - const uint32_t f16_allowed_negative_input = amdgpu::pseudo_scalar::execute_f16( + const uint32_t f16_allowed_negative_input = amdgpu::transcendental::execute_pseudo_f16( Operation::SQRT, f16_negative_minimum, false, false, 0, 1, 0, false, false); EXPECT_EQ(f16_allowed_negative_input & 0x7C00u, 0x7C00u); EXPECT_NE(f16_allowed_negative_input & 0x03FFu, 0u); - EXPECT_EQ(amdgpu::pseudo_scalar::execute_f16(Operation::RCP, f16_negative_maximum, false, false, - 0, 2, 0, false, false), + EXPECT_EQ(amdgpu::transcendental::execute_pseudo_f16(Operation::RCP, f16_negative_maximum, false, + false, 0, 2, 0, false, false), 0x00008100u); - EXPECT_EQ(amdgpu::pseudo_scalar::execute_f16(Operation::RCP, f16_negative_maximum, false, false, - 0, 0, 0, false, false), + EXPECT_EQ(amdgpu::transcendental::execute_pseudo_f16(Operation::RCP, f16_negative_maximum, false, + false, 0, 0, 0, false, false), 0x00008000u); } @@ -948,7 +1090,7 @@ TEST(PseudoScalarModeIntegrationTest, SetregInstructionsUpdateModesConsumedByPse EXPECT_TRUE( fixture.compute_unit->execute_instruction(exp.get(), *fixture.wavefront).succeeded()); EXPECT_EQ(fixture.wavefront->mode_raw(), 1u); - EXPECT_EQ(fixture.compute_unit->read_sgpr(fixture.sgpr_base() + kDestinationSgpr), 0x3FB504F4u); + EXPECT_EQ(fixture.compute_unit->read_sgpr(fixture.sgpr_base() + kDestinationSgpr), 0x3FB504F3u); const BaseEncodingWords set_denorm_words = encode_sopk(profile.setreg_imm_op, 0, encode_hwreg(kModeHwreg, 6, 2), 1u); diff --git a/emulation/rocjitsu/tests/race-detector/CMakeLists.txt b/emulation/rocjitsu/tests/race-detector/CMakeLists.txt index ef8b373b2d64..2d229814c4b9 100644 --- a/emulation/rocjitsu/tests/race-detector/CMakeLists.txt +++ b/emulation/rocjitsu/tests/race-detector/CMakeLists.txt @@ -141,11 +141,29 @@ endfunction() rj_add_gfx950_race_test_case(vgpr_waitcnt) rj_add_gfx950_race_test_case(vgpr_waitcnt_race) rj_add_gfx950_race_test_case(flat_global_wait_both) -rj_add_gfx950_race_test_case(flat_global_vmcnt_only_race) +rj_add_gfx950_race_test_case(flat_global_vmcnt_only) rj_add_gfx950_race_test_case(flat_global_lgkmcnt_only_race) rj_add_gfx950_race_test_case(flat_lds_wait_both) rj_add_gfx950_race_test_case(flat_lds_vmcnt_only_race) -rj_add_gfx950_race_test_case(flat_lds_lgkmcnt_only_race) +rj_add_gfx950_race_test_case(flat_lds_lgkmcnt_only) +rj_add_gfx950_race_test_case(flat_global_copy_vmcnt) +rj_add_gfx950_race_test_case(flat_global_copy_missing_vmcnt_race) +rj_add_gfx950_race_test_case(flat_global_does_not_advance_lgkmcnt_race) +rj_add_gfx950_race_test_case(flat_global_full_lgkmcnt_waited) +rj_add_gfx950_race_test_case(flat_lds_advances_lgkmcnt_at_capacity) +rj_add_gfx950_race_test_case(flat_lds_does_not_advance_vmcnt_race) +rj_add_gfx950_race_test_case(flat_lds_full_vmcnt_waited) +rj_add_gfx950_race_test_case(flat_global_advances_vmcnt_at_capacity) +rj_add_gfx950_race_test_case(vmcnt_no_wait_race) +rj_add_gfx950_race_test_case(lgkmcnt_no_wait_race) +rj_add_gfx950_race_test_case(vmcnt_capacity_boundary) +rj_add_gfx950_race_test_case(vmcnt_below_capacity_race) +rj_add_gfx950_race_test_case(lgkmcnt_capacity_boundary) +rj_add_gfx950_race_test_case(lgkmcnt_below_capacity_race) +rj_add_gfx950_race_test_case(mixed_lgkmcnt_capacity_boundary) +rj_add_gfx950_race_test_case(mixed_lgkmcnt_below_capacity_race) +rj_add_gfx950_race_test_case(wide_scalar_lgkmcnt_capacity_boundary) +rj_add_gfx950_race_test_case(wide_scalar_lgkmcnt_below_capacity_race) rj_add_gfx950_race_test_case(sgpr_waitcnt) rj_add_gfx950_race_test_case(sgpr_waitcnt_race) rj_add_gfx950_race_test_case(sgpr_waw_load_then_mov) @@ -186,6 +204,19 @@ rj_add_gfx950_race_test_case(f64_vgpr_waw) rj_add_gfx1151_race_test_case(vgpr_waitcnt) rj_add_gfx1151_race_test_case(vgpr_waitcnt_race) +rj_add_gfx1151_race_test_case(standalone_vmcnt_waits_for_vmem) +rj_add_gfx1151_race_test_case(standalone_vmcnt_does_not_wait_for_lds) +rj_add_gfx1151_race_test_case(standalone_lgkmcnt_waits_for_lds) +rj_add_gfx1151_race_test_case(standalone_lgkmcnt_does_not_wait_for_vmem) +rj_add_gfx1151_race_test_case(vmcnt_capacity_boundary) +rj_add_gfx1151_race_test_case(vmcnt_below_capacity_race) +rj_add_gfx1151_race_test_case(vscnt_pressure_does_not_advance_vmcnt) +rj_add_gfx1151_race_test_case(lgkmcnt_capacity_boundary) +rj_add_gfx1151_race_test_case(lgkmcnt_below_capacity_race) +rj_add_gfx1151_race_test_case(mixed_lgkmcnt_capacity_boundary) +rj_add_gfx1151_race_test_case(mixed_lgkmcnt_below_capacity_race) +rj_add_gfx1151_race_test_case(wide_scalar_lgkmcnt_capacity_boundary) +rj_add_gfx1151_race_test_case(wide_scalar_lgkmcnt_below_capacity_race) rj_add_gfx1151_race_test_case(d16_global_load_merge_no_race) rj_add_gfx1151_race_test_case(d16_global_load_hi_merge_no_race) rj_add_gfx1151_race_test_case(d16_global_load_missing_waitcnt_race) diff --git a/emulation/rocjitsu/tests/race-detector/hip_race_gfx1151_test.hip b/emulation/rocjitsu/tests/race-detector/hip_race_gfx1151_test.hip index 0e2dc78ca377..e51c2ee1241f 100644 --- a/emulation/rocjitsu/tests/race-detector/hip_race_gfx1151_test.hip +++ b/emulation/rocjitsu/tests/race-detector/hip_race_gfx1151_test.hip @@ -23,6 +23,10 @@ namespace { using rocjitsu::test::FindingCount; using rocjitsu::test::RaceExpectation; +using rocjitsu::test::lgkmcnt_capacity_kernel; +using rocjitsu::test::mixed_lgkmcnt_capacity_kernel; +using rocjitsu::test::vmcnt_capacity_kernel; +using rocjitsu::test::wide_scalar_lgkmcnt_capacity_kernel; using Gfx1151RaceTest = rocjitsu::test::RaceTestBase; @@ -36,21 +40,246 @@ __global__ void vgpr_no_race_kernel(const float *src, float *dst) { dst[tid] = val; } +TEST_F(Gfx1151RaceTest, vgpr_waitcnt) { + auto *src = allocWithData(32); + auto *dst = alloc(32); + vgpr_no_race_kernel<<<1, 32>>>(src, dst); + sync(); + ExpectNoRace(); +} + +__global__ void vgpr_race_kernel(const float *src, float *dst) { + int tid = threadIdx.x; + float val; + asm volatile("global_load_dword %0, %1, off\n" + "s_nop 0\n" + : "=v"(val) + : "v"(&src[tid])); + dst[tid] = val; +} + static constexpr RaceExpectation kVgprWaitcntRaceExpectation{ .findings = FindingCount::OneOrMore, .type = "VGPR", + .producer = "global_load_b32", + .consumer = "global_store_b32", }; -__global__ void vgpr_race_kernel(const float *src, float *dst) { +TEST_F(Gfx1151RaceTest, vgpr_waitcnt_race) { + auto *src = allocWithData(32); + auto *dst = alloc(32); + vgpr_race_kernel<<<1, 32>>>(src, dst); + sync(); + ExpectRace(kVgprWaitcntRaceExpectation); +} + +// --- GFX11.5 standalone wait-counter instructions --- + +// GFX11/11.5 retain the combined s_waitcnt instruction and also provide +// standalone forms for individual VMCNT, VSCNT, LGKMCNT, and EXPCNT domains. +// These pairs verify that VMCNT and LGKMCNT waits drain only their own domain. + +__global__ void standalone_vmcnt_waits_for_vmem_kernel(const float *src, float *dst) { int tid = threadIdx.x; float val; asm volatile("global_load_dword %0, %1, off\n" - "s_nop 0\n" + "s_waitcnt_vmcnt null, 0\n" : "=v"(val) : "v"(&src[tid])); dst[tid] = val; } +TEST_F(Gfx1151RaceTest, standalone_vmcnt_waits_for_vmem) { + auto *src = allocWithData(32); + auto *dst = alloc(32); + standalone_vmcnt_waits_for_vmem_kernel<<<1, 32>>>(src, dst); + sync(); + ExpectNoRace(); +} + +__global__ void standalone_vmcnt_does_not_wait_for_lds_kernel(int *dst) { + __shared__ int lds[32]; + int tid = threadIdx.x; + lds[tid] = tid + 1; + __syncthreads(); + + int val; + asm volatile("ds_read_b32 %0, %1\n" + "s_waitcnt_vmcnt null, 0\n" + : "=v"(val) + : "v"(tid * 4) + : "memory"); + dst[tid] = val; +} + +static constexpr RaceExpectation kStandaloneVmcntDoesNotWaitForLdsExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "ds_load_b32", + .consumer = "global_store_b32", +}; + +TEST_F(Gfx1151RaceTest, standalone_vmcnt_does_not_wait_for_lds) { + standalone_vmcnt_does_not_wait_for_lds_kernel<<<1, 32>>>(alloc(32)); + sync(); + ExpectRace(kStandaloneVmcntDoesNotWaitForLdsExpectation); +} + +__global__ void standalone_lgkmcnt_waits_for_lds_kernel(int *dst) { + __shared__ int lds[32]; + int tid = threadIdx.x; + lds[tid] = tid + 1; + __syncthreads(); + + int val; + asm volatile("ds_read_b32 %0, %1\n" + "s_waitcnt_lgkmcnt null, 0\n" + : "=v"(val) + : "v"(tid * 4) + : "memory"); + dst[tid] = val; +} + +TEST_F(Gfx1151RaceTest, standalone_lgkmcnt_waits_for_lds) { + standalone_lgkmcnt_waits_for_lds_kernel<<<1, 32>>>(alloc(32)); + sync(); + ExpectNoRace(); +} + +__global__ void standalone_lgkmcnt_does_not_wait_for_vmem_kernel(const float *src, float *dst) { + int tid = threadIdx.x; + float val; + asm volatile("global_load_dword %0, %1, off\n" + "s_waitcnt_lgkmcnt null, 0\n" + : "=v"(val) + : "v"(&src[tid])); + dst[tid] = val; +} + +static constexpr RaceExpectation kStandaloneLgkmcntDoesNotWaitForVmemExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "global_load_b32", + .consumer = "global_store_b32", +}; + +TEST_F(Gfx1151RaceTest, standalone_lgkmcnt_does_not_wait_for_vmem) { + auto *src = allocWithData(32); + auto *dst = alloc(32); + standalone_lgkmcnt_does_not_wait_for_vmem_kernel<<<1, 32>>>(src, dst); + sync(); + ExpectRace(kStandaloneLgkmcntDoesNotWaitForVmemExpectation); +} + +// --- GFX11 counter-capacity backpressure --- + +static constexpr int kGfx11CounterCapacity = 63; + +// The shared kernels consume the oldest result after a configurable number of +// younger operations. At capacity it is safe; one operation below still races. +static constexpr RaceExpectation kVmcntBelowCapacityRaceExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "global_load_b32", + .consumer = "v_mov_b32", +}; + +TEST_F(Gfx1151RaceTest, vmcnt_capacity_boundary) { + auto *src = allocWithData(32); + auto *dst = alloc(32); + vmcnt_capacity_kernel<<<1, 32>>>(src, dst); + sync(); + ExpectNoRace(); +} + +TEST_F(Gfx1151RaceTest, vmcnt_below_capacity_race) { + auto *src = allocWithData(32); + auto *dst = alloc(32); + vmcnt_capacity_kernel<<<1, 32>>>(src, dst); + sync(); + ExpectRace(kVmcntBelowCapacityRaceExpectation); +} + +__global__ void vscnt_pressure_does_not_advance_vmcnt_kernel(const float *src, float *scratch, + float *dst) { + int tid = threadIdx.x; + float producer, result; + asm volatile( + "global_load_dword %[producer], %[load_address], off\n" + ".rept %c[stores]\n" + "global_store_dword %[store_address], %[store_value], off\n" + ".endr\n" + "v_mov_b32 %[result], %[producer]\n" + : [producer] "=&v"(producer), [result] "=&v"(result) + : [load_address] "v"(&src[tid]), [store_address] "v"(&scratch[tid]), + [store_value] "v"(1.0f), [stores] "n"(kGfx11CounterCapacity) + : "memory"); + dst[tid] = result; +} + +static constexpr RaceExpectation kVscntPressureDoesNotAdvanceVmcntExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "global_load_b32", + .consumer = "v_mov_b32", +}; + +TEST_F(Gfx1151RaceTest, vscnt_pressure_does_not_advance_vmcnt) { + auto *src = allocWithData(32); + auto *scratch = alloc(32); + auto *dst = alloc(32); + vscnt_pressure_does_not_advance_vmcnt_kernel<<<1, 32>>>(src, scratch, dst); + sync(); + ExpectRace(kVscntPressureDoesNotAdvanceVmcntExpectation); +} + +static constexpr RaceExpectation kLgkmcntBelowCapacityRaceExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "ds_load_b32", + .consumer = "v_mov_b32", +}; + +TEST_F(Gfx1151RaceTest, lgkmcnt_capacity_boundary) { + lgkmcnt_capacity_kernel<<<1, 32>>>(alloc(32)); + sync(); + ExpectNoRace(); +} + +TEST_F(Gfx1151RaceTest, lgkmcnt_below_capacity_race) { + lgkmcnt_capacity_kernel<<<1, 32>>>(alloc(32)); + sync(); + ExpectRace(kLgkmcntBelowCapacityRaceExpectation); +} + +TEST_F(Gfx1151RaceTest, mixed_lgkmcnt_capacity_boundary) { + mixed_lgkmcnt_capacity_kernel + <<<1, 32>>>(allocWithData(1), alloc(32)); + sync(); + ExpectNoRace(); +} + +TEST_F(Gfx1151RaceTest, mixed_lgkmcnt_below_capacity_race) { + mixed_lgkmcnt_capacity_kernel + <<<1, 32>>>(allocWithData(1), alloc(32)); + sync(); + ExpectRace(kLgkmcntBelowCapacityRaceExpectation); +} + +TEST_F(Gfx1151RaceTest, wide_scalar_lgkmcnt_capacity_boundary) { + wide_scalar_lgkmcnt_capacity_kernel + <<<1, 32>>>(allocWithData(1), alloc(32)); + sync(); + ExpectNoRace(); +} + +TEST_F(Gfx1151RaceTest, wide_scalar_lgkmcnt_below_capacity_race) { + wide_scalar_lgkmcnt_capacity_kernel + <<<1, 32>>>(allocWithData(1), alloc(32)); + sync(); + ExpectRace(kLgkmcntBelowCapacityRaceExpectation); +} + // D16 loads write only one half of the destination dword on RDNA. This kernel // initializes the high half, performs a low-half global_load_d16_u8, waits for // vmcnt, and then stores the merged dword. The test below checks two things: @@ -146,22 +375,6 @@ lds_wave32_cross_wave_race_kernel(int *out) { out[tid] = lds[tid - 32]; } -TEST_F(Gfx1151RaceTest, vgpr_waitcnt) { - auto *src = allocWithData(32); - auto *dst = alloc(32); - vgpr_no_race_kernel<<<1, 32>>>(src, dst); - sync(); - ExpectNoRace(); -} - -TEST_F(Gfx1151RaceTest, vgpr_waitcnt_race) { - auto *src = allocWithData(32); - auto *dst = alloc(32); - vgpr_race_kernel<<<1, 32>>>(src, dst); - sync(); - ExpectRace(kVgprWaitcntRaceExpectation); -} - TEST_F(Gfx1151RaceTest, d16_global_load_merge_no_race) { auto *src = allocWithData(32); auto *dst = alloc(32); diff --git a/emulation/rocjitsu/tests/race-detector/hip_race_gfx950_test.hip b/emulation/rocjitsu/tests/race-detector/hip_race_gfx950_test.hip index a6fe3ea26058..05e06b08cfeb 100644 --- a/emulation/rocjitsu/tests/race-detector/hip_race_gfx950_test.hip +++ b/emulation/rocjitsu/tests/race-detector/hip_race_gfx950_test.hip @@ -27,6 +27,10 @@ using rocjitsu::test::FindingCount; using rocjitsu::test::RaceExpectation; +using rocjitsu::test::lgkmcnt_capacity_kernel; +using rocjitsu::test::mixed_lgkmcnt_capacity_kernel; +using rocjitsu::test::vmcnt_capacity_kernel; +using rocjitsu::test::wide_scalar_lgkmcnt_capacity_kernel; using Gfx950RaceTest = rocjitsu::test::RaceTestBase; @@ -83,12 +87,12 @@ TEST_F(Gfx950RaceTest, vgpr_waitcnt_race) { ExpectRace(kVgprWaitcntRaceExpectation); } -// --- Generic FLAT dual-counter waits --- +// --- Generic FLAT result waits --- -// Generic FLAT carries simultaneous VMCNT and LGKMCNT obligations, regardless -// of whether the uniformly resolved address uses the global or LDS route. The -// all-ones operands below leave one counter unchanged so each negative test -// demonstrates that draining only the other counter is insufficient. +// Generic FLAT uses the counters for its resolved memory domains: VMCNT for +// global/scratch requests and LGKMCNT for LDS requests. An unused domain must +// neither delay the result nor advance older requests through counter pressure. +// https://github.com/ROCm/rocm-systems/issues/12237 static constexpr int kFlatVmcntNoWait = 63; static constexpr int kFlatLgkmcntNoWait = 15; @@ -135,11 +139,11 @@ TEST_F(Gfx950RaceTest, flat_global_wait_both) { ExpectNoRace(); } -TEST_F(Gfx950RaceTest, flat_global_vmcnt_only_race) { +TEST_F(Gfx950RaceTest, flat_global_vmcnt_only) { auto *src = allocWithData(64), *dst = alloc(64); flat_global_wait_kernel<0, kFlatLgkmcntNoWait><<<1, 64>>>(src, dst); sync(); - ExpectRace(kFlatPartialWaitRaceExpectation); + ExpectNoRace(); } TEST_F(Gfx950RaceTest, flat_global_lgkmcnt_only_race) { @@ -161,10 +165,292 @@ TEST_F(Gfx950RaceTest, flat_lds_vmcnt_only_race) { ExpectRace(kFlatPartialWaitRaceExpectation); } -TEST_F(Gfx950RaceTest, flat_lds_lgkmcnt_only_race) { +TEST_F(Gfx950RaceTest, flat_lds_lgkmcnt_only) { flat_lds_wait_kernel<<<1, 64>>>(alloc(64)); sync(); - ExpectRace(kFlatPartialWaitRaceExpectation); + ExpectNoRace(); +} + +// Match the runtime copy sequence: both the load and its consumer are FLAT. +// Keep the consumer inside the assembly so compiler-inserted waits cannot hide +// the missing-wait control. The final wait is after the hazardous consumption. +template +__global__ void flat_global_copy_kernel(const int *src, int *dst) { + const int tid = static_cast(threadIdx.x); + int value; + asm volatile("flat_load_dword %[value], %[src]\n" + "s_waitcnt vmcnt(%c[vmcnt])\n" + "flat_store_dword %[dst], %[value]\n" + "s_waitcnt vmcnt(0) lgkmcnt(0)\n" + : [value] "=&v"(value) + : [src] "v"(&src[tid]), [dst] "v"(&dst[tid]), [vmcnt] "n"(Vmcnt) + : "memory"); +} + +TEST_F(Gfx950RaceTest, flat_global_copy_vmcnt) { + auto *src = allocWithData(64), *dst = alloc(64); + flat_global_copy_kernel<0><<<1, 64>>>(src, dst); + ASSERT_EQ(hipDeviceSynchronize(), hipSuccess); + ExpectNoRace(); + + std::vector result(64); + ASSERT_EQ(hipMemcpy(result.data(), dst, result.size() * sizeof(int), hipMemcpyDeviceToHost), + hipSuccess); + for (int lane = 0; lane < 64; ++lane) + EXPECT_EQ(result[lane], lane) << "lane " << lane; +} + +TEST_F(Gfx950RaceTest, flat_global_copy_missing_vmcnt_race) { + auto *src = allocWithData(64), *dst = alloc(64); + flat_global_copy_kernel<<<1, 64>>>(src, dst); + ASSERT_EQ(hipDeviceSynchronize(), hipSuccess); + ExpectRace({.findings = FindingCount::OneOrMore, + .kernel = "flat_global_copy_kernel", + .type = "VGPR", + .access = "read", + .producer = "flat_load_dword", + .consumer = "flat_store_dword"}); +} + +// TODO(newling): Add mixed LDS/global HIP cases once functional routing is fixed +// (https://github.com/ROCm/rocm-systems/issues/11456). The active C++ regression +// FlatLoadReadyLanesKnownFalsePositives records the remaining ready-lane false +// reports in https://github.com/ROCm/rocm-systems/issues/12237 without relying on +// the broken mixed-space transfers. + +// --- All-ones wait operands --- + +// CDNA1-CDNA4 use the all-ones wait operand as both the no-wait encoding and +// the maximum number of outstanding counter tokens. +static constexpr int kCdnaLgkmcntCapacity = 15; +static constexpr int kCdnaVmcntCapacity = 63; + +__global__ void vmcnt_no_wait_race_kernel(const float *src, float *dst) { + int tid = threadIdx.x + blockIdx.x * blockDim.x; + float val; + asm volatile( + "global_load_dword %[value], %[address], off\n" + "s_waitcnt vmcnt(%c[no_wait])\n" + : [value] "=v"(val) + : [address] "v"(&src[tid]), [no_wait] "n"(kCdnaVmcntCapacity)); + dst[tid] = val; +} + +static constexpr RaceExpectation kVmcntNoWaitRaceExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "global_load_dword", + .consumer = "global_store_dword", +}; + +TEST_F(Gfx950RaceTest, vmcnt_no_wait_race) { + auto *src = allocWithData(64), *dst = alloc(64); + vmcnt_no_wait_race_kernel<<<1, 64>>>(src, dst); + sync(); + ExpectRace(kVmcntNoWaitRaceExpectation); +} + +__global__ void lgkmcnt_no_wait_race_kernel(int *dst) { + __shared__ int lds[64]; + int tid = threadIdx.x; + lds[tid] = tid + 1; + __syncthreads(); + + int val; + asm volatile( + "ds_read_b32 %[value], %[address]\n" + "s_waitcnt lgkmcnt(%c[no_wait])\n" + : [value] "=v"(val) + : [address] "v"(tid * 4), [no_wait] "n"(kCdnaLgkmcntCapacity) + : "memory"); + dst[tid] = val; +} + +static constexpr RaceExpectation kLgkmcntNoWaitRaceExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "ds_read_b32", + .consumer = "global_store_dword", +}; + +TEST_F(Gfx950RaceTest, lgkmcnt_no_wait_race) { + auto *dst = alloc(64); + lgkmcnt_no_wait_race_kernel<<<1, 64>>>(dst); + sync(); + ExpectRace(kLgkmcntNoWaitRaceExpectation); +} + +// --- Counter-capacity backpressure --- + +// The shared kernels consume the oldest result after a configurable number of +// younger operations. At capacity it is safe; one operation below still races. + +static constexpr RaceExpectation kLgkmcntBelowCapacityRaceExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "ds_read_b32", + .consumer = "v_mov_b32", +}; + +TEST_F(Gfx950RaceTest, lgkmcnt_capacity_boundary) { + lgkmcnt_capacity_kernel<<<1, 64>>>(alloc(64)); + sync(); + ExpectNoRace(); +} + +TEST_F(Gfx950RaceTest, lgkmcnt_below_capacity_race) { + lgkmcnt_capacity_kernel<<<1, 64>>>(alloc(64)); + sync(); + ExpectRace(kLgkmcntBelowCapacityRaceExpectation); +} + +TEST_F(Gfx950RaceTest, mixed_lgkmcnt_capacity_boundary) { + mixed_lgkmcnt_capacity_kernel + <<<1, 64>>>(allocWithData(1), alloc(64)); + sync(); + ExpectNoRace(); +} + +TEST_F(Gfx950RaceTest, mixed_lgkmcnt_below_capacity_race) { + mixed_lgkmcnt_capacity_kernel + <<<1, 64>>>(allocWithData(1), alloc(64)); + sync(); + ExpectRace(kLgkmcntBelowCapacityRaceExpectation); +} + +TEST_F(Gfx950RaceTest, wide_scalar_lgkmcnt_capacity_boundary) { + wide_scalar_lgkmcnt_capacity_kernel + <<<1, 64>>>(allocWithData(1), alloc(64)); + sync(); + ExpectNoRace(); +} + +TEST_F(Gfx950RaceTest, wide_scalar_lgkmcnt_below_capacity_race) { + wide_scalar_lgkmcnt_capacity_kernel + <<<1, 64>>>(allocWithData(1), alloc(64)); + sync(); + ExpectRace(kLgkmcntBelowCapacityRaceExpectation); +} + +static constexpr RaceExpectation kVmcntBelowCapacityRaceExpectation{ + .findings = FindingCount::OneOrMore, + .type = "VGPR", + .producer = "global_load_dword", + .consumer = "v_mov_b32", +}; + +TEST_F(Gfx950RaceTest, vmcnt_capacity_boundary) { + auto *src = allocWithData(64), *dst = alloc(64); + vmcnt_capacity_kernel<<<1, 64>>>(src, dst); + sync(); + ExpectNoRace(); +} + +TEST_F(Gfx950RaceTest, vmcnt_below_capacity_race) { + auto *src = allocWithData(64), *dst = alloc(64); + vmcnt_capacity_kernel<<<1, 64>>>(src, dst); + sync(); + ExpectRace(kVmcntBelowCapacityRaceExpectation); +} + +// --- FLAT participation under counter pressure (#12237) --- + +// Fill one counter exactly, then issue a FLAT load. Only a FLAT using that +// domain needs a free slot and therefore proves the oldest result complete. +// A FLAT in the other domain must leave the missing-wait race detectable. +// Explicit-wait controls use the same addresses and instruction sequence. +template +__global__ void flat_after_full_lgkmcnt_kernel(const int *src, int *dst) { + __shared__ int lds[64]; + const int tid = static_cast(threadIdx.x); + lds[tid] = tid + 1; + __syncthreads(); + const uint64_t flat_address = FlatIsLds ? reinterpret_cast(&lds[tid]) + : reinterpret_cast(&src[tid]); + // Reserve a different VGPR for each filler to avoid unrelated LDS WAW hazards. + int producer, flat_result, result; + asm volatile("s_waitcnt vmcnt(0) lgkmcnt(0)\n" + "ds_read_b32 %[producer], %[lds_address]\n" + ".set .Lflat_lds_filler%=, 16\n" + ".rept %c[younger_reads]\n" + "ds_read_b32 v[.Lflat_lds_filler%=], %[lds_address]\n" + ".set .Lflat_lds_filler%=, .Lflat_lds_filler%= + 1\n" + ".endr\n" + "flat_load_dword %[flat_result], %[flat_address]\n" + "s_waitcnt lgkmcnt(%c[lgkmcnt])\n" + "v_mov_b32 %[result], %[producer]\n" + "s_waitcnt vmcnt(0) lgkmcnt(0)\n" + : [producer] "=&v"(producer), [flat_result] "=&v"(flat_result), + [result] "=&v"(result) + : [lds_address] "v"(tid * 4), [flat_address] "v"(flat_address), + [younger_reads] "n"(kCdnaLgkmcntCapacity - 1), + [lgkmcnt] "n"(WaitForLds ? 0 : kFlatLgkmcntNoWait) + : "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", + "v26", "v27", "v28", "v29", "memory"); + dst[tid] = result; +} + +template +__global__ void flat_after_full_vmcnt_kernel(const int *src, int *dst) { + __shared__ int lds[64]; + const int tid = static_cast(threadIdx.x); + lds[tid] = tid + 1; + __syncthreads(); + const uint64_t flat_address = FlatIsLds ? reinterpret_cast(&lds[tid]) + : reinterpret_cast(&src[tid]); + int producer, younger, flat_result, result; + asm volatile("s_waitcnt vmcnt(0) lgkmcnt(0)\n" + "global_load_dword %[producer], %[global_address], off\n" + ".rept %c[younger_loads]\n" + "global_load_dword %[younger], %[global_address], off\n" + ".endr\n" + "flat_load_dword %[flat_result], %[flat_address]\n" + "s_waitcnt vmcnt(%c[vmcnt])\n" + "v_mov_b32 %[result], %[producer]\n" + "s_waitcnt vmcnt(0) lgkmcnt(0)\n" + : [producer] "=&v"(producer), [younger] "=&v"(younger), + [flat_result] "=&v"(flat_result), [result] "=&v"(result) + : [global_address] "v"(&src[tid]), [flat_address] "v"(flat_address), + [younger_loads] "n"(kCdnaVmcntCapacity - 1), + [vmcnt] "n"(WaitForVmem ? 0 : kFlatVmcntNoWait) + : "memory"); + dst[tid] = result; +} + +TEST_F(Gfx950RaceTest, flat_global_does_not_advance_lgkmcnt_race) { + flat_after_full_lgkmcnt_kernel<<<1, 64>>>(allocWithData(64), alloc(64)); + ASSERT_EQ(hipDeviceSynchronize(), hipSuccess); + ExpectRace(kLgkmcntBelowCapacityRaceExpectation); +} + +TEST_F(Gfx950RaceTest, flat_global_full_lgkmcnt_waited) { + flat_after_full_lgkmcnt_kernel<<<1, 64>>>(allocWithData(64), alloc(64)); + ASSERT_EQ(hipDeviceSynchronize(), hipSuccess); + ExpectNoRace(); +} + +TEST_F(Gfx950RaceTest, flat_lds_advances_lgkmcnt_at_capacity) { + flat_after_full_lgkmcnt_kernel<<<1, 64>>>(allocWithData(64), alloc(64)); + ASSERT_EQ(hipDeviceSynchronize(), hipSuccess); + ExpectNoRace(); +} + +TEST_F(Gfx950RaceTest, flat_lds_does_not_advance_vmcnt_race) { + flat_after_full_vmcnt_kernel<<<1, 64>>>(allocWithData(64), alloc(64)); + ASSERT_EQ(hipDeviceSynchronize(), hipSuccess); + ExpectRace(kVmcntBelowCapacityRaceExpectation); +} + +TEST_F(Gfx950RaceTest, flat_lds_full_vmcnt_waited) { + flat_after_full_vmcnt_kernel<<<1, 64>>>(allocWithData(64), alloc(64)); + ASSERT_EQ(hipDeviceSynchronize(), hipSuccess); + ExpectNoRace(); +} + +TEST_F(Gfx950RaceTest, flat_global_advances_vmcnt_at_capacity) { + flat_after_full_vmcnt_kernel<<<1, 64>>>(allocWithData(64), alloc(64)); + ASSERT_EQ(hipDeviceSynchronize(), hipSuccess); + ExpectNoRace(); } // --- SGPR waitcnt --- diff --git a/emulation/rocjitsu/tests/race-detector/race_detector_counter_capacity_tests.cpp b/emulation/rocjitsu/tests/race-detector/race_detector_counter_capacity_tests.cpp new file mode 100644 index 000000000000..26602f466291 --- /dev/null +++ b/emulation/rocjitsu/tests/race-detector/race_detector_counter_capacity_tests.cpp @@ -0,0 +1,384 @@ +// Copyright Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "race_test_builder.h" + +#include + +#include +#include + +using namespace rocjitsu::plugins::race_detector; +namespace amdgpu = rocjitsu::amdgpu; + +namespace { + +constexpr int kWave = 0; +constexpr int kLane = 0; +constexpr int kScalarRegisters = 8; +constexpr int kDwordBytes = 4; + +// For CDNA1-CDNA4 combined counters, the all-ones no-wait encoding is also the +// maximum number of outstanding tokens. Issuing another token forces older +// work to complete before the new operation can issue. +constexpr auto kCdnaCounterCapacities = counterCapacitiesForArch(ROCJITSU_CODE_ARCH_CDNA4); +constexpr int kLgkmcntCapacity = kCdnaCounterCapacities.lgkmcnt; +constexpr int kVmcntCapacity = kCdnaCounterCapacities.vmcnt; +constexpr int kLgkmcntOverflow = 5; +constexpr int kVmcntOverflow = 7; +constexpr int kClearlyBelowCapacity = 10; + +RaceTestBuilder makeLgkmcntTest() { + return RaceTestBuilder(/*numWaves=*/1, /*vgprs=*/kLgkmcntCapacity + kLgkmcntOverflow, + /*sgprs=*/kScalarRegisters); +} + +RaceTestBuilder makeVmcntTest() { + return RaceTestBuilder(/*numWaves=*/1, /*vgprs=*/kVmcntCapacity + kVmcntOverflow, + /*sgprs=*/kScalarRegisters); +} + +void issueOrderedLdsReads(RaceTestBuilder &builder, int count, int firstVgpr = 0) { + for (int offset = 0; offset < count; ++offset) { + const int vgpr = firstVgpr + offset; + builder.ldsRead(kWave, kLane, /*addr=*/vgpr * kDwordBytes, /*bytes=*/kDwordBytes, + /*vgprDst=*/vgpr); + } +} + +void issueOrderedVmemLoads(RaceTestBuilder &builder, int count, int firstVgpr = 0) { + for (int offset = 0; offset < count; ++offset) { + builder.globalLoad(kWave, /*vgprBase=*/firstVgpr + offset, /*numRegs=*/1); + } +} + +void issueUnorderedVmcntLoads(RaceTestBuilder &builder, int count, int firstVgpr = 0) { + for (int offset = 0; offset < count; ++offset) { + builder.globalLoad(kWave, /*vgprBase=*/firstVgpr + offset, /*numRegs=*/1, /*exec=*/0, + /*byteMask=*/0xF, MemoryOrderClass::UNORDERED); + } +} + +void issueOrderedVmemStores(RaceTestBuilder &builder, int count) { + for (int index = 0; index < count; ++index) + builder.globalStore(kWave); +} + +void expectVgprReady(RaceTestBuilder &builder, int vgpr) { + builder.checkVgprRead(kWave, vgpr, kLane); + EXPECT_FALSE(builder.hasVgprRace(vgpr)); +} + +void expectVgprPending(RaceTestBuilder &builder, int vgpr) { + builder.checkVgprRead(kWave, vgpr, kLane); + EXPECT_TRUE(builder.hasVgprRace(vgpr)); +} + +void expectLdsReady(RaceTestBuilder &builder) { + builder.checkLdsRead(kWave, kLane, /*addr=*/0, /*bytes=*/kDwordBytes); + EXPECT_FALSE(builder.hasLdsRace(0)); +} + +void expectLdsPending(RaceTestBuilder &builder) { + builder.checkLdsRead(kWave, kLane, /*addr=*/0, /*bytes=*/kDwordBytes); + EXPECT_TRUE(builder.hasLdsRace(0)); +} + +} // namespace + +// ---- All-ones no-wait operands ---- + +TEST(RaceDetector, CounterCapacity_LgkmcntAllOnesWaitIsNoOpBelowCapacity) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kClearlyBelowCapacity); + + builder.waitcnt(kWave, /*vmcnt=*/-1, /*lgkmcnt=*/kLgkmcntCapacity); + expectVgprPending(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_VmcntAllOnesWaitIsNoOpBelowCapacity) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, kClearlyBelowCapacity); + + builder.waitcnt(kWave, /*vmcnt=*/kVmcntCapacity, /*lgkmcnt=*/-1); + expectVgprPending(builder, /*vgpr=*/0); +} + +// ---- Issue-time capacity backpressure ---- + +TEST(RaceDetector, CounterCapacity_MissingConfiguredCapacityFailsLoudly) { + RaceDetector detector(/*nWaves=*/ + 1, /*vgprCount=*/1, /*sgprCount=*/1, Dim3d(0), [](RaceViolation) {}, + CounterCapacities{.vmcnt = 0, .lgkmcnt = 0}); + + EXPECT_THROW( + detector.getWaveRaceState(0).prepareForCounterIncrement(amdgpu::WaitCounterType::VMCNT), + std::logic_error); +} + +TEST(RaceDetector, CounterCapacity_LgkmcntOverflowRetiresOldestOrderedReads) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity + kLgkmcntOverflow); + + expectVgprReady(builder, /*vgpr=*/kLgkmcntOverflow - 1); + expectVgprPending(builder, /*vgpr=*/kLgkmcntOverflow); +} + +TEST(RaceDetector, CounterCapacity_VmcntOverflowRetiresOldestOrderedLoads) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, kVmcntCapacity + kVmcntOverflow); + + expectVgprReady(builder, /*vgpr=*/kVmcntOverflow - 1); + expectVgprPending(builder, /*vgpr=*/kVmcntOverflow); +} + +TEST(RaceDetector, CounterCapacity_FullLgkmcntRetiresBeforeCurrentOperandRead) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity); + + builder.prepareCounterIncrement(kWave, amdgpu::WaitCounterType::LGKMCNT); + expectVgprReady(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_LgkmcntBelowCapacityKeepsCurrentOperandPending) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity - 1); + + builder.prepareCounterIncrement(kWave, amdgpu::WaitCounterType::LGKMCNT); + expectVgprPending(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_FullVmcntRetiresBeforeCurrentOperandRead) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, kVmcntCapacity); + + builder.prepareCounterIncrement(kWave, amdgpu::WaitCounterType::VMCNT); + expectVgprReady(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_VmcntBelowCapacityKeepsCurrentOperandPending) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, kVmcntCapacity - 1); + + builder.prepareCounterIncrement(kWave, amdgpu::WaitCounterType::VMCNT); + expectVgprPending(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_LoadcntAliasUsesVmcntCapacity) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, kVmcntCapacity); + + builder.prepareCounterIncrement(kWave, amdgpu::WaitCounterType::LOADCNT); + expectVgprReady(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_SplitLgkmAliasesUseCombinedCapacity) { + for (const auto counter : {amdgpu::WaitCounterType::DSCNT, amdgpu::WaitCounterType::KMCNT}) { + SCOPED_TRACE(static_cast(counter)); + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity); + + builder.prepareCounterIncrement(kWave, counter); + expectVgprReady(builder, /*vgpr=*/0); + } +} + +TEST(RaceDetector, CounterCapacity_StorecntDoesNotAdvanceVmcnt) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, kVmcntCapacity); + + builder.prepareCounterIncrement(kWave, amdgpu::WaitCounterType::STORECNT); + expectVgprPending(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_GenericFlatAdvancesBothCounterDomains) { + amdgpu::MemoryIssueInfo flatIssue; + flatIssue.counter_obligations_[0] = {amdgpu::WaitCounterType::LOADCNT, + amdgpu::MemoryCompletionClass::VMEM}; + flatIssue.counter_obligations_[1] = {amdgpu::WaitCounterType::DSCNT, + amdgpu::MemoryCompletionClass::LDS}; + flatIssue.num_counter_obligations_ = 2; + + auto vmemBuilder = makeVmcntTest(); + issueOrderedVmemLoads(vmemBuilder, kVmcntCapacity); + vmemBuilder.prepareMemoryIssue(kWave, flatIssue); + expectVgprReady(vmemBuilder, /*vgpr=*/0); + + auto ldsBuilder = makeLgkmcntTest(); + issueOrderedLdsReads(ldsBuilder, kLgkmcntCapacity); + ldsBuilder.prepareMemoryIssue(kWave, flatIssue); + expectVgprReady(ldsBuilder, /*vgpr=*/0); +} + +// ---- Mixed completion-order classes ---- + +TEST(RaceDetector, CounterCapacity_FullLgkmcntDoesNotProveScalarLoadComplete) { + auto builder = makeLgkmcntTest(); + builder.scalarLoad(kWave, /*sgprBase=*/0, /*numRegs=*/1); + issueOrderedLdsReads(builder, kLgkmcntCapacity); + + builder.checkSgprRead(kWave, /*reg=*/0); + EXPECT_TRUE(builder.hasSgprRace(0)); +} + +TEST(RaceDetector, CounterCapacity_ScalarIssueAdvancesFullOrderedLgkmcntClass) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity); + + builder.scalarLoad(kWave, /*sgprBase=*/0, /*numRegs=*/1); + expectVgprReady(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_WideScalarIssueContributesTwoLgkmcntTokens) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity - 1); + + builder.scalarLoad(kWave, /*sgprBase=*/0, /*numRegs=*/2); + expectVgprReady(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_MixedLgkmcntDoesNotGuessWhichEventCompleted) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity - 1); + builder.scalarLoad(kWave, /*sgprBase=*/0, /*numRegs=*/1); + + // The next scalar load needs one slot, but the completed event could be the + // first scalar load. It does not prove that the oldest LDS read completed. + builder.scalarLoad(kWave, /*sgprBase=*/1, /*numRegs=*/1); + expectVgprPending(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_InterleavedLgkmcntSoakRetiresOnlyProvableLdsPrefix) { + auto builder = makeLgkmcntTest(); + + // Build a deterministic, irregular stream of 49 LGKMCNT events: 15 ordered + // LDS reads interleaved with 34 unordered scalar stores. The detector's + // candidate set can exceed the physical capacity because it represents all + // events that might remain pending, not one concrete hardware state. Since + // it cannot identify which unordered events completed, all 15 LDS candidates + // remain until one more LGKMCNT operation issues. + constexpr std::array unorderedBursts = { + 3, 0, 4, 1, 2, 5, 0, 3, 1, 4, 2, 0, 5, 1, 3, + }; + for (int ordered = 0; ordered < kLgkmcntCapacity; ++ordered) { + for (int unordered = 0; unordered < unorderedBursts[ordered]; ++unordered) + builder.scalarStore(kWave); + builder.ldsRead(kWave, kLane, /*addr=*/ordered * kDwordBytes, + /*bytes=*/kDwordBytes, /*vgprDst=*/ordered); + } + + // Before the decisive issue, even the oldest LDS result may still be + // pending. The next scalar store requires a free LGKMCNT slot. Since 15 LDS + // events may remain, LDS FIFO ordering proves exactly the oldest one (v0) + // complete, while the next one (v1) remains a possible hazard. + builder.checkVgprRead(kWave, /*reg=*/0, kLane); + ASSERT_EQ(builder.raceCount(), 1); + + builder.scalarStore(kWave); + builder.checkVgprRead(kWave, /*reg=*/0, kLane); + EXPECT_EQ(builder.raceCount(), 1); + + builder.checkVgprRead(kWave, /*reg=*/1, kLane); + EXPECT_EQ(builder.raceCount(), 2); + EXPECT_TRUE(builder.hasVgprRace(1)); +} + +TEST(RaceDetector, CounterCapacity_FullVmcntDoesNotProveUnorderedLoadComplete) { + auto builder = makeVmcntTest(); + issueUnorderedVmcntLoads(builder, /*count=*/1); + issueOrderedVmemLoads(builder, kVmcntCapacity, /*firstVgpr=*/1); + + expectVgprPending(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_UnorderedLoadsDoNotAdvanceOrderedVmcntClass) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, /*count=*/1); + issueUnorderedVmcntLoads(builder, kVmcntCapacity, /*firstVgpr=*/1); + + expectVgprPending(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_UnorderedIssueAdvancesFullOrderedVmcntClass) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, kVmcntCapacity); + + issueUnorderedVmcntLoads(builder, /*count=*/1, /*firstVgpr=*/kVmcntCapacity); + expectVgprReady(builder, /*vgpr=*/0); +} + +// ---- Explicit partial waits after capacity backpressure ---- + +TEST(RaceDetector, CounterCapacity_PartialLgkmcntKeepsUnorderedScalarLoadPending) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity); + builder.scalarLoad(kWave, /*sgprBase=*/0, /*numRegs=*/1); + + builder.waitcnt(kWave, /*vmcnt=*/-1, /*lgkmcnt=*/kLgkmcntCapacity - 1); + expectVgprReady(builder, /*vgpr=*/0); + builder.checkSgprRead(kWave, /*reg=*/0); + EXPECT_TRUE(builder.hasSgprRace(0)); +} + +TEST(RaceDetector, CounterCapacity_PartialLgkmcntPreservesAmbiguousEventsWithinLimit) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity - 1); + builder.scalarLoad(kWave, /*sgprBase=*/0, /*numRegs=*/1); + + builder.waitcnt(kWave, /*vmcnt=*/-1, /*lgkmcnt=*/kLgkmcntCapacity - 1); + expectVgprPending(builder, /*vgpr=*/0); + builder.checkSgprRead(kWave, /*reg=*/0); + EXPECT_TRUE(builder.hasSgprRace(0)); +} + +TEST(RaceDetector, CounterCapacity_PartialLgkmcntRetiresNextOrderedRead) { + auto builder = makeLgkmcntTest(); + issueOrderedLdsReads(builder, kLgkmcntCapacity + kLgkmcntOverflow); + + builder.waitcnt(kWave, /*vmcnt=*/-1, /*lgkmcnt=*/kLgkmcntCapacity - 1); + expectVgprReady(builder, /*vgpr=*/kLgkmcntOverflow); + expectVgprPending(builder, /*vgpr=*/kLgkmcntOverflow + 1); +} + +TEST(RaceDetector, CounterCapacity_PartialVmcntRetiresNextOrderedLoad) { + auto builder = makeVmcntTest(); + issueOrderedVmemLoads(builder, kVmcntCapacity + kVmcntOverflow); + + builder.waitcnt(kWave, /*vmcnt=*/kVmcntCapacity - 1, /*lgkmcnt=*/-1); + expectVgprReady(builder, /*vgpr=*/kVmcntOverflow); + expectVgprPending(builder, /*vgpr=*/kVmcntOverflow + 1); +} + +// ---- Other VMCNT producers ---- + +TEST(RaceDetector, CounterCapacity_CombinedVmemStoresAdvanceVmcntCapacity) { + RaceTestBuilder builder(/*numWaves=*/1, /*vgprs=*/8, /*sgprs=*/kScalarRegisters); + builder.globalLoad(kWave, /*vgprBase=*/0, /*numRegs=*/1); + issueOrderedVmemStores(builder, kVmcntCapacity); + + expectVgprReady(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_CombinedVmemLoadRemainsPendingBeforeStoreCapacity) { + RaceTestBuilder builder(/*numWaves=*/1, /*vgprs=*/8, /*sgprs=*/kScalarRegisters); + builder.globalLoad(kWave, /*vgprBase=*/0, /*numRegs=*/1); + issueOrderedVmemStores(builder, kVmcntCapacity - 1); + + expectVgprPending(builder, /*vgpr=*/0); +} + +TEST(RaceDetector, CounterCapacity_VmcntCompletesDirectToLdsForOwningWave) { + auto builder = makeVmcntTest(); + builder.globalToLds(kWave, /*ldsAddrs=*/{0}, /*bytesPerLane=*/kDwordBytes, /*exec=*/1); + issueOrderedVmemLoads(builder, kVmcntCapacity); + + expectLdsReady(builder); +} + +TEST(RaceDetector, CounterCapacity_DirectToLdsRemainsPendingBelowVmcntCapacity) { + auto builder = makeVmcntTest(); + builder.globalToLds(kWave, /*ldsAddrs=*/{0}, /*bytesPerLane=*/kDwordBytes, /*exec=*/1); + issueOrderedVmemLoads(builder, kVmcntCapacity - 1); + + expectLdsPending(builder); +} diff --git a/emulation/rocjitsu/tests/race-detector/race_detector_event_registry_test.cpp b/emulation/rocjitsu/tests/race-detector/race_detector_event_registry_test.cpp index 2ffe2dd8551b..7168ec1b53f8 100644 --- a/emulation/rocjitsu/tests/race-detector/race_detector_event_registry_test.cpp +++ b/emulation/rocjitsu/tests/race-detector/race_detector_event_registry_test.cpp @@ -27,8 +27,9 @@ TEST(RaceDetector, EventRegistry_TrimmedWaveLocalEventsAreNotBarrierQueued) { // already-trimmed id, causing heap corruption and potentially corrupting LDS // race bookkeeping. int N = EventRegistry::kTrimAttemptInterval; - RaceDetector detector(/*nWaves=*/1, /*vgprCount=*/2, /*sgprCount=*/2, Dim3d(0), - [](RaceViolation) {}); + RaceDetector detector(/*nWaves=*/ + 1, /*vgprCount=*/2, /*sgprCount=*/2, Dim3d(0), [](RaceViolation) {}, + counterCapacitiesForArch(ROCJITSU_CODE_ARCH_CDNA4)); auto &wave = detector.getWaveRaceState(0); // Allocate enough wave-local loads to trigger EventRegistry trimming. Each diff --git a/emulation/rocjitsu/tests/race-detector/race_detector_tests.cpp b/emulation/rocjitsu/tests/race-detector/race_detector_tests.cpp index eb1ec8bd6c6d..51780c72eae0 100644 --- a/emulation/rocjitsu/tests/race-detector/race_detector_tests.cpp +++ b/emulation/rocjitsu/tests/race-detector/race_detector_tests.cpp @@ -1326,9 +1326,11 @@ TEST(RaceDetector, DualOffset_CrossWaveRace) { // Uses RaceDetector/WaveRaceState directly since the builder doesn't // expose registerDualOffsetLdsEvent. std::vector violations; - RaceDetector detector(/*nWaves=*/2, /*vgprCount=*/4, + RaceDetector detector(/*nWaves=*/ + 2, /*vgprCount=*/4, /*sgprCount=*/4, Dim3d(0), - [&](RaceViolation v) { violations.push_back(v); }); + [&](RaceViolation v) { violations.push_back(v); }, + counterCapacitiesForArch(ROCJITSU_CODE_ARCH_CDNA4)); auto &rs = detector.getWaveRaceState(0); std::vector ldsAddrs(64, 0); ldsAddrs[0] = 100; diff --git a/emulation/rocjitsu/tests/race-detector/race_test_builder.h b/emulation/rocjitsu/tests/race-detector/race_test_builder.h index 7d1ff817f676..ab07dad2f5ea 100644 --- a/emulation/rocjitsu/tests/race-detector/race_test_builder.h +++ b/emulation/rocjitsu/tests/race-detector/race_test_builder.h @@ -25,7 +25,8 @@ class RaceTestBuilder { RaceTestBuilder(int numWaves, int vgprs, int sgprs, int waveSize = 64, Dim3d wgId = Dim3d(0)) : waveSize_(waveSize), defaultExec_(waveSize == 64 ? ~0ULL : (1ULL << waveSize) - 1) { detector_ = std::make_unique( - numWaves, vgprs, sgprs, wgId, [this](RaceViolation v) { violations_.push_back(v); }); + numWaves, vgprs, sgprs, wgId, [this](RaceViolation v) { violations_.push_back(v); }, + counterCapacitiesForArch(ROCJITSU_CODE_ARCH_CDNA4)); for (int w = 0; w < numWaves; ++w) { waves_.push_back(&detector_->getWaveRaceState(w)); } @@ -41,6 +42,9 @@ class RaceTestBuilder { if (!exec) { exec = defaultExec_; } + waves_[wave]->prepareForCounterIncrement(amdgpu::WaitCounterType::VMCNT); + if (additionalWaitCounterType) + waves_[wave]->prepareForCounterIncrement(*additionalWaitCounterType); std::vector regs(numRegs); for (int i = 0; i < numRegs; ++i) { regs[i] = vgprBase + i; @@ -76,6 +80,7 @@ class RaceTestBuilder { exec = defaultExec_; } constexpr MemoryOrderClass memoryOrder = MemoryOrderClass::VMEM; + waves_[wave]->prepareForCounterIncrement(amdgpu::WaitCounterType::VMCNT); ldsAddrs.resize(waveSize_, 0); waves_[wave]->registerLdsEvent(pc_++, MemoryEventType::GLOBAL_TO_LDS, /*registers=*/{}, exec, waveSize_, ldsAddrs, bytesPerLane, @@ -89,6 +94,7 @@ class RaceTestBuilder { exec = defaultExec_; } constexpr MemoryOrderClass memoryOrder = MemoryOrderClass::VMEM; + waves_[wave]->prepareForCounterIncrement(amdgpu::WaitCounterType::VMCNT); waves_[wave]->registerEvent(pc_++, MemoryEventType::VGPR_TO_GLOBAL, /*registers=*/{}, exec, /*byteMask=*/0xF, amdgpu::WaitCounterType::VMCNT, memoryOrder); @@ -97,33 +103,54 @@ class RaceTestBuilder { /// Register a scalar load into SGPRs with its architecture-specific counter. void scalarLoad(int wave, int sgprBase, int numRegs, amdgpu::WaitCounterType waitCounterType = amdgpu::WaitCounterType::LGKMCNT) { + const uint8_t increment = numRegs > 1 ? 2 : 1; + waves_[wave]->prepareForCounterIncrement(waitCounterType, increment); + const std::array obligations{ + amdgpu::MemoryCounterObligation{waitCounterType, MemoryOrderClass::UNORDERED, increment}}; waves_[wave]->registerScalarLoad( pc_++, RegisterRef{RegClass::SGPR, static_cast(sgprBase), static_cast(numRegs)}, - defaultExec_, waitCounterType, MemoryOrderClass::UNORDERED); + defaultExec_, obligations, MemoryOrderClass::UNORDERED); } /// Register a scalar load into TTMPs with its architecture-specific counter. void ttmpLoad(int wave, int ttmpBase, int numRegs, amdgpu::WaitCounterType waitCounterType = amdgpu::WaitCounterType::LGKMCNT) { + const uint8_t increment = numRegs > 1 ? 2 : 1; + waves_[wave]->prepareForCounterIncrement(waitCounterType, increment); + const std::array obligations{ + amdgpu::MemoryCounterObligation{waitCounterType, MemoryOrderClass::UNORDERED, increment}}; waves_[wave]->registerScalarLoad( pc_++, RegisterRef{RegClass::TTMP, static_cast(ttmpBase), static_cast(numRegs)}, - defaultExec_, waitCounterType, MemoryOrderClass::UNORDERED); + defaultExec_, obligations, MemoryOrderClass::UNORDERED); } /// Register a scalar store so partial waits retain counter ordering. void scalarStore(int wave, amdgpu::WaitCounterType waitCounterType = amdgpu::WaitCounterType::LGKMCNT) { + waves_[wave]->prepareForCounterIncrement(waitCounterType); waves_[wave]->registerEvent(pc_++, MemoryEventType::SCALAR_TO_GLOBAL, {}, defaultExec_, 0xF, waitCounterType, MemoryOrderClass::UNORDERED); } + /// Apply issue-time backpressure before checking an operation's accesses. + void prepareCounterIncrement(int wave, amdgpu::WaitCounterType counter) { + waves_[wave]->prepareForCounterIncrement(counter); + } + + void prepareMemoryIssue(int wave, const amdgpu::MemoryIssueInfo &info) { + waves_[wave]->prepareForMemoryIssue(info); + } + /// Register an LDS write and validate against outstanding reads. void ldsWrite(int wave, int lane, int addr, int bytes, MemoryOrderClass memoryOrder = MemoryOrderClass::LDS, amdgpu::WaitCounterType waitCounterType = amdgpu::WaitCounterType::LGKMCNT, std::optional additionalWaitCounterType = std::nullopt) { + waves_[wave]->prepareForCounterIncrement(waitCounterType); + if (additionalWaitCounterType) + waves_[wave]->prepareForCounterIncrement(*additionalWaitCounterType); detector_->validateWrite(addr, WaveId{wave}, lane, bytes, memoryOrder); std::vector ldsAddrs(waveSize_, 0); ldsAddrs[lane] = addr; @@ -141,6 +168,9 @@ class RaceTestBuilder { amdgpu::WaitCounterType waitCounterType = amdgpu::WaitCounterType::LGKMCNT, MemoryOrderClass memoryOrder = MemoryOrderClass::LDS, std::optional additionalWaitCounterType = std::nullopt) { + waves_[wave]->prepareForCounterIncrement(waitCounterType); + if (additionalWaitCounterType) + waves_[wave]->prepareForCounterIncrement(*additionalWaitCounterType); detector_->validateRead(addr, WaveId{wave}, lane, bytes, memoryOrder); std::vector ldsAddrs(waveSize_, 0); ldsAddrs[lane] = addr; diff --git a/emulation/rocjitsu/tests/race-detector/race_test_support.hpp b/emulation/rocjitsu/tests/race-detector/race_test_support.hpp index 94c940ffd6b8..59e689dbfd68 100644 --- a/emulation/rocjitsu/tests/race-detector/race_test_support.hpp +++ b/emulation/rocjitsu/tests/race-detector/race_test_support.hpp @@ -13,6 +13,91 @@ namespace rocjitsu::test { +/// Common issue-capacity probes. Target suites instantiate these with their +/// architectural counter capacities and keep target-specific expectations. +template __global__ void vmcnt_capacity_kernel(const float *src, float *dst) { + int tid = threadIdx.x + blockIdx.x * blockDim.x; + float producer, younger, result; + asm volatile("global_load_dword %[producer], %[address], off\n" + ".rept %c[younger_loads]\n" + "global_load_dword %[younger], %[address], off\n" + ".endr\n" + "v_mov_b32 %[result], %[producer]\n" + : [producer] "=&v"(producer), [younger] "=&v"(younger), [result] "=&v"(result) + : [address] "v"(&src[tid]), [younger_loads] "n"(YoungerLoads) + : "memory"); + dst[tid] = result; +} + +template __global__ void lgkmcnt_capacity_kernel(int *dst) { + __shared__ int lds[64]; + int tid = threadIdx.x; + lds[tid] = tid + 1; + __syncthreads(); + + int producer, younger, result; + asm volatile("ds_read_b32 %[producer], %[address]\n" + ".rept %c[younger_reads]\n" + "ds_read_b32 %[younger], %[address]\n" + ".endr\n" + "v_mov_b32 %[result], %[producer]\n" + : [producer] "=&v"(producer), [younger] "=&v"(younger), [result] "=&v"(result) + : [address] "v"(tid * 4), [younger_reads] "n"(YoungerReads) + : "memory"); + dst[tid] = result; +} + +/// Fill LGKMCNT with ordered LDS reads, then issue an unordered scalar load. +/// At capacity, the scalar issue proves that the oldest LDS read completed; +/// one token below capacity, consuming that LDS result must still race. +template +__global__ void mixed_lgkmcnt_capacity_kernel(const int *scalar_src, int *dst) { + __shared__ int lds[64]; + int tid = threadIdx.x; + lds[tid] = tid + 1; + __syncthreads(); + + int producer, younger, scalar_result, result; + asm volatile("ds_read_b32 %[producer], %[lds_address]\n" + ".rept %c[younger_reads]\n" + "ds_read_b32 %[younger], %[lds_address]\n" + ".endr\n" + "s_load_dword %[scalar_result], %[scalar_address], 0x0\n" + "v_mov_b32 %[result], %[producer]\n" + : [producer] "=&v"(producer), [younger] "=&v"(younger), + [scalar_result] "=s"(scalar_result), [result] "=&v"(result) + : [lds_address] "v"(tid * 4), [scalar_address] "s"(scalar_src), + [younger_reads] "n"(YoungerReads) + : "memory"); + dst[tid] = result; +} + +/// Exercise the decoded two-token count for a wide scalar load. With one +/// fewer LDS token than the one-DWORD case, the wide issue still forces the +/// oldest ordered LDS read to complete. +template +__global__ void wide_scalar_lgkmcnt_capacity_kernel(const uint64_t *scalar_src, int *dst) { + __shared__ int lds[64]; + int tid = threadIdx.x; + lds[tid] = tid + 1; + __syncthreads(); + + int producer, younger, result; + uint64_t scalar_result; + asm volatile("ds_read_b32 %[producer], %[lds_address]\n" + ".rept %c[younger_reads]\n" + "ds_read_b32 %[younger], %[lds_address]\n" + ".endr\n" + "s_load_dwordx2 %[scalar_result], %[scalar_address], 0x0\n" + "v_mov_b32 %[result], %[producer]\n" + : [producer] "=&v"(producer), [younger] "=&v"(younger), + [scalar_result] "=s"(scalar_result), [result] "=&v"(result) + : [lds_address] "v"(tid * 4), [scalar_address] "s"(scalar_src), + [younger_reads] "n"(YoungerReads) + : "memory"); + dst[tid] = result; +} + class RaceTestBase : public ::testing::Test { protected: template T *alloc(int count) { diff --git a/emulation/rocjitsu/tests/readlane_selector_test.cpp b/emulation/rocjitsu/tests/readlane_selector_test.cpp index c845188c5348..d07624763d2b 100644 --- a/emulation/rocjitsu/tests/readlane_selector_test.cpp +++ b/emulation/rocjitsu/tests/readlane_selector_test.cpp @@ -12,12 +12,9 @@ #include -#include #include -#include #include #include -#include namespace { @@ -25,6 +22,32 @@ using namespace rocjitsu; class ReadlaneSelectorTest : public ::testing::TestWithParam { protected: + void SetUp() override { + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = GetParam(); + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = gfx9() ? 102 : 106; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + cu = amdgpu::ComputeUnitCore::create("readlane", cfg, &mem, &l2); + decoder = Decoder::create(GetParam()); + wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16); + ASSERT_NE(wf, nullptr); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) + cu->write_vgpr(wf->vgpr_alloc().base, lane, 0xa5000000u + lane); + } + + void TearDown() override { + if (wf) + wf->halt(); + } + + amdgpu::GpuMemory mem{"readlane_mem"}; + amdgpu::L2Cache l2{"readlane_l2"}; + std::unique_ptr cu; + std::unique_ptr decoder; + amdgpu::Wavefront *wf = nullptr; + bool gfx9() const { const auto arch = GetParam(); return arch == ROCJITSU_CODE_ARCH_CDNA1 || arch == ROCJITSU_CODE_ARCH_CDNA2 || @@ -40,36 +63,155 @@ INSTANTIATE_TEST_SUITE_P(AllArchitectures, ReadlaneSelectorTest, ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA5)); TEST_P(ReadlaneSelectorTest, ReadlaneMasksInline64AndLiteralIndices) { - for (auto arch : {GetParam()}) { + wf->set_exec(0); // READLANE ignores EXEC, even for inactive source lanes. + for (uint32_t selector : {192u, 255u}) { + if (selector == 255 && GetParam() != ROCJITSU_CODE_ARCH_RDNA3 && + GetParam() != ROCJITSU_CODE_ARCH_RDNA4) + continue; // Literal lane indices are only qualified on these two targets. + // v_readlane_b32 s4, v0, 64 / literal 95, followed by s_endpgm. + std::array words{gfx9() ? 0xd2890004u : 0xd7600004u, 256u | (selector << 9), 95, + 0xbfb00000}; + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + EXPECT_EQ(inst->size(), selector == 255 ? 12u : 8u); + EXPECT_EQ(inst->disassemble(), + selector == 255 ? "v_readlane_b32 s4, v0, 0x5f" : "v_readlane_b32 s4, v0, 64"); + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 4), + 0xa5000000u + ((selector == 255 ? 95u : 64u) & (wf->wf_size() - 1))); + } +} + +TEST_P(ReadlaneSelectorTest, ReadlaneWritesBothVccWordsWithEmptyExec) { + wf->set_exec(0); + // The lane index comes from s4, rather than its selector encoding. + cu->write_sgpr(wf->sgpr_alloc().base + 4, 5); + for (uint32_t dst : {106u, 107u}) { + SCOPED_TRACE(dst); + constexpr uint64_t initial = 0x1122334455667788ull; + wf->set_vcc_raw(initial); + std::array words{(gfx9() ? 0xd2890000u : 0xd7600000u) | dst, 256u | (4u << 9)}; + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + EXPECT_EQ( + inst->disassemble(), + GetParam() == ROCJITSU_CODE_ARCH_CDNA5 + ? (dst == 106 ? "v_readlane_b32 VCC_LO, v0, s4" : "v_readlane_b32 VCC_HI, v0, s4") + : (dst == 106 ? "v_readlane_b32 vcc_lo, v0, s4" : "v_readlane_b32 vcc_hi, v0, s4")); + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + const uint64_t value = 0xa5000005u; + EXPECT_EQ(wf->vcc(), dst == 106 ? (initial & 0xffffffff00000000ull) | value + : (initial & 0xffffffffull) | (value << 32)); + EXPECT_EQ(wf->exec(), 0u); + } +} + +TEST_P(ReadlaneSelectorTest, ReadfirstlaneWritesBothVccWords) { + for (bool e64 : {false, true}) { + for (uint64_t exec : {uint64_t{0}, uint64_t{1} << 5 | uint64_t{1} << 7}) { + for (uint32_t dst : {106u, 107u}) { + SCOPED_TRACE(e64); + SCOPED_TRACE(exec); + SCOPED_TRACE(dst); + constexpr uint64_t initial = 0x1122334455667788ull; + wf->set_vcc_raw(initial); + wf->set_exec(exec); + // VOP1 and VOP3 forms from the architecture encoding tables. + std::array words = + e64 ? std::array{(gfx9() ? 0xd1420000u : 0xd5820000u) | dst, 256u} + : std::array{0x7e000500u | (dst << 17), 0u}; + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + EXPECT_EQ(inst->size(), e64 ? 8u : 4u); + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + const uint64_t value = 0xa5000000u + (exec ? 5u : 0u); + EXPECT_EQ(wf->vcc(), dst == 106 ? (initial & 0xffffffff00000000ull) | value + : (initial & 0xffffffffull) | (value << 32)); + EXPECT_EQ(wf->exec(), exec); + } + } + } +} + +TEST_P(ReadlaneSelectorTest, WritelaneUsesVccSourceAndLaneIndexWithEmptyExec) { + wf->set_exec(0); + for (uint32_t src : {106u, 107u}) { + SCOPED_TRACE(src); + constexpr uint64_t initial = 0xdeadbeefcafebabeull; + wf->set_vcc_raw(initial); + // v_writelane_b32 v0, vcc_lo/hi, 5. + std::array words{gfx9() ? 0xd28a0000u : 0xd7610000u, src | (133u << 9)}; + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, lane), + lane == 5 ? (src == 106 ? 0xcafebabeu : 0xdeadbeefu) : 0xa5000000u + lane); + EXPECT_EQ(wf->vcc(), initial); + // v_writelane_b32 v0, 42, vcc_lo/hi. Each encoding uses one scalar source. + const uint64_t index = wf->wf_size() + 5u; + wf->set_vcc_raw(src == 106 ? index : index << 32); + words[1] = 170u | (src << 9); + inst.reset(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, lane), lane == 5 ? 42u : 0xa5000000u + lane); + EXPECT_EQ(wf->vcc(), src == 106 ? index : index << 32); + EXPECT_EQ(wf->exec(), 0u); + } +} + +TEST_P(ReadlaneSelectorTest, LaneReadsRejectUnqualifiedScalarDestinations) { + const bool rdna12 = + GetParam() == ROCJITSU_CODE_ARCH_RDNA1 || GetParam() == ROCJITSU_CODE_ARCH_RDNA2; + const uint32_t m0 = gfx9() || rdna12 ? 124u : 125u; + // LLVM assembles EXEC destinations, but their write behavior is not qualified. + // Keep them explicitly unsupported alongside the architectural M0 exclusion. + for (uint32_t dst : {m0, 126u, 127u}) { + SCOPED_TRACE(dst); + const std::array readlane{(gfx9() ? 0xd2890000u : 0xd7600000u) | dst, + 256u | (133u << 9)}; + const std::array firstlane_e32{0x7e000500u | (dst << 17), 0u}; + const std::array firstlane_e64{(gfx9() ? 0xd1420000u : 0xd5820000u) | dst, 256u}; + EXPECT_TRUE(decode_fails(*decoder, readlane.data())); + EXPECT_TRUE(decode_fails(*decoder, firstlane_e32.data())); + EXPECT_TRUE(decode_fails(*decoder, firstlane_e64.data())); + } +} + +TEST(CdnaReadlaneTest, RestoresBothVccHalvesFromInactiveSpillLanes) { + for (auto arch : {ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA4}) { + SCOPED_TRACE(arch); amdgpu::GpuMemory mem("readlane_mem"); amdgpu::L2Cache l2("readlane_l2"); amdgpu::ComputeUnitCore::Config cfg{}; cfg.arch = arch; cfg.num_wf_slots = 1; - cfg.sgprs_per_wf = gfx9() ? 102 : 106; - cfg.vgprs_per_wf = 16; + cfg.sgprs_per_wf = 102; + cfg.vgprs_per_wf = 128; cfg.lds_size_kb = 64; auto cu = amdgpu::ComputeUnitCore::create("readlane", cfg, &mem, &l2); auto decoder = Decoder::create(arch); - auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16); + auto *wf = cu->dispatch_wf(0, 0, 102, 128); ASSERT_NE(wf, nullptr); - wf->set_exec(0); // READLANE ignores EXEC, even for inactive source lanes. - for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) - cu->write_vgpr(wf->vgpr_alloc().base, lane, 1000 + lane); - for (uint32_t selector : {192u, 255u}) { - if (selector == 255 && arch != ROCJITSU_CODE_ARCH_RDNA3 && arch != ROCJITSU_CODE_ARCH_RDNA4) - continue; // Literal lane indices are only qualified on these two targets. - // v_readlane_b32 s4, v0, 64 / literal 95, followed by s_endpgm. - std::array words{gfx9() ? 0xd2890004u : 0xd7600004u, 256u | (selector << 9), 95, - 0xbfb00000}; - std::unique_ptr inst(decode_valid(*decoder, words.data())); + wf->set_exec(0); + wf->set_vcc(0xaabbccdd11223344ull); + cu->write_vgpr(wf->vgpr_alloc().base + 126, 33, 0x76543210u); + cu->write_vgpr(wf->vgpr_alloc().base + 126, 34, 0xfedcba98u); + // LLVM's tinygrad cumprod gradient restores VCC from VGPR spill lanes. + const std::array, 2> words{{ + {0xd289006au, 0x0001437eu}, // v_readlane_b32 vcc_lo, v126, 33 + {0xd289006bu, 0x0001457eu}, // v_readlane_b32 vcc_hi, v126, 34 + }}; + for (unsigned i = 0; i < words.size(); ++i) { + std::unique_ptr inst(decode_valid(*decoder, words[i].data())); ASSERT_NE(inst, nullptr); - EXPECT_EQ(inst->size(), selector == 255 ? 12u : 8u); EXPECT_EQ(inst->disassemble(), - selector == 255 ? "v_readlane_b32 s4, v0, 0x5f" : "v_readlane_b32 s4, v0, 64"); + i == 0 ? "v_readlane_b32 vcc_lo, v126, 33" : "v_readlane_b32 vcc_hi, v126, 34"); ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); - EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 4), - 1000u + ((selector == 255 ? 95u : 64u) & (wf->wf_size() - 1))); + EXPECT_EQ(wf->vcc(), i == 0 ? 0xaabbccdd76543210ull : 0xfedcba9876543210ull); + EXPECT_EQ(wf->exec(), 0u); } wf->halt(); } diff --git a/emulation/rocjitsu/tests/register_access_test.cpp b/emulation/rocjitsu/tests/register_access_test.cpp index 955d568e819d..84463cf366d2 100644 --- a/emulation/rocjitsu/tests/register_access_test.cpp +++ b/emulation/rocjitsu/tests/register_access_test.cpp @@ -1269,7 +1269,7 @@ TEST(RegisterAccessTest, WriteViewsCannotOutliveTheirAcquiredWriteMask) { // or bytes 2-3. Reading or preserving the other half inside the 32-bit // register file must neither expose its value nor create another callback. TEST(RegisterAccessTest, Packed16ReadsAndWritesObserveSelectedByteHalves) { - Fixture fx(ROCJITSU_CODE_ARCH_RDNA4); + Fixture fx(ROCJITSU_CODE_ARCH_RDNA4, kSgprsPerWave, 1, 8); ASSERT_NE(fx.wf, nullptr); uint32_t reg = fx.vgpr_base() + 5; constexpr uint32_t lane = 3; @@ -1306,6 +1306,122 @@ TEST(RegisterAccessTest, Packed16ReadsAndWritesObserveSelectedByteHalves) { EXPECT_EQ(fx.cu->read_vgpr(reg, lane), 0x33441122u); } +TEST(RegisterAccessTest, Packed16DppObservesSelectedPhysicalLanesAndHalves) { + Fixture fx(ROCJITSU_CODE_ARCH_RDNA4, kSgprsPerWave, 1, 8); + ASSERT_NE(fx.wf, nullptr); + fx.wf->set_exec(0b0101); + const uint32_t base = fx.vgpr_base(); + for (uint32_t lane = 0; lane < 4; ++lane) + fx.cu->write_vgpr(base + 1, lane, ((0xc000u + lane) << 16) | 0x1234u); + rdna4::Operand source(16, rdna4::OperandType::OPR_VGPR, 129, true); + RegisterAccess regs(*fx.wf); + std::optional staged; + // Swap adjacent lanes: active destinations 0 and 2 read inactive sources 1 and 3. + const auto plan = + dpp::make_dpp_plan(fx.wf->wf_size(), 0xb1, 0xf, 0xf, false, true, fx.wf->exec(), true); + dpp::apply_dpp(source, plan, fx.wf->exec(), staged, *fx.wf); + ASSERT_TRUE(staged.has_value()); + EXPECT_EQ(regs.read_lane(*staged, 0), 0xc001u); + EXPECT_EQ(regs.read_lane(*staged, 2), 0xc003u); + ASSERT_EQ(fx.plugin->reads.size(), 2u); + for (size_t i = 0; i < 2; ++i) { + EXPECT_EQ(fx.plugin->reads[i].physical_reg, base + 1); + EXPECT_EQ(fx.plugin->reads[i].lane_mask, uint64_t{1} << (2 * i + 1)); + EXPECT_EQ(fx.plugin->reads[i].byte_mask, ExecutionPlugin::kHighHalfByteMask); + } + staged.reset(); + fx.plugin->reads.clear(); + dpp::apply_source_modifiers(source, staged, *fx.wf, 0x8000, {.absolute = 1}, 0); + ASSERT_TRUE(staged.has_value()); + EXPECT_EQ(regs.read_lane(*staged, 0), 0x4000u); + EXPECT_EQ(regs.read_lane(*staged, 2), 0x4002u); + ASSERT_EQ(fx.plugin->reads.size(), 2u); + for (size_t i = 0; i < 2; ++i) { + EXPECT_EQ(fx.plugin->reads[i].physical_reg, base + 1); + EXPECT_EQ(fx.plugin->reads[i].lane_mask, uint64_t{1} << (2 * i)); + EXPECT_EQ(fx.plugin->reads[i].byte_mask, ExecutionPlugin::kHighHalfByteMask); + } + fx.plugin->reads.clear(); + fx.plugin->writes.clear(); + rdna4::Operand outside_src(16, rdna4::OperandType::OPR_VGPR, 128 + 8, true); + rdna4::Operand outside_dst(16, rdna4::OperandType::OPR_VGPR, 128 + 8, false, true); + EXPECT_EQ(regs.read_lane(outside_src, 0), 0u); + regs.write_lane(outside_dst, 0, 0xbeefu); + EXPECT_TRUE(fx.plugin->reads.empty()); + EXPECT_TRUE(fx.plugin->writes.empty()); +} + +TEST(RegisterAccessTest, Packed16DppObservesBroadcastSourcesOnceAndSkipsSuppressedReads) { + Fixture fx(ROCJITSU_CODE_ARCH_RDNA4, kSgprsPerWave, 1, 8); + ASSERT_NE(fx.wf, nullptr); + fx.wf->set_exec(0xf); + const uint32_t reg = fx.vgpr_base() + 1; + fx.cu->write_vgpr(reg, 3, 0xc2001234); + rdna4::Operand source(16, rdna4::OperandType::OPR_VGPR, 129, true); + RegisterAccess regs(*fx.wf); + std::optional staged; + // All four destinations broadcast lane 3. + const auto broadcast = + dpp::make_dpp_plan(fx.wf->wf_size(), 0xff, 0xf, 0xf, false, true, fx.wf->exec(), true); + dpp::apply_dpp(source, broadcast, fx.wf->exec(), staged, *fx.wf); + ASSERT_TRUE(staged.has_value()); + for (uint32_t lane = 0; lane < 4; ++lane) + EXPECT_EQ(regs.read_lane(*staged, lane), 0xc200u); + ASSERT_EQ(fx.plugin->reads.size(), 1u); + EXPECT_EQ(fx.plugin->reads[0].physical_reg, reg); + EXPECT_EQ(fx.plugin->reads[0].lane_mask, 1u << 3); + EXPECT_EQ(fx.plugin->reads[0].byte_mask, ExecutionPlugin::kHighHalfByteMask); + + fx.wf->set_exec(1); + for (bool bound : {false, true}) { + fx.plugin->reads.clear(); + const auto missing = dpp::make_dpp_plan(fx.wf->wf_size(), dpp::ROW_SHR1, 0xf, 0xf, bound, true, + fx.wf->exec(), true); + dpp::apply_dpp(source, missing, fx.wf->exec(), staged, *fx.wf); + EXPECT_TRUE(fx.plugin->reads.empty()); + EXPECT_EQ(regs.read_lane(*staged, 0), 0u); + } + for (uint32_t fi : {0u, 1u}) { + fx.plugin->reads.clear(); + // DPP8 broadcasts inactive lane 3. FI controls whether it is read. + dpp::apply_dpp8(source, 3, fi, staged, *fx.wf); + EXPECT_EQ(regs.read_lane(*staged, 0), fi ? 0xc200u : 0u); + ASSERT_EQ(fx.plugin->reads.size(), fi); + if (fi) { + EXPECT_EQ(fx.plugin->reads[0].physical_reg, reg); + EXPECT_EQ(fx.plugin->reads[0].lane_mask, 1u << 3); + EXPECT_EQ(fx.plugin->reads[0].byte_mask, ExecutionPlugin::kHighHalfByteMask); + } + } +} + +TEST(RegisterAccessTest, Packed16PhysicalMappingIncludesSourceAndDestinationMsb) { + Fixture fx(ROCJITSU_CODE_ARCH_CDNA5, kSgprsPerWave, 1, 1024); + ASSERT_NE(fx.wf, nullptr); + fx.wf->set_exec(1); + fx.wf->set_vgpr_msb_mode(1 | (2 << 2) | (3 << 6)); + const uint32_t base = fx.vgpr_base(); + RegisterAccess regs(*fx.wf); + for (auto role : {VgprMsbRole::Src0, VgprMsbRole::Src1}) { + const uint32_t reg = base + (fx.wf->vgpr_msb_for_role(role) << 8) + 1; + fx.cu->write_vgpr(reg, 0, 0xc2001234); + cdna5::Operand source(16, cdna5::OperandType::OPR_VGPR, 129, true); + source.set_vgpr_msb_role(role); + EXPECT_EQ(regs.read_lane(source, 0), 0xc200u); + ASSERT_EQ(fx.plugin->reads.back().physical_reg, reg); + EXPECT_EQ(fx.plugin->reads.back().byte_mask, ExecutionPlugin::kHighHalfByteMask); + } + const uint32_t dst_reg = base + (3u << 8) + 1; + fx.cu->write_vgpr(dst_reg, 0, 0xc2001234); + cdna5::Operand dst(16, cdna5::OperandType::OPR_VGPR, 129, false, true); + dst.set_vgpr_msb_role(VgprMsbRole::Dst); + regs.write_lane(dst, 0, 0x4000); + EXPECT_EQ(fx.cu->read_vgpr(dst_reg, 0), 0x40001234u); + ASSERT_EQ(fx.plugin->writes.size(), 1u); + EXPECT_EQ(fx.plugin->writes[0].physical_reg, dst_reg); + EXPECT_EQ(fx.plugin->writes[0].byte_mask, ExecutionPlugin::kHighHalfByteMask); +} + TEST(RegisterAccessTest, OperandWrite64ViewObservesBothPhysicalRegisters) { if constexpr (!util::has_stdx_simd) { GTEST_SKIP() << " unavailable"; diff --git a/emulation/rocjitsu/tests/register_file_test.cpp b/emulation/rocjitsu/tests/register_file_test.cpp index 8f5317b67f21..46ae92c9dc8f 100644 --- a/emulation/rocjitsu/tests/register_file_test.cpp +++ b/emulation/rocjitsu/tests/register_file_test.cpp @@ -36,6 +36,39 @@ concept HasContiguousData = requires(File &file) { file.data(); }; using SoftwareLazyUint32File = SoftwareLazyTestFile; static_assert(!HasContiguousData); +template class RegisterBlockCountTest : public ::testing::Test {}; +using RegisterBlockCountTypes = + ::testing::Types, SoftwareLazyUint32File, + RegisterFile>; +TYPED_TEST_SUITE(RegisterBlockCountTest, RegisterBlockCountTypes); + +TYPED_TEST(RegisterBlockCountTest, CountsAvailableBlocksAcrossExhaustionAndReuse) { + TypeParam file("registers"); + EXPECT_EQ(file.free_block_count(), 0u); + file.init(/*total_regs=*/35, /*regs_per_block=*/8); + EXPECT_EQ(file.free_block_count(), 4u); + EXPECT_EQ(file.allocate(0), -1); + EXPECT_EQ(file.free_block_count(), 4u); + for (uint32_t block = 0; block < 4; ++block) { + ASSERT_EQ(file.allocate(8), static_cast(block * 8)); + EXPECT_EQ(file.free_block_count(), 3 - block); + } + EXPECT_EQ(file.allocate(1), -1); + file.free(3); // Misaligned and out-of-range frees leave allocation unchanged. + file.free(40); + EXPECT_EQ(file.free_block_count(), 0u); + file.free(8); + file.free(24); + EXPECT_EQ(file.free_block_count(), 2u); + EXPECT_EQ(file.allocate(1), 8); + EXPECT_EQ(file.free_block_count(), 1u); + EXPECT_EQ(file.allocate(8), 24); + EXPECT_EQ(file.free_block_count(), 0u); + for (uint32_t block = 0; block < 4; ++block) + file.free(block * 8); + EXPECT_EQ(file.free_block_count(), 4u); +} + TEST(RegisterFileTest, ContiguousStorageClearsReusedBlock) { RegisterFile file("contiguous"); file.init(/*total_regs=*/16, /*regs_per_block=*/8); @@ -82,6 +115,41 @@ TEST(RegisterFileTest, SoftwareLazyStorageMaterializesOnlyMutableChunks) { EXPECT_EQ(const_storage[regs_per_chunk][0], 0u); } +TEST(RegisterFileTest, SoftwareLazyRecycledChunksAreZeroBeforeEveryMaterializationPath) { + using Storage = SoftwareLazyTestStorage; + constexpr uint32_t count = Storage::registers_per_chunk(); + Storage storage; + storage.init(count); + const auto &read_only = storage; + for (unsigned path = 0; path < 4; ++path) { + SCOPED_TRACE(path); + storage.for_each(0, count, [](uint32_t &value) { value = 0xfeed1234; }); + storage.reset(0, count); + EXPECT_EQ(storage.materialized_chunk_count(), 0u); + EXPECT_EQ(read_only[0], 0u); + EXPECT_EQ(read_only[count - 1], 0u); + constexpr uint32_t value = 42; + const auto bytes = std::as_bytes(std::span(&value, 1)); + switch (path) { + case 0: + storage[0] = value; + break; + case 1: + storage.for_each(0, 1, [](uint32_t ®) { reg = value; }); + break; + case 2: + storage.copy_from(0, bytes); + break; + case 3: + storage.copy_nonzero_from(0, bytes); + break; + } + EXPECT_EQ(read_only[0], value); + for (uint32_t index = 1; index < count; ++index) + EXPECT_EQ(read_only[index], 0u) << index; + } +} + TEST(RegisterFileTest, SoftwareLazyStorageSupportsFixedCapacityBoundary) { using Vgpr = simdojo::VectorReg<64, uint32_t>; using Storage = simdojo::detail::SoftwareLazyRegisterStorage; diff --git a/emulation/rocjitsu/tests/rocgdb/abort_waves.hip b/emulation/rocjitsu/tests/rocgdb/abort_waves.hip new file mode 100644 index 000000000000..e38784cff4d6 --- /dev/null +++ b/emulation/rocjitsu/tests/rocgdb/abort_waves.hip @@ -0,0 +1,24 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT +// +// Multi-wave abort workload for the ROCgdb CI harness (debug_kernel.sh). +// Each wave reports an abort. The debugger must be able to inspect the first +// stop and forward every remaining stop after ROCr has handled the queue error. + +#include +#include + +__global__ void abort_waves() { asm volatile("s_trap 2"); } + +int main() { + if (hipSetDevice(0) != hipSuccess) + return 1; + abort_waves<<<1, 128>>>(); + const hipError_t status = hipDeviceSynchronize(); + if (status == hipSuccess) { + std::fprintf(stderr, "abort_waves incorrectly synchronized successfully\n"); + return 2; + } + std::printf("abort_waves done: hipDeviceSynchronize reported %s\n", hipGetErrorName(status)); + return 0; +} diff --git a/emulation/rocjitsu/tests/rocgdb/debug_kernel.sh b/emulation/rocjitsu/tests/rocgdb/debug_kernel.sh index cf1002f526f2..abd8063130fd 100755 --- a/emulation/rocjitsu/tests/rocgdb/debug_kernel.sh +++ b/emulation/rocjitsu/tests/rocgdb/debug_kernel.sh @@ -5,12 +5,12 @@ # # rocgdb wave-debugging demo + CI harness for the rocjitsu emulator. # -# Compiles the demo HIP kernel with device debug info, then drives real ROCgdb -# through `mirage run` to: set a breakpoint on the GPU kernel `add_one`, run, -# stop at the wave, read PC/EXEC and the instruction at PC, single-step, and -# continue the kernel to completion. It asserts the expected markers so it can -# double as a CI smoke test for the full mirage + rocjitsu + rocm-dbgapi + ROCgdb -# stack. +# Compiles HIP workloads with device debug info, then drives real ROCgdb +# through `mirage run` to exercise breakpoints, stepping, watchpoints, device +# faults, scratch reads, multi-wave inspection, and fatal abort forwarding. +# Checks wave state at debugger stops and host results after continuation. +# Expected output markers make this a CI smoke test for the full mirage + +# rocjitsu + rocm-dbgapi + ROCgdb stack. # # Exit codes: # 0 success (all debug markers observed) OR a required tool is missing and @@ -42,7 +42,7 @@ fi # CTest matches the whole output against SKIP_REGULAR_EXPRESSION "SKIP:" (see # the RocgdbDebug.KernelBreakpoint registration in tests/CMakeLists.txt), and a -# match reports the entire seven-scenario run skipped regardless of the exit +# match reports the entire eight-scenario run skipped regardless of the exit # status. So the literal string "SKIP:" must be emitted only from here, where it # always means a test-wide prerequisite is missing and the run stops # immediately. A mid-run scenario that cannot proceed sets fail=1 instead -- @@ -545,6 +545,65 @@ else fail=1 fi +# --- Eighth scenario: multi-wave abort forwarding ----------------------------- +# A fatal exception can be reported by several waves in one queue. ROCr's +# exception callback is removed after the first report, but later debugger +# forwards must still succeed. Inspect the first stop, then pass all remaining +# SIGABRT stops to the runtime and require the host to observe a HIP error. +# ROCr handlers may leave PC on the abort or advance it by four bytes. Inspect +# both addresses; the instruction at PC-4 has no current-PC arrow in GDB output. +abort_app="$workdir/abort_waves" +if hipcc --offload-arch="$arch" -g -O0 -o "$abort_app" "$here/abort_waves.hip" 2>"$workdir/abort-build.log"; then + outfile8="$workdir/abort.out" + echo "running rocgdb (multi-wave abort forwarding) ..." + ( + ulimit -c 0 + timeout 180 "$mirage_bin" run --profile "$profile" "${mirage_runtime_args[@]}" \ + --env HIP_SKIP_ABORT_ON_GPU_ERROR=1 -- \ + rocgdb --batch \ + -ex 'set print thread-events off' \ + -ex 'run' \ + -ex 'info registers pc exec' \ + -ex 'x/i $pc' \ + -ex 'x/i $pc-4' \ + -ex 'handle SIGABRT nostop print pass' \ + -ex 'continue' \ + "$abort_app" "$outfile8" 2>&1 + status8=$? + out8="$(cat "$outfile8")" + echo "--------------------------------------------------------------------" + echo "$out8" + echo "--------------------------------------------------------------------" + if [[ $status8 -ne 0 ]]; then + echo "FAIL: multi-wave abort run exited with status $status8" >&2 + fail=1 + fi + check8() { + if grep -qaE "$1" <<<"$out8"; then + echo " ok: $2" + else + echo " MISSING: $2 (/$1/)" >&2 + fail=1 + fi + } + check8 'SIGABRT, Aborted' 'stopped at a device abort' + check8 '^pc +0x[0-9a-f]+ +0x[0-9a-f]+ <.*abort_waves' 'read the aborted wave PC' + check8 "$exec_regex" 'read the aborted wave EXEC mask' + check8 '^[[:space:]]*(=> )?0x[0-9a-f]+ .*:[[:space:]]+s_trap +2([[:space:]]|$)' \ + 'found the abort instruction at PC or PC-4' + check8 'abort_waves done: hipDeviceSynchronize reported hipError' 'host synchronization received a HIP error' + check8 'Inferior 1 .*exited normally' 'inferior exited after forwarding every abort' + if grep -qaEi 'fatal error:|wave_resume .* failed|send_exceptions failed|internal-error' <<<"$out8"; then + echo "FAIL: debugger failed while forwarding a device abort" >&2 + fail=1 + fi +else + cat "$workdir/abort-build.log" >&2 + echo "FAIL: could not build abort_waves.hip for $arch" >&2 + fail=1 +fi + if [[ $fail -ne 0 ]]; then echo "FAIL: one or more debug markers were missing" >&2 exit 1 diff --git a/emulation/rocjitsu/tests/sdma_packet_processor_test.cpp b/emulation/rocjitsu/tests/sdma_packet_processor_test.cpp index d91eb5d70e5b..1aa76adf3509 100644 --- a/emulation/rocjitsu/tests/sdma_packet_processor_test.cpp +++ b/emulation/rocjitsu/tests/sdma_packet_processor_test.cpp @@ -29,17 +29,17 @@ static_assert(std::same_as().con class RetryMemory final : public AddressSpaceTranslator, public PhysicalMemoryAccess { public: - RetryMemory() : bytes_(0x10000) {} + explicit RetryMemory(std::size_t size = 0x10000, uint64_t segment_size = 16) + : bytes_(size), segment_size_(segment_size) {} VmTranslationResult translate(uint64_t address, std::size_t size, VmAccessKind) const override { if (size == 0 || address > bytes_.size() || size > bytes_.size() - address) return {.outcome = VmAccessOutcome::Faulted, .translation = {}}; - constexpr uint64_t kSegment = 16; return { .outcome = VmAccessOutcome::Complete, .translation = {.domain = VmMemoryDomain::System, .address = address, - .contiguous_bytes = kSegment - (address % kSegment), + .contiguous_bytes = segment_size_ - (address % segment_size_), .mtype = Mtype::RW, .permissions = {.readable = true, .writable = true, .executable = true}}}; } @@ -119,6 +119,7 @@ class RetryMemory final : public AddressSpaceTranslator, public PhysicalMemoryAc private: std::vector bytes_; + uint64_t segment_size_; std::map write_calls_; uint64_t unavailable_write_address_ = UINT64_MAX; uint64_t unavailable_atomic_store_address_ = UINT64_MAX; @@ -129,7 +130,8 @@ class RetryMemory final : public AddressSpaceTranslator, public PhysicalMemoryAc class PacketProcessorFixture { public: - PacketProcessorFixture() : memory(std::make_shared()) { + explicit PacketProcessorFixture(std::size_t size = 0x10000, uint64_t segment_size = 16) + : memory(std::make_shared(size, segment_size)) { handle = vm.register_translated(7, memory, memory); access = vm.snapshot(handle); } @@ -150,6 +152,39 @@ std::array copy_packet(uint64_t source, uint64_t destination, uint3 static_cast(destination >> 32)}; } +TEST(SdmaPacketProcessorTest, ExtendedCountCopiesBeyondFourMiB) { + constexpr uint64_t kSource = 0x1000; + constexpr uint64_t kDestination = 0x500000; + constexpr uint32_t kBytes = 0x400010; + for (const auto dialect : {SdmaPacketDialect::Legacy, SdmaPacketDialect::LegacyExtendedCount, + SdmaPacketDialect::Gfx11Plus, SdmaPacketDialect::Gfx1250}) { + SCOPED_TRACE(static_cast(dialect)); + PacketProcessorFixture fixture(kDestination + kBytes, 0x1000); + ASSERT_TRUE(fixture.access); + // Check both ends of the transfer and the first byte beyond the legacy limit. + fixture.memory->store(kSource, 0x12345678); + fixture.memory->store(kSource + 0x400000, 0x87654321); + fixture.memory->store(kSource + kBytes - 4, 0xabcdef01); + SdmaPacketProcessor processor(dialect); + auto packet = copy_packet(kSource, kDestination, kBytes); + packet[1] |= 0xc0000000; // Reserved high bits must not extend the transfer. + const SdmaPacketProcessResult result = + processor.process({.available_dwords = packet, + .access = *fixture.access, + .continuation = fixture.continuation}); + ASSERT_EQ(result.packet.status, PacketProcessStatus::Complete); + EXPECT_FALSE(fixture.continuation.pending()); + EXPECT_EQ(fixture.memory->load(kDestination), 0x12345678u); + if (dialect == SdmaPacketDialect::Legacy) { + EXPECT_EQ(fixture.memory->load(kDestination + 0x400000), 0u); + EXPECT_EQ(fixture.memory->load(kDestination + kBytes - 4), 0u); + } else { + EXPECT_EQ(fixture.memory->load(kDestination + 0x400000), 0x87654321u); + EXPECT_EQ(fixture.memory->load(kDestination + kBytes - 4), 0xabcdef01u); + } + } +} + TEST(SdmaPacketProcessorTest, CopyResumesAtFirstUncommittedPhysicalSpan) { PacketProcessorFixture fixture; ASSERT_TRUE(fixture.access); diff --git a/emulation/rocjitsu/tests/shared_infra_test.cpp b/emulation/rocjitsu/tests/shared_infra_test.cpp index ca5fe2a80369..4bd0ea227b25 100644 --- a/emulation/rocjitsu/tests/shared_infra_test.cpp +++ b/emulation/rocjitsu/tests/shared_infra_test.cpp @@ -205,6 +205,13 @@ static_assert(isa_properties(ROCJITSU_CODE_ARCH_RDNA4).descriptor_vgpr_count_gra static_assert(isa_properties(ROCJITSU_CODE_ARCH_CDNA5).descriptor_vgpr_count_granule_wave32 == 16); static_assert(isa_properties(ROCJITSU_CODE_ARCH_CDNA5).descriptor_vgpr_count_granule_wave64 == 0); +static_assert(isa_properties(ROCJITSU_CODE_ARCH_CDNA4).vmcnt_capacity == 63); +static_assert(isa_properties(ROCJITSU_CODE_ARCH_CDNA4).lgkmcnt_capacity == 15); +static_assert(isa_properties(ROCJITSU_CODE_ARCH_RDNA3_5).vmcnt_capacity == 63); +static_assert(isa_properties(ROCJITSU_CODE_ARCH_RDNA3_5).lgkmcnt_capacity == 63); +static_assert(isa_properties(ROCJITSU_CODE_ARCH_RDNA4).vmcnt_capacity == 0); +static_assert(isa_properties(ROCJITSU_CODE_ARCH_RDNA4).lgkmcnt_capacity == 0); + // RDNA3/3.5 retain monolithic S_WAITCNT (GFX11 layout). static_assert(HasMonolithicWaitcnt); @@ -764,6 +771,66 @@ void fill_vgprs(amdgpu::ComputeUnitCore &cu, uint32_t base, uint32_t regs, uint3 cu.write_vgpr(base + reg, lane, value); } +TEST(MfmaExecTest, SwmmacI32K64U4VaryingSelectorsMatchSparseReference) { + amdgpu::GpuMemory gpu_mem("rdna4_swmmac_u4_exec_mem"); + amdgpu::L2Cache l2("rdna4_swmmac_u4_exec_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_RDNA4; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 106; + cfg.vgprs_per_wf = 256; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("rdna4_swmmac_u4_exec", cfg, &gpu_mem, &l2); + ASSERT_NE(cu, nullptr); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + ASSERT_NE(wf, nullptr); + ASSERT_EQ(wf->wf_size(), 32u); + wf->set_exec((1ULL << wf->wf_size()) - 1ULL); + + const uint32_t vb = wf->vgpr_alloc().base; + const uint32_t a_base = vb, b_base = vb + 4, c_base = vb + 12; + const uint32_t d_base = vb + 20, index_base = vb + 32; + fill_vgprs(*cu, vb, 40, 32, 0); + constexpr std::array, 6> pairs = { + {{0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}}}; + auto write_nibble = [&](uint32_t base, const amdgpu::InputLoc &loc, uint32_t value) { + const uint32_t reg = base + loc.vgpr_offset; + const uint32_t shift = 4u * loc.sub_element; + const uint32_t old = cu->read_vgpr(reg, loc.lane); + cu->write_vgpr(reg, loc.lane, old | (value << shift)); + }; + + for (uint32_t row = 0; row < 16; ++row) + for (uint32_t group = 0; group < 16; ++group) { + const auto pair = pairs[(row + group + 73u) % pairs.size()]; + // K=64 selectors follow the same interleaved lane halves as sparse A. + const uint32_t lane = row + 16u * ((group / 4u) & 1u); + const uint32_t shift = 4u * (4u * (group / 8u) + group % 4u); + const uint32_t old = cu->read_vgpr(index_base, lane); + cu->write_vgpr(index_base, lane, old | ((pair[0] | (pair[1] << 2u)) << shift)); + for (uint32_t slot = 0; slot < 2; ++slot) + write_nibble(a_base, amdgpu::swmmac_a_input_loc(32, 16, 64, row, 2u * group + slot, 4), 1u); + } + for (uint32_t k = 0; k < 64; ++k) + for (uint32_t col = 0; col < 16; ++col) + write_nibble(b_base, amdgpu::swmmac_b_input_loc(32, 16, 64, col, k, 4), (k % 15u) + 1u); + + amdgpu::exec_swmmac_i32(*cu, 16, 16, 64, 4, d_base, a_base, b_base, c_base, index_base, 16, + /*index_key=*/0, amdgpu::extract_u4, amdgpu::extract_u4, + /*clamp=*/false); + for (uint32_t row = 0; row < 16; ++row) + for (uint32_t col = 0; col < 16; ++col) { + uint32_t expected = 0; + for (uint32_t group = 0; group < 16; ++group) { + const auto pair = pairs[(row + group + 73u) % pairs.size()]; + expected += ((4u * group + pair[0]) % 15u) + 1u; + expected += ((4u * group + pair[1]) % 15u) + 1u; + } + const auto out = amdgpu::wmma_output_loc_32(16, 16, row, col); + EXPECT_EQ(cu->read_vgpr(d_base + out.reg, out.lane), expected) << row << "," << col; + } +} + TEST(MfmaExecTest, SwmmacF32K32Fp8MatchesSparseReference) { amdgpu::GpuMemory gpu_mem("rdna4_swmmac_fp8_exec_mem"); amdgpu::L2Cache l2("rdna4_swmmac_fp8_exec_l2"); @@ -2634,6 +2701,169 @@ TEST_P(CuFactoryTest, CreatesSuccessfully) { EXPECT_EQ(cu->arch(), arch); } +TEST_P(CuFactoryTest, ActivityQueriesTrackOverlappingPauseReasonsAndSlotReuse) { + amdgpu::GpuMemory memory("activity_memory"); + amdgpu::L2Cache l2("activity_l2"); + amdgpu::ComputeUnitCore::Config config{}; + config.arch = GetParam(); + config.num_wf_slots = 2; + config.sgprs_per_wf = 106; + config.vgprs_per_wf = 32; + config.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("activity_cu", config, &memory, &l2); + ASSERT_NE(cu, nullptr); + EXPECT_FALSE(cu->has_active_wfs()); + EXPECT_FALSE(cu->has_runnable_wfs()); + auto launch = [&] { return cu->dispatch_wf(0, 0, config.sgprs_per_wf, config.vgprs_per_wf); }; + auto *first = launch(); + ASSERT_NE(first, nullptr); + EXPECT_TRUE(cu->has_active_wfs()); + EXPECT_TRUE(cu->has_runnable_wfs()); + first->set_debug_halted(true); + EXPECT_TRUE(cu->has_active_wfs()); + EXPECT_FALSE(cu->has_runnable_wfs()); + auto *second = launch(); + ASSERT_NE(second, nullptr); + EXPECT_TRUE(cu->has_runnable_wfs()); + + // Clearing one pause reason cannot make a wave runnable while another holds + // it. Repeated assignments must not change the aggregate a second time. + for (uint32_t mask = 0; mask < 8; ++mask) { + SCOPED_TRACE(mask); + for (unsigned repeat = 0; repeat < 2; ++repeat) { + second->set_debug_halted(mask & 1); + second->set_debug_suspended(mask & 2); + second->set_runtime_suspended(mask & 4); + EXPECT_TRUE(cu->has_active_wfs()); + EXPECT_EQ(cu->has_runnable_wfs(), mask == 0); + } + } + second->set_debug_halted(false); + second->set_debug_suspended(false); + second->set_runtime_suspended(false); + const auto saved = second->debug_stop_state(); + second->debug_trap(1); + EXPECT_FALSE(cu->has_runnable_wfs()); + second->restore_debug_stop_state(saved); + EXPECT_TRUE(cu->has_runnable_wfs()); + second->set_state(amdgpu::WfState::VM_RETRY); + EXPECT_TRUE(cu->has_runnable_wfs()); + second->set_state(amdgpu::WfState::WAITCNT); + EXPECT_TRUE(cu->has_runnable_wfs()); + second->halt(); + EXPECT_TRUE(cu->has_active_wfs()); + EXPECT_FALSE(cu->has_runnable_wfs()); + first->halt(); + EXPECT_FALSE(cu->has_active_wfs()); + EXPECT_FALSE(cu->has_runnable_wfs()); + first = launch(); + ASSERT_NE(first, nullptr); + EXPECT_TRUE(cu->has_active_wfs()); + EXPECT_TRUE(cu->has_runnable_wfs()); + first->halt(); + EXPECT_FALSE(cu->has_active_wfs()); + EXPECT_FALSE(cu->has_runnable_wfs()); +} + +TEST_P(CuFactoryTest, WorkgroupAdmissionTracksResidentSlotsAcrossFailureAndReuse) { + amdgpu::GpuMemory memory("admission_memory"); + amdgpu::L2Cache l2("admission_l2"); + amdgpu::ComputeUnitCore::Config config{}; + config.arch = GetParam(); + config.num_wf_slots = 4; + config.sgprs_per_wf = 106; + config.vgprs_per_wf = 32; + config.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("admission_cu", config, &memory, &l2); + ASSERT_NE(cu, nullptr); + const auto capacity = [&](uint32_t free) { + for (uint32_t count = 0; count <= config.num_wf_slots + 1; ++count) { + EXPECT_EQ(cu->can_accept_workgroup(count), count <= free) << count; + } + }; + capacity(4); + // Rejected creation, SGPR allocation, and VGPR allocation leave an idle slot. + EXPECT_EQ(cu->dispatch_wf_at(1, 0, 0, 106, 32, 1), nullptr); + EXPECT_EQ(cu->dispatch_wf_at(1, 0, 0, 0, 32), nullptr); + EXPECT_EQ(cu->dispatch_wf_at(1, 0, 0, 106, 0), nullptr); + capacity(4); + // A restored nonzero slot can coexist with untouched holes. + auto *last = cu->dispatch_wf_at(3, 7, 0, 106, 32); + ASSERT_NE(last, nullptr); + last->set_dispatch_id(11); + capacity(3); + for (const auto state : {amdgpu::WfState::WAITCNT, amdgpu::WfState::VM_RETRY, + amdgpu::WfState::ENDING, amdgpu::WfState::RUNNING}) { + last->set_state(state); + capacity(3); + } + last->set_debug_halted(true); + last->set_debug_suspended(true); + last->set_runtime_suspended(true); + capacity(3); + auto *first = cu->dispatch_wf(8, 0, 106, 32); + ASSERT_NE(first, nullptr); + EXPECT_EQ(first->wf_id(), 0u); + first->set_dispatch_id(12); + capacity(2); + cu->abort_workgroup(11, 7); + capacity(3); + cu->free_wavefront_resources(*last); + capacity(3); + cu->abort_dispatch(12); + capacity(4); + for (uint32_t slot = 0; slot < config.num_wf_slots; ++slot) { + auto *wave = cu->dispatch_wf(20 + slot, 0, 106, 32); + ASSERT_NE(wave, nullptr); + EXPECT_EQ(wave->wf_id(), slot); + capacity(config.num_wf_slots - slot - 1); + } + EXPECT_EQ(cu->dispatch_wf(30, 0, 106, 32), nullptr); + for (uint32_t slot = 0; slot < config.num_wf_slots; ++slot) { + cu->wf(slot)->halt(); + capacity(slot + 1); + } + EXPECT_FALSE(cu->can_accept_workgroup(1, 64 * 1024 + 1)); + EXPECT_TRUE(cu->can_accept_workgroup(4, 64 * 1024)); +} + +TEST_P(CuFactoryTest, WorkgroupAdmissionRetainsScratchPrefixAndScoreboardBounds) { + amdgpu::GpuMemory memory("scratch_admission_memory"); + amdgpu::L2Cache l2("scratch_admission_l2"); + amdgpu::ComputeUnitCore::Config config{}; + config.arch = GetParam(); + config.num_wf_slots = 4; + config.sgprs_per_wf = 106; + config.vgprs_per_wf = 32; + config.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("scratch_admission_cu", config, &memory, &l2); + ASSERT_NE(cu, nullptr); + cu->set_scratch_slots_per_cu(1); + EXPECT_TRUE(cu->can_accept_workgroup(4)); + EXPECT_TRUE(cu->can_accept_workgroup(1, 0, 1)); + EXPECT_FALSE(cu->can_accept_workgroup(2, 0, 1)); + auto *first = cu->dispatch_wf(0, 0, 106, 32); + ASSERT_NE(first, nullptr); + EXPECT_EQ(first->wf_id(), 0u); + EXPECT_TRUE(cu->can_accept_workgroup(3)); + EXPECT_FALSE(cu->can_accept_workgroup(1, 0, 1)); + first->halt(); + // Even UINT32_MAX is an exclusive physical scoreboard bound. + cu->set_shader_engine_location(0, UINT32_MAX - 1); + EXPECT_TRUE(cu->can_accept_workgroup(1)); + EXPECT_FALSE(cu->can_accept_workgroup(2)); + first = cu->dispatch_wf(1, 0, 106, 32); + ASSERT_NE(first, nullptr); + EXPECT_EQ(first->wf_id(), 0u); + EXPECT_FALSE(cu->can_accept_workgroup(1)); + EXPECT_EQ(cu->dispatch_wf(2, 0, 106, 32), nullptr); + first->halt(); + cu->set_shader_engine_location(0, UINT32_MAX); + EXPECT_FALSE(cu->can_accept_workgroup(1)); + cu->set_shader_engine_location(0, 0); + EXPECT_TRUE(cu->can_accept_workgroup(4)); +} + TEST_P(CuFactoryTest, LdsContentsSurviveWorkgroupAllocationReuse) { amdgpu::GpuMemory mem("test_mem"); amdgpu::L2Cache l2("test_l2"); @@ -8065,4 +8295,46 @@ TEST(AluExceptionTest, OutputModifierDoesNotFabricateInexact) { EXPECT_EQ(amdgpu::classify_mul_f32(nan, 3.0f, 2.0f) & kInexact, 0u); } +TEST(AluExceptionTest, MultiplyExactProductCoversF32RangeAndRounding) { + struct Case { + uint32_t lhs; + uint32_t rhs; + float scale; + uint32_t causes; + }; + constexpr Case cases[] = { + {0x3f800000, 0x40000000, 4.0f, 0}, + {0x3f800001, 0x3f800001, 1.0f, 1u << 5}, + {0x00800000, 0x3f800000, 0.5f, 1u << 4}, + {0x00800000, 0x00800000, 1.0f, (1u << 4) | (1u << 5)}, + {0x00000001, 0x3f800000, 1.0f, (1u << 1) | (1u << 4)}, + {0x00000001, 0x7f000000, 4.0f, 1u << 1}, + }; + for (uint32_t round = 0; round < 4; ++round) { + const amdgpu::fp_mode::ScopedEnvironment environment(round); + for (const auto &sample : cases) { + SCOPED_TRACE(testing::Message() << "round " << round << " lhs " << sample.lhs << " rhs " + << sample.rhs << " scale " << sample.scale); + volatile float lhs = std::bit_cast(sample.lhs); + volatile float rhs = std::bit_cast(sample.rhs); + EXPECT_EQ(amdgpu::classify_mul_f32(lhs, rhs, sample.scale), sample.causes); + } + volatile float largest = std::bit_cast(0x7f7fffffu); + // Directed rounding toward a finite result still reports INEXACT. + EXPECT_EQ(amdgpu::classify_mul_f32(largest, 2.0f), (1u << 5) | (round < 2 ? 1u << 3 : 0u)); + } +} + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) +TEST(AluExceptionTest, HostDazPreservesExactSubnormalOutputClassification) { + const amdgpu::fp_mode::ScopedEnvironment environment(0); + _mm_setcsr(_mm_getcsr() | (1u << 6)); + volatile float lhs = std::bit_cast(0x00800000u); + volatile float rhs = 1.0f; + // OMOD produces an exact subnormal. Widening that result through a host + // conversion would flush it and falsely report INEXACT. + EXPECT_EQ(amdgpu::classify_mul_f32(lhs, rhs, 0.5f), 1u << 4); +} +#endif + } // namespace diff --git a/emulation/rocjitsu/tests/simd_correctness/simd_coverage_test.cpp b/emulation/rocjitsu/tests/simd_correctness/simd_coverage_test.cpp index 0af1db09a533..4e467bee05af 100644 --- a/emulation/rocjitsu/tests/simd_correctness/simd_coverage_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/simd_coverage_test.cpp @@ -227,10 +227,10 @@ template void check_packed_half(Wavefront &wf) { expected = fp_mode::packed_select_bf16(af, bf, Op == PackedFloatOp::MIN); expected = fp_mode::clamp_bf16(expected, iteration & 1, floating_clamp_nan_to_zero(wf)); } else if constexpr (Op == PackedFloatOp::FMA) { - expected = - fp_mode::fma_f16(a[i], b[i], c[i], false, false, false, false, false, false, - wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, - iteration & 1, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + expected = fp_mode::fma_f16(a[i], b[i], c[i], false, false, false, false, false, false, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), 0, + iteration & 1, wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf), + fp_mode::quiets_nan(wf.cu().arch(), wf.ieee_mode())); } else { constexpr auto operation = Op == PackedFloatOp::ADD ? fp_mode::PackedBinaryOp::ADD : Op == PackedFloatOp::MUL ? fp_mode::PackedBinaryOp::MUL @@ -241,7 +241,8 @@ template void check_packed_half(Wavefront &wf) { : fp_mode::PackedBinaryOp::MAXIMUM; expected = fp_mode::packed_binary_f16(operation, a[i], b[i], wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64(), iteration & 1, - wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf)); + wf.fp16_ovfl(), floating_clamp_nan_to_zero(wf), + wf.cu().arch(), wf.ieee_mode()); } ASSERT_EQ(actual[i], expected) << mode << ":" << iteration << " inputs=" << a[i] << "," << b[i] << "," << c[i]; @@ -580,21 +581,31 @@ TEST_F(SimdCoverage, MixedFmaUsesProfileSemanticsAndCompilerGuard) { WordInst inst; inst.inst_.opsel_hi = 0; inst.inst_.opsel_hi_2 = 0; + constexpr uint32_t exec = 0xa5a5f0f0u; + constexpr uint32_t sentinel = 0xdeadbeefu; + wf->set_exec(exec); const float a = std::bit_cast(0x3f800001u); const float b = std::bit_cast(0x3f7fffffu); for (uint32_t lane = 0; lane < 32; ++lane) { put(0, lane, 0x3f800001u); put(1, lane, 0x3f7fffffu); put(2, lane, 0xbf800000u); + put(3, lane, sentinel); } -#if defined(__clang__) && defined(__FMA__) +#if defined(__FMA__) ASSERT_FALSE((try_execute_vop3p_fma_mix_simd(inst, *wf))); + for (uint32_t lane = 0; lane < 32; ++lane) + EXPECT_EQ(get(3, lane), sentinel); #else ASSERT_TRUE((try_execute_vop3p_fma_mix_simd(inst, *wf))); - EXPECT_EQ(get(3, 0), std::bit_cast(a * b - 1.0f)); + for (uint32_t lane = 0; lane < 32; ++lane) + EXPECT_EQ(get(3, lane), + (exec & (1u << lane)) ? std::bit_cast(a * b - 1.0f) : sentinel); #endif ASSERT_TRUE((try_execute_vop3p_fma_mix_simd(inst, *wf))); - EXPECT_EQ(get(3, 0), std::bit_cast(std::fma(a, b, -1.0f))); + for (uint32_t lane = 0; lane < 32; ++lane) + EXPECT_EQ(get(3, lane), + (exec & (1u << lane)) ? std::bit_cast(std::fma(a, b, -1.0f)) : sentinel); } // Opt-in simulator host timing. Alternate scalar/SIMD order with matched inputs diff --git a/emulation/rocjitsu/tests/simd_correctness/smfmac_simd_exact_test.cpp b/emulation/rocjitsu/tests/simd_correctness/smfmac_simd_exact_test.cpp new file mode 100644 index 000000000000..85d4ad70cdfe --- /dev/null +++ b/emulation/rocjitsu/tests/simd_correctness/smfmac_simd_exact_test.cpp @@ -0,0 +1,551 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file smfmac_simd_exact_test.cpp +/// @brief Decoded CDNA3/CDNA4 F32 SMFMAC SIMD-vs-scalar checks. + +#include "mma_exact_test_support.h" + +#include "../decode_test_util.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/opcodes.h" +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" + +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace rocjitsu; +using namespace mma_exact; + +constexpr uint32_t WF_SIZE = 64; +constexpr uint32_t A_OFF = 0; +constexpr uint32_t B_OFF = 32; +constexpr uint32_t INDEX_OFF = 64; +constexpr uint32_t DST_OFF = 96; +constexpr uint32_t STATE_REGS = 128; +constexpr uint32_t MAX_DST_REGS = 16; + +// A nibble is (high two bits: second K position, low two bits: first K +// position). Exercise all six legal ordered 2-of-4 selector pairs. +constexpr std::array LEGAL_PAIRS{0x4u, 0x8u, 0xCu, 0x9u, 0xDu, 0xEu}; + +enum class Alias { None, A, B, Index }; + +struct SmfmacCase { + rj_code_arch_t arch; + uint16_t opcode; + const char *name; + Fmt a_fmt; + Fmt b_fmt; + uint8_t a_regs; + uint8_t b_regs; + uint8_t dst_regs; +}; + +#define C3(OP, NAME, A, B, D, AFMT, BFMT) \ + SmfmacCase { ROCJITSU_CODE_ARCH_CDNA3, cdna3::k##OP##Vop3pMfma, NAME, AFMT, BFMT, A, B, D } +#define C4(OP, NAME, A, B, D, AFMT, BFMT) \ + SmfmacCase { ROCJITSU_CODE_ARCH_CDNA4, cdna4::k##OP##Vop3pMfma, NAME, AFMT, BFMT, A, B, D } + +// Every generated, implemented F32 SMFMAC form. The I32 forms currently +// report UnimplementedInstruction and are intentionally outside this suite. +constexpr std::array CDNA3_CASES{ + C3(VSmfmacF3216x16x32F16, "v_smfmac_f32_16x16x32_f16", 2, 4, 4, Fmt::F16, Fmt::F16), + C3(VSmfmacF3232x32x16F16, "v_smfmac_f32_32x32x16_f16", 2, 4, 16, Fmt::F16, Fmt::F16), + C3(VSmfmacF3216x16x32Bf16, "v_smfmac_f32_16x16x32_bf16", 2, 4, 4, Fmt::BF16, Fmt::BF16), + C3(VSmfmacF3232x32x16Bf16, "v_smfmac_f32_32x32x16_bf16", 2, 4, 16, Fmt::BF16, Fmt::BF16), + C3(VSmfmacF3216x16x64Bf8Bf8, "v_smfmac_f32_16x16x64_bf8_bf8", 2, 4, 4, Fmt::BF8, Fmt::BF8), + C3(VSmfmacF3216x16x64Bf8Fp8, "v_smfmac_f32_16x16x64_bf8_fp8", 2, 4, 4, Fmt::BF8, Fmt::FP8), + C3(VSmfmacF3216x16x64Fp8Bf8, "v_smfmac_f32_16x16x64_fp8_bf8", 2, 4, 4, Fmt::FP8, Fmt::BF8), + C3(VSmfmacF3216x16x64Fp8Fp8, "v_smfmac_f32_16x16x64_fp8_fp8", 2, 4, 4, Fmt::FP8, Fmt::FP8), + C3(VSmfmacF3232x32x32Bf8Bf8, "v_smfmac_f32_32x32x32_bf8_bf8", 2, 4, 16, Fmt::BF8, Fmt::BF8), + C3(VSmfmacF3232x32x32Bf8Fp8, "v_smfmac_f32_32x32x32_bf8_fp8", 2, 4, 16, Fmt::BF8, Fmt::FP8), + C3(VSmfmacF3232x32x32Fp8Bf8, "v_smfmac_f32_32x32x32_fp8_bf8", 2, 4, 16, Fmt::FP8, Fmt::BF8), + C3(VSmfmacF3232x32x32Fp8Fp8, "v_smfmac_f32_32x32x32_fp8_fp8", 2, 4, 16, Fmt::FP8, Fmt::FP8), +}; + +constexpr std::array CDNA4_CASES{ + C4(VSmfmacF3216x16x64F16, "v_smfmac_f32_16x16x64_f16", 4, 8, 4, Fmt::F16, Fmt::F16), + C4(VSmfmacF3232x32x32F16, "v_smfmac_f32_32x32x32_f16", 4, 8, 16, Fmt::F16, Fmt::F16), + C4(VSmfmacF3216x16x64Bf16, "v_smfmac_f32_16x16x64_bf16", 4, 8, 4, Fmt::BF16, Fmt::BF16), + C4(VSmfmacF3232x32x32Bf16, "v_smfmac_f32_32x32x32_bf16", 4, 8, 16, Fmt::BF16, Fmt::BF16), + C4(VSmfmacF3216x16x128Bf8Bf8, "v_smfmac_f32_16x16x128_bf8_bf8", 4, 8, 4, Fmt::BF8, Fmt::BF8), + C4(VSmfmacF3216x16x128Bf8Fp8, "v_smfmac_f32_16x16x128_bf8_fp8", 4, 8, 4, Fmt::BF8, Fmt::FP8), + C4(VSmfmacF3216x16x128Fp8Bf8, "v_smfmac_f32_16x16x128_fp8_bf8", 4, 8, 4, Fmt::FP8, Fmt::BF8), + C4(VSmfmacF3216x16x128Fp8Fp8, "v_smfmac_f32_16x16x128_fp8_fp8", 4, 8, 4, Fmt::FP8, Fmt::FP8), + C4(VSmfmacF3232x32x64Bf8Bf8, "v_smfmac_f32_32x32x64_bf8_bf8", 4, 8, 16, Fmt::BF8, Fmt::BF8), + C4(VSmfmacF3232x32x64Bf8Fp8, "v_smfmac_f32_32x32x64_bf8_fp8", 4, 8, 16, Fmt::BF8, Fmt::FP8), + C4(VSmfmacF3232x32x64Fp8Bf8, "v_smfmac_f32_32x32x64_fp8_bf8", 4, 8, 16, Fmt::FP8, Fmt::BF8), + C4(VSmfmacF3232x32x64Fp8Fp8, "v_smfmac_f32_32x32x64_fp8_fp8", 4, 8, 16, Fmt::FP8, Fmt::FP8), + // gfx950 also exposes the CDNA3 shapes and encodings. + C4(VSmfmacF3216x16x32F16, "v_smfmac_f32_16x16x32_f16", 2, 4, 4, Fmt::F16, Fmt::F16), + C4(VSmfmacF3232x32x16F16, "v_smfmac_f32_32x32x16_f16", 2, 4, 16, Fmt::F16, Fmt::F16), + C4(VSmfmacF3216x16x32Bf16, "v_smfmac_f32_16x16x32_bf16", 2, 4, 4, Fmt::BF16, Fmt::BF16), + C4(VSmfmacF3232x32x16Bf16, "v_smfmac_f32_32x32x16_bf16", 2, 4, 16, Fmt::BF16, Fmt::BF16), + C4(VSmfmacF3216x16x64Bf8Bf8, "v_smfmac_f32_16x16x64_bf8_bf8", 2, 4, 4, Fmt::BF8, Fmt::BF8), + C4(VSmfmacF3216x16x64Bf8Fp8, "v_smfmac_f32_16x16x64_bf8_fp8", 2, 4, 4, Fmt::BF8, Fmt::FP8), + C4(VSmfmacF3216x16x64Fp8Bf8, "v_smfmac_f32_16x16x64_fp8_bf8", 2, 4, 4, Fmt::FP8, Fmt::BF8), + C4(VSmfmacF3216x16x64Fp8Fp8, "v_smfmac_f32_16x16x64_fp8_fp8", 2, 4, 4, Fmt::FP8, Fmt::FP8), + C4(VSmfmacF3232x32x32Bf8Bf8, "v_smfmac_f32_32x32x32_bf8_bf8", 2, 4, 16, Fmt::BF8, Fmt::BF8), + C4(VSmfmacF3232x32x32Bf8Fp8, "v_smfmac_f32_32x32x32_bf8_fp8", 2, 4, 16, Fmt::BF8, Fmt::FP8), + C4(VSmfmacF3232x32x32Fp8Bf8, "v_smfmac_f32_32x32x32_fp8_bf8", 2, 4, 16, Fmt::FP8, Fmt::BF8), + C4(VSmfmacF3232x32x32Fp8Fp8, "v_smfmac_f32_32x32x32_fp8_fp8", 2, 4, 16, Fmt::FP8, Fmt::FP8), +}; + +#undef C3 +#undef C4 + +struct SmfmacFixture : ExactFixture { + explicit SmfmacFixture(rj_code_arch_t arch) : ExactFixture(arch, WF_SIZE) {} +}; + +void restore_window(SmfmacFixture &fx, uint32_t off, const std::vector &words) { + const uint32_t regs = static_cast(words.size() / WF_SIZE); + for (uint32_t reg = 0; reg < regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + off + reg, lane, + words[static_cast(reg) * WF_SIZE + lane]); +} + +void restore(SmfmacFixture &fx, const std::vector &words) { + restore_window(fx, 0, words); +} + +uint8_t f8_element(Fmt fmt, Mode mode, uint32_t i, std::mt19937 &rng, bool fnuz) { + if (mode == Mode::NaN) { + if (fnuz) + return 0x80u; + if (fmt == Fmt::FP8) + return (i & 1) ? 0xFFu : 0x7Fu; + return static_cast((i & 1 ? 0x80u : 0u) | (0x7Du + i % 3)); + } + if (mode == Mode::Inf) + return static_cast((i & 1 ? 0x80u : 0u) | 0x7Cu); + if (mode == Mode::Denorm) + return static_cast((i & 1 ? 0x80u : 0u) | 0x01u); + if (mode == Mode::MaxFinite) + return fnuz ? static_cast((i & 1 ? 0x80u : 0u) | 0x7Fu) + : static_cast((i & 1 ? 0x80u : 0u) | (fmt == Fmt::FP8 ? 0x7Eu : 0x7Bu)); + if (mode == Mode::Zeros) + return 0; + if (mode == Mode::SignedZero) + return fnuz ? 0u : static_cast((i & 1) ? 0x80u : 0u); + + float value = 0; + if (mode == Mode::Cancel) + value = (i & 1) ? -1.0f : 1.0f; + else + value = static_cast(std::uniform_int_distribution(-4, 4)(rng)); + if (fnuz) + return fmt == Fmt::FP8 ? util::f32_to_fp8_e4m3_fnuz_rne(value) + : util::f32_to_bf8_e5m2_fnuz_rne(value); + return fmt == Fmt::FP8 ? util::f32_to_fp8_e4m3_rne(value) : util::f32_to_bf8_e5m2_rne(value); +} + +void seed_input(SmfmacFixture &fx, uint32_t off, uint32_t regs, Fmt fmt, Mode mode, uint32_t seed, + bool fnuz) { + if (fmt != Fmt::FP8 && fmt != Fmt::BF8) { + fx.seed(off, regs, fmt, mode, seed); + return; + } + std::mt19937 rng(seed); + uint32_t i = 0; + for (uint32_t reg = 0; reg < regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + uint32_t word = 0; + for (uint32_t byte = 0; byte < 4; ++byte) + word |= static_cast(f8_element(fmt, mode, i++, rng, fnuz)) << (8 * byte); + fx.cu->write_vgpr(fx.vbase + off + reg, lane, word); + } +} + +void seed_indices(SmfmacFixture &fx, uint32_t off, uint32_t phase) { + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + uint32_t word = 0; + for (uint32_t nibble = 0; nibble < 8; ++nibble) + word |= LEGAL_PAIRS[(lane + 5 * nibble + phase) % LEGAL_PAIRS.size()] << (4 * nibble); + fx.cu->write_vgpr(fx.vbase + off, lane, word); + } +} + +std::unique_ptr decode(const SmfmacCase &test, uint32_t a_off, uint32_t b_off, + uint32_t index_off, bool acc_dst = false, + bool acc_sources = false) { + const auto fields = cdna4::Vop3pMfmaBuilderFields{ + .vdst = DST_OFF, + .acc_cd = static_cast(acc_dst), + .src0 = static_cast(256 + a_off), + .src1 = static_cast(256 + b_off), + .src2 = static_cast(256 + index_off), + .acc = static_cast(acc_sources ? 3 : 0), + }; + auto decoder = Decoder::create(test.arch); + if (!decoder) + return nullptr; + if (test.arch == ROCJITSU_CODE_ARCH_CDNA3) { + const auto words = cdna3::build_vop3p_mfma(test.opcode, {.vdst = fields.vdst, + .acc_cd = fields.acc_cd, + .src0 = fields.src0, + .src1 = fields.src1, + .src2 = fields.src2, + .acc = fields.acc}); + return std::unique_ptr(decode_valid(*decoder, words.data())); + } + const auto words = cdna4::build_vop3p_mfma(test.opcode, fields); + return std::unique_ptr(decode_valid(*decoder, words.data())); +} + +void compare_state(const SmfmacCase &test, const std::vector &initial, + const std::vector &scalar, const std::vector &simd) { + for (uint32_t reg = 0; reg < STATE_REGS; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + const size_t i = static_cast(reg) * WF_SIZE + lane; + ASSERT_EQ(simd[i], scalar[i]) + << test.name << " reg=" << reg << " lane=" << lane << " scalar=0x" << std::hex + << scalar[i] << " simd=0x" << simd[i]; + if (reg < DST_OFF || reg >= DST_OFF + test.dst_regs) { + ASSERT_EQ(scalar[i], initial[i]) + << test.name << ": scalar path changed source/canary reg=" << reg << " lane=" << lane; + ASSERT_EQ(simd[i], initial[i]) + << test.name << ": SIMD path changed source/canary reg=" << reg << " lane=" << lane; + } + } +} + +void execute_both(SmfmacFixture &fx, const SmfmacCase &test, Instruction &instruction, + const std::vector &initial) { + ForceScalarGuard force_scalar_guard; + restore(fx, initial); + util::set_force_scalar_for_testing(true); + const bool scalar_ok = fx.cu->execute_instruction(&instruction, *fx.wf).succeeded(); + const auto scalar = fx.snapshot(0, STATE_REGS); + + restore(fx, initial); + util::set_force_scalar_for_testing(false); + const bool simd_ok = fx.cu->execute_instruction(&instruction, *fx.wf).succeeded(); + const auto simd = fx.snapshot(0, STATE_REGS); + + ASSERT_TRUE(scalar_ok) << test.name << ": scalar execution failed"; + ASSERT_TRUE(simd_ok) << test.name << ": SIMD execution failed"; + compare_state(test, initial, scalar, simd); +} + +void run_case(const SmfmacCase &test, Mode mode, uint32_t seed, uint32_t phase, + Alias alias = Alias::None) { + SmfmacFixture fx(test.arch); + ASSERT_NE(fx.wf, nullptr); + const uint32_t a_off = alias == Alias::A ? DST_OFF : A_OFF; + const uint32_t b_off = alias == Alias::B ? DST_OFF : B_OFF; + const uint32_t index_off = alias == Alias::Index ? DST_OFF : INDEX_OFF; + + restore(fx, std::vector(static_cast(STATE_REGS) * WF_SIZE, 0)); + // D is also C, so every case starts with a real accumulator. For overlap + // cases, overwrite its initial words with the aliased source afterwards. + fx.seed(DST_OFF, test.dst_regs, Fmt::F32, Mode::RandomInt, seed + 3); + const bool fnuz = test.arch == ROCJITSU_CODE_ARCH_CDNA3; + seed_input(fx, a_off, test.a_regs, test.a_fmt, mode, seed + 1, fnuz); + seed_input(fx, b_off, test.b_regs, test.b_fmt, mode, seed + 2, fnuz); + seed_indices(fx, index_off, phase); + fx.seed_words(DST_OFF - 1, 1, seed + 4); + fx.seed_words(DST_OFF + MAX_DST_REGS, 1, seed + 5); + + auto instruction = decode(test, a_off, b_off, index_off); + ASSERT_NE(instruction, nullptr) << test.name; + const auto initial = fx.snapshot(0, STATE_REGS); + execute_both(fx, test, *instruction, initial); +} + +void run_acc_case(const SmfmacCase &test, bool acc_sources, uint32_t seed) { + ASSERT_EQ(test.arch, ROCJITSU_CODE_ARCH_CDNA3); + SmfmacFixture fx(test.arch); + ASSERT_NE(fx.wf, nullptr); + restore(fx, std::vector(static_cast(STATE_REGS) * WF_SIZE, 0)); + restore_window(fx, amdgpu::ACC_VGPR_OFFSET, + std::vector(static_cast(STATE_REGS) * WF_SIZE, 0)); + + const uint32_t src_bank = acc_sources ? amdgpu::ACC_VGPR_OFFSET : 0; + seed_input(fx, src_bank + A_OFF, test.a_regs, test.a_fmt, Mode::RandomInt, seed + 1, true); + seed_input(fx, src_bank + B_OFF, test.b_regs, test.b_fmt, Mode::RandomInt, seed + 2, true); + // src2 is an ordinary VGPR operand; only A/B use the MFMA acc selector. + seed_indices(fx, INDEX_OFF, seed); + fx.seed(amdgpu::ACC_VGPR_OFFSET + DST_OFF, test.dst_regs, Fmt::F32, Mode::RandomInt, seed + 3); + fx.seed_words(amdgpu::ACC_VGPR_OFFSET + DST_OFF - 1, 1, seed + 4); + fx.seed_words(amdgpu::ACC_VGPR_OFFSET + DST_OFF + MAX_DST_REGS, 1, seed + 5); + + auto instruction = decode(test, A_OFF, B_OFF, INDEX_OFF, true, acc_sources); + ASSERT_NE(instruction, nullptr) << test.name; + const auto initial_v = fx.snapshot(0, STATE_REGS); + const auto initial_acc = fx.snapshot(amdgpu::ACC_VGPR_OFFSET, STATE_REGS); + ForceScalarGuard force_scalar_guard; + restore(fx, initial_v); + restore_window(fx, amdgpu::ACC_VGPR_OFFSET, initial_acc); + util::set_force_scalar_for_testing(true); + const bool scalar_ok = fx.cu->execute_instruction(instruction.get(), *fx.wf).succeeded(); + const auto scalar_v = fx.snapshot(0, STATE_REGS); + const auto scalar_acc = fx.snapshot(amdgpu::ACC_VGPR_OFFSET, STATE_REGS); + + restore(fx, initial_v); + restore_window(fx, amdgpu::ACC_VGPR_OFFSET, initial_acc); + util::set_force_scalar_for_testing(false); + const bool simd_ok = fx.cu->execute_instruction(instruction.get(), *fx.wf).succeeded(); + const auto simd_v = fx.snapshot(0, STATE_REGS); + const auto simd_acc = fx.snapshot(amdgpu::ACC_VGPR_OFFSET, STATE_REGS); + + ASSERT_TRUE(scalar_ok) << test.name << ": AccVGPR scalar execution failed"; + ASSERT_TRUE(simd_ok) << test.name << ": AccVGPR SIMD execution failed"; + for (uint32_t reg = 0; reg < STATE_REGS; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + const size_t i = static_cast(reg) * WF_SIZE + lane; + ASSERT_EQ(scalar_v[i], initial_v[i]) << test.name << ": scalar changed VGPR " << reg; + ASSERT_EQ(simd_v[i], initial_v[i]) << test.name << ": SIMD changed VGPR " << reg; + ASSERT_EQ(simd_acc[i], scalar_acc[i]) + << test.name << ": AccVGPR result mismatch at reg=" << reg << " lane=" << lane; + if (reg < DST_OFF || reg >= DST_OFF + test.dst_regs) { + ASSERT_EQ(scalar_acc[i], initial_acc[i]) + << test.name << ": scalar changed AccVGPR source/canary reg=" << reg; + ASSERT_EQ(simd_acc[i], initial_acc[i]) + << test.name << ": SIMD changed AccVGPR source/canary reg=" << reg; + } + } +} + +void run_nan_priority_case(const SmfmacCase &test, bool nan_in_b, uint32_t seed) { + SmfmacFixture fx(test.arch); + ASSERT_NE(fx.wf, nullptr); + restore(fx, std::vector(static_cast(STATE_REGS) * WF_SIZE, 0)); + const bool fnuz = test.arch == ROCJITSU_CODE_ARCH_CDNA3; + seed_input(fx, A_OFF, test.a_regs, test.a_fmt, Mode::Cancel, seed + 1, fnuz); + seed_input(fx, B_OFF, test.b_regs, test.b_fmt, nan_in_b ? Mode::NaN : Mode::Cancel, seed + 2, + fnuz); + fx.seed(DST_OFF, test.dst_regs, Fmt::F32, Mode::RandomInt, seed + 3); + if (!nan_in_b) + for (uint32_t reg = 0; reg < test.dst_regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + // Signaling NaN with a payload: replay must follow tied C as well as + // the packed A and B inputs. + fx.cu->write_vgpr(fx.vbase + DST_OFF + reg, lane, 0x7FA12345u); + seed_indices(fx, INDEX_OFF, seed); + fx.seed_words(DST_OFF - 1, 1, seed + 4); + fx.seed_words(DST_OFF + MAX_DST_REGS, 1, seed + 5); + auto instruction = decode(test, A_OFF, B_OFF, INDEX_OFF); + ASSERT_NE(instruction, nullptr) << test.name; + const auto initial = fx.snapshot(0, STATE_REGS); + execute_both(fx, test, *instruction, initial); +} + +template void run_all_forms(const std::array &cases) { + constexpr std::array MODES{Mode::RandomInt, Mode::Zeros, Mode::SignedZero, Mode::Cancel, + Mode::NaN, Mode::Inf, Mode::Denorm, Mode::MaxFinite}; + for (size_t case_index = 0; case_index < cases.size(); ++case_index) { + const auto &test = cases[case_index]; + for (size_t mode_index = 0; mode_index < MODES.size(); ++mode_index) { + const Mode mode = MODES[mode_index]; + // FP8 has no infinity, and CDNA3 FNUZ BF8 has none either. + if (mode == Mode::Inf && (test.a_fmt == Fmt::FP8 || test.b_fmt == Fmt::FP8 || + (test.arch == ROCJITSU_CODE_ARCH_CDNA3 && test.a_fmt == Fmt::BF8))) + continue; + SCOPED_TRACE(::testing::Message() << test.name << " arch=" << static_cast(test.arch) + << " mode=" << static_cast(mode)); + run_case(test, mode, 101 + 17 * case_index + mode_index, case_index + mode_index); + if (testing::Test::HasFatalFailure()) + return; + } + } +} + +template void run_all_aliases(const std::array &cases) { + constexpr std::array ALIASES{Alias::A, Alias::B, Alias::Index}; + for (size_t case_index = 0; case_index < cases.size(); ++case_index) + for (size_t alias_index = 0; alias_index < ALIASES.size(); ++alias_index) { + const auto &test = cases[case_index]; + SCOPED_TRACE(::testing::Message() << test.name << " arch=" << static_cast(test.arch) + << " alias=" << static_cast(ALIASES[alias_index])); + run_case(test, Mode::RandomInt, 701 + 17 * case_index + alias_index, case_index + alias_index, + ALIASES[alias_index]); + if (testing::Test::HasFatalFailure()) + return; + } +} + +template void run_all_nan_sources(const std::array &cases) { + for (size_t case_index = 0; case_index < cases.size(); ++case_index) + for (bool nan_in_b : {true, false}) { + const auto &test = cases[case_index]; + SCOPED_TRACE(::testing::Message() << test.name << " arch=" << static_cast(test.arch) + << (nan_in_b ? " NaN in B" : " signaling NaN in tied C")); + run_nan_priority_case(test, nan_in_b, 1901 + 17 * case_index + !nan_in_b); + if (testing::Test::HasFatalFailure()) + return; + } +} + +void check_fused_overflow_cancellation(const SmfmacCase &test, bool negate) { + SmfmacFixture fx(test.arch); + ASSERT_NE(fx.wf, nullptr); + restore(fx, std::vector(static_cast(STATE_REGS) * WF_SIZE, 0)); + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + INDEX_OFF, lane, 0x44444444u); + + // One selected BF16 product is (+/-2^64) * 2^64. Fusing it with + // (-/+FLT_MAX) yields +/-2^104. A rounded multiply would overflow to Inf. + fx.cu->write_vgpr(fx.vbase + A_OFF, 0, negate ? 0x0000DF80u : 0x00005F80u); + fx.cu->write_vgpr(fx.vbase + B_OFF, 0, 0x00005F80u); + fx.cu->write_vgpr(fx.vbase + DST_OFF, 0, negate ? 0x7F7FFFFFu : 0xFF7FFFFFu); + auto instruction = decode(test, A_OFF, B_OFF, INDEX_OFF); + ASSERT_NE(instruction, nullptr); + const auto initial = fx.snapshot(0, STATE_REGS); + execute_both(fx, test, *instruction, initial); + if (testing::Test::HasFatalFailure()) + return; + const auto result = fx.snapshot(DST_OFF, test.dst_regs); + EXPECT_EQ(result[0], negate ? 0xF3800000u : 0x73800000u) + << test.name << ": SMFMAC must use fused accumulation"; +} + +} // namespace + +TEST(SmfmacSimdExact, Cdna3DecodedForms) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "SMFMAC AVX-512 fast path requires 16-wide native SIMD"; + run_all_forms(CDNA3_CASES); +} + +TEST(SmfmacSimdExact, Cdna4DecodedForms) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "SMFMAC AVX-512 fast path requires 16-wide native SIMD"; + run_all_forms(CDNA4_CASES); +} + +TEST(SmfmacSimdExact, SparseGatherMatchesIndependentPhysicalLayoutOracle) { + for (const auto &test : {CDNA3_CASES[0], CDNA4_CASES[0]}) { + SCOPED_TRACE(::testing::Message() << test.name << " arch=" << static_cast(test.arch)); + SmfmacFixture fx(test.arch); + ASSERT_NE(fx.wf, nullptr); + restore(fx, std::vector(static_cast(STATE_REGS) * WF_SIZE, 0)); + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + INDEX_OFF, lane, 0x44444444u); + + // Use raw physical locations, independent of the implementation's layout + // helpers. A reg1/lane21/high is row5, second compressed element in its + // group (q3 on CDNA3, q5 on CDNA4). The nibble at metadata lane21 selects + // dense position 3, so B reg3/high at lane23 or lane39 is the only matching + // nonzero element. D reg1/lane23 is row5, column7: 2 * 3 = 6. + fx.cu->write_vgpr(fx.vbase + A_OFF + 1, 21, 0x40000000u); + fx.cu->write_vgpr(fx.vbase + INDEX_OFF, 21, 0x444444C4u); + const uint32_t b_lane = test.arch == ROCJITSU_CODE_ARCH_CDNA3 ? 23u : 39u; + fx.cu->write_vgpr(fx.vbase + B_OFF + 3, b_lane, 0x42000000u); + + auto instruction = decode(test, A_OFF, B_OFF, INDEX_OFF); + ASSERT_NE(instruction, nullptr); + const auto initial = fx.snapshot(0, STATE_REGS); + std::vector expected(static_cast(test.dst_regs) * WF_SIZE, 0); + expected[WF_SIZE + 23] = 0x40C00000u; + ForceScalarGuard force_scalar_guard; + for (bool force_scalar : {true, false}) { + restore(fx, initial); + util::set_force_scalar_for_testing(force_scalar); + ASSERT_TRUE(fx.cu->execute_instruction(instruction.get(), *fx.wf).succeeded()) + << test.name + << (force_scalar ? ": scalar execution failed" : ": default execution failed"); + EXPECT_EQ(fx.snapshot(DST_OFF, test.dst_regs), expected) + << test.name << (force_scalar ? ": scalar result" : ": default result"); + } + } +} + +TEST(SmfmacSimdExact, Cdna4IgnoresPartialExecAndUpdatesFullWave) { + const auto &test = CDNA4_CASES[0]; + SmfmacFixture fx(test.arch); + ASSERT_NE(fx.wf, nullptr); + restore(fx, std::vector(static_cast(STATE_REGS) * WF_SIZE, 0)); + + constexpr uint32_t packed_f16_ones = 0x3C003C00u; + for (uint32_t reg = 0; reg < test.a_regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + A_OFF + reg, lane, packed_f16_ones); + for (uint32_t reg = 0; reg < test.b_regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + B_OFF + reg, lane, packed_f16_ones); + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + INDEX_OFF, lane, 0x44444444u); + + auto instruction = decode(test, A_OFF, B_OFF, INDEX_OFF); + ASSERT_NE(instruction, nullptr); + const auto initial = fx.snapshot(0, STATE_REGS); + const std::vector expected(static_cast(test.dst_regs) * WF_SIZE, + std::bit_cast(32.0f)); + constexpr uint64_t partial_exec = 0xAAAAAAAAAAAAAAAAULL; + ForceScalarGuard force_scalar_guard; + for (bool force_scalar : {true, false}) { + restore(fx, initial); + fx.wf->set_exec(partial_exec); + util::set_force_scalar_for_testing(force_scalar); + ASSERT_TRUE(fx.cu->execute_instruction(instruction.get(), *fx.wf).succeeded()) + << (force_scalar ? "scalar execution failed" : "default execution failed"); + EXPECT_EQ(fx.snapshot(DST_OFF, test.dst_regs), expected) + << (force_scalar ? "scalar result" : "default result"); + EXPECT_EQ(fx.wf->exec(), partial_exec); + } +} + +TEST(SmfmacSimdExact, Cdna3DestinationAliasesEachSource) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "SMFMAC AVX-512 fast path requires 16-wide native SIMD"; + run_all_aliases(CDNA3_CASES); +} + +TEST(SmfmacSimdExact, Cdna4DestinationAliasesEachSource) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "SMFMAC AVX-512 fast path requires 16-wide native SIMD"; + run_all_aliases(CDNA4_CASES); +} + +TEST(SmfmacSimdExact, Cdna3AccVgprDestinationAndSources) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "SMFMAC AVX-512 fast path requires 16-wide native SIMD"; + for (size_t i = 0; i < CDNA3_CASES.size(); ++i) + for (bool acc_sources : {false, true}) { + const auto &test = CDNA3_CASES[i]; + SCOPED_TRACE(::testing::Message() << test.name << " acc_cd=1 acc_sources=" << acc_sources); + run_acc_case(test, acc_sources, 1201 + 17 * i + acc_sources); + if (testing::Test::HasFatalFailure()) + return; + } +} + +TEST(SmfmacSimdExact, Cdna3NaNSourcePriority) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "SMFMAC AVX-512 fast path requires 16-wide native SIMD"; + run_all_nan_sources(CDNA3_CASES); +} + +TEST(SmfmacSimdExact, Cdna4NaNSourcePriority) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "SMFMAC AVX-512 fast path requires 16-wide native SIMD"; + run_all_nan_sources(CDNA4_CASES); +} + +TEST(SmfmacSimdExact, Bf16OverflowCancellationRequiresFusion) { + for (const auto &test : {CDNA3_CASES[2], CDNA4_CASES[2]}) + for (bool negate : {false, true}) { + SCOPED_TRACE(::testing::Message() + << test.name << " arch=" << static_cast(test.arch) << " negate=" << negate); + check_fused_overflow_cancellation(test, negate); + } +} diff --git a/emulation/rocjitsu/tests/simd_correctness/swmmac_k128_simd_exact_test.cpp b/emulation/rocjitsu/tests/simd_correctness/swmmac_k128_simd_exact_test.cpp new file mode 100644 index 000000000000..2ece215f09ae --- /dev/null +++ b/emulation/rocjitsu/tests/simd_correctness/swmmac_k128_simd_exact_test.cpp @@ -0,0 +1,527 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file swmmac_k128_simd_exact_test.cpp +/// @brief Decoded scalar-vs-AVX-512 checks for gfx1250 K=128 FP8/BF8 SWMMAC. + +#include "mma_exact_test_support.h" + +#include "../decode_test_util.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna5/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna5/opcodes.h" +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" + +#include +#include +#include +#include +#include + +namespace { + +using namespace rocjitsu; +using namespace mma_exact; + +constexpr uint32_t WF_SIZE = mma_test::WMMA_WF_SIZE; +constexpr uint32_t A_OFF = 0; +constexpr uint32_t B_OFF = 32; +constexpr uint32_t D_OFF = 64; +constexpr uint32_t INDEX_OFF = 96; +constexpr uint32_t STATE_REGS = 128; +constexpr uint32_t A_REGS = 8; +constexpr uint32_t B_REGS = 16; +constexpr uint32_t INDEX_REGS = 2; +constexpr uint32_t F32_D_REGS = 8; +constexpr uint32_t F16_D_REGS = 4; +constexpr std::array LEGAL_PAIRS{0x4u, 0x8u, 0xCu, 0x9u, 0xDu, 0xEu}; + +enum class DstAlias { None, A, B, Index }; + +struct SwmmacCase { + uint16_t opcode; + const char *name; + Fmt a_fmt; + Fmt b_fmt; + bool f32_result; + + uint32_t dst_regs() const { return f32_result ? F32_D_REGS : F16_D_REGS; } +}; + +constexpr std::array CASES{ + SwmmacCase{cdna5::kVSwmmacF3216x16x128Fp8Fp8Vop3p, "v_swmmac_f32_16x16x128_fp8_fp8", Fmt::FP8, + Fmt::FP8, true}, + SwmmacCase{cdna5::kVSwmmacF3216x16x128Fp8Bf8Vop3p, "v_swmmac_f32_16x16x128_fp8_bf8", Fmt::FP8, + Fmt::BF8, true}, + SwmmacCase{cdna5::kVSwmmacF3216x16x128Bf8Fp8Vop3p, "v_swmmac_f32_16x16x128_bf8_fp8", Fmt::BF8, + Fmt::FP8, true}, + SwmmacCase{cdna5::kVSwmmacF3216x16x128Bf8Bf8Vop3p, "v_swmmac_f32_16x16x128_bf8_bf8", Fmt::BF8, + Fmt::BF8, true}, + SwmmacCase{cdna5::kVSwmmacF1616x16x128Fp8Fp8Vop3p, "v_swmmac_f16_16x16x128_fp8_fp8", Fmt::FP8, + Fmt::FP8, false}, + SwmmacCase{cdna5::kVSwmmacF1616x16x128Fp8Bf8Vop3p, "v_swmmac_f16_16x16x128_fp8_bf8", Fmt::FP8, + Fmt::BF8, false}, + SwmmacCase{cdna5::kVSwmmacF1616x16x128Bf8Fp8Vop3p, "v_swmmac_f16_16x16x128_bf8_fp8", Fmt::BF8, + Fmt::FP8, false}, + SwmmacCase{cdna5::kVSwmmacF1616x16x128Bf8Bf8Vop3p, "v_swmmac_f16_16x16x128_bf8_bf8", Fmt::BF8, + Fmt::BF8, false}, +}; + +struct SwmmacFixture : ExactFixture { + SwmmacFixture() : ExactFixture(ROCJITSU_CODE_ARCH_CDNA5, WF_SIZE) {} +}; + +struct Observation { + bool succeeded; + std::vector state; +}; + +void restore(SwmmacFixture &fx, const std::vector &words) { + ASSERT_EQ(words.size(), static_cast(STATE_REGS) * WF_SIZE); + for (uint32_t reg = 0; reg < STATE_REGS; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + reg, lane, words[static_cast(reg) * WF_SIZE + lane]); +} + +void clear_state(SwmmacFixture &fx) { + restore(fx, std::vector(static_cast(STATE_REGS) * WF_SIZE, 0)); +} + +void seed_indices(SwmmacFixture &fx, uint32_t off, uint32_t phase) { + // A 32-entry index set occupies two VGPRs per lane. Exercise every legal + // ordered 2:4 pair in each half, including the upper register. + for (uint32_t reg = 0; reg < INDEX_REGS; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + uint32_t word = 0; + for (uint32_t group = 0; group < 8; ++group) + word |= LEGAL_PAIRS[(phase + lane + reg * 3 + group) % LEGAL_PAIRS.size()] << (4 * group); + fx.cu->write_vgpr(fx.vbase + off + reg, lane, word); + } +} + +void seed_constant_indices(SwmmacFixture &fx) { + for (uint32_t reg = 0; reg < INDEX_REGS; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + INDEX_OFF + reg, lane, 0x44444444u); +} + +void seed_constant_operand(SwmmacFixture &fx, uint32_t off, uint32_t regs, uint8_t value) { + const uint32_t word = static_cast(value) * 0x01010101u; + for (uint32_t reg = 0; reg < regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + off + reg, lane, word); +} + +void write_byte(SwmmacFixture &fx, uint32_t off, uint32_t reg, uint32_t lane, uint32_t byte, + uint8_t value) { + const uint32_t address = fx.vbase + off + reg; + const uint32_t shift = 8 * byte; + const uint32_t old = fx.cu->read_vgpr(address, lane); + fx.cu->write_vgpr(address, lane, + (old & ~(0xFFu << shift)) | (static_cast(value) << shift)); +} + +std::unique_ptr decode_case(const SwmmacCase &test, uint32_t a_off = A_OFF, + uint32_t b_off = B_OFF, uint32_t index_off = INDEX_OFF) { + const auto words = + cdna5::build_vop3p(test.opcode, {.vdst = D_OFF, + .src0 = static_cast(256 + a_off), + .src1 = static_cast(256 + b_off), + .src2 = static_cast(256 + index_off)}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + if (!decoder) + return nullptr; + return std::unique_ptr(decode_valid(*decoder, words.data())); +} + +Observation execute_from_state(SwmmacFixture &fx, Instruction &instruction, + const std::vector &initial, bool force_scalar) { + restore(fx, initial); + util::set_force_scalar_for_testing(force_scalar); + const bool succeeded = fx.cu->execute_instruction(&instruction, *fx.wf).succeeded(); + return {succeeded, fx.snapshot(0, STATE_REGS)}; +} + +void expect_equal_and_bounded(const SwmmacCase &test, const std::vector &initial, + const Observation &scalar, const Observation &simd) { + ASSERT_TRUE(scalar.succeeded) << test.name << ": forced-scalar execution failed"; + ASSERT_TRUE(simd.succeeded) << test.name << ": default execution failed"; + for (uint32_t reg = 0; reg < STATE_REGS; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + const size_t i = static_cast(reg) * WF_SIZE + lane; + ASSERT_EQ(simd.state[i], scalar.state[i]) + << test.name << ": SIMD mismatch at reg=" << reg << " lane=" << lane; + if (reg < D_OFF || reg >= D_OFF + test.dst_regs()) { + ASSERT_EQ(scalar.state[i], initial[i]) + << test.name << ": scalar changed source or canary reg=" << reg << " lane=" << lane; + ASSERT_EQ(simd.state[i], initial[i]) + << test.name << ": SIMD changed source or canary reg=" << reg << " lane=" << lane; + } + } +} + +void expect_output(const SwmmacCase &test, const Observation &observation, + const std::vector &expected) { + const auto first = observation.state.begin() + static_cast(D_OFF) * WF_SIZE; + const std::vector actual(first, first + static_cast(test.dst_regs()) * WF_SIZE); + EXPECT_EQ(actual, expected) << test.name << ": independent expected output"; +} + +uint32_t first_output(const SwmmacCase &test, const Observation &observation) { + const uint32_t word = observation.state[static_cast(D_OFF) * WF_SIZE]; + return test.f32_result ? word : word & 0xFFFFu; +} + +void run_case(const SwmmacCase &test, Mode mode, uint32_t seed, uint32_t phase, + DstAlias alias = DstAlias::None) { + SwmmacFixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + clear_state(fx); + + const uint32_t a_off = alias == DstAlias::A ? D_OFF : A_OFF; + const uint32_t b_off = alias == DstAlias::B ? D_OFF : B_OFF; + const uint32_t index_off = alias == DstAlias::Index ? D_OFF : INDEX_OFF; + + // The decoded instruction reads C from its tied D tuple. Seed it first so + // source-overlap trials can replace the same physical words afterward. + fx.seed(D_OFF, test.dst_regs(), test.f32_result ? Fmt::F32 : Fmt::F16, Mode::RandomInt, seed + 3); + fx.seed(a_off, A_REGS, test.a_fmt, mode, seed + 1); + fx.seed(b_off, B_REGS, test.b_fmt, mode, seed + 2); + seed_indices(fx, index_off, phase); + fx.seed_words(D_OFF - 1, 1, seed + 4); + fx.seed_words(D_OFF + B_REGS, 1, seed + 5); + + auto instruction = decode_case(test, a_off, b_off, index_off); + ASSERT_NE(instruction, nullptr) << test.name; + const auto initial = fx.snapshot(0, STATE_REGS); + ForceScalarGuard guard; + const auto scalar = execute_from_state(fx, *instruction, initial, true); + const auto simd = execute_from_state(fx, *instruction, initial, false); + expect_equal_and_bounded(test, initial, scalar, simd); +} + +uint8_t one(Fmt fmt) { return fmt == Fmt::FP8 ? 0x38u : 0x3Cu; } +uint8_t nan(Fmt fmt, bool negative) { + const uint8_t positive = fmt == Fmt::FP8 ? 0x7Fu : 0x7Eu; + return negative ? static_cast(positive | 0x80u) : positive; +} + +float decode_eight_bit(Fmt fmt, uint8_t raw) { + return fmt == Fmt::FP8 ? util::fp8_e4m3_ocp_to_f32(raw) : util::bf8_e5m2_ocp_to_f32(raw); +} + +uint32_t result_bits(const SwmmacCase &test, float value) { + return test.f32_result ? std::bit_cast(value) : util::f32_to_f16(value); +} + +void expect_uniform_result_with_overflow_modes(const SwmmacCase &test, uint8_t a, uint8_t b, + uint32_t clear_word, uint32_t set_word) { + SwmmacFixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + clear_state(fx); + seed_constant_indices(fx); + seed_constant_operand(fx, A_OFF, A_REGS, a); + seed_constant_operand(fx, B_OFF, B_REGS, b); + fx.seed_words(D_OFF - 1, 1, 1801); + fx.seed_words(D_OFF + test.dst_regs(), 1, 1802); + + auto instruction = decode_case(test); + ASSERT_NE(instruction, nullptr) << test.name; + const auto initial = fx.snapshot(0, STATE_REGS); + ForceScalarGuard guard; + for (bool fp16_ovfl : {false, true}) { + SCOPED_TRACE(::testing::Message() << "FP16_OVFL=" << fp16_ovfl); + const uint32_t mode = fp16_ovfl ? amdgpu::Wavefront::FP16_OVFL_BIT : 0u; + fx.wf->set_mode_raw(mode); + const auto scalar = execute_from_state(fx, *instruction, initial, true); + fx.wf->set_mode_raw(mode); + const auto simd = execute_from_state(fx, *instruction, initial, false); + expect_equal_and_bounded(test, initial, scalar, simd); + const std::vector expected(static_cast(test.dst_regs()) * WF_SIZE, + fp16_ovfl ? set_word : clear_word); + expect_output(test, scalar, expected); + expect_output(test, simd, expected); + if (testing::Test::HasFatalFailure()) + return; + } +} + +} // namespace + +TEST(SwmmacK128SimdExact, AllEightDecodedFormsMatchForcedScalar) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "K=128 SWMMAC requires 16-lane native SIMD"; + + constexpr std::array MODES{Mode::RandomInt, Mode::Zeros, Mode::SignedZero, Mode::Cancel, + Mode::NaN, Mode::Inf, Mode::Denorm, Mode::MaxFinite}; + for (size_t case_index = 0; case_index < CASES.size(); ++case_index) { + const auto &test = CASES[case_index]; + for (size_t mode_index = 0; mode_index < MODES.size(); ++mode_index) { + const Mode mode = MODES[mode_index]; + if (mode == Mode::Inf && (!fmt_has_inf(test.a_fmt) || !fmt_has_inf(test.b_fmt))) + continue; + SCOPED_TRACE(::testing::Message() << test.name << " mode=" << static_cast(mode)); + run_case(test, mode, 101 + 17 * case_index + mode_index, case_index + mode_index); + if (testing::Test::HasFatalFailure()) + return; + } + } +} + +TEST(SwmmacK128SimdExact, InfinityInEitherOperandHonorsFp16Ovfl) { + for (const auto &test : CASES) + for (bool infinity_in_a : {true, false}) { + if ((infinity_in_a ? test.a_fmt : test.b_fmt) != Fmt::BF8) + continue; + for (bool negative : {false, true}) { + SCOPED_TRACE(::testing::Message() + << test.name << " infinity source=" << (infinity_in_a ? "A" : "B") + << " negative=" << negative); + // FP8 has no infinity. Fill only the BF8 operand with signed infinity + // and the other operand with +1 so every selected product is infinity. + const uint8_t infinity = negative ? 0xFCu : 0x7Cu; + const uint8_t a = infinity_in_a ? infinity : one(test.a_fmt); + const uint8_t b = infinity_in_a ? one(test.b_fmt) : infinity; + const uint32_t clear_word = test.f32_result ? (negative ? 0xFF800000u : 0x7F800000u) + : (negative ? 0xFC00FC00u : 0x7C007C00u); + const uint32_t set_word = + test.f32_result ? clear_word : (negative ? 0xFBFFFBFFu : 0x7BFF7BFFu); + expect_uniform_result_with_overflow_modes(test, a, b, clear_word, set_word); + if (testing::Test::HasFatalFailure()) + return; + } + } +} + +TEST(SwmmacK128SimdExact, AllF16FormsHonorFp16OvflForFiniteOverflow) { + for (const auto &test : CASES) { + if (test.f32_result) + continue; + for (bool negative : {false, true}) { + SCOPED_TRACE(::testing::Message() << test.name << " negative=" << negative); + // Both formats represent 256 exactly. The 64 selected products sum to + // signed 2^22, overflowing F16 even for FP8/FP8 without infinite inputs. + const uint8_t positive_a = test.a_fmt == Fmt::FP8 ? 0x78u : 0x5Cu; + const uint8_t a = negative ? static_cast(positive_a | 0x80u) : positive_a; + const uint8_t b = test.b_fmt == Fmt::FP8 ? 0x78u : 0x5Cu; + expect_uniform_result_with_overflow_modes(test, a, b, negative ? 0xFC00FC00u : 0x7C007C00u, + negative ? 0xFBFFFBFFu : 0x7BFF7BFFu); + if (testing::Test::HasFatalFailure()) + return; + } + } +} + +TEST(SwmmacK128SimdExact, DestinationAliasesA_BAndBothMetadataWords) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "K=128 SWMMAC requires 16-lane native SIMD"; + + constexpr std::array ALIASES{DstAlias::A, DstAlias::B, DstAlias::Index}; + for (size_t case_index = 0; case_index < CASES.size(); ++case_index) + for (size_t alias_index = 0; alias_index < ALIASES.size(); ++alias_index) { + const auto &test = CASES[case_index]; + SCOPED_TRACE(::testing::Message() + << test.name << " alias=" << static_cast(ALIASES[alias_index])); + run_case(test, Mode::RandomInt, 701 + 17 * case_index + alias_index, case_index + alias_index, + ALIASES[alias_index]); + if (testing::Test::HasFatalFailure()) + return; + } +} + +TEST(SwmmacK128SimdExact, TiedCIsReadFromD) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "K=128 SWMMAC requires 16-lane native SIMD"; + + for (const auto &test : CASES) { + SCOPED_TRACE(test.name); + SwmmacFixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + clear_state(fx); + seed_constant_indices(fx); + const uint32_t c_word = test.f32_result ? 0x3F800000u : 0x3C003C00u; + for (uint32_t reg = 0; reg < test.dst_regs(); ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + D_OFF + reg, lane, c_word); + auto instruction = decode_case(test); + ASSERT_NE(instruction, nullptr); + const auto initial = fx.snapshot(0, STATE_REGS); + ForceScalarGuard guard; + const auto scalar = execute_from_state(fx, *instruction, initial, true); + const auto simd = execute_from_state(fx, *instruction, initial, false); + expect_equal_and_bounded(test, initial, scalar, simd); + const std::vector expected(static_cast(test.dst_regs()) * WF_SIZE, c_word); + expect_output(test, scalar, expected); + expect_output(test, simd, expected); + } +} + +TEST(SwmmacK128SimdExact, IgnoresPartialExecAndUpdatesFullWave) { + constexpr uint64_t partial_exec = 0xAAAAAAAAu; + for (const auto &test : CASES) { + SCOPED_TRACE(test.name); + SwmmacFixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + clear_state(fx); + seed_constant_indices(fx); + + const uint32_t a_word = static_cast(one(test.a_fmt)) * 0x01010101u; + const uint32_t b_word = static_cast(one(test.b_fmt)) * 0x01010101u; + for (uint32_t reg = 0; reg < A_REGS; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + A_OFF + reg, lane, a_word); + for (uint32_t reg = 0; reg < B_REGS; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + fx.cu->write_vgpr(fx.vbase + B_OFF + reg, lane, b_word); + + auto instruction = decode_case(test); + ASSERT_NE(instruction, nullptr); + const auto initial = fx.snapshot(0, STATE_REGS); + const uint32_t expected_word = + test.f32_result ? std::bit_cast(64.0f) : uint32_t{0x54005400u}; + const std::vector expected(static_cast(test.dst_regs()) * WF_SIZE, + expected_word); + ForceScalarGuard guard; + for (bool force_scalar : {true, false}) { + restore(fx, initial); + fx.wf->set_exec(partial_exec); + util::set_force_scalar_for_testing(force_scalar); + ASSERT_TRUE(fx.cu->execute_instruction(instruction.get(), *fx.wf).succeeded()) + << (force_scalar ? "scalar execution failed" : "default execution failed"); + EXPECT_EQ(fx.snapshot(D_OFF, test.dst_regs()), expected) + << (force_scalar ? "scalar result" : "default result"); + EXPECT_EQ(fx.wf->exec(), partial_exec); + } + } +} + +TEST(SwmmacK128SimdExact, UpperMetadataAndPhysicalGatherMatchIndependentOracle) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "K=128 SWMMAC requires 16-lane native SIMD"; + + for (const auto &test : CASES) { + SCOPED_TRACE(test.name); + SwmmacFixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + clear_state(fx); + seed_constant_indices(fx); + + // Raw physical locations, deliberately independent of the layout helpers: + // A reg7/lane21/byte1 is row5, compressed K=61. Its selector is entry29 + // of index lane21, stored in the upper metadata register at nibble6. + // Changing that pair to (0,3) selects dense K=123. B reg14/lane23/byte3 + // is dense K=123, column7. The only product is D[5,7] = 2 * 3 = 6. + write_byte(fx, A_OFF, 7, 21, 1, 0x40u); + write_byte(fx, B_OFF, 14, 23, 3, test.b_fmt == Fmt::FP8 ? 0x44u : 0x42u); + const uint32_t old = fx.cu->read_vgpr(fx.vbase + INDEX_OFF + 1, 21); + fx.cu->write_vgpr(fx.vbase + INDEX_OFF + 1, 21, (old & ~(0xFu << 24)) | (0xCu << 24)); + + auto instruction = decode_case(test); + ASSERT_NE(instruction, nullptr); + const auto initial = fx.snapshot(0, STATE_REGS); + ForceScalarGuard guard; + const auto scalar = execute_from_state(fx, *instruction, initial, true); + const auto simd = execute_from_state(fx, *instruction, initial, false); + expect_equal_and_bounded(test, initial, scalar, simd); + std::vector expected(static_cast(test.dst_regs()) * WF_SIZE, 0); + // gfx1250 wave32: F32 row5/col7 is D reg5/lane7; packed F16 is + // D reg2/lane7/high half. These coordinates are intentionally literal. + if (test.f32_result) + expected[5 * WF_SIZE + 7] = 0x40C00000u; + else + expected[2 * WF_SIZE + 7] = 0x46000000u; + expect_output(test, scalar, expected); + expect_output(test, simd, expected); + } +} + +TEST(SwmmacK128SimdExact, NaNSourcePriorityIsBitExact) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "K=128 SWMMAC requires 16-lane native SIMD"; + + enum class Source { A, B, C }; + for (const auto &test : CASES) + for (Source source : {Source::A, Source::B, Source::C}) { + SCOPED_TRACE(::testing::Message() << test.name << " NaN source=" << static_cast(source)); + SwmmacFixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + clear_state(fx); + seed_constant_indices(fx); + // Distinct signs and a C payload make A, then B, then C priority + // observable after exceptional-lane replay. + const uint8_t a = source == Source::A ? nan(test.a_fmt, true) : one(test.a_fmt); + const uint8_t b = source == Source::A ? nan(test.b_fmt, false) + : source == Source::B ? nan(test.b_fmt, true) + : one(test.b_fmt); + write_byte(fx, A_OFF, 0, 0, 0, a); + write_byte(fx, B_OFF, 0, 0, 0, b); + fx.cu->write_vgpr(fx.vbase + D_OFF, 0, test.f32_result ? 0x7FC12345u : 0x00007E05u); + + float selected = 0.0f; + if (source == Source::A) + selected = decode_eight_bit(test.a_fmt, a); + else if (source == Source::B) + selected = decode_eight_bit(test.b_fmt, b); + else + selected = test.f32_result ? std::bit_cast(0x7FC12345u) : util::f16_to_f32(0x7E05u); + selected = std::bit_cast(std::bit_cast(selected) | 0x00400000u); + const uint32_t expected = result_bits(test, selected); + + auto instruction = decode_case(test); + ASSERT_NE(instruction, nullptr); + const auto initial = fx.snapshot(0, STATE_REGS); + ForceScalarGuard guard; + const auto scalar = execute_from_state(fx, *instruction, initial, true); + const auto simd = execute_from_state(fx, *instruction, initial, false); + expect_equal_and_bounded(test, initial, scalar, simd); + EXPECT_EQ(first_output(test, scalar), expected) << test.name << ": scalar NaN payload"; + EXPECT_EQ(first_output(test, simd), expected) << test.name << ": SIMD NaN payload"; + } +} + +TEST(SwmmacK128SimdExact, PackedF16RoundsHalfwayToEven) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "K=128 SWMMAC requires 16-lane native SIMD"; + + for (size_t case_index = 4; case_index < CASES.size(); ++case_index) { + const auto &test = CASES[case_index]; + SCOPED_TRACE(test.name); + SwmmacFixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + clear_state(fx); + seed_constant_indices(fx); + + // Row0 starts at 1.0 (even retained bit), row1 at the next half value + // (odd retained bit). One product of 2^-3 * 2^-8 = 2^-11 puts each result + // exactly halfway between adjacent F16 values. Both share D reg0/lane0. + fx.cu->write_vgpr(fx.vbase + D_OFF, 0, 0x3C013C00u); + const uint8_t a = test.a_fmt == Fmt::FP8 ? 0x20u : 0x30u; + const uint8_t b = test.b_fmt == Fmt::FP8 ? 0x02u : 0x1Cu; + write_byte(fx, A_OFF, 0, 0, 0, a); + write_byte(fx, A_OFF, 0, 1, 0, a); + write_byte(fx, B_OFF, 0, 0, 0, b); + + auto instruction = decode_case(test); + ASSERT_NE(instruction, nullptr); + const auto initial = fx.snapshot(0, STATE_REGS); + ForceScalarGuard guard; + const auto scalar = execute_from_state(fx, *instruction, initial, true); + const auto simd = execute_from_state(fx, *instruction, initial, false); + expect_equal_and_bounded(test, initial, scalar, simd); + EXPECT_EQ(scalar.state[static_cast(D_OFF) * WF_SIZE], 0x3C023C00u); + EXPECT_EQ(simd.state[static_cast(D_OFF) * WF_SIZE], 0x3C023C00u); + } +} diff --git a/emulation/rocjitsu/tests/simd_correctness/swmmac_k64_simd_exact_test.cpp b/emulation/rocjitsu/tests/simd_correctness/swmmac_k64_simd_exact_test.cpp index de6f981b02e1..b8c6847118c2 100644 --- a/emulation/rocjitsu/tests/simd_correctness/swmmac_k64_simd_exact_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/swmmac_k64_simd_exact_test.cpp @@ -552,11 +552,12 @@ TEST(SwmmacK64SimdExact, F16NaNPayloadPriorityIsBitExact) { }; constexpr std::array CASES{ NanCase{"A qNaN wins over B and C", 0x7E01u, 0x7E03u, 0x7E05u, 0x7FC02000u, 0x7E01u}, + NanCase{"A qNaN keeps an even payload", 0x7E02u, 0x3C00u, 0, 0x7FC04000u, 0x7E02u}, NanCase{"B qNaN wins over C", 0x3C00u, 0x7E03u, 0x7E05u, 0x7FC06000u, 0x7E03u}, NanCase{"C qNaN is retained", 0, 0, 0x7E05u, 0x7FC0A000u, 0x7E05u}, NanCase{"A sNaN is quieted", 0x7C01u, 0x3C00u, 0, 0x7FC02000u, 0x7E01u}, NanCase{"signed B qNaN is retained", 0x3C00u, 0xFE07u, 0, 0xFFC0E000u, 0xFE07u}, - NanCase{"Inf times zero is canonical", 0x7C00u, 0, 0, 0xFFC00000u, 0xFE01u}, + NanCase{"Inf times zero is canonical", 0x7C00u, 0, 0, 0xFFC00000u, 0xFE00u}, }; for (const auto &nan : CASES) { diff --git a/emulation/rocjitsu/tests/simd_correctness/vop2_fma_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop2_fma_simd_correctness_test.cpp index a345d212760e..d4d05d96ef07 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop2_fma_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop2_fma_simd_correctness_test.cpp @@ -201,13 +201,12 @@ void check_case(const FmaCase &c, uint64_t exec) { const auto scalar_out = run_mode(/*force_scalar=*/true); const auto simd_out = run_mode(/*force_scalar=*/false); - // Core A/B equivalence per active, non-skipped lane. NaN-input lanes carry an - // accepted NaN-payload divergence and are excluded identically in both runs - // (the skip condition is input-derived). + // F32 compares every active lane, including the selected NaN payload. + // Legacy F16 forms retain their separate host-arithmetic behavior. for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { const bool active = (exec >> lane) & 1ULL; if (active) { - if (nan_lane[lane]) + if (c.is_f16 && nan_lane[lane]) continue; EXPECT_EQ(scalar_out[lane], simd_out[lane]) << c.label << " lane " << lane << ": SIMD path diverged from scalar body"; diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_binary_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_binary_simd_correctness_test.cpp index b5ecbbfb8bda..7f5f9b748379 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_binary_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_binary_simd_correctness_test.cpp @@ -25,14 +25,19 @@ #include "rocjitsu/vm/amdgpu/l2_cache.h" #include "rocjitsu/vm/amdgpu/wavefront.h" +#include "rocjitsu/vm/plugins/execution_plugin.h" +#include "rocjitsu/vm/plugins/execution_plugin_group.h" #include "util/simd.h" #include #include +#include #include #include #include #include +#include +#include namespace { @@ -197,16 +202,20 @@ struct Fixture { std::unique_ptr decoder; amdgpu::Wavefront *wf = nullptr; - Fixture() : gpu_mem("vop3_bin_simd_mem"), l2("vop3_bin_simd_l2") { + explicit Fixture(rj_code_arch_t arch = ROCJITSU_CODE_ARCH_CDNA4, uint32_t wave_size = 0, + std::shared_ptr plugins = {}) + : gpu_mem("vop3_bin_simd_mem"), l2("vop3_bin_simd_l2") { amdgpu::ComputeUnitCore::Config cfg{}; - cfg.arch = ROCJITSU_CODE_ARCH_CDNA4; + cfg.arch = arch; cfg.num_wf_slots = 1; cfg.sgprs_per_wf = SGPRS_PER_WF; cfg.vgprs_per_wf = VGPRS_PER_WF; cfg.lds_size_kb = 64; cu = amdgpu::ComputeUnitCore::create("cu_vop3_bin_simd", cfg, &gpu_mem, &l2); - decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4); - wf = cu->dispatch_wf(0, 0, SGPRS_PER_WF, VGPRS_PER_WF); + decoder = Decoder::create(arch); + if (plugins) + cu->set_plugin_group(std::move(plugins)); + wf = cu->dispatch_wf(0, 0, SGPRS_PER_WF, VGPRS_PER_WF, wave_size); } void seed_inputs(Kind k, uint64_t exec) { @@ -246,6 +255,162 @@ struct ForceScalarGuard { ~ForceScalarGuard() { util::set_force_scalar_for_testing(orig); } }; +// Hostile host state must survive both the scalar reference and SIMD execution. +struct SubtractionHostEnvironment { + std::fenv_t saved{}; +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + uint32_t saved_mxcsr = _mm_getcsr(); +#endif + SubtractionHostEnvironment() { + EXPECT_EQ(std::feholdexcept(&saved), 0); + EXPECT_EQ(std::fesetround(FE_UPWARD), 0); + EXPECT_EQ(std::feraiseexcept(FE_DIVBYZERO), 0); +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + _mm_setcsr(_mm_getcsr() | (1u << 6) | (1u << 15)); +#endif + } + ~SubtractionHostEnvironment() { + std::fesetenv(&saved); +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + _mm_setcsr(saved_mxcsr); +#endif + } + static std::array snapshot() { + int control = 0; +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + control = static_cast(_mm_getcsr()); +#endif + return {std::fegetround(), std::fetestexcept(FE_ALL_EXCEPT), control}; + } +}; + +class SubtractionObserver : public ExecutionPlugin { +public: + SubtractionObserver() : ExecutionPlugin("subtraction_fp_state") {} + void onAmdgpuReadVgprLanes(const amdgpu::Wavefront *, uint32_t reg, uint64_t lanes, + uint8_t bytes) override { + record(false, reg, lanes, bytes); + } + void onAmdgpuWriteVgprLanes(const amdgpu::Wavefront *, uint32_t reg, uint64_t lanes, + uint8_t bytes) override { + record(true, reg, lanes, bytes); + } + void record(bool write, uint32_t reg, uint64_t lanes, uint8_t bytes) { + const auto state = SubtractionHostEnvironment::snapshot(); + events.push_back({write, reg, lanes, bytes, static_cast(state[0]), + static_cast(state[1]), static_cast(state[2])}); + } + std::vector> events; +}; + +TEST(Vop3BinarySimdCorrectness, SubtractionModesAndHostStateAcrossTargets) { + ForceScalarGuard gate_guard; + // Cancellation, signed zeros, rounding boundaries, overflow, and distinct + // signaling/quiet NaN payloads. Compare all bits; no exceptional-lane skips. + constexpr std::array, 16> inputs{{ + {0x3f800000u, 0x33000000u}, + {0xbf800000u, 0x33800000u}, + {0x3f800000u, 0x3f800000u}, + {0u, 0x80000000u}, + {0x80000000u, 0u}, + {0x7f7fffffu, 0xff7fffffu}, + {0x00800000u, 1u}, + {0x007fffffu, 0x80000001u}, + {1u, 0x00800000u}, + {0x7f800000u, 0x7f800000u}, + {0xff800000u, 0x7f800000u}, + {0x7fc01234u, 0xffc05678u}, + {0x7f801234u, 0xffc05678u}, + {0x7fc01234u, 0xff805678u}, + {0x7f801234u, 0xff805678u}, + {0x3f800000u, 0xff801234u}, + }}; + for (auto arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA4}) { + const bool rdna = arch != ROCJITSU_CODE_ARCH_CDNA4; + for (unsigned form = 0; form < 4; ++form) { + const bool e64 = form & 2, reverse = form & 1; + // All 16 MODE combinations; rotate masks, aliases and modifiers rather + // than forming an additional Cartesian product. Finish with empty EXEC, + // register observers, and an attached debugger. + for (unsigned variant = 0; variant < 19; ++variant) { + const uint32_t mode = variant % 16; + SCOPED_TRACE(testing::Message() + << "arch=" << arch << " form=" << form << " variant=" << variant); + auto group = std::make_shared(PluginSinkConfig{}); + auto observer_owner = std::make_unique(); + auto *observer = observer_owner.get(); + group->add(std::move(observer_owner)); + Fixture fx(arch, rdna ? 32 : 64, variant == 17 ? group : nullptr); + ASSERT_NE(fx.wf, nullptr); + fx.cu->set_debug_active(variant == 18); + const uint32_t width = fx.wf->wf_size(), base = fx.wf->vgpr_alloc().base; + const uint64_t full = width == 64 ? ~uint64_t{0} : 0xffffffffu; + const uint64_t exec = variant == 16 ? 0 + : (variant & 1) ? full & 0xa55af00f87654321ULL + : full; + const uint32_t dst = std::array{4, 0, 1}[variant % 3]; + uint32_t words[2]{}; + const uint32_t op = (rdna ? 4u : 2u) + reverse; + if (e64) { + vop3_encode(256 + op, dst, 256, 257, variant % 4, (variant / 4) % 4, mode % 4, + variant % 2, words); + if (rdna) { + words[0] |= 1u << 26; + words[1] |= 1u << 25; + } + } else { + words[0] = (op << 25) | (dst << 17) | (1u << 9) | 256u; + } + std::unique_ptr inst(decode_valid(*fx.decoder, words)); + ASSERT_NE(inst, nullptr); + auto run = [&](bool scalar) { + util::set_force_scalar_for_testing(scalar); + fx.wf->set_exec(exec); + fx.wf->set_mode_raw((mode & 3u) | ((mode >> 2) << 4)); + std::array out{}; + for (uint32_t lane = 0; lane < width; ++lane) { + fx.cu->write_vgpr(base + 4, lane, DST_SENTINEL); + fx.cu->write_vgpr(base, lane, inputs[lane % inputs.size()][0]); + fx.cu->write_vgpr(base + 1, lane, inputs[lane % inputs.size()][1]); + } + observer->events.clear(); + { + SubtractionHostEnvironment host; + const auto before = SubtractionHostEnvironment::snapshot(); + const bool succeeded = fx.cu->execute_instruction(inst.get(), *fx.wf).succeeded(); + const auto after = SubtractionHostEnvironment::snapshot(); + EXPECT_TRUE(succeeded); + EXPECT_EQ(after, before); + } + // Snapshotting storage must not add events to the observed execution. + const auto events = observer->events; + for (uint32_t lane = 0; lane < width; ++lane) { + out[lane] = fx.cu->read_vgpr(base + dst, lane); + if (!(exec & (uint64_t{1} << lane))) { + EXPECT_EQ(out[lane], dst == 4 ? DST_SENTINEL : inputs[lane % inputs.size()][dst]); + } + } + return std::pair{out, events}; + }; + const auto reference = run(true), actual = run(false); + EXPECT_EQ(actual, reference); + if (!e64 && (exec & 1u)) { + // 1 - 2^-25 is exactly halfway below 1. This independent rounding + // witness also catches GCC moving the SIMD subtraction across MODE. + const uint32_t round = mode & 3u; + const uint32_t expected = reverse + ? ((round == 0 || round == 2) ? 0xbf800000u : 0xbf7fffffu) + : (round < 2 ? 0x3f800000u : 0x3f7fffffu); + EXPECT_EQ(actual.first[0], expected); + } + if (variant == 17) { + EXPECT_FALSE(actual.second.empty()); + } + } + } + } +} + void check(const Case &c, uint32_t abs, uint32_t neg, uint32_t omod, uint32_t clamp, uint64_t exec) { ForceScalarGuard gate_guard; diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_cvt_pk_f32_rdna_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_cvt_pk_f32_rdna_correctness_test.cpp index aa748ccb01b0..d7e0aa40bbda 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_cvt_pk_f32_rdna_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_cvt_pk_f32_rdna_correctness_test.cpp @@ -17,8 +17,8 @@ /// with src0 -> low half, src1 -> high half of the dst. /// /// This file also pins the normalized packed signed i16 behavior: -/// v_cvt_pk_norm_i16_f32 scales by 32767, clamps to [-32768, 32767], rounds to nearest-even, -/// and packs the signed 16-bit results. The no-underscore signed spelling +/// v_cvt_pk_norm_i16_f32 scales by 32767 without intermediate rounding, clamps to [-32767, 32767], +/// rounds to nearest-even, and packs the signed 16-bit results. The no-underscore signed spelling /// (v_cvt_pknorm_i16_f32) is CDNA-only and not decodable here. /// /// Each case is executed twice in the same process (force-scalar and force-SIMD @@ -111,18 +111,18 @@ uint32_t golden(bool is_u16, float f0, float f1) { // --- Normalized pack-convert ------------------------------------------------ // Both arch-generated lambdas scale by K, clamp, map NaN->0, then round to // nearest-even (mirrors execute_shared.h / util::cvt_pknorm_*_f32_simd). -float round_nearest_even_golden(float value) { - float lower = std::floor(value); - float fraction = value - lower; - if (fraction > 0.5f || (fraction == 0.5f && (static_cast(lower) & int32_t{1}) != 0)) - lower += 1.0f; +double round_nearest_even_golden(double value) { + double lower = std::floor(value); + double fraction = value - lower; + if (fraction > 0.5 || (fraction == 0.5 && (static_cast(lower) & int32_t{1}) != 0)) + lower += 1.0; return lower; } uint16_t cvt_norm_i16(float f) { if (std::isnan(f)) return 0; return static_cast(static_cast( - round_nearest_even_golden(std::clamp(f * 32767.0f, -32768.0f, 32767.0f)))); + round_nearest_even_golden(std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0)))); } uint32_t golden_norm_i16(float f0, float f1) { return (static_cast(cvt_norm_i16(f1)) << 16) | static_cast(cvt_norm_i16(f0)); @@ -130,15 +130,10 @@ uint32_t golden_norm_i16(float f0, float f1) { // Normalized-domain inputs: in-range, fractional/truncation, the i16/u16 // divergence points (±1.0), out-of-range (clamped), and NaN (-> 0). -const std::array kNormInputs = {{ - 0.0f, - 1.0f, - 0.5f, - -1.0f, - -0.5f, - 2.0f, - 0.25f, - std::numeric_limits::quiet_NaN(), +const std::array kNormInputs = {{ + 0.0f, 1.0f, 0.5f, -1.0f, -0.5f, 2.0f, 0.25f, std::numeric_limits::quiet_NaN(), + std::bit_cast(0xbf800080u), // Below -1: symmetric saturation. + std::bit_cast(0x3f002000u), // FP32 scaling creates a false midpoint. }}; float norm_f0_for(uint32_t lane) { return kNormInputs[lane % kNormInputs.size()]; } float norm_f1_for(uint32_t lane) { return kNormInputs[(lane + 3) % kNormInputs.size()]; } diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_fp16_transcendental_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_fp16_transcendental_simd_correctness_test.cpp index 093b2e384df6..3f61843e46a3 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_fp16_transcendental_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_fp16_transcendental_simd_correctness_test.cpp @@ -2,19 +2,13 @@ // SPDX-License-Identifier: MIT /// @file vop3_fp16_transcendental_simd_correctness_test.cpp -/// @brief Bit-identity check (SIMD fast path vs scalar body) for the FTZ-bearing -/// f16 VOP3 transcendentals on CDNA4: v_rcp/v_rsq/v_exp/v_log_f16. The scalar -/// body widens f16 -> f32, applies src0 abs/neg, calls the -/// `transcendental::*_f32` reference (which flushes input denormals to ±0, -/// has NaN/±0/±Inf carve-outs, and re-flushes the result), applies dst -/// omod/clamp, and narrows back via f32_to_f16. The SIMD glue runs the matching -/// `util::*_f32_simd` helpers in-vector. Each case runs TWICE in the same process -/// -- once forcing the scalar body, once the SIMD fast path, with identical -/// inputs/EXEC -- and the results are asserted equal per active, non-skipped lane -/// (util::set_force_scalar_for_testing flips the gate in-process). NaN-result -/// lanes carry an accepted payload divergence and are excluded from the -/// comparison — NaN-ness is deterministic from the inputs, so both runs skip the -/// same lanes. In-process inactive lanes must keep the sentinel. +/// @brief SIMD/scalar comparison for the CDNA4 F16 VOP3 transcendentals. +/// Inputs are promoted to F32, modified, evaluated, and narrowed to F16. +/// RSQ honors the F16 input-denormal mode and preserves quieted NaN payloads. +/// Each case runs with scalar execution forced and then with SIMD allowed, +/// using identical inputs and EXEC. RSQ compares every active lane exactly; +/// the other operations retain their existing NaN-result exclusions. +/// Inactive lanes must keep the sentinel. #include "decode_test_util.h" #include "util/simd_test_hooks.h" @@ -63,9 +57,11 @@ struct Case { uint32_t opcode; }; +constexpr uint32_t kRsqOpcode = 383; + const std::array kCases = {{ {"v_rcp_f16_vop3", 381}, - {"v_rsq_f16_vop3", 383}, + {"v_rsq_f16_vop3", kRsqOpcode}, {"v_exp_f16_vop3", 385}, {"v_log_f16_vop3", 384}, }}; @@ -82,8 +78,8 @@ const std::array kF16Edges = {{ 0xDEADFC00u, // -Inf 0xDEAD7E00u, // +qNaN 0xDEADFD00u, // -sNaN - 0xDEAD0001u, // +smallest denormal (gets FTZ-flushed to +0) - 0xDEAD8001u, // -smallest denormal (gets FTZ-flushed to -0) + 0xDEAD0001u, // +smallest denormal + 0xDEAD8001u, // -smallest denormal 0xDEAD03FFu, // +largest denormal 0xDEAD0400u, // +smallest normal 0xDEAD7BFFu, // +HALF_MAX @@ -142,13 +138,15 @@ struct ForceScalarGuard { template void check_case(const Case &c, uint32_t abs, uint32_t neg, uint32_t omod, uint32_t clamp, - uint64_t exec, InputFn lane_input, const std::string &extra = "") { + uint64_t exec, InputFn lane_input, const std::string &extra = "", + uint32_t mode = 0) { (void)extra; ForceScalarGuard gate_guard; auto run_mode = [&](bool force_scalar) -> std::array { util::set_force_scalar_for_testing(force_scalar); Fixture fx; + fx.wf->set_mode_raw(mode); EXPECT_NE(fx.cu, nullptr); EXPECT_NE(fx.wf, nullptr); uint32_t words[4] = {0u, 0u, 0u, 0u}; @@ -164,11 +162,11 @@ void check_case(const Case &c, uint32_t abs, uint32_t neg, uint32_t omod, uint32 const auto simd_out = run_mode(/*force_scalar=*/false); // Core A/B equivalence per active, non-skipped lane. NaN-result lanes carry an - // accepted f16 NaN-payload divergence and are excluded identically in both runs. + // accepted payload divergence outside RSQ and are excluded in those operations. for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { const bool active = (exec >> lane) & 1ULL; if (active) { - if (is_f16_nan(scalar_out[lane]) || is_f16_nan(simd_out[lane])) + if (c.opcode != kRsqOpcode && (is_f16_nan(scalar_out[lane]) || is_f16_nan(simd_out[lane]))) continue; EXPECT_EQ(scalar_out[lane], simd_out[lane]) << c.name << " abs=" << abs << " neg=" << neg << " omod=" << omod << " clamp=" << clamp @@ -211,6 +209,19 @@ TEST(Vop3Fp16TranscendentalSimdCorrectness, EdgeInputs_PartialExec) { /*exec=*/0xA5A5'F0F0'1234'8001ULL, edge_in); } +TEST(Vop3Fp16TranscendentalSimdCorrectness, ReciprocalSquareRootModes) { + if constexpr (!util::has_stdx_simd) { + GTEST_SKIP() << " unavailable"; + return; + } + const Case rsq{"v_rsq_f16_vop3", kRsqOpcode}; + auto edge_in = [](uint32_t lane) { return kF16Edges[lane % kF16Edges.size()]; }; + for (uint32_t mode = 0; mode < 16; ++mode) { + SCOPED_TRACE(mode); + check_case(rsq, 0, 0, 0, 0, ~0ULL, edge_in, "", ((mode & 3u) << 2) | ((mode >> 2) << 6)); + } +} + /// Exhaustive 65536-input sweep: every f16 bit pattern, batched 64 lanes per /// dispatch (1024 dispatches per op). Catches FTZ/±0/Inf/normal-boundary /// divergences that the small edge set may miss. Modifiers default off — the diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_modifier_simd_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_modifier_simd_test.cpp index 0e0793cd0cb5..e3cfb3ccfca2 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_modifier_simd_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_modifier_simd_test.cpp @@ -98,6 +98,10 @@ double ref_src(double x, bool abs, bool neg) { } float ref_dst(float v, uint32_t omod, bool clamp, bool clamp_nan_to_zero) { + // Output scaling preserves the operation's selected NaN payload, including + // signaling NaNs passed through with IEEE mode disabled on older targets. + if (std::isnan(v)) + return clamp && clamp_nan_to_zero ? 0.0f : v; if (omod == 1) v *= 2.0f; else if (omod == 2) diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3p_dot_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3p_dot_simd_correctness_test.cpp index bcfa6bb7f4b7..03182688394b 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3p_dot_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3p_dot_simd_correctness_test.cpp @@ -195,9 +195,10 @@ TEST(Vop3pDotIntSimdCorrectness, PartialExec) { check_int_case(c, /*exec=*/0xA5A5'F0F0'1234'8001ULL, clamp); } -// GFX11 uses the ordinary integer CLAMP behavior for DOT4. GFX12 and gfx1250 -// explicitly ignore the same encoded bit. These fixed overflow vectors are an -// ISA golden rather than a scalar/SIMD self-comparison. +// The RDNA4 ISA manual says integer DOT4 ignores CLAMP, but gfx1201 hardware +// experiments show saturation. GFX11 also saturates; gfx1250 ignores CLAMP. +// These fixed overflow vectors check both scalar and SIMD paths against the +// observed ISA behavior rather than only comparing the paths with each other. uint32_t run_modern_dot4(rj_code_arch_t arch, uint32_t opcode, uint32_t src0_value, uint32_t src1_value, uint32_t accumulator, uint32_t neg, bool force_scalar) { @@ -252,7 +253,7 @@ TEST(Vop3pDotIntSimdCorrectness, Dot4ClampPolicyMatchesArchitecture) { std::numeric_limits::max()); EXPECT_EQ(run_modern_dot4(ROCJITSU_CODE_ARCH_RDNA4, 23, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFF00u, 0, force_scalar), - kUnsignedWrapped); + std::numeric_limits::max()); EXPECT_EQ(run_modern_dot4(ROCJITSU_CODE_ARCH_CDNA5, 23, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFF00u, 0, force_scalar), kUnsignedWrapped); @@ -263,7 +264,7 @@ TEST(Vop3pDotIntSimdCorrectness, Dot4ClampPolicyMatchesArchitecture) { static_cast(std::numeric_limits::max())); EXPECT_EQ(run_modern_dot4(ROCJITSU_CODE_ARCH_RDNA4, 22, 0x7F7F7F7Fu, 0x7F7F7F7Fu, 0x7FFFFFF5u, 3, force_scalar), - kMixedSignedWrapped); + static_cast(std::numeric_limits::max())); EXPECT_EQ(run_modern_dot4(ROCJITSU_CODE_ARCH_CDNA5, 22, 0x7F7F7F7Fu, 0x7F7F7F7Fu, 0x7FFFFFF5u, 3, force_scalar), kMixedSignedWrapped); diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3p_fma_mix_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3p_fma_mix_simd_correctness_test.cpp index edfa6705d614..5d1768ff31f5 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3p_fma_mix_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3p_fma_mix_simd_correctness_test.cpp @@ -1,42 +1,16 @@ // Copyright (c) 2026 Advanced Micro Devices, Inc. // SPDX-License-Identifier: MIT -/// @file vop3p_fma_mix_simd_correctness_test.cpp -/// @brief Bit-identity check (SIMD fast path vs scalar body) for the six VOP3P -/// fma_mix / mad_mix mixed-precision ops. Each case runs TWICE in the same -/// process -- once forcing the scalar body, once the SIMD fast path, with -/// identical inputs/EXEC -- and the results are asserted equal per active, -/// non-skipped lane (util::set_force_scalar_for_testing flips the gate -/// in-process). NaN-input lanes carry an accepted NaN-payload divergence and are -/// excluded from the comparison — the skip condition is computed from the -/// inputs, identical in both runs, so both skip the same lanes. In-process -/// inactive lanes must keep the dst seed. The six ops: -/// - RDNA3+ : v_fma_mix_f32 / v_fma_mixlo_f16 / v_fma_mixhi_f16 -/// - CDNA1-4: v_mad_mix_f32 / v_mad_mixlo_f16 / v_mad_mixhi_f16 -/// -/// All six implement the same scalar formula (`a * b + c` plain, no std::fma, -/// then optional clamp to [0, 1]) and differ only in (a) the per-source f16 -/// widening shape selected by `op_sel_hi` + `op_sel_hi_2` (widen f16 half to -/// f32 vs read raw f32) and the f16-half pick selected by `op_sel`, and -/// (b) the destination narrowing shape (f32 / f16-lo / f16-hi). The field -/// named `neg_hi` in the shared VOP3P layout carries per-source abs for this -/// mix-family encoding, `neg` applies per-source sign xor, and `clamp` -/// saturates the result to [0, 1]. -/// -/// The test covers every per-source neg combination, several op_sel / -/// op_sel_hi patterns (so all four widen-modes per source are exercised), -/// clamp on/off, full and partial EXEC masks. Input lanes carry an f32 edge -/// list packed both as raw f32 (when read in the no-widen mode) and as two -/// f16 halves (when the corresponding op_sel_hi bit is set), so every -/// per-source data fetch path is hit. NaN-input lanes are skipped because -/// the scalar `a * b + c` and the SIMD chain can quiet sNaN to qNaN with -/// different payloads (accepted divergence, mirroring the existing VOP3 -/// ternary tests). +/// Compare decoded MIX instructions with SIMD enabled and forced scalar, +/// including source selectors, modifiers, sparse EXEC and destination aliasing. +/// RDNA uses fused arithmetic; CDNA1-4 retains the legacy MAD implementation. #include "decode_test_util.h" #include "util/simd_test_hooks.h" #include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3p.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3p.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" @@ -54,6 +28,7 @@ #include #include #include +#include #include namespace { @@ -276,7 +251,7 @@ template struct Fixture { cfg.lds_size_kb = 64; cu = amdgpu::ComputeUnitCore::create("cu_vop3p_fma_mix", cfg, &gpu_mem, &l2); decoder = Decoder::create(cfg.arch); - wf = cu->dispatch_wf(0, 0, SGPRS_PER_WF, VGPRS_PER_WF); + wf = cu->dispatch_wf(0, 0, SGPRS_PER_WF, VGPRS_PER_WF, WF_SIZE); } void seed_vgprs(uint32_t rot, bool widen0, bool widen1, bool widen2, uint64_t exec, @@ -498,4 +473,158 @@ TEST(Vop3pFmaMixSimdCorrectness, CdnaMadMixHiF16_PartialExec) { check_op<64, 1>(kVop3pOpMadMixHiF16, kPartialExecCdna, DST_SENTINEL, "v_mad_mixhi_f16"); } +TEST(Vop3pFmaMixSimdCorrectness, FusedHalfRoundingMatchesScalar) { + using F = util::native; + // Exact ties with tiny addends, cancellation, overflow, denormals and NaNs. + constexpr std::array, 12> boundaries{{ + {0x3f801000, 0x3f800000, 0x17800000}, + {0x3f801000, 0x3f800000, 0x97800000}, + {0x33000000, 0x3f800000, 0x00000001}, + {0x33000000, 0x3f800000, 0x80000001}, + {0x477ff000, 0x3f800000, 0x97800000}, + {0xc77ff000, 0x3f800000, 0x17800000}, + {0x80000000, 0x3f800000, 0x80000000}, + {0x00000000, 0x3f800000, 0x80000000}, + {0x3f800000, 0x3f800000, 0xbf800000}, + {0x00000001, 0x00000001, 0x80000000}, + {0x7f800000, 0x00000000, 0x3f800000}, + {0x7fc12345, 0x7fc54321, 0x7f823456}, + }}; + std::mt19937 random(0x164fa); + for (uint32_t round = 0; round < 4; ++round) + for (bool clamp : {false, true}) + for (bool overflow : {false, true}) { + for (unsigned batch = 0; batch < boundaries.size() + 128; ++batch) { + const auto source = [&](unsigned index) { + return F([&](auto lane) { + return std::bit_cast( + batch < boundaries.size() ? boundaries[(batch + lane) % boundaries.size()][index] + : static_cast(random())); + }); + }; + const F a = source(0), b = source(1), c = source(2); + const auto actual = + amdgpu::mixed_fma_f16_simd(a, b, c, round, clamp, overflow, true); + amdgpu::fp_mode::ScopedEnvironment environment(0); + for (std::size_t lane = 0; lane < F::size(); ++lane) + ASSERT_EQ(actual[lane], amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + a[lane], b[lane], c[lane], round, clamp, overflow, true)) + << "round=" << round << " clamp=" << clamp << " overflow=" << overflow + << " batch=" << batch << " lane=" << lane; + } + } +} + +template +void check_direct_mix_modes(uint32_t denorm = 3) { + constexpr auto destination = F32 ? amdgpu::FmaMixDst::F32 + : High ? amdgpu::FmaMixDst::F16_HI + : amdgpu::FmaMixDst::F16_LO; + using HalfInst = std::conditional_t< + ArchTag == 0, std::conditional_t, + std::conditional_t>; + using Inst = std::conditional_t; + Fixture<32, ArchTag> fixture; + ForceScalarGuard guard; + util::set_force_scalar_for_testing(false); + for (uint32_t round = 0; round < 4; ++round) + for (bool alias : {false, true}) { + uint32_t words[2]; + const auto encode = ArchTag == 0 ? vop3p_encode_rdna3 : vop3p_encode_cdna4; + encode(F32 ? 32 + : High ? 34 + : 33, + kDstVgpr, 0, 0, 0, 0, alias ? 256 + kDstVgpr : 256, 257, 258, 0, 0, words); + std::unique_ptr instruction(decode_valid(*fixture.decoder, words)); + auto *typed = dynamic_cast(instruction.get()); + ASSERT_NE(typed, nullptr); + fixture.wf->set_mode_raw((round << (F32 ? 0 : 2)) | (denorm << 4) | 0xc0u); + const auto seed = [&] { + fixture.seed_vgprs(1, false, false, false, kPartialExecRdna, 0x3f801000u); + }; + seed(); + util::set_force_scalar_for_testing(true); + ASSERT_TRUE(fixture.cu->execute_instruction(instruction.get(), *fixture.wf).succeeded()); + std::array expected; + const uint32_t base = fixture.wf->vgpr_alloc().base + kDstVgpr; + for (uint32_t lane = 0; lane < 32; ++lane) + expected[lane] = fixture.cu->read_vgpr(base, lane); + seed(); + util::set_force_scalar_for_testing(false); + ASSERT_TRUE( + (amdgpu::try_execute_vop3p_fma_mix_simd(*typed, *fixture.wf))); + for (uint32_t lane = 0; lane < 32; ++lane) + EXPECT_EQ(fixture.cu->read_vgpr(base, lane), expected[lane]) + << "round=" << round << " alias=" << alias << " lane=" << lane; + } +} + +TEST(Vop3pFmaMixSimdCorrectness, HalfResultsStayVectorizedInEveryRoundingMode) { + check_direct_mix_modes(); + check_direct_mix_modes(); + check_direct_mix_modes(); + check_direct_mix_modes(); +} + +TEST(Vop3pFmaMixSimdCorrectness, F32ResultsStayVectorizedAcrossModes) { + for (uint32_t denorm = 0; denorm < 4; ++denorm) + check_direct_mix_modes(denorm); +} + +// Directly require admission: result parity alone would also pass if the new +// vector path declined every directed-rounding wave. +TEST(Vop3pFmaMixSimdCorrectness, DirectedSimdPreservesModifiersAndWave64Aliases) { + ForceScalarGuard guard; + auto check = []() { + Fixture fx; + const uint64_t all = util::mask(Size); + for (uint32_t round = 1; round < 4; ++round) { + for (bool high : {false, true}) { + for (uint32_t destination : {0u, kDstVgpr}) { + for (uint64_t mask : + {all, all & uint64_t{0x8000000180000004}, uint64_t{1} << (Size - 1)}) { + uint32_t words[4]{}; + vop3p_encode_rdna3(high ? kVop3pOpFmaMixHiF16 : kVop3pOpFmaMixLoF16, destination, 2, 5, + 1, 0, 256, 257, 258, 1, 4, words); + std::unique_ptr inst(decode_valid(*fx.decoder, words)); + ASSERT_NE(inst, nullptr); + auto seed = [&] { + fx.seed_vgprs(round, true, false, true, mask, DST_SENTINEL); + fx.wf->set_mode_raw(0xc0 | (round << 2)); + }; + seed(); + util::set_force_scalar_for_testing(true); + ASSERT_TRUE(fx.cu->execute_instruction(inst.get(), *fx.wf).succeeded()); + std::array expected{}; + for (uint32_t lane = 0; lane < Size; ++lane) + expected[lane] = fx.cu->read_vgpr(fx.wf->vgpr_alloc().base + destination, lane); + seed(); + util::set_force_scalar_for_testing(false); + bool admitted; + if (high) { + auto *typed = dynamic_cast(inst.get()); + ASSERT_NE(typed, nullptr); + admitted = amdgpu::try_execute_vop3p_fma_mix_simd( + *typed, *fx.wf); + } else { + auto *typed = dynamic_cast(inst.get()); + ASSERT_NE(typed, nullptr); + admitted = amdgpu::try_execute_vop3p_fma_mix_simd( + *typed, *fx.wf); + } + ASSERT_TRUE(admitted); + for (uint32_t lane = 0; lane < Size; ++lane) + EXPECT_EQ(fx.cu->read_vgpr(fx.wf->vgpr_alloc().base + destination, lane), + expected[lane]) + << "round=" << round << " high=" << high << " destination=" << destination + << " lane=" << lane; + } + } + } + } + }; + check.template operator()<32>(); + check.template operator()<64>(); +} + } // namespace diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3p_pk_f32_selection_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3p_pk_f32_selection_test.cpp index 7464e7d44b01..70c96023820d 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3p_pk_f32_selection_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3p_pk_f32_selection_test.cpp @@ -4,7 +4,7 @@ /// @file vop3p_pk_f32_selection_test.cpp /// @brief Compare packed FP32 ADD/MUL/FMA source selection and floating-point /// policies between scalar and SIMD execution on CDNA2 through CDNA5. -/// Only FMA NaN sign/payload selection may differ; both results must be quiet NaNs. +/// NaN selection and quieting must also agree with the architectural policy. #include "decode_test_util.h" #include "util/simd_test_hooks.h" @@ -184,14 +184,11 @@ class PackedF32Selection : public testing::TestWithParamwf_size(); ++lane) { for (uint32_t half = 0; half < 2; ++half) { - // As in the existing FMA SIMD contract, NaN payload choice can differ. - // Both paths must produce a quiet NaN; non-NaN results must agree exactly. + ASSERT_EQ(simd[lane][half], scalar[lane][half]) << "lane=" << lane << " half=" << half; + // CDNA2-4 can preserve signaling NaNs with IEEE mode disabled. if ((exec & (uint64_t{1} << lane)) && operation == 2 && is_nan(scalar[lane][half]) && - is_nan(simd[lane][half])) { + (arch == ROCJITSU_CODE_ARCH_CDNA5 || (mode & 0x200u))) { EXPECT_NE(scalar[lane][half] & 0x00400000, 0u); - EXPECT_NE(simd[lane][half] & 0x00400000, 0u); - } else { - ASSERT_EQ(simd[lane][half], scalar[lane][half]) << "lane=" << lane << " half=" << half; } if (dst == 6 && !(exec & (uint64_t{1} << lane))) { ASSERT_EQ(simd[lane][half], kSentinel); @@ -211,8 +208,9 @@ TEST_P(PackedF32Selection, AllSelectionsPreservePolicies) { for (uint32_t denorm = 0; denorm < 4; ++denorm) for (uint8_t clamp : {0, 1}) for (uint32_t dx10_clamp : {0u, 0x100u}) - compare(machine, lo, hi, 5, 2, 0xc0 | round | (denorm << 4) | dx10_clamp, clamp, 6, - SourceKind::Vgpr, 0xd6a53cf0a5c3697bULL); + for (uint32_t ieee : {0u, 0x200u}) + compare(machine, lo, hi, 5, 2, 0xc0 | round | (denorm << 4) | dx10_clamp | ieee, + clamp, 6, SourceKind::Vgpr, 0xd6a53cf0a5c3697bULL); } TEST_P(PackedF32Selection, ScalarSourcesAliasingAndEmptyExec) { diff --git a/emulation/rocjitsu/tests/simd_correctness/wmma_simd_exact_test.cpp b/emulation/rocjitsu/tests/simd_correctness/wmma_simd_exact_test.cpp index 4f22f444d155..fc7239327af3 100644 --- a/emulation/rocjitsu/tests/simd_correctness/wmma_simd_exact_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/wmma_simd_exact_test.cpp @@ -517,7 +517,8 @@ TEST(WmmaSimdExact, SparseK128NaNPayloadsMatchScalar) { amdgpu::extract_bf8, amdgpu::extract_bf8); }, ACC, ACC_REGS); - EXPECT_EQ(fx.cu->read_vgpr(fx.vbase + ACC, 0) & 0xFFFFu, 0x7E01u); + // Narrowing 0x7FC00000 retains the quiet bit without adding a payload bit. + EXPECT_EQ(fx.cu->read_vgpr(fx.vbase + ACC, 0) & 0xFFFFu, 0x7E00u); } // --- integer WMMA/SWMMAC, signed/unsigned, clamp on and off --- diff --git a/emulation/rocjitsu/tests/simdojo_sim_test.cpp b/emulation/rocjitsu/tests/simdojo_sim_test.cpp index d98ad2d5c603..cb2aa7ef69f4 100644 --- a/emulation/rocjitsu/tests/simdojo_sim_test.cpp +++ b/emulation/rocjitsu/tests/simdojo_sim_test.cpp @@ -1764,6 +1764,33 @@ TEST(CacheMaintenanceTest, FullInvalidationClearsRetainedMetadataAfterRefill) { } } +TEST(CacheMaintenanceTest, FullInvalidationVisitsSparseAndBoundarySets) { + Cache<6, 32, 2> cache; + constexpr std::array sets{0, 7, 8, 24, 31}; + std::array retained{}; + for (uint32_t round = 0; round != 3; ++round) { + for (size_t index = 0; index != sets.size(); ++index) { + const uint64_t address = uint64_t{sets[index]} * 64; + auto *tag = cache.allocate(address, round); + tag->dirty = true; + tag->coherence_epoch = 17; + tag->coherence = CoherenceState::MODIFIED; + retained[index] = tag; + if (index % 2 == 0) + cache.invalidate(address, round); + } + cache.invalidate_all(); + for (size_t index = 0; index != sets.size(); ++index) { + EXPECT_FALSE(cache.lookup(uint64_t{sets[index]} * 64, nullptr, round)); + EXPECT_FALSE(retained[index]->valid); + EXPECT_FALSE(retained[index]->dirty); + EXPECT_EQ(retained[index]->coherence_epoch, 0u); + EXPECT_EQ(retained[index]->coherence, CoherenceState::INVALID); + } + cache.invalidate_all(); + } +} + TEST(CacheMaintenanceTest, DirtyWalkPreservesOrderAcrossReplacementAndInvalidation) { TestCache cache; constexpr uint64_t kLineSize = TestCache::LINE_SIZE; diff --git a/emulation/rocjitsu/tests/simulated_kfd_test.cpp b/emulation/rocjitsu/tests/simulated_kfd_test.cpp index 98e4c2163c35..883016911811 100644 --- a/emulation/rocjitsu/tests/simulated_kfd_test.cpp +++ b/emulation/rocjitsu/tests/simulated_kfd_test.cpp @@ -1494,6 +1494,8 @@ TEST_F(SimulatedKfdTest, WakeDoesNotDisturbPendingAutoResetEventPage) { create.event_type = 0; create.auto_reset = 1; ASSERT_EQ(drv->ioctl(AMDKFD_IOC_CREATE_EVENT, &create), 0); + kfd_ioctl_create_event_args unsignaled{}; + ASSERT_EQ(drv->ioctl(AMDKFD_IOC_CREATE_EVENT, &unsignaled), 0); // Register a waiter deterministically: poll the event's waiter count rather than // sleeping, so the SET_EVENT below is guaranteed to take the "waiters present" @@ -1502,11 +1504,14 @@ TEST_F(SimulatedKfdTest, WakeDoesNotDisturbPendingAutoResetEventPage) { std::atomic wait_rc{-1}; std::atomic wait_result{0}; std::thread waiter([&] { - kfd_event_data ev{}; - ev.event_id = create.event_id; + // Keep wait-all parked after the first event signals. A legacy single-event + // wait now correctly completes, which would race the cancellation below. + std::array ev{}; + ev[0].event_id = create.event_id; + ev[1].event_id = unsignaled.event_id; kfd_ioctl_wait_events_args wait{}; - wait.events_ptr = reinterpret_cast(&ev); - wait.num_events = 1; + wait.events_ptr = reinterpret_cast(ev.data()); + wait.num_events = ev.size(); wait.wait_for_all = 1; wait.timeout = 0xFFFFFFFFu; wait_rc.store(drv->ioctl(AMDKFD_IOC_WAIT_EVENTS, &wait), std::memory_order_release); diff --git a/emulation/rocjitsu/tests/smfmac_simd_benchmark.cpp b/emulation/rocjitsu/tests/smfmac_simd_benchmark.cpp new file mode 100644 index 000000000000..eea547b17edf --- /dev/null +++ b/emulation/rocjitsu/tests/smfmac_simd_benchmark.cpp @@ -0,0 +1,344 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file smfmac_simd_benchmark.cpp +/// @brief Decoded-instruction A/B benchmark for the F32 sparse MFMA family. +/// +/// Covers every F32 SMFMAC helper shape on CDNA3 and CDNA4, including mixed +/// FP8/BF8 inputs. Each opcode is decoded once. Forced-scalar and default +/// execution must produce identical output bits before timings are reported. +/// The default mode selects AVX-512 when the fast path is present. The D +/// register window is restored outside each timed interval so every call +/// starts from the same accumulator state. + +#include "decode_test_util.h" +#include "mma_test_util.h" +#include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/opcodes.h" +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" +#include "rocjitsu/vm/amdgpu/compute_unit.h" +#include "rocjitsu/vm/amdgpu/gpu_memory.h" +#include "rocjitsu/vm/amdgpu/l2_cache.h" +#include "rocjitsu/vm/amdgpu/wavefront.h" +#include "util/data_types.h" +#include "util/simd.h" +#include "util/simd_test_hooks.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace rocjitsu; +using Clock = std::chrono::steady_clock; + +constexpr uint32_t WF_SIZE = mma_test::MFMA_WF_SIZE; +constexpr uint32_t A_OFF = 0; +constexpr uint32_t B_OFF = 32; +constexpr uint32_t INDEX_OFF = 64; +constexpr uint32_t D_OFF = 96; +constexpr int ITERATIONS = mma_test::BENCH_ITERATIONS / 20; + +enum class Format { F16, BF16, FP8, BF8 }; + +struct Shape { + rj_code_arch_t arch; + uint32_t m, n, k; + uint32_t a_regs, b_regs, d_regs; + bool fp8; + // For 16-bit inputs: F16, BF16. For 8-bit inputs: BF8/BF8, + // BF8/FP8, FP8/BF8, FP8/FP8. + std::array opcodes; +}; + +constexpr std::array SHAPES{ + Shape{ROCJITSU_CODE_ARCH_CDNA3, + 16, + 16, + 32, + 2, + 4, + 4, + false, + {cdna3::kVSmfmacF3216x16x32F16Vop3pMfma, cdna3::kVSmfmacF3216x16x32Bf16Vop3pMfma, 0, 0}}, + Shape{ROCJITSU_CODE_ARCH_CDNA3, + 32, + 32, + 16, + 2, + 4, + 16, + false, + {cdna3::kVSmfmacF3232x32x16F16Vop3pMfma, cdna3::kVSmfmacF3232x32x16Bf16Vop3pMfma, 0, 0}}, + Shape{ROCJITSU_CODE_ARCH_CDNA4, + 16, + 16, + 64, + 4, + 8, + 4, + false, + {cdna4::kVSmfmacF3216x16x64F16Vop3pMfma, cdna4::kVSmfmacF3216x16x64Bf16Vop3pMfma, 0, 0}}, + Shape{ROCJITSU_CODE_ARCH_CDNA4, + 32, + 32, + 32, + 4, + 8, + 16, + false, + {cdna4::kVSmfmacF3232x32x32F16Vop3pMfma, cdna4::kVSmfmacF3232x32x32Bf16Vop3pMfma, 0, 0}}, + Shape{ROCJITSU_CODE_ARCH_CDNA3, + 16, + 16, + 64, + 2, + 4, + 4, + true, + {cdna3::kVSmfmacF3216x16x64Bf8Bf8Vop3pMfma, cdna3::kVSmfmacF3216x16x64Bf8Fp8Vop3pMfma, + cdna3::kVSmfmacF3216x16x64Fp8Bf8Vop3pMfma, cdna3::kVSmfmacF3216x16x64Fp8Fp8Vop3pMfma}}, + Shape{ROCJITSU_CODE_ARCH_CDNA3, + 32, + 32, + 32, + 2, + 4, + 16, + true, + {cdna3::kVSmfmacF3232x32x32Bf8Bf8Vop3pMfma, cdna3::kVSmfmacF3232x32x32Bf8Fp8Vop3pMfma, + cdna3::kVSmfmacF3232x32x32Fp8Bf8Vop3pMfma, cdna3::kVSmfmacF3232x32x32Fp8Fp8Vop3pMfma}}, + Shape{ROCJITSU_CODE_ARCH_CDNA4, + 16, + 16, + 128, + 4, + 8, + 4, + true, + {cdna4::kVSmfmacF3216x16x128Bf8Bf8Vop3pMfma, cdna4::kVSmfmacF3216x16x128Bf8Fp8Vop3pMfma, + cdna4::kVSmfmacF3216x16x128Fp8Bf8Vop3pMfma, cdna4::kVSmfmacF3216x16x128Fp8Fp8Vop3pMfma}}, + Shape{ROCJITSU_CODE_ARCH_CDNA4, + 32, + 32, + 64, + 4, + 8, + 16, + true, + {cdna4::kVSmfmacF3232x32x64Bf8Bf8Vop3pMfma, cdna4::kVSmfmacF3232x32x64Bf8Fp8Vop3pMfma, + cdna4::kVSmfmacF3232x32x64Fp8Bf8Vop3pMfma, cdna4::kVSmfmacF3232x32x64Fp8Fp8Vop3pMfma}}, +}; + +class ForceScalarGuard { +public: + ForceScalarGuard() : original_(util::force_scalar()) {} + ~ForceScalarGuard() { util::set_force_scalar_for_testing(original_); } + + ForceScalarGuard(const ForceScalarGuard &) = delete; + ForceScalarGuard &operator=(const ForceScalarGuard &) = delete; + +private: + bool original_; +}; + +struct BenchFixture { + amdgpu::GpuMemory gpu_mem{"smfmac_simd_bench_mem"}; + amdgpu::L2Cache l2{"smfmac_simd_bench_l2"}; + std::unique_ptr cu; + amdgpu::Wavefront *wf = nullptr; + uint32_t vbase = 0; + + explicit BenchFixture(rj_code_arch_t arch) { + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = arch; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = mma_test::SGPRS_PER_WF; + cfg.vgprs_per_wf = mma_test::VGPRS_PER_WF; + cfg.lds_size_kb = 64; + cu = amdgpu::ComputeUnitCore::create("cu_smfmac_simd_bench", cfg, &gpu_mem, &l2); + if (cu) { + wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + if (wf) + vbase = wf->vgpr_alloc().base; + } + } + + void seed(uint32_t off, uint32_t regs, Format fmt, uint32_t seed, rj_code_arch_t arch) { + mma_test::SmallGen gen(seed); + for (uint32_t reg = 0; reg < regs; ++reg) { + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + uint32_t word = 0; + if (fmt == Format::F16 || fmt == Format::BF16) { + const auto pack = [&](float value) { + return fmt == Format::F16 ? util::f32_to_f16(value) : util::f32_to_bf16(value); + }; + word = pack(gen()) | (static_cast(pack(gen())) << 16); + } else { + const auto pack = [&](float value) { + if (fmt == Format::FP8) + return arch == ROCJITSU_CODE_ARCH_CDNA3 ? util::f32_to_fp8_e4m3_fnuz_rne(value) + : util::f32_to_fp8_e4m3_rne(value); + return arch == ROCJITSU_CODE_ARCH_CDNA3 ? util::f32_to_bf8_e5m2_fnuz_rne(value) + : util::f32_to_bf8_e5m2_rne(value); + }; + for (uint32_t byte = 0; byte < 4; ++byte) + word |= static_cast(pack(gen())) << (8 * byte); + } + cu->write_vgpr(vbase + off + reg, lane, word); + } + } + } + + void seed_accumulator(uint32_t regs, uint32_t seed) { + mma_test::SmallGen gen(seed); + for (uint32_t reg = 0; reg < regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + cu->write_vgpr(vbase + D_OFF + reg, lane, std::bit_cast(gen())); + } + + void seed_index() { + // All six legal 2-of-4 selector pairs are used across lanes and nibbles. + constexpr std::array pairs{0x4, 0x8, 0xC, 0x9, 0xD, 0xE}; + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + uint32_t word = 0; + for (uint32_t nibble = 0; nibble < 8; ++nibble) + word |= pairs[(lane + nibble) % pairs.size()] << (4 * nibble); + cu->write_vgpr(vbase + INDEX_OFF, lane, word); + } + } + + std::vector snapshot(uint32_t regs) const { + std::vector words(static_cast(regs) * WF_SIZE); + for (uint32_t reg = 0; reg < regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + words[static_cast(reg) * WF_SIZE + lane] = cu->read_vgpr(vbase + D_OFF + reg, lane); + return words; + } + + void restore(const std::vector &words) { + for (size_t i = 0; i < words.size(); ++i) + cu->write_vgpr(vbase + D_OFF + static_cast(i / WF_SIZE), + static_cast(i % WF_SIZE), words[i]); + } +}; + +std::array build_words(const Shape &shape, uint16_t opcode) { + constexpr uint32_t a = 256 + A_OFF, b = 256 + B_OFF, idx = 256 + INDEX_OFF; + if (shape.arch == ROCJITSU_CODE_ARCH_CDNA3) + return cdna3::build_vop3p_mfma(opcode, + {.vdst = D_OFF, .acc_cd = 0, .src0 = a, .src1 = b, .src2 = idx}); + return cdna4::build_vop3p_mfma(opcode, + {.vdst = D_OFF, .acc_cd = 0, .src0 = a, .src1 = b, .src2 = idx}); +} + +void benchmark_case(const Shape &shape, size_t variant) { + const Format fmt_a = shape.fp8 ? (variant < 2 ? Format::BF8 : Format::FP8) + : (variant == 0 ? Format::F16 : Format::BF16); + const Format fmt_b = shape.fp8 ? (variant % 2 == 0 ? Format::BF8 : Format::FP8) : fmt_a; + const char *fmt_name[] = {"f16", "bf16", "bf8_bf8", "bf8_fp8", "fp8_bf8", "fp8_fp8"}; + const char *suffix = shape.fp8 ? fmt_name[variant + 2] : fmt_name[variant]; + const std::string label = "v_smfmac_f32_" + std::to_string(shape.m) + "x" + + std::to_string(shape.n) + "x" + std::to_string(shape.k) + "_" + suffix; + SCOPED_TRACE(label); + + BenchFixture fx(shape.arch); + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + const uint32_t seed = 101 + static_cast(variant) + 17 * shape.k; + fx.seed(A_OFF, shape.a_regs, fmt_a, seed, shape.arch); + fx.seed(B_OFF, shape.b_regs, fmt_b, seed + 1, shape.arch); + fx.seed_index(); + fx.seed_accumulator(shape.d_regs, seed + 2); + const auto initial = fx.snapshot(shape.d_regs); + + const auto words = build_words(shape, shape.opcodes[variant]); + auto decoder = Decoder::create(shape.arch); + ASSERT_NE(decoder, nullptr); + std::unique_ptr instruction(decode_valid(*decoder, words.data())); + ASSERT_NE(instruction, nullptr); + + bool execution_failed = false; + const auto run = [&] { + execution_failed |= !fx.cu->execute_instruction(instruction.get(), *fx.wf).succeeded(); + }; + + ForceScalarGuard force_scalar_guard; + util::set_force_scalar_for_testing(true); + run(); + ASSERT_FALSE(execution_failed) << label; + const auto scalar = fx.snapshot(shape.d_regs); + ASSERT_NE(scalar, initial) << label << ": instruction did not update D"; + + fx.restore(initial); + util::set_force_scalar_for_testing(false); + run(); + ASSERT_FALSE(execution_failed) << label; + const auto simd = fx.snapshot(shape.d_regs); + ASSERT_EQ(scalar.size(), simd.size()); + for (size_t i = 0; i < scalar.size(); ++i) + ASSERT_EQ(scalar[i], simd[i]) << label << ": raw F32 mismatch at word " << i; + + const auto time_block = [&](bool force_scalar) { + util::set_force_scalar_for_testing(force_scalar); + for (int i = 0; i < 8; ++i) { + fx.restore(initial); + run(); + } + std::chrono::nanoseconds elapsed{}; + for (int i = 0; i < ITERATIONS / 2; ++i) { + fx.restore(initial); + const auto start = Clock::now(); + run(); + elapsed += std::chrono::duration_cast(Clock::now() - start); + } + return elapsed.count(); + }; + + // ABBA ordering balances changes in clock rate and thermal state. + const auto scalar_first = time_block(true); + const auto simd_first = time_block(false); + const auto simd_second = time_block(false); + const auto scalar_second = time_block(true); + util::set_force_scalar_for_testing(false); + + const double scalar_ns = static_cast(scalar_first + scalar_second) / ITERATIONS; + const double simd_ns = static_cast(simd_first + simd_second) / ITERATIONS; + const double macs = static_cast(shape.m) * shape.n * shape.k / 2; + std::printf("\n === decoded %s (%s, wave64, fresh D) ===\n" + " MACs/op: %.0f scalar: %9.1f ns default: %9.1f ns speedup: %5.2fx\n", + label.c_str(), shape.arch == ROCJITSU_CODE_ARCH_CDNA3 ? "gfx942" : "gfx950", macs, + scalar_ns, simd_ns, simd_ns > 0.0 ? scalar_ns / simd_ns : 0.0); + EXPECT_GT(scalar_ns, 0.0); + EXPECT_GT(simd_ns, 0.0); + EXPECT_FALSE(execution_failed) << label; +} + +} // namespace + +TEST(SmfmacSimdBenchmark, DecodedF32Forms) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "SMFMAC AVX-512 paths require 16-lane native SIMD"; + + for (const auto &shape : SHAPES) { + const size_t variants = shape.fp8 ? 4 : 2; + for (size_t variant = 0; variant < variants; ++variant) { + benchmark_case(shape, variant); + if (testing::Test::HasFatalFailure()) + return; + } + } +} diff --git a/emulation/rocjitsu/tests/swmmac_k128_simd_benchmark.cpp b/emulation/rocjitsu/tests/swmmac_k128_simd_benchmark.cpp new file mode 100644 index 000000000000..41d9797bb7a4 --- /dev/null +++ b/emulation/rocjitsu/tests/swmmac_k128_simd_benchmark.cpp @@ -0,0 +1,269 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +/// @file swmmac_k128_simd_benchmark.cpp +/// @brief Decoded-instruction A/B benchmark for the eight gfx1250 K=128 +/// FP8/BF8 SWMMAC forms. +/// +/// The forced-scalar and default paths use the same decoded instruction and +/// must agree bit-for-bit before timing. The default path uses AVX-512 when +/// the fast path is present. Each timed call starts with the same C state; +/// restoring it is outside the timed interval. + +#include "decode_test_util.h" +#include "mma_test_util.h" +#include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna5/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna5/opcodes.h" +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" +#include "rocjitsu/vm/amdgpu/compute_unit.h" +#include "rocjitsu/vm/amdgpu/gpu_memory.h" +#include "rocjitsu/vm/amdgpu/l2_cache.h" +#include "rocjitsu/vm/amdgpu/wavefront.h" +#include "util/data_types.h" +#include "util/simd.h" +#include "util/simd_test_hooks.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace rocjitsu; +using Clock = std::chrono::steady_clock; + +constexpr uint32_t WF_SIZE = mma_test::WMMA_WF_SIZE; +constexpr uint32_t A_OFF = 0; +constexpr uint32_t B_OFF = 32; +constexpr uint32_t ACC_OFF = 64; +constexpr uint32_t INDEX_OFF = 96; +constexpr uint32_t A_REGS = 8; +constexpr uint32_t B_REGS = 16; +constexpr uint32_t INDEX_REGS = 2; +constexpr int ITERATIONS = mma_test::BENCH_ITERATIONS / 10; +constexpr double SPARSE_MACS = 16.0 * 16.0 * (128.0 / 2.0); + +enum class Format { FP8, BF8, F32, F16 }; + +struct BenchCase { + uint16_t opcode; + const char *name; + const char *label; + Format a_format; + Format b_format; + Format accumulator_format; + uint32_t accumulator_regs; + uint32_t seed; +}; + +constexpr std::array BENCH_CASES{ + BenchCase{cdna5::kVSwmmacF3216x16x128Fp8Fp8Vop3p, "F32Fp8Fp8", "v_swmmac_f32_16x16x128_fp8_fp8", + Format::FP8, Format::FP8, Format::F32, 8, 101}, + BenchCase{cdna5::kVSwmmacF3216x16x128Fp8Bf8Vop3p, "F32Fp8Bf8", "v_swmmac_f32_16x16x128_fp8_bf8", + Format::FP8, Format::BF8, Format::F32, 8, 111}, + BenchCase{cdna5::kVSwmmacF3216x16x128Bf8Fp8Vop3p, "F32Bf8Fp8", "v_swmmac_f32_16x16x128_bf8_fp8", + Format::BF8, Format::FP8, Format::F32, 8, 121}, + BenchCase{cdna5::kVSwmmacF3216x16x128Bf8Bf8Vop3p, "F32Bf8Bf8", "v_swmmac_f32_16x16x128_bf8_bf8", + Format::BF8, Format::BF8, Format::F32, 8, 131}, + BenchCase{cdna5::kVSwmmacF1616x16x128Fp8Fp8Vop3p, "F16Fp8Fp8", "v_swmmac_f16_16x16x128_fp8_fp8", + Format::FP8, Format::FP8, Format::F16, 4, 141}, + BenchCase{cdna5::kVSwmmacF1616x16x128Fp8Bf8Vop3p, "F16Fp8Bf8", "v_swmmac_f16_16x16x128_fp8_bf8", + Format::FP8, Format::BF8, Format::F16, 4, 151}, + BenchCase{cdna5::kVSwmmacF1616x16x128Bf8Fp8Vop3p, "F16Bf8Fp8", "v_swmmac_f16_16x16x128_bf8_fp8", + Format::BF8, Format::FP8, Format::F16, 4, 161}, + BenchCase{cdna5::kVSwmmacF1616x16x128Bf8Bf8Vop3p, "F16Bf8Bf8", "v_swmmac_f16_16x16x128_bf8_bf8", + Format::BF8, Format::BF8, Format::F16, 4, 171}, +}; + +class ForceScalarGuard { +public: + ForceScalarGuard() : original_(util::force_scalar()) {} + ~ForceScalarGuard() { util::set_force_scalar_for_testing(original_); } + + ForceScalarGuard(const ForceScalarGuard &) = delete; + ForceScalarGuard &operator=(const ForceScalarGuard &) = delete; + +private: + bool original_; +}; + +struct BenchFixture { + amdgpu::GpuMemory gpu_mem{"swmmac_k128_bench_mem"}; + amdgpu::L2Cache l2{"swmmac_k128_bench_l2"}; + std::unique_ptr cu; + amdgpu::Wavefront *wf = nullptr; + uint32_t vbase = 0; + + BenchFixture() { + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = mma_test::SGPRS_PER_WF; + cfg.vgprs_per_wf = mma_test::VGPRS_PER_WF; + cfg.lds_size_kb = 64; + cu = amdgpu::ComputeUnitCore::create("cu_swmmac_k128_bench", cfg, &gpu_mem, &l2); + if (cu) { + wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + if (wf) + vbase = wf->vgpr_alloc().base; + } + } + + void seed(uint32_t off, uint32_t regs, Format fmt, uint32_t seed) { + mma_test::SmallGen gen(seed); + for (uint32_t reg = 0; reg < regs; ++reg) { + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + uint32_t word = 0; + switch (fmt) { + case Format::F32: + word = std::bit_cast(gen()); + break; + case Format::F16: + word = util::f32_to_f16(gen()); + word |= static_cast(util::f32_to_f16(gen())) << 16; + break; + case Format::FP8: + case Format::BF8: + for (uint32_t byte = 0; byte < 4; ++byte) { + const uint8_t packed = fmt == Format::FP8 ? util::f32_to_fp8_e4m3_rne(gen()) + : util::f32_to_bf8_e5m2_rne(gen()); + word |= static_cast(packed) << (8 * byte); + } + break; + } + cu->write_vgpr(vbase + off + reg, lane, word); + } + } + } + + void seed_indices() { + // The six legal index0 < index1 pairs, varied across both K=128 words. + constexpr std::array pairs{0x4u, 0x8u, 0xCu, 0x9u, 0xDu, 0xEu}; + for (uint32_t reg = 0; reg < INDEX_REGS; ++reg) { + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) { + uint32_t word = 0; + for (uint32_t nibble = 0; nibble < 8; ++nibble) + word |= pairs[(lane + nibble + 3 * reg) % pairs.size()] << (4 * nibble); + cu->write_vgpr(vbase + INDEX_OFF + reg, lane, word); + } + } + } + + std::vector snapshot(uint32_t off, uint32_t regs) const { + std::vector words(static_cast(regs) * WF_SIZE); + for (uint32_t reg = 0; reg < regs; ++reg) + for (uint32_t lane = 0; lane < WF_SIZE; ++lane) + words[static_cast(reg) * WF_SIZE + lane] = cu->read_vgpr(vbase + off + reg, lane); + return words; + } + + void restore(uint32_t off, const std::vector &words) { + for (size_t i = 0; i < words.size(); ++i) + cu->write_vgpr(vbase + off + static_cast(i / WF_SIZE), + static_cast(i % WF_SIZE), words[i]); + } +}; + +void benchmark_case(const BenchCase &test) { + BenchFixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + + fx.seed(A_OFF, A_REGS, test.a_format, test.seed); + fx.seed(B_OFF, B_REGS, test.b_format, test.seed + 1); + fx.seed_indices(); + fx.seed(ACC_OFF, test.accumulator_regs, test.accumulator_format, test.seed + 2); + const auto initial = fx.snapshot(ACC_OFF, test.accumulator_regs); + + const auto words = cdna5::build_vop3p( + test.opcode, + {.vdst = ACC_OFF, .src0 = 256 + A_OFF, .src1 = 256 + B_OFF, .src2 = 256 + INDEX_OFF}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr instruction(decode_valid(*decoder, words.data())); + ASSERT_NE(instruction, nullptr); + EXPECT_EQ(instruction->mnemonic(), std::string_view(test.label)); + + bool execution_failed = false; + const auto run = [&] { + execution_failed |= !fx.cu->execute_instruction(instruction.get(), *fx.wf).succeeded(); + }; + const auto reset_accumulator = [&] { fx.restore(ACC_OFF, initial); }; + + ForceScalarGuard force_scalar_guard; + util::set_force_scalar_for_testing(true); + run(); + ASSERT_FALSE(execution_failed) << test.label; + const auto scalar = fx.snapshot(ACC_OFF, test.accumulator_regs); + ASSERT_NE(scalar, initial) << test.label << ": instruction did not update D"; + + reset_accumulator(); + util::set_force_scalar_for_testing(false); + run(); + ASSERT_FALSE(execution_failed) << test.label; + const auto simd = fx.snapshot(ACC_OFF, test.accumulator_regs); + ASSERT_EQ(scalar.size(), simd.size()); + for (size_t i = 0; i < scalar.size(); ++i) + ASSERT_EQ(scalar[i], simd[i]) << test.label << ": raw output mismatch at word " << i; + + const auto time_block = [&](bool force_scalar) { + util::set_force_scalar_for_testing(force_scalar); + for (int i = 0; i < 12; ++i) { + reset_accumulator(); + run(); + } + std::chrono::nanoseconds elapsed{}; + for (int i = 0; i < ITERATIONS / 2; ++i) { + reset_accumulator(); + const auto start = Clock::now(); + run(); + elapsed += std::chrono::duration_cast(Clock::now() - start); + } + return elapsed.count(); + }; + + // ABBA block order balances clock and thermal drift between the modes. + const auto scalar_first = time_block(true); + const auto default_first = time_block(false); + const auto default_second = time_block(false); + const auto scalar_second = time_block(true); + util::set_force_scalar_for_testing(false); + + const double scalar_ns = static_cast(scalar_first + scalar_second) / ITERATIONS; + const double default_ns = static_cast(default_first + default_second) / ITERATIONS; + std::printf("\n === decoded %s (gfx1250, wave32, fresh C) ===\n" + " iterations/mode: %d MACs/op: %.0f\n" + " scalar: %9.1f ns default: %9.1f ns speedup: %5.2fx\n", + test.label, ITERATIONS, SPARSE_MACS, scalar_ns, default_ns, + default_ns > 0.0 ? scalar_ns / default_ns : 0.0); + EXPECT_GT(scalar_ns, 0.0); + EXPECT_GT(default_ns, 0.0); + EXPECT_FALSE(execution_failed) << test.label; +} + +class SwmmacK128SimdBenchmark : public testing::TestWithParam {}; + +TEST_P(SwmmacK128SimdBenchmark, DecodedInstruction) { + SKIP_IF_NO_SIMD(); + if (util::native::size() != 16) + GTEST_SKIP() << "the K=128 SWMMAC fast paths require 16-lane native SIMD"; + benchmark_case(BENCH_CASES[GetParam()]); +} + +INSTANTIATE_TEST_SUITE_P(Forms, SwmmacK128SimdBenchmark, + testing::Range(0, BENCH_CASES.size()), + [](const testing::TestParamInfo &info) { + return std::string(BENCH_CASES[info.param].name); + }); + +} // namespace diff --git a/emulation/rocjitsu/tests/sysfs_test.cpp b/emulation/rocjitsu/tests/sysfs_test.cpp index 27ade44a74fe..f35347180586 100644 --- a/emulation/rocjitsu/tests/sysfs_test.cpp +++ b/emulation/rocjitsu/tests/sysfs_test.cpp @@ -2,15 +2,17 @@ // SPDX-License-Identifier: MIT /// @file sysfs_test.cpp -/// @brief Golden tests for the synthetic KFD topology's debug capability bits. +/// @brief Tests for synthetic GPU topology, IP discovery, and debug capabilities. /// /// @details Verifies that Sysfs::generate() advertises the KFD debugger API /// (HSA_CAP_TRAP_DEBUG_*) capability/debug_prop bits that rocdbgapi's /// os_driver_kfd.cpp reads to decide whether an agent is debuggable, and that -/// architecture-specific "precise" debug bits are gated correctly. +/// architecture-specific "precise" debug bits are gated correctly. Also checks +/// that IP discovery publishes the configured GC, SDMA, and NBIF versions. #include "rocjitsu/kmd/linux/sysfs.h" +#include "rocjitsu/config/config_common.h" #include "rocjitsu/config/config_loader.h" #include "rocjitsu/kmd/linux/amdgpu_properties.h" #include "rocjitsu/kmd/linux/cwsr.h" @@ -27,10 +29,12 @@ RJ_DIAGNOSTIC_POP #include +#include #include #include #include #include +#include #include #include @@ -81,6 +85,132 @@ TEST(SysfsTopologyTest, DefaultGpuInfoHasCoherentSdmaCounts) { EXPECT_EQ(props["num_sdma_queues_per_engine"], 0u); } +TEST(SysfsTopologyTest, W7900PublishesComputeIpDiscoveryVersions) { + Sysfs sysfs; + const auto identity = config::load_device_identity( + std::string(CONFIG_DIR) + "/gfx1100_w7900.json", kEmbeddedSchema); + auto gpu = gpu_info_from_config(identity.device, 1); + gpu.drm_render_minor = 129; + ASSERT_FALSE(sysfs.generate(gpu).empty()); + + // Captured W7900 sysfs values used by direct KFD/PM4 clients such as tinygrad. + for (const std::string node : {"card0", "renderD129"}) { + const std::string base = sysfs.drm_path() + "/" + node + "/device/ip_discovery/die/0/"; + EXPECT_EQ(read_sysfs_file(base + "11/0/major"), "11\n"); + EXPECT_EQ(read_sysfs_file(base + "11/0/minor"), "0\n"); + EXPECT_EQ(read_sysfs_file(base + "11/0/revision"), "0\n"); + EXPECT_EQ(read_sysfs_file(base + "42/0/major"), "6\n"); + EXPECT_EQ(read_sysfs_file(base + "42/0/minor"), "0\n"); + EXPECT_EQ(read_sysfs_file(base + "42/0/revision"), "0\n"); + EXPECT_EQ(read_sysfs_file(base + "108/0/major"), "4\n"); + EXPECT_EQ(read_sysfs_file(base + "108/0/minor"), "3\n"); + EXPECT_EQ(read_sysfs_file(base + "108/0/revision"), "0\n"); + } +} + +TEST(SysfsTopologyTest, R9700PublishesComputeIpDiscoveryVersions) { + Sysfs sysfs; + const auto identity = config::load_device_identity( + std::string(CONFIG_DIR) + "/gfx1201_r9700.json", kEmbeddedSchema); + auto gpu = gpu_info_from_config(identity.device, 1); + gpu.drm_render_minor = 128; + ASSERT_FALSE(sysfs.generate(gpu).empty()); + + // Captured R9700 sysfs values used by direct KFD/PM4 clients such as tinygrad. + for (const std::string node : {"card0", "renderD128"}) { + const std::string base = sysfs.drm_path() + "/" + node + "/device/ip_discovery/die/0/"; + EXPECT_EQ(read_sysfs_file(base + "11/0/major"), "12\n"); + EXPECT_EQ(read_sysfs_file(base + "11/0/minor"), "0\n"); + EXPECT_EQ(read_sysfs_file(base + "11/0/revision"), "1\n"); + EXPECT_EQ(read_sysfs_file(base + "42/0/major"), "7\n"); + EXPECT_EQ(read_sysfs_file(base + "42/0/minor"), "0\n"); + EXPECT_EQ(read_sysfs_file(base + "42/0/revision"), "1\n"); + EXPECT_EQ(read_sysfs_file(base + "108/0/major"), "6\n"); + EXPECT_EQ(read_sysfs_file(base + "108/0/minor"), "3\n"); + EXPECT_EQ(read_sysfs_file(base + "108/0/revision"), "1\n"); + } +} + +TEST(SysfsTopologyTest, CdnaAndRdna35PublishComputeIpDiscoveryVersions) { + struct Profile { + const char *filename; + uint32_t num_xcc; + std::array, 3> blocks; + }; + const Profile profiles[] = { + {"gfx942_cdna3_kmd.json", 8, {{{11, 9, 4, 3}, {42, 4, 4, 2}, {108, 7, 9, 0}}}}, + {"gfx950_mi355x_kmd.json", 8, {{{11, 9, 5, 0}, {42, 4, 4, 5}, {108, 7, 9, 0}}}}, + {"gfx1151.json", 1, {{{11, 11, 5, 1}, {42, 6, 1, 1}, {108, 7, 11, 1}}}}, + }; + for (const auto &profile : profiles) { + SCOPED_TRACE(profile.filename); + Sysfs sysfs; + const auto identity = config::load_device_identity( + std::string(CONFIG_DIR) + "/" + profile.filename, kEmbeddedSchema); + auto gpu = gpu_info_from_config(identity.device, profile.num_xcc); + gpu.drm_render_minor = 128; + ASSERT_FALSE(sysfs.generate(gpu).empty()); + for (const std::string node : {"card0", "renderD128"}) { + const std::string base = sysfs.drm_path() + "/" + node + "/device/ip_discovery/die/0/"; + for (const auto &block : profile.blocks) { + const std::string path = base + std::to_string(block[0]) + "/0/"; + EXPECT_EQ(read_sysfs_file(path + "major"), std::to_string(block[1]) + "\n"); + EXPECT_EQ(read_sysfs_file(path + "minor"), std::to_string(block[2]) + "\n"); + EXPECT_EQ(read_sysfs_file(path + "revision"), std::to_string(block[3]) + "\n"); + } + } + } +} + +TEST(SysfsTopologyTest, IpDiscoveryUsesOwnedConfigVersions) { + const auto device = config::with_parsed_simulation_config_json( + R"({"vm":{"gpu":{"device":{ + "gfx_target_version":110000, "num_sdma_engines":0, + "ip_versions":[{"hardware_id":11,"major":11,"minor":0,"revision":7}] + }}}})", + kEmbeddedSchema, [](const fb::SimulationConfig *parsed) { + return config::kfd_device_from_fb(parsed->vm()->gpu()->device(), "vm.gpu.device"); + }); + // The parser and its storage have been destroyed before sysfs consumes the data. + Sysfs sysfs; + ASSERT_FALSE(sysfs.generate(gpu_info_from_config(device, 1)).empty()); + const std::string base = sysfs.drm_path() + "/card0/device/ip_discovery/die/0/"; + EXPECT_EQ(read_sysfs_file(base + "11/0/revision"), "7\n"); + EXPECT_FALSE(std::filesystem::exists(base + "42")); + EXPECT_FALSE(std::filesystem::exists(base + "108")); +} + +TEST(SysfsTopologyTest, OmittedIpVersionsDoNotInventDiscoveryBlocks) { + Sysfs sysfs; + ASSERT_FALSE(sysfs.generate(make_gpu_info(110000)).empty()); + EXPECT_FALSE(std::filesystem::exists(sysfs.drm_path() + "/card0/device/ip_discovery")); +} + +TEST(SysfsTopologyTest, RejectsInvalidIpVersionsInVmAndGuestConfigs) { + for (const std::string entries : + {R"({"hardware_id":0,"major":11})", R"({"hardware_id":11,"major":0})", + R"({"hardware_id":11,"major":256})", R"({"hardware_id":11,"major":11,"revision":256})", + R"({"hardware_id":11,"major":11},{"hardware_id":11,"major":12})"}) { + SCOPED_TRACE(entries); + for (bool guest : {false, true}) { + SCOPED_TRACE(guest); + const std::string json = + (guest ? R"({"dbt_guest":{"guest_device":)" : R"({"vm":{"gpu":{"device":)") + + std::string(R"({"num_sdma_engines":0,"ip_versions":[)") + entries + + (guest ? "]}}}" : "]}}}}"); + EXPECT_THROW(config::with_parsed_simulation_config_json( + json, kEmbeddedSchema, + [guest](const fb::SimulationConfig *parsed) { + const auto *device = guest ? parsed->dbt_guest()->guest_device() + : parsed->vm()->gpu()->device(); + (void)config::kfd_device_from_fb(device, guest ? "dbt_guest.guest_device" + : "vm.gpu.device"); + }), + std::runtime_error); + } + } +} + // Golden per-GFXIP expectations. Each row mirrors what // kfd_topology_set_capabilities() in drivers/gpu/drm/amd/amdkfd/kfd_topology.c // programs for the corresponding GC hardware IP version. The watch-mask lo/hi diff --git a/emulation/rocjitsu/tests/transcendental_test.cpp b/emulation/rocjitsu/tests/transcendental_test.cpp index 914cb6167bc1..3fde6db2155f 100644 --- a/emulation/rocjitsu/tests/transcendental_test.cpp +++ b/emulation/rocjitsu/tests/transcendental_test.cpp @@ -31,6 +31,33 @@ TEST(TranscendentalTest, RcpF32SpecialCases) { EXPECT_FLOAT_EQ(rcp_f32(2.0f), 0.5f); } +TEST(TranscendentalTest, RcpF32MatchesPhysicalRdna3AndRdna4) { + const uint32_t cases[][2] = { + {0x3f800000u, 0x3f800000u}, {0x3f800001u, 0x3f7ffffeu}, {0x3f81ffffu, 0x3f7c0fc3u}, + {0x3f820000u, 0x3f7c0fc1u}, {0x3f83ffffu, 0x3f783e12u}, {0x3f840000u, 0x3f783e10u}, + {0x3fc00000u, 0x3f2aaaaau}, {0x3fffffffu, 0x3f000001u}, {0x3f802922u, 0x3f7fadd6u}, + {0x3f932eddu, 0x3f5ea262u}, {0x3fb0333cu, 0x3f39f868u}, {0x3ffff486u, 0x3f0005beu}, + {0x00800000u, 0x7e800000u}, {0x7f000000u, 0x00000000u}, {0x00000001u, 0x7f800000u}, + {0x7fa12345u, 0x7fe12345u}, + }; + for (const auto &test : cases) + for (uint32_t sign : {0u, 0x80000000u}) + EXPECT_EQ(std::bit_cast(rcp_f32(std::bit_cast(test[0] | sign))), + test[1] | sign) + << std::hex << test[0] << " sign=" << sign; +} + +TEST(TranscendentalTest, RcpF32CompleteNormalizedHardwareDigest) { + // FNV-style hash of raw result words captured independently on gfx1100 and gfx1201. + // Check the complete mantissa domain without storing the 32 MiB capture. + uint64_t digest = 14695981039346656037ull; + for (uint32_t mantissa = 0; mantissa < (1u << 23); ++mantissa) { + const float input = std::bit_cast(0x3f800000u | mantissa); + digest = (digest ^ std::bit_cast(rcp_f32(input))) * 1099511628211ull; + } + EXPECT_EQ(digest, 0xd54ec24992572df9ull); +} + TEST(TranscendentalTest, RsqF32SpecialCases) { EXPECT_EQ(rsq_f32(0.0f), std::numeric_limits::infinity()); EXPECT_TRUE(std::isnan(rsq_f32(-1.0f))); @@ -39,12 +66,74 @@ TEST(TranscendentalTest, RsqF32SpecialCases) { EXPECT_FLOAT_EQ(rsq_f32(4.0f), 0.5f); } +TEST(TranscendentalTest, RsqF32MatchesPhysicalRdna3AndRdna4) { + const uint32_t cases[][2] = { + {0x3f800000u, 0x3f800000u}, {0x3f800001u, 0x3f7fffffu}, {0x3f83ffffu, 0x3f7c1765u}, + {0x3f840000u, 0x3f7c1764u}, {0x3f8040c4u, 0x3f7fbf55u}, {0x3f80cab3u, 0x3f7f363du}, + {0x3fbfffffu, 0x3f5105ecu}, {0x3fc00000u, 0x3f5105ecu}, {0x3fffffffu, 0x3f3504f3u}, + {0x40000000u, 0x3f3504f3u}, {0x40000001u, 0x3f3504f2u}, {0x4003ffffu, 0x3f32416au}, + {0x40040000u, 0x3f32416au}, {0x403fffffu, 0x3f13cd3bu}, {0x40400000u, 0x3f13cd3au}, + {0x407fffffu, 0x3f000000u}, {0x00000000u, 0x7f800000u}, {0x80000000u, 0xff800000u}, + {0x00000001u, 0x7f800000u}, {0x80000001u, 0xff800000u}, {0xbf800000u, 0xffc00000u}, + {0xff800000u, 0xffc00000u}, {0x7f800000u, 0x00000000u}, {0x7fa12345u, 0x7fe12345u}, + {0xffa12345u, 0xffe12345u}, + }; + for (const auto &test : cases) + EXPECT_EQ(std::bit_cast(rsq_f32(std::bit_cast(test[0]))), test[1]) + << std::hex << test[0]; +} + +TEST(TranscendentalTest, RsqF32CompleteNormalizedHardwareDigest) { + // FNV-style hash of raw result words captured independently on gfx1100 and gfx1201. + // Cover every mantissa in both exponent parities without storing the 64 MiB capture. + uint64_t digest = 14695981039346656037ull; + for (uint32_t index = 0; index < (1u << 24); ++index) { + const float input = std::bit_cast(0x3f800000u + index); + digest = (digest ^ std::bit_cast(rsq_f32(input))) * 1099511628211ull; + } + EXPECT_EQ(digest, 0x010bc79eb6e48cafull); +} + TEST(TranscendentalTest, SqrtF32SpecialCases) { EXPECT_TRUE(std::isnan(sqrt_f32(-1.0f))); EXPECT_EQ(sqrt_f32(0.0f), 0.0f); EXPECT_EQ(sqrt_f32(std::numeric_limits::infinity()), std::numeric_limits::infinity()); EXPECT_FLOAT_EQ(sqrt_f32(4.0f), 2.0f); + EXPECT_EQ(std::bit_cast(sqrt_f32(std::bit_cast(0x7fa12345u), false)), + 0x7fa12345u); +} + +TEST(TranscendentalTest, SqrtF32MatchesPhysicalRdna3AndRdna4) { + // Raw words captured on gfx1100 and gfx1201. Rows exercise, in order: a Crysis shader + // input, round-half-even ties, square truncation, the dropped square-input bits, the + // linear rounding shift, the cubic-product bias, product normalization, sparse residuals, + // exponent edges, and special values. + const uint32_t cases[][2] = { + {0x3f7a2707u, 0x3f7d0f30u}, {0x3f800001u, 0x3f800000u}, {0x3f800005u, 0x3f800002u}, + {0x3f80005fu, 0x3f800030u}, {0x3f8015eeu, 0x3f800af7u}, {0x4005a8f7u, 0x3fb8fa71u}, + {0x3f823902u, 0x3f811b48u}, {0x4006e699u, 0x3fb9d5bau}, {0x3f8339a8u, 0x3f819a42u}, + {0x400659f1u, 0x3fb974bfu}, {0x00800001u, 0x20000000u}, {0x00ffffffu, 0x203504f3u}, + {0x01000001u, 0x203504f4u}, {0x3fffffffu, 0x3fb504f3u}, {0x407fffffu, 0x3fffffffu}, + {0x7e800001u, 0x5f000000u}, {0x7f7fffffu, 0x5f7fffffu}, {0x00000000u, 0x00000000u}, + {0x80000000u, 0x80000000u}, {0x00000001u, 0x00000000u}, {0x80000001u, 0x80000000u}, + {0xbf800000u, 0xffc00000u}, {0xff800000u, 0xffc00000u}, {0x7f800000u, 0x7f800000u}, + {0x7fa12345u, 0x7fe12345u}, {0xffa12345u, 0xffe12345u}, + }; + for (const auto &test : cases) + EXPECT_EQ(std::bit_cast(sqrt_f32(std::bit_cast(test[0]))), test[1]) + << std::hex << test[0]; +} + +TEST(TranscendentalTest, SqrtF32CompleteNormalizedHardwareDigest) { + // FNV-style hash of raw result words captured independently on gfx1100 and gfx1201. + // Cover every mantissa in both exponent parities without storing the 64 MiB capture. + uint64_t digest = 14695981039346656037ull; + for (uint32_t index = 0; index < (1u << 24); ++index) { + const float input = std::bit_cast(0x3f800000u + index); + digest = (digest ^ std::bit_cast(sqrt_f32(input))) * 1099511628211ull; + } + EXPECT_EQ(digest, 0x514d49d31f12176bull); } TEST(TranscendentalTest, LogF32SpecialCases) { @@ -56,6 +145,37 @@ TEST(TranscendentalTest, LogF32SpecialCases) { EXPECT_FLOAT_EQ(log_f32(4.0f), 2.0f); } +TEST(TranscendentalTest, LogF32MatchesPhysicalRdna3AndRdna4) { + // Raw witnesses distinguish near-one row boundaries, the reflected inner + // coordinate, intermediate product precision and exact-grid packing. + const uint32_t captured[][2] = { + {0x3f000000u, 0xbf800000u}, {0x3f7bffffu, 0xbcba1fa3u}, {0x3f7c0000u, 0xbcba1f74u}, + {0x3f7c0001u, 0xbcba1f46u}, {0x3f7dffecu, 0xbc396b23u}, {0x3f7dffffu, 0xbc39643au}, + {0x3f7e0000u, 0xbc3963ddu}, {0x3f7e0001u, 0xbc396380u}, {0x3f7ff5cbu, 0xb96ba0e4u}, + {0x3f7ffffdu, 0xb48a7faeu}, {0x3f7ffffeu, 0xb438aa3cu}, {0x3f7fffffu, 0xb3b8aa3cu}, + {0x3f800000u, 0x00000000u}, {0x3f800001u, 0x3438aa3bu}, {0x3f800005u, 0x3566d4c6u}, + {0x3f800006u, 0x358a7faau}, {0x3f800043u, 0x37415204u}, {0x3f800c30u, 0x3a0ca2fau}, + {0x3f80ffffu, 0x3c37f1ceu}, {0x3f810000u, 0x3c37f286u}, {0x3f810001u, 0x3c37f33du}, + {0x3f81ffffu, 0x3cb73c5au}, {0x3f820000u, 0x3cb73cb4u}, {0x3f820001u, 0x3cb73d0fu}, + {0x3f835d48u, 0x3d19508bu}, {0x3f83ffffu, 0x3d35d66fu}, {0x3f840000u, 0x3d35d69cu}, + {0x3f85ffffu, 0x3d8759afu}, {0x3f860000u, 0x3d8759c5u}, {0x3f860001u, 0x3d8759dbu}, + {0x3fffffffu, 0x3f7fffffu}, + }; + for (const auto &sample : captured) + EXPECT_EQ(std::bit_cast(log_f32(std::bit_cast(sample[0]))), sample[1]) + << std::hex << sample[0]; +} + +TEST(TranscendentalTest, LogF32CompleteMantissaHardwareDigest) { + // Independently captured on both cards over [0.5,2), including both + // near-one paths and every ordinary polynomial interval. + uint64_t digest = 14695981039346656037ull; + for (uint32_t bits = 0x3f000000u; bits < 0x40000000u; ++bits) + digest = + (digest ^ std::bit_cast(log_f32(std::bit_cast(bits)))) * 1099511628211ull; + EXPECT_EQ(digest, 0xc33a54efbae098beull); +} + TEST(TranscendentalTest, ExpF32SpecialCases) { EXPECT_EQ(exp_f32(-std::numeric_limits::infinity()), 0.0f); EXPECT_EQ(exp_f32(std::numeric_limits::infinity()), @@ -64,6 +184,39 @@ TEST(TranscendentalTest, ExpF32SpecialCases) { EXPECT_FLOAT_EQ(exp_f32(1.0f), 2.0f); } +TEST(TranscendentalTest, ExpF32MatchesPhysicalRdna3AndRdna4) { + // Captures cover staged polynomial rounding, signed reduction, the small + // input cutoff, and finite overflow/underflow boundaries. + const uint32_t captured[][2] = { + {0x337fffffu, 0x3f800000u}, {0x33800000u, 0x3f800000u}, {0x33800001u, 0x3f800000u}, + {0xb37fffffu, 0x3f800000u}, {0xb3800000u, 0x3f7fffffu}, {0xb3800001u, 0x3f7fffffu}, + {0x42ffffffu, 0x7f7fffa7u}, {0x43000000u, 0x7f800000u}, {0xc2fc0000u, 0x00800000u}, + {0xc2fc0001u, 0x00000000u}, {0x3f0567ecu, 0x3fb7b03du}, {0xc114ed44u, 0x3acecc1eu}, + {0x35500000u, 0x3f800004u}, {0x3e000090u, 0x3f8b95d0u}, {0x3e80001eu, 0x3f9837f6u}, + {0x3ec0001au, 0x3fa5fedcu}, {0x3f000013u, 0x3fb504fcu}, {0x3f20002au, 0x3fc56740u}, + {0x3f40006fu, 0x3fd7453eu}, {0x3f600022u, 0x3feac0dcu}, + }; + for (const auto &sample : captured) + EXPECT_EQ(std::bit_cast(exp_f32(std::bit_cast(sample[0]))), sample[1]) + << std::hex << sample[0]; +} + +TEST(TranscendentalTest, ExpF32CompleteFractionHardwareDigests) { + // Independent raw captures of 2^24 equally spaced inputs in each signed + // unit interval, covering all 32 polynomial intervals and their boundaries. + const uint64_t expected[] = {0x68959bb684f42e5full, 0x88b75684494db8d3ull}; + for (uint32_t sign = 0; sign < 2; ++sign) { + uint64_t digest = 14695981039346656037ull; + for (uint32_t index = 0; index < (1u << 24); ++index) { + const float positive = static_cast(index) * 0x1p-24f; + const uint32_t bits = std::bit_cast(positive) | (sign << 31); + digest = (digest ^ std::bit_cast(exp_f32(std::bit_cast(bits)))) * + 1099511628211ull; + } + EXPECT_EQ(digest, expected[sign]) << "sign=" << sign; + } +} + TEST(TranscendentalTest, SinCosF32SpecialCases) { // sin(2*pi*0) = 0, cos(2*pi*0) = 1 EXPECT_NEAR(sin_f32(0.0f), 0.0f, 1e-6f); @@ -76,6 +229,51 @@ TEST(TranscendentalTest, SinCosF32SpecialCases) { EXPECT_TRUE(std::isnan(cos_f32(std::numeric_limits::infinity()))); } +TEST(TranscendentalTest, SinCosF32CapturedRangeBoundaries) { + // Raw outputs captured independently on gfx1100 and gfx1201 with denormals + // preserved. Include small-input and polynomial boundaries, reflected + // intervals, exact quadrants, and finite inputs too large for integer casts. + const uint32_t cases[][3] = { + {0x00000000u, 0x00000000u, 0x3f800000u}, {0x80000000u, 0x80000000u, 0x3f800000u}, + {0x00000001u, 0x00000006u, 0x3f800000u}, {0x80000001u, 0x80000006u, 0x3f800000u}, + {0x007fffffu, 0x01c90fd3u, 0x3f800000u}, {0x00800000u, 0x01c90fd5u, 0x3f800000u}, + {0x39bfffffu, 0x3b16cbdfu, 0x3f7fffd3u}, {0x39c00000u, 0x3b16cbdfu, 0x3f7fffd3u}, + {0x39ffffffu, 0x3b490fbeu, 0x3f7fffb1u}, {0x3a000000u, 0x3b490fcbu, 0x3f7fffb1u}, + {0x3e000000u, 0x3f3504f4u, 0x3f3504f3u}, {0x3e7fffffu, 0x3f800000u, 0x33c90fd9u}, + {0x3e800000u, 0x3f800000u, 0x00000000u}, {0x3f000000u, 0x00000000u, 0xbf800000u}, + {0x3f800000u, 0x00000000u, 0x3f800000u}, {0x7f7fffffu, 0x00000000u, 0x3f800000u}, + {0xff7fffffu, 0x00000000u, 0x3f800000u}, {0x7f800000u, 0xffc00000u, 0xffc00000u}, + {0xff800000u, 0xffc00000u, 0xffc00000u}, + }; + for (const auto &test : cases) { + SCOPED_TRACE(test[0]); + EXPECT_EQ(std::bit_cast(sin_f32(std::bit_cast(test[0]))), test[1]); + EXPECT_EQ(std::bit_cast(cos_f32(std::bit_cast(test[0]))), test[2]); + } +} + +TEST(TranscendentalTest, SinCosF32CapturedArithmeticStages) { + // Captured on gfx1100 and gfx1201. These inputs distinguish intermediate + // coordinate/product rounding, normalized linear saturation and the + // reflected cosine cutoff from a single wide polynomial evaluation. + const uint32_t cases[][3] = { + {0x3aab9885u, 0x3c06c500u, 0x3f7ffdc8u}, {0x3b8419b1u, 0x3ccf7b0du, 0x3f7feaf9u}, + {0x3bcf188bu, 0x3d229c2du, 0x3f7fcc55u}, {0x3c98be2du, 0x3def6125u, 0x3f7e3ec8u}, + {0x3d2001fbu, 0x3e78d2d0u, 0x3f7853c7u}, {0x3d6002f3u, 0x3eac7f01u, 0x3f7108a3u}, + {0x3dc084adu, 0x3f0e9072u, 0x3f54a13bu}, {0x3df0010du, 0x3f2bebe5u, 0x3f3dae6bu}, + {0x3e7f2800u, 0x3f7fff1fu, 0x3ba9a533u}, {0x3e7ff600u, 0x3f7fffffu, 0x397b53d7u}, + {0x3e7fff08u, 0x3f800000u, 0x37c2c761u}, {0x3e7ffff1u, 0x3f800000u, 0x35bc7ee2u}, + {0x3e7ffa50u, 0x3f800000u, 0x390ef149u}, {0x3efffd28u, 0x390ef149u, 0xbf800000u}, + {0x3f3ffe94u, 0xbf800000u, 0xb90ef149u}, {0x3f7ffe94u, 0xb90ef149u, 0x3f800000u}, + }; + for (const auto &test : cases) + for (unsigned op = 0; op < 2; ++op) { + const float input = std::bit_cast(test[0]); + const uint32_t actual = std::bit_cast(op ? cos_f32(input) : sin_f32(input)); + EXPECT_EQ(actual, test[op + 1]) << std::hex << test[0]; + } +} + TEST(TranscendentalTest, RcpF64SpecialCases) { EXPECT_EQ(rcp_f64(0.0), std::numeric_limits::infinity()); EXPECT_EQ(rcp_f64(-0.0), -std::numeric_limits::infinity()); @@ -116,4 +314,65 @@ TEST(TranscendentalTest, SqrtF32Ulp) { } } +TEST(TranscendentalTest, HalfLogExpCompleteHardwareDigests) { + struct Captured { + uint32_t denorm_mode; + bool overflow; + uint64_t digest[4]; + }; + // FNV hashes of raw F16 hardware captures over all 65536 input encodings. + // Order: LOG signaling/quiet policy, then EXP signaling/quiet policy. + const Captured captured[] = { + {0u, + false, + {0x7ec126a2efb7cc29ull, 0x7b139780c1becc29ull, 0xc64da58f553026d8ull, + 0xa9b68cfe7b8372d8ull}}, + {0u, + true, + {0xf5160d9ac46aec29ull, 0x3999dfce791d6c29ull, 0x689733bef391b6d8ull, + 0xbdc5546b38d182d8ull}}, + {1u, + false, + {0x1a7dc0df37544143ull, 0xab561022e31ba143ull, 0xc64da58f553026d8ull, + 0xa9b68cfe7b8372d8ull}}, + {1u, + true, + {0x07029e1e69dff32bull, 0xab21f15c2191eb2bull, 0x689733bef391b6d8ull, + 0xbdc5546b38d182d8ull}}, + {2u, + false, + {0x7ec126a2efb7cc29ull, 0x7b139780c1becc29ull, 0x590d5c6485394589ull, + 0x7a49d8175530c589ull}}, + {2u, + true, + {0xf5160d9ac46aec29ull, 0x3999dfce791d6c29ull, 0x610235de125ed589ull, + 0x36d64dcb7d64d589ull}}, + {3u, + false, + {0x1a7dc0df37544143ull, 0xab561022e31ba143ull, 0x590d5c6485394589ull, + 0x7a49d8175530c589ull}}, + {3u, + true, + {0x07029e1e69dff32bull, 0xab21f15c2191eb2bull, 0x610235de125ed589ull, + 0x36d64dcb7d64d589ull}}, + }; + for (const auto &sample : captured) { + for (bool logarithm : {true, false}) { + for (bool quiet : {false, true}) { + uint64_t digest = 14695981039346656037ull; + for (uint32_t input = 0; input < 65536; ++input) { + const float value = util::f16_to_f32(static_cast(input)); + const float result = + logarithm ? log_exp_f16(value, sample.denorm_mode, sample.overflow, quiet) + : log_exp_f16(value, sample.denorm_mode, sample.overflow, quiet); + digest = (digest ^ util::f32_to_f16(result)) * 1099511628211ull; + } + EXPECT_EQ(digest, sample.digest[(logarithm ? 0 : 2) + quiet]) + << "log=" << logarithm << " quiet=" << quiet << " denorm=" << sample.denorm_mode + << " overflow=" << sample.overflow; + } + } + } +} + } // namespace diff --git a/emulation/rocjitsu/tests/util_simd_test.cpp b/emulation/rocjitsu/tests/util_simd_test.cpp index 06010f305440..f7bc26f26b0b 100644 --- a/emulation/rocjitsu/tests/util_simd_test.cpp +++ b/emulation/rocjitsu/tests/util_simd_test.cpp @@ -29,6 +29,12 @@ namespace { +struct RestoreEnvironment { + std::fenv_t saved; + RestoreEnvironment() { std::fegetenv(&saved); } + ~RestoreEnvironment() { std::fesetenv(&saved); } +}; + // Toolchain-guard sweep length. The bit-exactness guards below were validated // over a 250k-iteration full-range sweep, but running that on every CI build // adds significant wall time (5 sweeps x 250k x SIMD width). Default to a @@ -437,14 +443,8 @@ TEST(UtilSimd, ForceScalarOverrideDoesNotReachADlopenedModule) { ::dlclose(module); } -// Toolchain guard for the SIMD fast path of v_exp_f32 (stdx::exp2) and -// v_log_f32 (stdx::log2). Those VOP1 kernels take the SIMD path whenever -// `util::has_stdx_simd`, with no runtime value check — so the vector libm -// MUST be bit-identical to the scalar std::* used by the generated body, or -// the fast path silently returns wrong results. These tests sweep full-range -// random bit patterns (covering NaN/Inf/denormal/negative) and fail loudly if -// any lane diverges, blocking a divergent toolchain at CI time. If one fails, -// drop the corresponding row from SIMD_VOP1_UNARY rather than ship wrong math. +// Compare scalar and SIMD paths over full-range random encodings, including +// NaN payloads. LOG and EXP use the shared hardware mappings in both paths. template void expect_simd_bit_exact(ScalarFn scalar_fn, VectorFn vector_fn) { using V = util::native; @@ -468,14 +468,18 @@ void expect_simd_bit_exact(ScalarFn scalar_fn, VectorFn vector_fn) { TEST(UtilSimd, Exp2_VectorMatchesScalar_BitExact) { SKIP_IF_NO_SIMD(); - expect_simd_bit_exact([](float x) { return std::exp2(x); }, - [](util::native v) { return util::stdx::exp2(v); }); + for (bool quiet_snan : {false, true}) + expect_simd_bit_exact( + [quiet_snan](float x) { return util::amdgpu_exp_f32(x, quiet_snan); }, + [quiet_snan](util::native v) { return util::exp_f32_simd(v, quiet_snan); }); } TEST(UtilSimd, Log2_VectorMatchesScalar_BitExact) { SKIP_IF_NO_SIMD(); - expect_simd_bit_exact([](float x) { return std::log2(x); }, - [](util::native v) { return util::stdx::log2(v); }); + for (bool quiet_snan : {false, true}) + expect_simd_bit_exact( + [quiet_snan](float x) { return util::amdgpu_log_f32(x, quiet_snan); }, + [quiet_snan](util::native v) { return util::log_f32_simd(v, quiet_snan); }); } // Toolchain guard for the ternary VOP2 FMA/MAC/MAD SIMD fast path @@ -622,6 +626,30 @@ TEST(UtilSimd, FractF64_VectorMatchesScalar_BitExact) { expect_f64_unary_bit_exact([](double x) { return x - std::floor(x); }, [](util::native x) { return x - util::floor_simd(x); }); } +TEST(UtilSimd, RcpF32MatchesPhysicalHardwareBits) { + SKIP_IF_NO_SIMD(); + RestoreEnvironment restore; + const uint32_t cases[][2] = { + {0x3FC00000u, 0x3F2AAAAAu}, {0x3F802922u, 0x3F7FADD6u}, {0xBFB0333Cu, 0xBF39F868u}, + {0x00800000u, 0x7E800000u}, {0xFF7FFFFFu, 0x80000000u}, {0x80000001u, 0xFF800000u}, + {0x7F800000u, 0x00000000u}, {0xFFA12345u, 0xFFE12345u}, + }; + using V = util::native; + for (int mode : {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO}) { + ASSERT_EQ(std::fesetround(mode), 0); + for (unsigned start = 0; start < std::size(cases); ++start) { + std::array input_bits; + for (unsigned lane = 0; lane < V::size(); ++lane) + input_bits[lane] = cases[(start + lane) % std::size(cases)][0]; + const V input = util::load(input_bits.data()); + const V result = util::rcp_f32_simd(input); + for (unsigned lane = 0; lane < V::size(); ++lane) + EXPECT_EQ(std::bit_cast(float(result[lane])), + cases[(start + lane) % std::size(cases)][1]); + } + } +} + TEST(UtilSimd, RcpF64_VectorMatchesScalar_BitExact) { SKIP_IF_NO_SIMD(); expect_f64_unary_bit_exact([](double x) { return 1.0f / x; }, @@ -724,6 +752,30 @@ TEST(UtilSimd, F16ToF32_VectorMatchesScalar_Exhaustive) { } } +TEST(UtilSimd, F16RoundTripPreservesEveryEncoding) { + SKIP_IF_NO_SIMD(); + RestoreEnvironment environment; + std::feclearexcept(FE_ALL_EXCEPT); + using V = util::native; + constexpr std::size_t W = util::native_width_v; + for (uint32_t base = 0; base < 65536u; base += static_cast(W)) { + uint32_t input[W]; + for (std::size_t i = 0; i < W; ++i) + input[i] = base + static_cast(i); + const V encoded(input, util::stdx::element_aligned); + const auto promoted = util::f16_to_f32_simd(encoded); + const V nearest = util::f32_to_f16_simd(promoted); + const V truncated = util::f32_to_f16_rtz_simd(promoted); + for (std::size_t i = 0; i < W; ++i) { + ASSERT_EQ(nearest[i], input[i]); + ASSERT_EQ(truncated[i], input[i]); + } + } + // Discarded lanes in the subnormal paths must still have valid shift counts. + // Some SIMD backends implement variable shifts with floating conversions. + EXPECT_EQ(std::fetestexcept(FE_INVALID), 0); +} + TEST(UtilSimd, F32ToF16_VectorMatchesScalar_Sweep) { SKIP_IF_NO_SIMD(); using V = util::native; @@ -938,110 +990,26 @@ TEST(UtilSimd, IeeeMaximum3_F32_BitExact) { } } -// --- Cubemap face ops (v_cube{id,ma,sc,tc}_f32) ------------------------------ -// -// util::cube_{id,sc,tc}_f32_simd back the v_cube* gap ops; they must be -// bit-identical to the generated scalar bodies (transcribed verbatim below). -// Sweep every sign/magnitude/tie/zero/Inf/NaN triple. - -float scalar_cubeid(float x, float y, float z) { - float ax = std::fabs(x), ay = std::fabs(y), az = std::fabs(z); - if (ax >= ay && ax >= az) - return x >= 0 ? 0.0f : 1.0f; - if (ay >= ax && ay >= az) - return y >= 0 ? 2.0f : 3.0f; - return z >= 0 ? 4.0f : 5.0f; -} -float scalar_cubesc(float x, float y, float z) { - float ax = std::fabs(x), ay = std::fabs(y), az = std::fabs(z); - if (ax >= ay && ax >= az) - return x >= 0 ? z : -z; - if (ay >= ax && ay >= az) - return x; - return z >= 0 ? -x : x; -} -float scalar_cubetc(float x, float y, float z) { - float ax = std::fabs(x), ay = std::fabs(y), az = std::fabs(z); - if (ax >= ay && ax >= az) - return -y; - if (ay >= ax && ay >= az) - return y >= 0 ? -z : z; - return -y; -} -float scalar_cubema(float x, float y, float z) { - float ax = std::fabs(x), ay = std::fabs(y), az = std::fabs(z); - return 2.0f * std::fmax(ax, std::fmax(ay, az)); -} - -template void check_cube(SimdFn simd_fn, ScalarFn scalar_fn) { - using V = util::native; - constexpr std::size_t W = V::size(); - const std::array grid = {{-2.0f, -1.0f, -0.0f, 0.0f, 0.5f, 1.0f, 2.0f, - std::numeric_limits::infinity(), - std::numeric_limits::quiet_NaN()}}; - for (float zv : grid) - for (float yv : grid) - for (float xv : grid) { - alignas(V) float xb[W], yb[W], zb[W]; - for (std::size_t i = 0; i < W; ++i) { - xb[i] = xv; - yb[i] = yv; - zb[i] = zv; - } - V x(xb, util::stdx::vector_aligned), y(yb, util::stdx::vector_aligned), - z(zb, util::stdx::vector_aligned); - V r = simd_fn(x, y, z); - const float expect = scalar_fn(xv, yv, zv); - for (std::size_t i = 0; i < W; ++i) { - const float rv = r[i]; - EXPECT_EQ(bits_of(rv), bits_of(expect)) - << "x=" << xv << " y=" << yv << " z=" << zv << " lane=" << i; - } - } -} - -TEST(UtilSimd, CubeId_F32_BitExact) { - SKIP_IF_NO_SIMD(); - check_cube([](auto x, auto y, auto z) { return util::cube_id_f32_simd(x, y, z); }, scalar_cubeid); -} -TEST(UtilSimd, CubeSc_F32_BitExact) { - SKIP_IF_NO_SIMD(); - check_cube([](auto x, auto y, auto z) { return util::cube_sc_f32_simd(x, y, z); }, scalar_cubesc); -} -TEST(UtilSimd, CubeTc_F32_BitExact) { - SKIP_IF_NO_SIMD(); - check_cube([](auto x, auto y, auto z) { return util::cube_tc_f32_simd(x, y, z); }, scalar_cubetc); -} -TEST(UtilSimd, CubeMa_F32_BitExact) { - SKIP_IF_NO_SIMD(); - check_cube( - [](auto x, auto y, auto z) { - return 2.0f * util::stdx::fmax(util::stdx::abs(x), - util::stdx::fmax(util::stdx::abs(y), util::stdx::abs(z))); - }, - scalar_cubema); -} - // --- Normalized pack-convert lanes (v_cvt_pk[_]norm_{i16,u16}_*) ------------- int16_t scalar_cvt_pknorm_i16(float f) { if (std::isnan(f)) return 0; - float scaled = std::clamp(f * 32767.0f, -32768.0f, 32767.0f); - float lower = std::floor(scaled); - float fraction = scaled - lower; - if (fraction > 0.5f || (fraction == 0.5f && (static_cast(lower) & int32_t{1}) != 0)) - lower += 1.0f; + double scaled = std::clamp(static_cast(f) * 32767.0, -32767.0, 32767.0); + double lower = std::floor(scaled); + double fraction = scaled - lower; + if (fraction > 0.5 || (fraction == 0.5 && (static_cast(lower) & int32_t{1}) != 0)) + lower += 1.0; return static_cast(lower); } uint16_t scalar_cvt_pknorm_u16(float f) { if (std::isnan(f)) return 0; - float scaled = std::clamp(f * 65535.0f, 0.0f, 65535.0f); - float lower = std::floor(scaled); - float fraction = scaled - lower; - if (fraction > 0.5f || (fraction == 0.5f && (static_cast(lower) & int32_t{1}) != 0)) - lower += 1.0f; + double scaled = std::clamp(static_cast(f) * 65535.0, 0.0, 65535.0); + double lower = std::floor(scaled); + double fraction = scaled - lower; + if (fraction > 0.5 || (fraction == 0.5 && (static_cast(lower) & int32_t{1}) != 0)) + lower += 1.0; return static_cast(lower); } @@ -1085,11 +1053,7 @@ template void check_rndne_host_independence() { std::mt19937_64 rng(0x11939); for (unsigned i = 0; i < 2048; ++i) inputs.push_back(static_cast(rng())); - struct RestoreEnvironment { - std::fenv_t saved; - RestoreEnvironment() { std::fegetenv(&saved); } - ~RestoreEnvironment() { std::fesetenv(&saved); } - } restore_environment; + RestoreEnvironment restore_environment; ASSERT_EQ(std::fesetround(FE_TONEAREST), 0); std::vector expected; for (Bits bits : inputs) { @@ -1153,6 +1117,37 @@ TEST(UtilSimd, CvtPkNormI16_F32_BitExact) { } } +TEST(UtilSimd, PackedNormalizedThresholdsIgnoreHostRounding) { + SKIP_IF_NO_SIMD(); + using V = util::native; + RestoreEnvironment restore_environment; + // Generate the neighborhood in nearest mode, then evaluate under every mode. + ASSERT_EQ(std::fesetround(FE_TONEAREST), 0); + std::vector inputs; + for (int i = 0; i < 65536; ++i) { + for (float center : {float((i + 0.5) / 65535), float((i - 32768 + 0.5) / 32767)}) { + inputs.push_back(std::nextafter(center, -std::numeric_limits::infinity())); + inputs.push_back(center); + inputs.push_back(std::nextafter(center, std::numeric_limits::infinity())); + } + } + for (int mode : {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO}) { + ASSERT_EQ(std::fesetround(mode), 0); + SCOPED_TRACE(mode); + for (size_t base = 0; base < inputs.size(); base += V::size()) { + V f([&](auto lane) { return inputs[(base + lane) % inputs.size()]; }); + const auto signed_result = util::cvt_pknorm_i16_f32_simd(f); + const auto unsigned_result = util::cvt_pknorm_u16_f32_simd(f); + for (size_t lane = 0; lane < V::size(); ++lane) { + EXPECT_EQ(static_cast(signed_result[lane]), scalar_cvt_pknorm_i16(f[lane])) + << "input bits=" << std::bit_cast(float(f[lane])); + EXPECT_EQ(static_cast(unsigned_result[lane]), scalar_cvt_pknorm_u16(f[lane])) + << "input bits=" << std::bit_cast(float(f[lane])); + } + } + } +} + #undef SKIP_IF_NO_SIMD } // namespace diff --git a/emulation/rocjitsu/tests/valu_floating_policy_test.cpp b/emulation/rocjitsu/tests/valu_floating_policy_test.cpp index 01d195d4eb67..177c12a6b920 100644 --- a/emulation/rocjitsu/tests/valu_floating_policy_test.cpp +++ b/emulation/rocjitsu/tests/valu_floating_policy_test.cpp @@ -21,6 +21,7 @@ #include "rocjitsu/isa/arch/amdgpu/generated/rdna3_5/opcodes.h" #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/builders.h" #include "rocjitsu/isa/arch/amdgpu/generated/rdna4/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h" #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" @@ -32,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -45,19 +47,20 @@ namespace { class InstructionPolicyMachine { public: - explicit InstructionPolicyMachine(rj_code_arch_t arch) + explicit InstructionPolicyMachine(rj_code_arch_t arch, uint32_t vgprs = 256, + uint32_t wave_size = 0) : memory_("policy_memory"), cache_("policy_cache") { amdgpu::ComputeUnitCore::Config config{}; config.arch = arch; config.num_wf_slots = 1; config.sgprs_per_wf = 106; - config.vgprs_per_wf = 256; + config.vgprs_per_wf = vgprs; config.lds_size_kb = 64; cache_.set_backing_memory(&memory_); compute_unit_ = amdgpu::ComputeUnitCore::create("instruction_policy", config, &memory_, &cache_); decoder_ = Decoder::create(arch); - wave_ = compute_unit_->dispatch_wf(0, 0, 106, 256); + wave_ = compute_unit_->dispatch_wf(0, 0, 106, vgprs, wave_size); wave_->set_exec(1); base_ = wave_->vgpr_alloc().base; } @@ -70,6 +73,39 @@ class InstructionPolicyMachine { compute_unit_->write_vgpr(base_ + 1, 0, b); compute_unit_->write_vgpr(base_ + 2, 0, c); compute_unit_->write_vgpr(base_ + 6, 0, 0xfacebeef); + execute(words); + return compute_unit_->read_vgpr(base_ + 6, 0); + } + std::array run_lanes(const std::array &words, + const std::array &a, + const std::array &b, uint64_t active) { + wave_->set_mode_raw(0xf0); + wave_->set_exec(active); + for (uint32_t lane = 0; lane < wave_->wf_size(); ++lane) { + compute_unit_->write_vgpr(base_, lane, a[lane]); + compute_unit_->write_vgpr(base_ + 1, lane, b[lane]); + compute_unit_->write_vgpr(base_ + 6, lane, 0xa55a5a5a); + } + execute(words); + std::array result{}; + for (uint32_t lane = 0; lane < wave_->wf_size(); ++lane) + result[lane] = compute_unit_->read_vgpr(base_ + 6, lane); + return result; + } + template + uint32_t run_scalar(const std::array &words, uint32_t a, uint32_t b, + uint32_t accumulator, uint32_t mode) { + wave_->set_mode_raw(mode); + const auto base = wave_->sgpr_alloc().base; + compute_unit_->write_sgpr(base, a); + compute_unit_->write_sgpr(base + 1, b); + compute_unit_->write_sgpr(base + 6, accumulator); + execute(words); + return compute_unit_->read_sgpr(base + 6); + } + +private: + template void execute(const std::array &words) { std::array padded{}; std::ranges::copy(words, padded.begin()); DecodeResult decoded = decoder_->decode(padded.data()); @@ -77,10 +113,7 @@ class InstructionPolicyMachine { throw std::runtime_error("Instruction encoding rejected by decoder_"); std::unique_ptr instruction(std::move(decoded).value()); EXPECT_TRUE(compute_unit_->execute_instruction(instruction.get(), *wave_).succeeded()); - return compute_unit_->read_vgpr(base_ + 6, 0); } - -private: amdgpu::GpuMemory memory_; amdgpu::L2Cache cache_; std::unique_ptr compute_unit_; @@ -185,69 +218,345 @@ std::array packed_words(rj_code_arch_t arch, uint16_t op, uint8_t c std::array unary_words(rj_code_arch_t arch, size_t op) { switch (arch) { case ROCJITSU_CODE_ARCH_CDNA1: { - const std::array ops = {cdna1::kVRcpF32Vop1, cdna1::kVSqrtF32Vop1, - cdna1::kVLogF32Vop1, cdna1::kVExpF32Vop1, - cdna1::kVRsqF32Vop1}; + const std::array ops = { + cdna1::kVRcpF32Vop1, cdna1::kVSqrtF32Vop1, cdna1::kVLogF32Vop1, cdna1::kVExpF32Vop1, + cdna1::kVRsqF32Vop1, cdna1::kVRsqF16Vop1, cdna1::kVSqrtF16Vop1}; return cdna1::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_CDNA2: { - const std::array ops = {cdna2::kVRcpF32Vop1, cdna2::kVSqrtF32Vop1, - cdna2::kVLogF32Vop1, cdna2::kVExpF32Vop1, - cdna2::kVRsqF32Vop1}; + const std::array ops = { + cdna2::kVRcpF32Vop1, cdna2::kVSqrtF32Vop1, cdna2::kVLogF32Vop1, cdna2::kVExpF32Vop1, + cdna2::kVRsqF32Vop1, cdna2::kVRsqF16Vop1, cdna2::kVSqrtF16Vop1}; return cdna2::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_CDNA3: { - const std::array ops = {cdna3::kVRcpF32Vop1, cdna3::kVSqrtF32Vop1, - cdna3::kVLogF32Vop1, cdna3::kVExpF32Vop1, - cdna3::kVRsqF32Vop1}; + const std::array ops = { + cdna3::kVRcpF32Vop1, cdna3::kVSqrtF32Vop1, cdna3::kVLogF32Vop1, cdna3::kVExpF32Vop1, + cdna3::kVRsqF32Vop1, cdna3::kVRsqF16Vop1, cdna3::kVSqrtF16Vop1}; return cdna3::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_CDNA4: { - const std::array ops = {cdna4::kVRcpF32Vop1, cdna4::kVSqrtF32Vop1, - cdna4::kVLogF32Vop1, cdna4::kVExpF32Vop1, - cdna4::kVRsqF32Vop1}; + const std::array ops = { + cdna4::kVRcpF32Vop1, cdna4::kVSqrtF32Vop1, cdna4::kVLogF32Vop1, cdna4::kVExpF32Vop1, + cdna4::kVRsqF32Vop1, cdna4::kVRsqF16Vop1, cdna4::kVSqrtF16Vop1}; return cdna4::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_CDNA5: { - const std::array ops = {cdna5::kVRcpF32Vop1, cdna5::kVSqrtF32Vop1, - cdna5::kVLogF32Vop1, cdna5::kVExpF32Vop1, - cdna5::kVRsqF32Vop1}; + const std::array ops = { + cdna5::kVRcpF32Vop1, cdna5::kVSqrtF32Vop1, cdna5::kVLogF32Vop1, cdna5::kVExpF32Vop1, + cdna5::kVRsqF32Vop1, cdna5::kVRsqF16Vop1, cdna5::kVSqrtF16Vop1}; return cdna5::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_RDNA1: { - const std::array ops = {rdna1::kVRcpF32Vop1, rdna1::kVSqrtF32Vop1, - rdna1::kVLogF32Vop1, rdna1::kVExpF32Vop1, - rdna1::kVRsqF32Vop1}; + const std::array ops = { + rdna1::kVRcpF32Vop1, rdna1::kVSqrtF32Vop1, rdna1::kVLogF32Vop1, rdna1::kVExpF32Vop1, + rdna1::kVRsqF32Vop1, rdna1::kVRsqF16Vop1, rdna1::kVSqrtF16Vop1}; return rdna1::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_RDNA2: { - const std::array ops = {rdna2::kVRcpF32Vop1, rdna2::kVSqrtF32Vop1, - rdna2::kVLogF32Vop1, rdna2::kVExpF32Vop1, - rdna2::kVRsqF32Vop1}; + const std::array ops = { + rdna2::kVRcpF32Vop1, rdna2::kVSqrtF32Vop1, rdna2::kVLogF32Vop1, rdna2::kVExpF32Vop1, + rdna2::kVRsqF32Vop1, rdna2::kVRsqF16Vop1, rdna2::kVSqrtF16Vop1}; return rdna2::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_RDNA3: { - const std::array ops = {rdna3::kVRcpF32Vop1, rdna3::kVSqrtF32Vop1, - rdna3::kVLogF32Vop1, rdna3::kVExpF32Vop1, - rdna3::kVRsqF32Vop1}; + const std::array ops = { + rdna3::kVRcpF32Vop1, rdna3::kVSqrtF32Vop1, rdna3::kVLogF32Vop1, rdna3::kVExpF32Vop1, + rdna3::kVRsqF32Vop1, rdna3::kVRsqF16Vop1, rdna3::kVSqrtF16Vop1}; return rdna3::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_RDNA3_5: { - const std::array ops = {rdna3_5::kVRcpF32Vop1, rdna3_5::kVSqrtF32Vop1, - rdna3_5::kVLogF32Vop1, rdna3_5::kVExpF32Vop1, - rdna3_5::kVRsqF32Vop1}; + const std::array ops = { + rdna3_5::kVRcpF32Vop1, rdna3_5::kVSqrtF32Vop1, rdna3_5::kVLogF32Vop1, rdna3_5::kVExpF32Vop1, + rdna3_5::kVRsqF32Vop1, rdna3_5::kVRsqF16Vop1, rdna3_5::kVSqrtF16Vop1}; return rdna3_5::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } case ROCJITSU_CODE_ARCH_RDNA4: { - const std::array ops = {rdna4::kVRcpF32Vop1, rdna4::kVSqrtF32Vop1, - rdna4::kVLogF32Vop1, rdna4::kVExpF32Vop1, - rdna4::kVRsqF32Vop1}; + const std::array ops = { + rdna4::kVRcpF32Vop1, rdna4::kVSqrtF32Vop1, rdna4::kVLogF32Vop1, rdna4::kVExpF32Vop1, + rdna4::kVRsqF32Vop1, rdna4::kVRsqF16Vop1, rdna4::kVSqrtF16Vop1}; return rdna4::build_vop1(ops[op], {.src0 = 256, .vdst = 6}); } default: return {}; } } +std::array root_f16_vop3_words(rj_code_arch_t arch, bool square_root = false) { + switch (arch) { + case ROCJITSU_CODE_ARCH_CDNA1: + return cdna1::build_vop3(square_root ? cdna1::kVSqrtF16Vop3 : cdna1::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_CDNA2: + return cdna2::build_vop3(square_root ? cdna2::kVSqrtF16Vop3 : cdna2::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_CDNA3: + return cdna3::build_vop3(square_root ? cdna3::kVSqrtF16Vop3 : cdna3::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_CDNA4: + return cdna4::build_vop3(square_root ? cdna4::kVSqrtF16Vop3 : cdna4::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_CDNA5: + return cdna5::build_vop3(square_root ? cdna5::kVSqrtF16Vop3 : cdna5::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_RDNA1: + return rdna1::build_vop3(square_root ? rdna1::kVSqrtF16Vop3 : rdna1::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_RDNA2: + return rdna2::build_vop3(square_root ? rdna2::kVSqrtF16Vop3 : rdna2::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_RDNA3: + return rdna3::build_vop3(square_root ? rdna3::kVSqrtF16Vop3 : rdna3::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_RDNA3_5: + return rdna3_5::build_vop3(square_root ? rdna3_5::kVSqrtF16Vop3 : rdna3_5::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + case ROCJITSU_CODE_ARCH_RDNA4: + return rdna4::build_vop3(square_root ? rdna4::kVSqrtF16Vop3 : rdna4::kVRsqF16Vop3, + {.vdst = 6, .src0 = 256}); + default: + return {}; + } +} +std::array fma_f32_words(rj_code_arch_t arch, uint8_t absolute, uint8_t negate, + uint8_t omod, uint8_t clamp) { + switch (arch) { + case ROCJITSU_CODE_ARCH_CDNA1: + return cdna1::build_vop3(cdna1::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_CDNA2: + return cdna2::build_vop3(cdna2::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_CDNA3: + return cdna3::build_vop3(cdna3::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_CDNA4: + return cdna4::build_vop3(cdna4::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_CDNA5: + return cdna5::build_vop3(cdna5::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_RDNA1: + return rdna1::build_vop3(rdna1::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_RDNA2: + return rdna2::build_vop3(rdna2::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_RDNA3: + return rdna3::build_vop3(rdna3::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_RDNA3_5: + return rdna3_5::build_vop3(rdna3_5::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + case ROCJITSU_CODE_ARCH_RDNA4: + return rdna4::build_vop3(rdna4::kVFmaF32Vop3, {.vdst = 6, + .abs = absolute, + .clamp = clamp, + .src0 = 256, + .src1 = 257, + .src2 = 258, + .omod = omod, + .neg = negate}); + default: + throw std::runtime_error("Unsupported FMA test architecture"); + } +} + +TEST(ValuFloatingPolicy, FmaF32OutputScalingFlushesSubnormalIntermediate) { + // Physical gfx1201: scaling a rounded subnormal by two still yields +0. + // Earlier targets disable OMOD when output denormals or IEEE mode are set. + for (auto arch : kAllArchitectures) { + InstructionPolicyMachine machine(arch); + const auto words = fma_f32_words(arch, 0, 0, 1, 0); + for (uint32_t mode : {0xf0u, 0x2f0u}) + for (uint32_t addend : {0x007fffffu, 0x807fffffu}) { + const uint32_t expected = + arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5 ? 0u : addend; + witness("fma_omod_subnormal", arch, machine.run(words, 0, 0, addend, mode), expected); + } + } +} + +TEST(ValuFloatingPolicy, FmaF32FlushesTinySignificandBeforePacking) { + // Raw gfx1100/gfx1201 captures at the minimum-normal boundary. A tiny + // product changes directed rounding even far below the addend's low bit. + struct Case { + uint32_t a, b, c; + std::array preserved, flushed; + }; + const Case cases[] = { + {1, 1, 0x007fffff, {0x007fffff, 0x00800000, 0x007fffff, 0x007fffff}, {0, 0, 0, 0}}, + {1, 0x3f000000, 0x007fffff, {0x00800000, 0x00800000, 0x007fffff, 0x007fffff}, {0, 0, 0, 0}}, + {0x80000001, + 1, + 0x00800000, + {0x00800000, 0x00800000, 0x007fffff, 0x007fffff}, + {0x00800000, 0x00800000, 0, 0}}, + {1, + 1, + 0x00800000, + {0x00800000, 0x00800001, 0x00800000, 0x00800000}, + {0x00800000, 0x00800001, 0x00800000, 0x00800000}}, + {1, + 0x80000001, + 0x807fffff, + {0x807fffff, 0x807fffff, 0x80800000, 0x807fffff}, + {0x80000000, 0x80000000, 0x80000000, 0x80000000}}, + }; + for (auto arch : kAllArchitectures) { + InstructionPolicyMachine machine(arch); + for (uint32_t ieee : {0u, 1u}) + for (uint32_t denorm : {1u, 3u}) + for (uint32_t round = 0; round < 4; ++round) + for (uint8_t omod : {0, 1}) { + const bool scaling = + omod && (arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5 || + (!ieee && !(denorm & 2u))); + const auto words = fma_f32_words(arch, 0, 0, omod, 0); + for (const auto &test : cases) { + uint32_t expected = + scaling || !(denorm & 2u) ? test.flushed[round] : test.preserved[round]; + if (scaling) + expected = (expected & 0x7fffffffu) == 0 ? 0 : expected + 0x00800000u; + SCOPED_TRACE(::testing::Message() + << "arch=" << arch << " IEEE=" << ieee << " denorm=" << denorm + << " round=" << round << " omod=" << unsigned(omod)); + EXPECT_EQ( + machine.run(words, test.a, test.b, test.c, (ieee << 9) | (denorm << 4) | round), + expected); + } + } + } +} + +TEST(ValuFloatingPolicy, ScalarFmaF32UsesModeAndNanPolicy) { + // CDNA5 section 6.8 gives SALU floating point the same MODE controls as VALU. + InstructionPolicyMachine machine(ROCJITSU_CODE_ARCH_CDNA5); + struct Case { + uint32_t a, b, c, mode, expected; + }; + const Case cases[] = { + {0x3f800001, 0x3f800001, 0, 0x31, 0x3f800003}, + {0x3f800001, 0x3f800001, 0, 0x30, 0x3f800002}, + {1, 0x3f800000, 0, 0, 0}, + {1, 0x3f800000, 0, 0x30, 1}, + {1, 1, 0x007fffff, 0x11, 0}, + {0x7f801abc, 0xff803def, 0x7fc01234, 0x30, 0x7fc01abc}, + }; + for (uint16_t opcode : {cdna5::kSFmacF32Sop2, cdna5::kSFmaakF32Sop2, cdna5::kSFmamkF32Sop2}) { + for (const auto &test : cases) { + const auto first = cdna5::build_sop2(opcode, {.ssrc0 = 0, .ssrc1 = 1, .sdst = 6}); + const bool multiply_literal = opcode == cdna5::kSFmamkF32Sop2; + const std::array words{first[0], multiply_literal ? test.b : test.c}; + EXPECT_EQ( + machine.run_scalar(words, test.a, multiply_literal ? test.c : test.b, test.c, test.mode), + test.expected) + << opcode; + } + } +} + +TEST(ValuFloatingPolicy, FmaF32MatchesPhysicalNanBits) { + // Captured V_FMA_F32 outputs on gfx1100/gfx1201, including both IEEE modes. + // Earlier RDNA/CDNA share the MODE.IEEE rule; CDNA5, like RDNA4, always quiets. + struct Case { + uint32_t a, b, c, flushed, preserved, modified; + }; + const Case cases[] = { + {0x7f801abc, 0xff803def, 0x7fc01234, 0x7f801abc, 0x7f801abc, 0xff801abc}, + {0xff803def, 0x7f801abc, 0x7fc01234, 0xff803def, 0xff803def, 0xff803def}, + {0x7fc01234, 0xff803def, 0x7f801abc, 0x7fc01234, 0x7fc01234, 0xffc01234}, + {0x3f800000, 0xff803def, 0x7fc01234, 0xff803def, 0xff803def, 0x7f803def}, + {0x3f800000, 0x3f800000, 0x7f801abc, 0x7f801abc, 0x7f801abc, 0x7f801abc}, + {0x3f800000, 0x3f800000, 0xff803def, 0xff803def, 0xff803def, 0x7f803def}, + {0x00000000, 0x7f800000, 0xff803def, 0xffc00000, 0xffc00000, 0xffc00000}, + {0x7f800000, 0x80000000, 0x7f801abc, 0xffc00000, 0xffc00000, 0xffc00000}, + {0x00000001, 0x7f800000, 0xff803def, 0xffc00000, 0xff803def, 0x7f803def}, + {0x807fffff, 0xff800000, 0x7f801abc, 0xffc00000, 0x7f801abc, 0x7f801abc}, + {0x7f800000, 0x3f800000, 0xff800000, 0xffc00000, 0xffc00000, 0x7f800000}, + }; + for (auto arch : kAllArchitectures) { + InstructionPolicyMachine machine(arch); + for (uint32_t ieee : {0u, 1u}) + for (uint32_t denorm : {0u, 1u, 2u, 3u}) + for (uint32_t round : {0u, 1u, 2u, 3u}) + for (unsigned variant = 0; variant < 4; ++variant) { + const bool modified = variant == 1, clamped = variant == 3; + const auto words = fma_f32_words(arch, modified ? 5 : 0, modified ? 3 : 0, + variant == 2 ? 1 : 0, clamped); + for (const auto &test : cases) { + uint32_t expected = (denorm & 1u) ? test.preserved : test.flushed; + if (modified && ((denorm & 1u) || test.flushed == test.preserved)) + expected = test.modified; + if ((expected & 0x7fffffffu) > 0x7f800000u && + (ieee || arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5)) + expected |= 0x00400000u; + if (clamped && (arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5)) + expected = 0; + SCOPED_TRACE(::testing::Message() + << "arch=" << arch << " IEEE=" << ieee << " denorm=" << denorm + << " round=" << round << " variant=" << variant); + EXPECT_EQ( + machine.run(words, test.a, test.b, test.c, (ieee << 9) | (denorm << 4) | round), + expected); + } + } + } +} + TEST(ValuFloatingPolicy, PackedMinimumPropagatesNaN) { for (rj_code_arch_t arch : {ROCJITSU_CODE_ARCH_CDNA5, ROCJITSU_CODE_ARCH_RDNA4}) { InstructionPolicyMachine machine(arch); @@ -370,6 +679,66 @@ TEST(ValuFloatingPolicy, F32UnaryDenormControls) { } } } +TEST(ValuFloatingPolicy, ReciprocalSquareRootUsesSharedMappingAcrossProfiles) { + // The shared one-ULP/FTZ contract applies to all profiles. Raw hardware + // qualification for these finite approximation witnesses covers RDNA3/4. + for (rj_code_arch_t arch : kAllArchitectures) { + InstructionPolicyMachine machine(arch); + const auto words = unary_words(arch, 4); + const bool quiet = arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + for (const auto &test : + {std::pair{0x3f80cab3u, 0x3f7f363du}, std::pair{0x80000001u, 0xff800000u}, + std::pair{0xbf800000u, 0xffc00000u}, + std::pair{0x7fa12345u, quiet ? 0x7fe12345u : 0x7fa12345u}}) + witness("rsq_f32_shared", arch, machine.run(words, test.first, 0, 0, 0xf0), test.second); + } +} + +TEST(ValuFloatingPolicy, LogarithmUsesSharedMappingAcrossProfiles) { + // All profiles use the shared approximation; these raw finite witnesses + // have hardware qualification on RDNA3/4. LOG ignores rounding/denormal MODE. + const uint32_t captured[][2] = { + {0x3f7c0000u, 0xbcba1f74u}, {0x3f7e0001u, 0xbc396380u}, {0x3f800005u, 0x3566d4c6u}, + {0x3f810000u, 0x3c37f286u}, {0x3f820001u, 0x3cb73d0fu}, {0x3f835d48u, 0x3d19508bu}, + {0x3f860001u, 0x3d8759dbu}, + }; + for (rj_code_arch_t arch : kAllArchitectures) { + InstructionPolicyMachine machine(arch); + const auto words = unary_words(arch, 2); + for (uint32_t ieee : {0u, 1u}) + for (uint32_t mode = 0; mode < 16; ++mode) { + const uint32_t raw_mode = 0xc0u | (mode & 3u) | ((mode >> 2) << 4) | (ieee << 9); + for (const auto &sample : captured) + witness("log_f32_shared", arch, machine.run(words, sample[0], 0, 0, raw_mode), sample[1]); + } + } +} + +TEST(ValuFloatingPolicy, HalfReciprocalSquareRootModesAndNaNPayloads) { + // All profiles support F16 input denormals; the shared mapping meets the + // stricter 0.51-ULP bound of older RDNA/CDNA profiles. + // Finite witness bits are physically qualified on RDNA3/4. + for (rj_code_arch_t arch : kAllArchitectures) { + InstructionPolicyMachine machine(arch); + const bool quiet = arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + for (uint32_t mode = 0; mode < 16; ++mode) { + const uint32_t raw_mode = 0x30u | ((mode & 3u) << 2) | ((mode >> 2) << 6); + for (const auto &test : + {std::pair{0x0401u, 0x57ffu}, std::pair{0x7c02u, quiet ? 0x7e02u : 0x7c02u}, + std::pair{0xfc02u, quiet ? 0xfe02u : 0xfc02u}, std::pair{0xbc00u, 0xfe00u}, + std::pair{0x0001u, (mode & 4u) ? 0x6c00u : 0x7c00u}, + std::pair{0x8001u, (mode & 4u) ? 0xfe00u : 0xfc00u}}) { + witness("rsq_f16_vop1", arch, + machine.run(unary_words(arch, 5), test.first, 0, 0, raw_mode) & 0xffffu, + test.second); + witness("rsq_f16_vop3", arch, + machine.run(root_f16_vop3_words(arch), test.first, 0, 0, raw_mode) & 0xffffu, + test.second); + } + } + } +} + TEST(ValuFloatingPolicy, F32UnaryQuietsSignalingNaNs) { for (rj_code_arch_t arch : kAllArchitectures) { InstructionPolicyMachine machine(arch); @@ -402,6 +771,43 @@ TEST(ValuFloatingPolicy, PackedBf16Clamp) { opcode == cdna5::kVPkMaxNumBf16Vop3p ? 0x3f803f80 : 0x00003f80); } } + +TEST(ValuFloatingPolicy, SquareRootModesAndNaNPayloads) { + // Finite words and MODE behavior captured on gfx1100/gfx1201. The mapping + // also meets the one-ULP F32 and 0.51-ULP F16 bounds of older profiles. + for (rj_code_arch_t arch : kAllArchitectures) { + InstructionPolicyMachine machine(arch); + for (uint32_t ieee : {0u, 1u}) { + const bool quiet = + ieee || arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + for (uint32_t mode = 0; mode < 16; ++mode) { + const uint32_t f32_mode = 0xc0u | (mode & 3u) | ((mode >> 2) << 4) | (ieee << 9); + for (const auto &sample : + {std::pair{0x3f7a2707u, 0x3f7d0f30u}, std::pair{0x3f8339a8u, 0x3f819a42u}, + std::pair{0xbf800000u, 0xffc00000u}, + std::pair{0x7f800001u, quiet ? 0x7fc00001u : 0x7f800001u}}) + witness("sqrt_f32", arch, machine.run(unary_words(arch, 1), sample.first, 0, 0, f32_mode), + sample.second); + const uint32_t f16_mode = 0x30u | ((mode & 3u) << 2) | ((mode >> 2) << 6) | (ieee << 9); + for (const auto &sample : + {std::pair{0x4200u, 0x3eeeu}, std::pair{0x0401u, 0x2000u}, + std::pair{0x0001u, (mode & 4u) ? 0x0c00u : 0x0000u}, + std::pair{0x8001u, (mode & 4u) ? 0xfe00u : 0x8000u}, std::pair{0xbc00u, 0xfe00u}, + std::pair{0x7c02u, quiet ? 0x7e02u : 0x7c02u}, + std::pair{0xfc02u, quiet ? 0xfe02u : 0xfc02u}}) { + witness("sqrt_f16_vop1", arch, + machine.run(unary_words(arch, 6), sample.first, 0, 0, f16_mode) & 0xffffu, + sample.second); + witness("sqrt_f16_vop3", arch, + machine.run(root_f16_vop3_words(arch, true), sample.first, 0, 0, f16_mode) & + 0xffffu, + sample.second); + } + } + } + } +} + TEST(ValuFloatingPolicy, Bf16UnaryQuietsSignalingNaNs) { InstructionPolicyMachine machine(ROCJITSU_CODE_ARCH_CDNA5); for (uint16_t opcode : {cdna5::kVRcpBf16Vop1, cdna5::kVSqrtBf16Vop1, cdna5::kVRsqBf16Vop1, @@ -577,4 +983,104 @@ TEST(ValuFloatingPolicy, Rdna4F32DotInlineSourcesMatchHardware) { opcode == rdna4::kVDot2F32F16Vop3p ? 0x3f800000u : 0x40000000u); } } +TEST(ValuFloatingPolicy, DppFloatingSourceAbsoluteAndNegate) { + for (const auto arch : kAllArchitectures) { + InstructionPolicyMachine machine(arch); + std::array words{}; +#define ADD_DPP(ARCH, ENUM) \ + case ENUM: \ + words[0] = \ + ARCH::build_vop2(ARCH::kVAddF32Vop2, {.src0 = amdgpu::SRC_DPP, .vsrc1 = 1, .vdst = 6})[0]; \ + break; + switch (arch) { + ADD_DPP(cdna1, ROCJITSU_CODE_ARCH_CDNA1) + ADD_DPP(cdna2, ROCJITSU_CODE_ARCH_CDNA2) + ADD_DPP(cdna3, ROCJITSU_CODE_ARCH_CDNA3) + ADD_DPP(cdna4, ROCJITSU_CODE_ARCH_CDNA4) + ADD_DPP(cdna5, ROCJITSU_CODE_ARCH_CDNA5) + ADD_DPP(rdna1, ROCJITSU_CODE_ARCH_RDNA1) + ADD_DPP(rdna2, ROCJITSU_CODE_ARCH_RDNA2) + ADD_DPP(rdna3, ROCJITSU_CODE_ARCH_RDNA3) + ADD_DPP(rdna3_5, ROCJITSU_CODE_ARCH_RDNA3_5) + ADD_DPP(rdna4, ROCJITSU_CODE_ARCH_RDNA4) + default: + FAIL(); + } +#undef ADD_DPP + for (uint32_t modifiers = 0; modifiers < 16; ++modifiers) { + SCOPED_TRACE(testing::Message() << "arch=" << unsigned(arch) << " modifiers=" << modifiers); + // Identity quad permutation. ABS precedes NEG for each floating source. + words[1] = 0xff000000u | (0xe4u << 8) | (modifiers << 20); + float a = modifiers & 2 ? 2.0f : -2.0f; + float b = modifiers & 8 ? 3.0f : -3.0f; + if (modifiers & 1) + a = -a; + if (modifiers & 4) + b = -b; + EXPECT_EQ(machine.run(words, 0xc0000000, 0xc0400000), std::bit_cast(a + b)); + } + } +} + +// The full-active wave32 matrix was captured bit-for-bit on gfx1100 and gfx1201. +// Every sum is exactly representable, so host rounding does not define the oracle. +TEST(ValuFloatingPolicy, DppTrue16SelectsHalvesBeforeSourceModifiers) { + std::array a{}, b{}; + for (uint32_t lane = 0; lane < 64; ++lane) { + const uint32_t step = (lane & 3) * 0x100; + a[lane] = ((0xc800 + step) << 16) | (0x4000 + step); + b[lane] = ((0x4600 + step) << 16) | (0xc000 + step); + } + for (auto arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, ROCJITSU_CODE_ARCH_RDNA4, + ROCJITSU_CODE_ARCH_CDNA5}) { + for (uint32_t wave_size : {32u, 64u}) { + if (arch == ROCJITSU_CODE_ARCH_CDNA5 && wave_size == 64) + continue; + for (uint32_t vgprs : {8u, 256u}) { + InstructionPolicyMachine machine(arch, vgprs, wave_size); + for (uint64_t active : {~uint64_t{0}, uint64_t{0x5555555555555555}}) { + for (uint32_t swap : {0u, 1u}) { + for (uint32_t halves = 0; halves < 8; ++halves) { + for (uint32_t modifiers = 0; modifiers < 16; ++modifiers) { + SCOPED_TRACE(testing::Message() + << "arch=" << unsigned(arch) << " vgprs=" << vgprs << " wave_size=" + << wave_size << " active=" << active << " swap=" << swap + << " halves=" << halves << " modifiers=" << modifiers); + const uint32_t a_shift = (halves & 1) ? 16 : 0; + const uint32_t b_shift = (halves & 2) ? 16 : 0; + const uint32_t dst_shift = (halves & 4) ? 16 : 0; + // V_ADD_F16 e32 and DPP16 have identical encodings on these targets. + const uint32_t dst = 6 + (dst_shift ? 128 : 0); + const uint32_t src1 = 1 + (b_shift ? 128 : 0); + const std::array words = {0x640000fau | (dst << 17) | (src1 << 9), + 0xff040000u | (modifiers << 20) | + ((swap ? 0xb1u : 0xe4u) << 8) | + (a_shift ? 128u : 0u)}; + const auto got = machine.run_lanes(words, a, b, active); + for (uint32_t lane = 0; lane < wave_size; ++lane) { + uint32_t expected = 0xa55a5a5a; + if (active & (uint64_t{1} << lane)) { + float av = util::f16_to_f32(a[lane ^ swap] >> a_shift); + float bv = util::f16_to_f32(b[lane] >> b_shift); + if (modifiers & 2) + av = std::abs(av); + if (modifiers & 8) + bv = std::abs(bv); + if (modifiers & 1) + av = -av; + if (modifiers & 4) + bv = -bv; + expected = (expected & ~(0xffffu << dst_shift)) | + (uint32_t{util::f32_to_f16(av + bv)} << dst_shift); + } + EXPECT_EQ(got[lane], expected) << "lane=" << lane; + } + } + } + } + } + } + } + } +} } // namespace diff --git a/emulation/rocjitsu/tests/valu_fp_mode_test.cpp b/emulation/rocjitsu/tests/valu_fp_mode_test.cpp index 1ce465d3ad1a..9ca22135d91d 100644 --- a/emulation/rocjitsu/tests/valu_fp_mode_test.cpp +++ b/emulation/rocjitsu/tests/valu_fp_mode_test.cpp @@ -1,6 +1,8 @@ // Copyright (c) 2026 Advanced Micro Devices, Inc. // SPDX-License-Identifier: MIT +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/opcodes.h" #include "rocjitsu/isa/arch/amdgpu/shared/fp_mode.h" #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" @@ -8,6 +10,7 @@ #include "rocjitsu/vm/amdgpu/gpu_memory.h" #include "rocjitsu/vm/amdgpu/l2_cache.h" #include "rocjitsu/vm/amdgpu/wavefront.h" +#include "util/simd_test_hooks.h" #include #include @@ -928,40 +931,312 @@ std::vector ldexp_f16_mode_cases() { } std::vector dx9_fma_cases() { + std::vector cases; + for (rj_code_arch_t arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) { + const bool has_accumulator_form = + arch == ROCJITSU_CODE_ARCH_RDNA3 || arch == ROCJITSU_CODE_ARCH_RDNA3_5; + for (uint32_t form = 0; form < (has_accumulator_form ? 3u : 1u); ++form) + for (uint32_t denorm = 0; denorm < 4; ++denorm) + for (uint32_t rounding = 0; rounding < 4; ++rounding) + for (bool ieee : {false, true}) { + const std::string prefix = "Arch" + std::to_string(arch) + "Form" + + std::to_string(form) + "Denorm" + std::to_string(denorm) + + "Round" + std::to_string(rounding) + "Ieee" + + std::to_string(ieee); + // v_fma_dx9_zero_f32 v6,v0,v1,v2 and v_fmac_dx9_zero_f32 v6,v0,v1, + // assembled with llvm-mc. RDNA4/CDNA5 expose only the three-source form. + const std::array words = + form == 0 ? std::array{0xd6090006u, 0x040a0300u, 0u} + : form == 1 ? std::array{0x0c0c0300u, 0u, 0u} + : std::array{0xd5060006u, 0x00020300u, 0u}; + const auto add = [&](const std::string &name, uint32_t a, uint32_t b, uint32_t c, + uint32_t expected, uint32_t omod = 0) { + auto encoding = words; + encoding[1] |= omod << 27; + cases.push_back({prefix + name, + arch, + encoding, + {{0, a}, {1, b}, {form == 0 ? 2u : 6u, c}}, + {{6, expected}}, + (uint32_t(ieee) << 9) | (denorm << 4) | rounding, + FE_TONEAREST}); + }; + add("ZeroInf", 0u, 0x7f800000u, 0x3f800000u, 0x3f800000u); + add("NanZero", 0x7fc01234u, 0x80000000u, 0x3f800000u, 0x3f800000u); + add("InputFlush", 1u, 0x4b000000u, 0u, 0u); + add("OutputFlush", 0x00800000u, 0x3f000000u, 0u, 0u); + const uint32_t zero_sum = rounding == 2 ? 0x80000000u : 0u; + add("AccumulatorFlush", 0u, 0x7f800000u, 0x80000001u, zero_sum); + add("NegativeZeroAccumulator", 0u, 0x7f800000u, 0x80000000u, zero_sum); + add("NegativeZeroProduct", 0x80000000u, 0x3f800000u, 0x80000000u, zero_sum); + add("NormalControl", 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x40000000u); + // Captured gfx1100/gfx1201 boundary and payload witnesses. + add("TinyBeforePacking", 0x80800000u, 0x33800000u, 0x00800000u, 0u); + add("NormalSignificand", 0x80800000u, 0x33000000u, 0x00800000u, + rounding < 2 ? 0x00800000u : 0u); + add("FirstNan", 0x7fc00111u, 0x7fc00222u, 0x7fc00333u, 0x7fc00111u); + const uint32_t nan = (ieee || !has_accumulator_form) ? 0x7fc00012u : 0x7f800012u; + add("SignalingFirst", 0x7f800012u, 0x3f800000u, 0x3f800000u, nan); + add("SignalingSecond", 0x3f800000u, 0x7f800012u, 0x3f800000u, nan); + add("SignalingAccumulator", 0x3f800000u, 0x3f800000u, 0x7f800012u, nan); + add("ZeroProductSignalingAccumulator", 0u, 0x7f800000u, 0x7f800012u, nan); + if (form != 1) + for (uint32_t omod = 1; omod < 4; ++omod) { + // Mandatory output flushing keeps OMOD enabled regardless of + // denormal MODE. RDNA3/3.5 still disable it in IEEE mode. + const bool enabled = !has_accumulator_form || !ieee; + const uint32_t scaled = !enabled ? 0x40000000u + : omod == 1 ? 0x40800000u + : omod == 2 ? 0x41000000u + : 0x3f800000u; + add("Omod" + std::to_string(omod), 0x3f800000u, 0x3f800000u, 0x3f800000u, scaled, + omod); + add("OmodZero" + std::to_string(omod), 0u, 0x7f800000u, 0x80000000u, + enabled ? 0u : zero_sum, omod); + if (omod == 3) + add("OmodUnderflow", 0x80800000u, 0x3f800000u, 0u, + enabled ? 0x80000000u : 0x80800000u, omod); + } + } + } + return cases; +} + +std::vector binary_f32_policy_cases() { std::vector cases; for (rj_code_arch_t arch : - {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) { - for (uint32_t denorm = 0; denorm < 4; ++denorm) { - const std::string prefix = "Arch" + std::to_string(arch) + "Denorm" + std::to_string(denorm); - const auto add = [&](const char *name, uint32_t a, uint32_t b, uint32_t c, - uint32_t expected) { - // v_fma_dx9_zero_f32 v6, v0, v1, v2, assembled with llvm-mc. - cases.push_back({prefix + name, - arch, - {0xd6090006u, 0x040a0300u}, - {{0, a}, {1, b}, {2, c}}, - {{6, expected}}, - denorm << 4, - FE_TONEAREST}); - }; - add("ZeroInf", 0u, 0x7f800000u, 0x3f800000u, 0x3f800000u); - add("NanZero", 0x7fc01234u, 0x80000000u, 0x3f800000u, 0x3f800000u); - add("InputFlush", 1u, 0x4b000000u, 0u, 0u); - add("OutputFlush", 0x00800000u, 0x3f000000u, 0u, 0u); - add("AccumulatorFlush", 0u, 0x7f800000u, 0x80000001u, 0x80000000u); - add("NegativeZeroAccumulator", 0u, 0x7f800000u, 0x80000000u, 0x80000000u); - add("NormalControl", 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x40000000u); - } + {ROCJITSU_CODE_ARCH_CDNA1, ROCJITSU_CODE_ARCH_CDNA2, ROCJITSU_CODE_ARCH_CDNA3, + ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA5, ROCJITSU_CODE_ARCH_RDNA1, + ROCJITSU_CODE_ARCH_RDNA2, ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4}) { + const bool rdna_encoding = arch >= ROCJITSU_CODE_ARCH_RDNA1 || arch == ROCJITSU_CODE_ARCH_CDNA5; + const bool always_quiets = arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + for (bool multiply : {false, true}) + for (bool e64 : {false, true}) + for (uint32_t denorm = 0; denorm < 4; ++denorm) + for (uint32_t rounding = 0; rounding < 4; ++rounding) + for (bool ieee : {false, true}) { + // Assembled for each target with llvm-mc. Expected raw bits were + // captured on gfx1100/gfx1201; the shared MODE policy also applies + // to earlier RDNA/CDNA arithmetic and CDNA5's always-quiet policy. + const uint32_t first = e64 ? (rdna_encoding ? (multiply ? 0xd5080006u : 0xd5030006u) + : (multiply ? 0xd1050006u : 0xd1010006u)) + : (rdna_encoding ? (multiply ? 0x100c0300u : 0x060c0300u) + : (multiply ? 0x0a0c0300u : 0x020c0300u)); + const std::array words{ + first, e64 ? (rdna_encoding ? 0x02020300u : 0x00020300u) : 0u, 0u}; + const std::string prefix = "Arch" + std::to_string(arch) + + (multiply ? "Mul" : "Add") + (e64 ? "E64" : "E32") + + "Denorm" + std::to_string(denorm) + "Round" + + std::to_string(rounding) + "Ieee" + std::to_string(ieee); + auto add = [&](const char *name, uint32_t a, uint32_t b, uint32_t expected) { + cases.push_back({prefix + name, + arch, + words, + {{0, a}, {1, b}}, + {{6, expected}}, + (uint32_t(ieee) << 9) | (denorm << 4) | rounding, + FE_UPWARD}); + }; + const uint32_t nan = ieee || always_quiets ? 0x7fc01abcu : 0x7f801abcu; + add("FirstNan", 0x7f801abcu, 0xffc00222u, nan); + add("SecondNan", 0x3f800000u, 0x7f801abcu, nan); + add("ZeroSign", 0x80000000u, multiply ? 0x3f800000u : 0x80000000u, 0x80000000u); + add("OppositeZeros", 0u, 0x80000000u, multiply || rounding == 2 ? 0x80000000u : 0u); + if (multiply) { + const uint32_t positive = !(denorm & 2u) ? 0u + : rounding < 2 ? 0x00800000u + : 0x007fffffu; + const uint32_t negative = !(denorm & 2u) ? 0x80000000u + : rounding == 0 || rounding == 2 ? 0x80800000u + : 0x807fffffu; + add("PositiveTinySignificand", 0x00800000u, 0x3f7fffffu, positive); + add("NegativeTinySignificand", 0x80800000u, 0x3f7fffffu, negative); + } + } } return cases; } +std::vector omod_underflow_cases() { + std::vector cases; + for (rj_code_arch_t arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) + for (uint32_t op = 0; op < 3; ++op) + for (uint32_t denorm = 0; denorm < 4; ++denorm) + for (uint32_t rounding = 0; rounding < 4; ++rounding) + for (bool ieee : {false, true}) { + // FMA, FMAC and ADD with div:2. Physical gfx1100/gfx1201 preserve + // the sign when scaling underflows a negative normal result. + const std::array words = op == 0 ? std::array{0xd6130006u, 0x1c0a0300u, 0u} + : op == 1 + ? std::array{0xd52b0006u, 0x18020300u, 0u} + : std::array{0xd5030006u, 0x18020300u, 0u}; + const bool enabled = arch == ROCJITSU_CODE_ARCH_RDNA4 || + arch == ROCJITSU_CODE_ARCH_CDNA5 || (!ieee && !(denorm & 2u)); + cases.push_back({"Arch" + std::to_string(arch) + "Op" + std::to_string(op) + "Denorm" + + std::to_string(denorm) + "Round" + std::to_string(rounding) + + "Ieee" + std::to_string(ieee), + arch, + words, + {{0, 0x80800000u}, {1, op == 2 ? 0u : 0x3f800000u}, {2, 0u}, {6, 0u}}, + {{6, enabled ? 0x80000000u : 0x80800000u}}, + (uint32_t(ieee) << 9) | (denorm << 4) | rounding, + FE_TONEAREST}); + auto boundary = cases.back(); + boundary.name += "BeforePacking"; + boundary.sources = {{0, 0x807fffffu}, + {1, op == 2 ? 0x80800000u : 0x3f800000u}, + {2, 0x80800000u}, + {6, 0x80800000u}}; + boundary.expected = {{6, enabled ? 0x80000000u + : (denorm & 1u) ? 0x80ffffffu + : 0x80800000u}}; + cases.push_back(std::move(boundary)); + } + return cases; +} + +std::vector f16_fma_nan_cases() { + std::vector cases; + for (rj_code_arch_t arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) + for (bool packed : {false, true}) + for (bool ieee : {false, true}) + for (bool clamp : {false, true}) + for (uint32_t denorm = 0; denorm < 4; ++denorm) { + const bool modern = + arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + const auto add = [&](const char *name, uint16_t a, uint16_t b, uint16_t c, + uint16_t nan) { + const uint32_t expected = + clamp && modern ? 0u : nan | ((modern || ieee) ? 0x0200u : 0u); + const auto pair = [packed](uint32_t value) { + return packed ? value * 0x10001u : value; + }; + cases.push_back({"Arch" + std::to_string(arch) + "Packed" + std::to_string(packed) + + "Ieee" + std::to_string(ieee) + "Clamp" + std::to_string(clamp) + + "Denorm" + std::to_string(denorm) + name, + arch, + {(packed ? 0xcc0e4006u : 0xd6480006u) | (clamp ? 0x8000u : 0u), + packed ? 0x1c0a0300u : 0x040a0300u, 0u}, + {{0, pair(a)}, {1, pair(b)}, {2, pair(c)}, {6, 0u}}, + {{6, pair(expected)}}, + (uint32_t(ieee) << 9) | (denorm << 6), + FE_TONEAREST}); + }; + // Raw witnesses captured on gfx1100/gfx1201, including the GCC + // optimization-dependent payload selection seen in packed FMA. + add("FirstPayload", 0x7fc1u, 0xff80u, 0xff80u, 0x7fc1u); + add("SignalingFirst", 0x7c01u, 0x3c00u, 0u, 0x7c01u); + add("SignalingSecond", 0x3c00u, 0xfc12u, 0x7e01u, 0xfc12u); + add("SignalingAddend", 0x3c00u, 0x3c00u, 0x7c01u, 0x7c01u); + add("InvalidProductFirst", 0u, 0x7c00u, 0x7e01u, 0xfe00u); + add("InvalidFiniteAddend", 0u, 0x7c00u, 0x3c00u, 0xfe00u); + add("OppositeInfinities", 0x7c00u, 0x3c00u, 0xfc00u, 0xfe00u); + add("FlushedProductFirst", 1u, 0x7c00u, 0x7e01u, denorm & 1u ? 0x7e01u : 0xfe00u); + } + return cases; +} + +std::vector f16_fma_omod_cases() { + std::vector cases; + for (rj_code_arch_t arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) + for (bool ieee : {false, true}) + for (bool overflow : {false, true}) + for (bool clamp : {false, true}) + for (uint32_t denorm = 0; denorm < 4; ++denorm) { + const bool enabled = arch == ROCJITSU_CODE_ARCH_RDNA4 || + arch == ROCJITSU_CODE_ARCH_CDNA5 || (!ieee && !(denorm & 2u)); + const auto add = [&](const char *name, uint16_t a, uint16_t b, uint16_t c, + uint32_t omod, uint16_t expected) { + cases.push_back( + {"Arch" + std::to_string(arch) + "Ieee" + std::to_string(ieee) + "Overflow" + + std::to_string(overflow) + "Clamp" + std::to_string(clamp) + "Denorm" + + std::to_string(denorm) + name, + arch, + {0xd6480006u | (clamp ? 0x8000u : 0u), 0x040a0300u | (omod << 27), 0u}, + {{0, a}, {1, b}, {2, c}, {6, 0u}}, + {{6, expected}}, + (uint32_t(ieee) << 9) | (uint32_t(overflow) << 23) | (denorm << 6), + FE_UPWARD}); + }; + // Both physical cards round the arithmetic before applying OMOD. + add("NegativeUnderflow", 0x0400u, 0xbc00u, 0u, 3, + clamp ? 0u + : enabled ? 0x8000u + : 0x8400u); + add("OverflowBeforeDivision", 0x3c00u, 0x7bffu, 0x7bffu, 3, + clamp ? 0x3c00u + : overflow ? (enabled ? 0x77ffu : 0x7bffu) + : 0x7c00u); + add("TinyBeforeMultiplication", 0x0400u, 0x3800u, 0u, 2, + enabled || !(denorm & 2u) ? 0u : 0x0200u); + add("NegativeTinyBeforeMultiplication", 0x0400u, 0xb800u, 0u, 2, + enabled || clamp ? 0u + : (denorm & 2u) ? 0x8200u + : 0x8000u); + } + // Captured FMA and FMAC agree on both cards for every rounding mode. + const uint16_t negative_tiny[] = {0x8000u, 0u, 0x8000u, 0u}; + const uint16_t positive_overflow[] = {0x77ffu, 0x7c00u, 0x77ffu, 0x77ffu}; + for (rj_code_arch_t arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) + for (bool accumulate : {false, true}) + for (uint32_t round = 0; round < 4; ++round) { + const auto add = [&](const char *name, uint16_t a, uint16_t b, uint16_t c, + uint16_t expected) { + cases.push_back( + {"DirectedArch" + std::to_string(arch) + "Accumulate" + std::to_string(accumulate) + + "Round" + std::to_string(round) + name, + arch, + {accumulate ? 0xd5360006u : 0xd6480006u, accumulate ? 0x1a020300u : 0x1c0a0300u, 0u}, + {{0, a}, {1, b}, {2, c}, {6, c}}, + {{6, expected}}, + 0x40u | (round << 2), + FE_TOWARDZERO}); + }; + // RNE retains a negative normal until div:2 makes it tiny; RTZ first + // flushes the arithmetic result, so the modifier receives a signed zero. + add("NegativeUnderflow", 1u, 0x3400u, 0x8400u, negative_tiny[round]); + add("OverflowBeforeDivision", 0x0400u, 0x0400u, 0x7bffu, positive_overflow[round]); + } + return cases; +} + std::vector host_mxcsr_cases() { std::vector cases; #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) constexpr uint32_t kDazMask = 1u << 6; constexpr uint32_t kFtzMask = 1u << 15; constexpr uint32_t kControlMask = _MM_ROUND_MASK | kDazMask | kFtzMask; + for (rj_code_arch_t arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) + for (bool packed : {false, true}) + for (bool trap_invalid : {false, true}) { + const auto pair = [packed](uint32_t value) { return packed ? value * 0x10001u : value; }; + const auto add = [&](const char *name, uint16_t a, uint16_t b, uint16_t c, + uint16_t expected) { + cases.push_back( + {"F16FmaArch" + std::to_string(arch) + "Packed" + std::to_string(packed) + "Trap" + + std::to_string(trap_invalid) + name, + arch, + {packed ? 0xcc0e4006u : 0xd6480006u, packed ? 0x1c0a0300u : 0x040a0300u, 0u}, + {{0, pair(a)}, {1, pair(b)}, {2, pair(c)}}, + {{6, packed ? pair(expected) : 0xdead0000u | expected}}, + 0xf0u, + FE_UPWARD, + 0xffffu, + trap_invalid ? 0x1f00u : 0x1f80u}); + }; + // The residual must not expose Inf-Inf to the caller's exception state. + add("InfiniteProduct", 0x7c00u, 0x3c00u, 0u, 0x7c00u); + add("InfiniteAddend", 0x3c00u, 0x3c00u, 0x7c00u, 0x7c00u); + add("InvalidProduct", 0u, 0x7c00u, 0x3c00u, 0xfe00u); + add("InvalidSum", 0x7c00u, 0x3c00u, 0xfc00u, 0xfe00u); + } // Change MXCSR rounding independently of the x87 rounding reported by fegetround(). for (uint32_t rounding : {_MM_ROUND_UP, _MM_ROUND_DOWN, _MM_ROUND_TOWARD_ZERO}) { const bool upward = rounding == _MM_ROUND_UP; @@ -1081,6 +1356,563 @@ std::vector modifier_environment_cases() { return cases; } +std::vector fma_mix_f32_cases() { + struct Sample { + const char *name; + uint32_t a, b, c, mode, expected; + }; + // Raw gfx1100/gfx1201 captures; the older targets share the RDNA3 MODE policy. + constexpr Sample samples[] = { + {"RoundUp", 0x3f800000, 0x3f800000, 0x33800000, 1, 0x3f800001}, + {"RoundDown", 0xbf800000, 0x3f800000, 0xb3800000, 2, 0xbf800001}, + {"Nearest", 0x3f800000, 0x3f800000, 0x33800000, 0, 0x3f800000}, + {"TowardZero", 0x3f800000, 0x3f800000, 0x33800000, 3, 0x3f800000}, + {"InputFlush", 0x00000001, 0x4b000000, 0, 0, 0}, + {"InputPreserve", 0x00000001, 0x4b000000, 0, 0x30, 0x00800000}, + {"OutputFlush", 0x00800000, 0x3f000000, 0, 0, 0}, + {"OutputPreserve", 0x00800000, 0x3f000000, 0, 0x30, 0x00400000}, + {"NegativeZero", 0x3f800000, 0x3f800000, 0xbf800000, 2, 0x80000000}, + {"FusedCancellation", 0x3f800001, 0x3f7ffffe, 0xbf800000, 0, 0xa8800000}, + {"SignalingNan", 0x7f800012, 0x3f800000, 0x3f800000, 0, 0x7f800012}, + {"IeeeNan", 0x7f800012, 0x3f800000, 0x3f800000, 0x200, 0x7fc00012}, + }; + std::vector cases; + for (auto arch : + {ROCJITSU_CODE_ARCH_RDNA1, ROCJITSU_CODE_ARCH_RDNA2, ROCJITSU_CODE_ARCH_RDNA3, + ROCJITSU_CODE_ARCH_RDNA3_5, ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) { + for (const auto &sample : samples) { + uint32_t expected = sample.expected; + if (expected == 0x7f800012 && + (arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5)) + expected = 0x7fc00012; + cases.push_back({"Arch" + std::to_string(arch) + sample.name, + arch, + {0xcc200006u, 0x040a0300u}, + {{0, sample.a}, {1, sample.b}, {2, sample.c}}, + {{6, expected}}, + sample.mode, + FE_DOWNWARD, + 0x9fc0u, + 0x9f60u}); + } + } + return cases; +} + +std::vector trig_fp_cases() { + struct Target { + const char *name; + rj_code_arch_t arch; + bool rdna_encoding; + bool always_quiets; + }; + const Target targets[] = { + {"Cdna1", ROCJITSU_CODE_ARCH_CDNA1, false, false}, + {"Cdna2", ROCJITSU_CODE_ARCH_CDNA2, false, false}, + {"Cdna3", ROCJITSU_CODE_ARCH_CDNA3, false, false}, + {"Cdna4", ROCJITSU_CODE_ARCH_CDNA4, false, false}, + {"Rdna1", ROCJITSU_CODE_ARCH_RDNA1, true, false}, + {"Rdna2", ROCJITSU_CODE_ARCH_RDNA2, true, false}, + {"Rdna3", ROCJITSU_CODE_ARCH_RDNA3, true, false}, + {"Rdna35", ROCJITSU_CODE_ARCH_RDNA3_5, true, false}, + {"Rdna4", ROCJITSU_CODE_ARCH_RDNA4, true, true}, + {"Cdna5", ROCJITSU_CODE_ARCH_CDNA5, true, true}, + }; + std::vector cases; + for (const auto &target : targets) + for (unsigned cosine = 0; cosine < 2; ++cosine) + for (unsigned e64 = 0; e64 < 2; ++e64) { + // Assembled with llvm-mc for each target. The hardware captures use + // gfx1100/gfx1201; the other targets check shared full-range and MODE + // execution without claiming their finite approximations are identical. + std::array words{}; + if (e64) { + words[0] = (target.rdna_encoding ? 0xd5b50006u : 0xd1690006u) + cosine * 0x10000u; + words[1] = target.rdna_encoding ? 0x02010100u : 0x00000100u; + } else { + words[0] = (target.rdna_encoding ? 0x7e0c6b00u : 0x7e0c5300u) + cosine * 0x200u; + } + const std::string prefix = + std::string(target.name) + (cosine ? "Cos" : "Sin") + (e64 ? "E64" : "E32"); + auto add = [&](const std::string &name, uint32_t input, uint32_t expected, uint32_t mode) { + cases.push_back({prefix + name, + target.arch, + words, + {{0, input}}, + {{6, expected}}, + mode, + FE_UPWARD, + 0x8040u, + 0x8040u}); + }; + add("LargeFinite", 0xff7fffffu, cosine ? 0x3f800000u : 0u, 240u); + for (uint32_t ieee = 0; ieee < 2; ++ieee) + add("SignalingNan" + std::to_string(ieee), 0xff812345u, + target.always_quiets || ieee ? 0xffc12345u : 0xff812345u, 240u | (ieee << 9)); + for (uint32_t denorm = 0; denorm < 4; ++denorm) + for (uint32_t rounding = 0; rounding < 4; ++rounding) { + uint32_t mode = 192u | (denorm << 4) | rounding; + add("Subnormal" + std::to_string(mode), 0x80000001u, + cosine ? 0x3f800000u + : denorm == 3 ? 0x80000006u + : 0x80000000u, + mode); + } + if (target.arch == ROCJITSU_CODE_ARCH_RDNA3 || target.arch == ROCJITSU_CODE_ARCH_RDNA4) { + for (uint32_t rounding = 0; rounding < 4; ++rounding) { + add("CapturedOctant" + std::to_string(rounding), 0x3e000000u, + cosine ? 0x3f3504f3u : 0x3f3504f4u, 240u | rounding); + // Raw captures distinguish the quadratic stages and the + // normalized/reflected boundaries in both instruction encodings. + const uint32_t captured[][3] = { + {0x3aab9885u, 0x3c06c500u, 0x3f7ffdc8u}, {0x3d2001fbu, 0x3e78d2d0u, 0x3f7853c7u}, + {0x3dc084adu, 0x3f0e9072u, 0x3f54a13bu}, {0x3e7fff08u, 0x3f800000u, 0x37c2c761u}, + {0x3e7ffa50u, 0x3f800000u, 0x390ef149u}, + }; + for (const auto &sample : captured) + add("CapturedStages" + std::to_string(sample[0]) + "Round" + std::to_string(rounding), + sample[0], sample[cosine + 1], 240u | rounding); + } + } + } + return cases; +} + +std::vector log_exp_policy_cases() { + std::vector cases; + for (rj_code_arch_t arch : + {ROCJITSU_CODE_ARCH_CDNA1, ROCJITSU_CODE_ARCH_CDNA2, ROCJITSU_CODE_ARCH_CDNA3, + ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA5, ROCJITSU_CODE_ARCH_RDNA1, + ROCJITSU_CODE_ARCH_RDNA2, ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4}) { + const bool gcn = arch == ROCJITSU_CODE_ARCH_CDNA1 || arch == ROCJITSU_CODE_ARCH_CDNA2 || + arch == ROCJITSU_CODE_ARCH_CDNA3 || arch == ROCJITSU_CODE_ARCH_CDNA4; + for (bool logarithm : {false, true}) + for (bool e64 : {false, true}) { + // Assembled independently for all ten profiles with llvm-mc. + std::array words{}; + if (e64) { + words[0] = + (gcn ? 0xd1600006u : 0xd5a50006u) + (logarithm ? (gcn ? 0x10000u : 0x20000u) : 0u); + words[1] = gcn ? 0x00000100u : 0x02010100u; + } else { + words[0] = (gcn ? 0x7e0c4100u : 0x7e0c4b00u) + (logarithm ? (gcn ? 0x200u : 0x400u) : 0u); + } + for (uint32_t ieee = 0; ieee < 2; ++ieee) + for (uint32_t rounding = 0; rounding < 4; ++rounding) { + const uint32_t mode = 240u | (ieee << 9) | rounding; + const std::string prefix = "Arch" + std::to_string(arch) + (logarithm ? "Log" : "Exp") + + (e64 ? "E64" : "E32") + "Mode" + std::to_string(mode); + const auto add = [&](const char *name, uint32_t input, uint32_t expected) { + // Preserve flags and FTZ/DAZ under upward rounding, with the + // host invalid-operation trap enabled. + cases.push_back({prefix + name, + arch, + words, + {{0, input}}, + {{6, expected}}, + mode, + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + }; + const bool quiet = + ieee || arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + add("SignalingNan", 0x7f812345u, quiet ? 0x7fc12345u : 0x7f812345u); + add("NegativeSignalingNan", 0xff812345u, quiet ? 0xffc12345u : 0xff812345u); + add("QuietNan", 0xffc12345u, 0xffc12345u); + add("Negative", 0xbf800000u, logarithm ? 0xffc00000u : 0x3f000000u); + add("NegativeInfinity", 0xff800000u, logarithm ? 0xffc00000u : 0u); + add("NegativeSubnormal", 0x80000001u, logarithm ? 0xff800000u : 0x3f800000u); + if (arch == ROCJITSU_CODE_ARCH_RDNA3 || arch == ROCJITSU_CODE_ARCH_RDNA4) + add("CapturedRounding", logarithm ? 0x3f174424u : 0x3f0567ecu, + logarithm ? 0xbf425164u : 0x3fb7b03du); + if (!logarithm && + (arch == ROCJITSU_CODE_ARCH_RDNA3 || arch == ROCJITSU_CODE_ARCH_RDNA4)) { + const uint32_t captured[][2] = { + {0x337fffffu, 0x3f800000u}, {0x33800000u, 0x3f800000u}, + {0x33800001u, 0x3f800000u}, {0xb37fffffu, 0x3f800000u}, + {0xb3800000u, 0x3f7fffffu}, {0xb3800001u, 0x3f7fffffu}, + {0x42ffffffu, 0x7f7fffa7u}, {0x43000000u, 0x7f800000u}, + {0xc2fc0000u, 0x00800000u}, {0xc2fc0001u, 0x00000000u}, + {0x3f0567ecu, 0x3fb7b03du}, {0xc114ed44u, 0x3acecc1eu}, + {0x35500000u, 0x3f800004u}, {0x3e000090u, 0x3f8b95d0u}, + {0x3e80001eu, 0x3f9837f6u}, {0x3ec0001au, 0x3fa5fedcu}, + {0x3f000013u, 0x3fb504fcu}, {0x3f20002au, 0x3fc56740u}, + {0x3f40006fu, 0x3fd7453eu}, {0x3f600022u, 0x3feac0dcu}, + }; + for (const auto &sample : captured) { + const std::string name = "CapturedExp" + std::to_string(sample[0]); + add(name.c_str(), sample[0], sample[1]); + } + } + if (logarithm && e64) { + // LOG always disables output denormals, so preservation MODE + // cannot suppress scaling. IEEE mode still gates older profiles. + for (uint32_t omod : {1u, 2u, 3u}) { + auto scaled_words = words; + scaled_words[1] |= omod << 27; + const bool active = + !ieee || arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + const uint32_t scaled[] = {0x40000000u, 0x40800000u, 0x41000000u, 0x3f800000u}; + for (uint32_t denorm = 0; denorm < 4; ++denorm) + cases.push_back({prefix + "LogScale" + std::to_string(omod) + "Denorm" + + std::to_string(denorm), + arch, + scaled_words, + {{0, 0x40800000u}}, + {{6, scaled[active ? omod : 0]}}, + 192u | (ieee << 9) | (denorm << 4) | rounding, + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + } + // Modifiers run after the integer LOG mapping. Exercise both + // preserved signaling NaNs and invalid-operation results while + // host invalid traps, flags, rounding and flush controls persist. + for (uint32_t dx10 : {0u, 1u}) + for (uint32_t modifier : {0u, 1u, 2u, 3u, 4u}) { + auto modified_words = words; + if (modifier == 4) + modified_words[0] |= 0x8000u; + else + modified_words[1] |= modifier << 27; + const bool nan_to_zero = + modifier == 4 && (dx10 || arch == ROCJITSU_CODE_ARCH_RDNA4 || + arch == ROCJITSU_CODE_ARCH_CDNA5); + for (uint32_t input : {0xbf800000u, 0x7f800123u}) { + const uint32_t result = input == 0xbf800000u ? 0xffc00000u + : quiet ? 0x7fc00123u + : 0x7f800123u; + cases.push_back({prefix + "LogModifier" + std::to_string(modifier) + "Dx10" + + std::to_string(dx10) + "Input" + std::to_string(input), + arch, + modified_words, + {{0, input}}, + {{6, nan_to_zero ? 0u : result}}, + mode | (dx10 << 8), + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + } + } + } + if (!logarithm && e64) { + // OMOD overflows after EXP returns. Its rounding and exception + // flags must remain independent of the host environment. + for (uint32_t omod : {1u, 2u, 3u}) { + auto scaled_words = words; + scaled_words[1] |= omod << 27; + for (uint32_t denorm = 0; denorm < 4; ++denorm) { + const bool active = + !ieee || arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + const uint32_t unscaled = omod == 3 ? 0x00800000u : 0x7f000000u; + const uint32_t expected = active ? (omod == 3 ? 0u : 0x7f800000u) : unscaled; + for (int host_round : {FE_DOWNWARD, FE_UPWARD}) + cases.push_back({prefix + "Scale" + std::to_string(omod) + "Denorm" + + std::to_string(denorm) + "Host" + + std::to_string(host_round), + arch, + scaled_words, + {{0, omod == 3 ? 0xc2fc0000u : 0x42fe0000u}}, + {{6, expected}}, + 192u | (ieee << 9) | (denorm << 4) | rounding, + host_round, + 0x9fc0u, + 0x9b60u}); + } + } + } + { + std::array half_words{}; + if (e64) { + half_words[0] = gcn ? (logarithm ? 0xd1800006u : 0xd1810006u) + : (logarithm ? 0xd5d70006u : 0xd5d80006u); + half_words[1] = gcn ? 0x00000100u : 0x02010100u; + } else { + half_words[0] = gcn ? (logarithm ? 0x7e0c8100u : 0x7e0c8300u) + : (logarithm ? 0x7e0caf00u : 0x7e0cb100u); + } + const bool preserves_high_half = + !gcn && arch != ROCJITSU_CODE_ARCH_RDNA1 && arch != ROCJITSU_CODE_ARCH_RDNA2; + const uint32_t half_cases[][2] = { + {0x7c01u, quiet ? 0x7e01u : 0x7c01u}, + {0xfc01u, quiet ? 0xfe01u : 0xfc01u}, + {0xbc00u, logarithm ? 0xfe00u : 0x3800u}, + {0xfc00u, logarithm ? 0xfe00u : 0u}, + }; + for (bool fp16_ovfl : {false, true}) + for (const auto &sample : half_cases) + cases.push_back({prefix + "Half" + std::to_string(sample[0]) + "Ovfl" + + std::to_string(fp16_ovfl), + arch, + half_words, + {{0, sample[0]}}, + {{6, sample[1] | (preserves_high_half ? 0xdead0000u : 0u)}}, + mode | (fp16_ovfl ? (1u << 23) : 0u), + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + } + } + } + } + return cases; +} + +std::vector sdwa_log_exp_policy_cases() { + std::vector cases; + for (rj_code_arch_t arch : + {ROCJITSU_CODE_ARCH_CDNA1, ROCJITSU_CODE_ARCH_CDNA2, ROCJITSU_CODE_ARCH_CDNA3, + ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA1, ROCJITSU_CODE_ARCH_RDNA2}) { + const bool gcn = arch != ROCJITSU_CODE_ARCH_RDNA1 && arch != ROCJITSU_CODE_ARCH_RDNA2; + // The older manuals specify SDWA OMOD by reference to VOP3. These exact + // values test that shared policy, not unmeasured hardware approximations. + // The SDWA words were assembled independently for each profile with llvm-mc. + for (bool logarithm : {false, true}) + for (uint32_t ieee = 0; ieee < 2; ++ieee) + for (uint32_t denorm = 0; denorm < 4; ++denorm) + for (uint32_t rounding = 0; rounding < 4; ++rounding) + for (uint32_t omod : {1u, 2u, 3u}) { + const uint32_t mode = 192u | (ieee << 9) | (denorm << 4) | rounding; + const uint32_t opcode = gcn ? (logarithm ? 0x7e0c42f9u : 0x7e0c40f9u) + : (logarithm ? 0x7e0c4ef9u : 0x7e0c4af9u); + const std::array words{opcode, 0x00060600u | (omod << 14), 0}; + const uint32_t log_results[] = {0x40000000u, 0x40800000u, 0x41000000u, 0x3f800000u}; + const uint32_t input = logarithm ? 0x40800000u + : omod == 3 ? 0xc2fc0000u + : 0x42fe0000u; + const uint32_t unscaled = omod == 3 ? 0x00800000u : 0x7f000000u; + const uint32_t expected = logarithm ? log_results[ieee ? 0 : omod] + : ieee ? unscaled + : omod == 3 ? 0u + : 0x7f800000u; + for (bool nan : {false, true}) + for (int host_round : {FE_DOWNWARD, FE_UPWARD}) + cases.push_back({"Arch" + std::to_string(arch) + (logarithm ? "Log" : "Exp") + + "Mode" + std::to_string(mode) + "Omod" + + std::to_string(omod) + "Nan" + std::to_string(nan) + "Host" + + std::to_string(host_round), + arch, + words, + {{0, nan ? 0x7f812345u : input}}, + {{6, nan ? (ieee ? 0x7fc12345u : 0x7f812345u) : expected}}, + mode, + host_round, + 0x9fc0u, + 0x9b60u}); + } + } + return cases; +} + +std::vector half_log_exp_arithmetic_cases() { + struct Captured { + const char *name; + bool logarithm; + uint32_t source; + uint32_t mode; + uint32_t modifier; + uint32_t rdna3; + uint32_t rdna4; + }; + // Raw gfx1100/gfx1201 captures. Modifier 4 is CLAMP; 1..3 are OMOD. + const Captured captured[] = { + {"LogPreserveInput", true, 0x0001u, 240u, 0u, 0xce00u, 0xce00u}, + {"LogFlushInput", true, 0x0001u, 48u, 0u, 0xfc00u, 0xfc00u}, + {"LogFlushSaturate", true, 0x0001u, 8388656u, 0u, 0xfbffu, 0xfbffu}, + {"LogZeroSaturate", true, 0x0000u, 8388848u, 0u, 0xfbffu, 0xfbffu}, + {"LogInputInfinity", true, 0x7c00u, 8388848u, 0u, 0x7c00u, 0x7c00u}, + {"ExpFiniteOverflow", false, 0x4c00u, 8388848u, 0u, 0x7bffu, 0x7bffu}, + {"ExpInputInfinity", false, 0x7c00u, 8388848u, 0u, 0x7c00u, 0x7c00u}, + {"ExpPreserveOutput", false, 0xcb80u, 240u, 0u, 0x0200u, 0x0200u}, + {"ExpFlushOutput", false, 0xcb80u, 112u, 0u, 0x0000u, 0x0000u}, + {"ExpScaleFlushedResult", false, 0xcb80u, 240u, 2u, 0x0200u, 0x0000u}, + {"ExpScaleSaturatedResult", false, 0x4c00u, 8388656u, 3u, 0x77ffu, 0x77ffu}, + {"ExpScaleOverflow", false, 0x4c00u, 48u, 3u, 0x7c00u, 0x7c00u}, + {"ExpScaleMode", false, 0x4c00u, 8388848u, 3u, 0x7bffu, 0x77ffu}, + {"LogNanClamp", true, 0xfc01u, 48u, 4u, 0xfc01u, 0x0000u}, + {"ExpNanScale", false, 0x7c01u, 48u, 1u, 0x7c01u, 0x7e01u}, + {"LogInvalidClamp", true, 0xbc00u, 48u, 4u, 0xfe00u, 0x0000u}, + {"ExpSingleRounding", false, 0x11c5u, 240u, 0u, 0x3c01u, 0x3c01u}, + }; + std::vector cases; + for (rj_code_arch_t arch : + {ROCJITSU_CODE_ARCH_CDNA1, ROCJITSU_CODE_ARCH_CDNA2, ROCJITSU_CODE_ARCH_CDNA3, + ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA5, ROCJITSU_CODE_ARCH_RDNA1, + ROCJITSU_CODE_ARCH_RDNA2, ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4}) { + const bool gcn = arch == ROCJITSU_CODE_ARCH_CDNA1 || arch == ROCJITSU_CODE_ARCH_CDNA2 || + arch == ROCJITSU_CODE_ARCH_CDNA3 || arch == ROCJITSU_CODE_ARCH_CDNA4; + const bool true16 = + !gcn && arch != ROCJITSU_CODE_ARCH_RDNA1 && arch != ROCJITSU_CODE_ARCH_RDNA2; + const bool always_omod = arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5; + for (const auto &sample : captured) { + // Other profiles check exact values and MODE policy, not unmeasured finite approximations. + if (sample.source == 0x11c5u && arch != ROCJITSU_CODE_ARCH_RDNA3 && + arch != ROCJITSU_CODE_ARCH_RDNA4) + continue; + for (bool e64 : {false, true}) { + if (!e64 && sample.modifier != 0) + continue; + std::array words{}; + if (e64) { + words[0] = gcn ? (sample.logarithm ? 0xd1800006u : 0xd1810006u) + : (sample.logarithm ? 0xd5d70006u : 0xd5d80006u); + words[1] = gcn ? 0x00000100u : 0x02010100u; + if (sample.modifier == 4) + words[0] |= 0x8000u; + else + words[1] |= sample.modifier << 27; + } else { + words[0] = gcn ? (sample.logarithm ? 0x7e0c8100u : 0x7e0c8300u) + : (sample.logarithm ? 0x7e0caf00u : 0x7e0cb100u); + } + for (uint32_t rounding = 0; rounding < 4; ++rounding) { + const uint32_t expected = + (always_omod ? sample.rdna4 : sample.rdna3) | (true16 ? 0xdead0000u : 0u); + cases.push_back({"Arch" + std::to_string(arch) + sample.name + (e64 ? "E64" : "E32") + + "Round" + std::to_string(rounding), + arch, + words, + {{0, sample.source}}, + {{6, expected}}, + sample.mode | (rounding << 2), + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + } + } + } + if (gcn || arch == ROCJITSU_CODE_ARCH_RDNA1 || arch == ROCJITSU_CODE_ARCH_RDNA2) { + // Assembled SDWA forms select the high source and destination halves. + // MODE and register-placement contracts also apply on these older profiles. + for (uint32_t rounding = 0; rounding < 4; ++rounding) { + for (bool signaling_nan : {false, true}) { + const uint32_t source = signaling_nan ? 0x7c01u : 0x3c00u; + const uint32_t result = signaling_nan ? 0x7c01u : 0x4400u; + cases.push_back({"Arch" + std::to_string(arch) + "SdwaExpNan" + + std::to_string(signaling_nan) + "Round" + std::to_string(rounding), + arch, + {gcn ? 0x7e0c82f9u : 0x7e0cb0f9u, 0x00055500u}, + {{0, (source << 16) | 0x1234u}}, + {{6, (result << 16) | 0xbeefu}}, + 48u | (rounding << 2), + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + } + cases.push_back( + {"Arch" + std::to_string(arch) + "SdwaLogClampRound" + std::to_string(rounding), + arch, + {gcn ? 0x7e0c80f9u : 0x7e0caef9u, 0x00053500u}, + {{0, 0x7c011234u}}, + {{6, 0x0000beefu}}, + 304u | (rounding << 2), + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + } + } + } + return cases; +} + +std::vector fma_mix_half_cases() { + std::vector cases; + for (auto arch : + {ROCJITSU_CODE_ARCH_RDNA1, ROCJITSU_CODE_ARCH_RDNA2, ROCJITSU_CODE_ARCH_RDNA3, + ROCJITSU_CODE_ARCH_RDNA3_5, ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) { + for (uint32_t mode = 0; mode < 4; ++mode) { + for (bool high : {false, true}) { + // Physical RDNA3/4 shader exports: the exact product is just below + // 0x340c, but rounding it to F32 first produces that exact F16 value. + const uint32_t result = mode < 2 ? 0x340cu : 0x340bu; + cases.push_back({"Arch" + std::to_string(arch) + "MixHalf" + std::to_string(high) + + "Round" + std::to_string(mode), + arch, + {high ? 0xcc224006u : 0xcc214006u, 0x820204ffu, 0x3b333333u}, + {{2, 0x42b90000u}}, + {{6, high ? (result << 16) | 0xbeefu : 0xdead0000u | result}}, + 240u | (mode << 2), + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + } + } + // Both cards prioritize an invalid product over the addend NaN. Ordinary + // multiple-NaN inputs still select the first source's payload. + for (bool high : {false, true}) { + for (bool clamp : {false, true}) { + for (uint32_t source = 0; source < 3; ++source) { + const uint32_t a = source == 0 ? 0 : source == 1 ? 0x7f800000u : 0x7fc12000u; + const uint32_t b = source == 0 ? 0x7f800000u : source == 1 ? 0 : 0x3f800000u; + uint32_t result = source < 2 ? 0xfe00u : 0x7e09u; + if (clamp && (arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5)) + result = 0; + cases.push_back( + {"Arch" + std::to_string(arch) + "MixHalf" + std::to_string(high) + "Nan" + + std::to_string(source) + "Clamp" + std::to_string(clamp), + arch, + {(high ? 0xcc220006u : 0xcc210006u) | (clamp ? 0x8000u : 0), 0x040a0300u}, + {{0, a}, {1, b}, {2, 0x7fe9a000u}}, + {{6, high ? (result << 16) | 0xbeefu : 0xdead0000u | result}}, + 0, + FE_UPWARD, + 0x9fc0u, + 0x9f60u}); + } + } + } + } + return cases; +} + +// Raw gfx1100/gfx1201 witnesses for instruction policies around the math mapping. +// The fixture checks sparse/full EXEC, inactive lanes and a hostile host environment. +std::vector transcendental_policy_cases() { + struct Captured { + const char *name; + uint32_t opcode, input, mode, omod, rdna3, rdna4; + bool half; + }; + const Captured captured[] = { + {"RcpZeroSaturate", 0xd5d40006u, 0x0000u, 0x8000c0u, 0, 0x7bffu, 0x7bffu, true}, + {"RsqNegativeZeroSaturate", 0xd5d60006u, 0x8000u, 0x8000c0u, 0, 0xfbffu, 0xfbffu, true}, + {"RcpFlushInput", 0xd5d40006u, 0x03ffu, 0, 0, 0x7c00u, 0x7c00u, true}, + {"RcpPreserveInput", 0xd5d40006u, 0x03ffu, 0xc0u, 0, 0x7401u, 0x7401u, true}, + {"RcpFlushOutput", 0xd5d40006u, 0x7bffu, 0, 0, 0x0000u, 0x0000u, true}, + {"RcpPreserveOutput", 0xd5d40006u, 0x7bffu, 0xc0u, 0, 0x0100u, 0x0100u, true}, + {"RcpHalfNan", 0xd5d40006u, 0x7c01u, 0, 0, 0x7c01u, 0x7e01u, true}, + {"RsqHalfNan", 0xd5d60006u, 0x7c01u, 0x200u, 0, 0x7e01u, 0x7e01u, true}, + {"SqrtHalfNanScale", 0xd5d50006u, 0x7c01u, 0, 1, 0x7c01u, 0x7e01u, true}, + {"SinHalfNan", 0xd5e00006u, 0x7c01u, 0, 0, 0x7c01u, 0x7e01u, true}, + {"CosHalfNan", 0xd5e10006u, 0x7c01u, 0, 0, 0x7c01u, 0x7e01u, true}, + {"SinHalfFlushInput", 0xd5e00006u, 0x0001u, 0, 0, 0x0000u, 0x0000u, true}, + {"SinHalfPreserveInput", 0xd5e00006u, 0x0001u, 0xc0u, 0, 0x0006u, 0x0006u, true}, + {"RcpNan", 0xd5aa0006u, 0x7f800001u, 0, 0, 0x7f800001u, 0x7fc00001u, false}, + {"RsqNan", 0xd5ae0006u, 0x7f800001u, 0, 0, 0x7f800001u, 0x7fc00001u, false}, + {"RcpScaleDenormMode", 0xd5aa0006u, 0x3f800000u, 0xf0u, 1, 0x40000000u, 0x40000000u, false}, + {"RsqScaleDenormMode", 0xd5ae0006u, 0x3f800000u, 0xf0u, 1, 0x40000000u, 0x40000000u, false}, + {"CosScaleDenormMode", 0xd5b60006u, 0x00000000u, 0xf0u, 1, 0x40000000u, 0x40000000u, false}, + {"RcpScaleOverflow", 0xd5aa0006u, 0x00800000u, 0, 2, 0x7f800000u, 0x7f800000u, false}, + }; + std::vector cases; + for (rj_code_arch_t arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA4}) + for (const auto &sample : captured) + cases.push_back({"Arch" + std::to_string(arch) + sample.name, + arch, + {sample.opcode, 0x00000100u | (sample.omod << 27)}, + {{0, sample.input}}, + {{6, (arch == ROCJITSU_CODE_ARCH_RDNA3 ? sample.rdna3 : sample.rdna4) | + (sample.half ? 0xdead0000u : 0u)}}, + sample.mode, + FE_DOWNWARD, + 0x9fc0u, + 0x9b60u}); + return cases; +} + class ValuFpModeTest : public testing::TestWithParam {}; TEST_P(ValuFpModeTest, HonorsModeAndPreservesInactiveLanes) { @@ -1157,11 +1989,48 @@ TEST_P(ValuFpModeTest, HonorsModeAndPreservesInactiveLanes) { wave->halt(); } +INSTANTIATE_TEST_SUITE_P(TranscendentalPolicy, ValuFpModeTest, + testing::ValuesIn(transcendental_policy_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(FmaMixF32, ValuFpModeTest, testing::ValuesIn(fma_mix_f32_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(FmaMixHalf, ValuFpModeTest, testing::ValuesIn(fma_mix_half_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + INSTANTIATE_TEST_SUITE_P(AllTargets, ValuFpModeTest, testing::ValuesIn(kCases), [](const testing::TestParamInfo &info) { return info.param.name; }); +INSTANTIATE_TEST_SUITE_P(Trigonometry, ValuFpModeTest, testing::ValuesIn(trig_fp_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(LogExp, ValuFpModeTest, testing::ValuesIn(log_exp_policy_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(SdwaLogExp, ValuFpModeTest, testing::ValuesIn(sdwa_log_exp_policy_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(HalfLogExp, ValuFpModeTest, + testing::ValuesIn(half_log_exp_arithmetic_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + INSTANTIATE_TEST_SUITE_P(Adjacent, ValuFpModeTest, testing::ValuesIn(adjacent_fp_cases()), [](const testing::TestParamInfo &info) { return info.param.name; @@ -1177,6 +2046,26 @@ INSTANTIATE_TEST_SUITE_P(Dx9Fma, ValuFpModeTest, testing::ValuesIn(dx9_fma_cases return info.param.name; }); +INSTANTIATE_TEST_SUITE_P(BinaryF32, ValuFpModeTest, testing::ValuesIn(binary_f32_policy_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(OmodUnderflow, ValuFpModeTest, testing::ValuesIn(omod_underflow_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(F16FmaOmod, ValuFpModeTest, testing::ValuesIn(f16_fma_omod_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + +INSTANTIATE_TEST_SUITE_P(F16FmaNan, ValuFpModeTest, testing::ValuesIn(f16_fma_nan_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + INSTANTIATE_TEST_SUITE_P(HostMxcsr, ValuFpModeTest, testing::ValuesIn(host_mxcsr_cases()), [](const testing::TestParamInfo &info) { return info.param.name; @@ -1188,6 +2077,40 @@ INSTANTIATE_TEST_SUITE_P(ModifierEnvironment, ValuFpModeTest, return info.param.name; }); +TEST(ValuFpModeHelpers, PackedF16NanSelection) { + using namespace amdgpu::fp_mode; + // Physical gfx1100/gfx1201 captures, with IEEE disabled/enabled. Entries + // give arithmetic and min/max outputs for legacy-off, legacy-on and newer. + struct Case { + uint16_t a, b; + std::array arithmetic, select; + }; + constexpr Case cases[] = { + {0x7fc1, 0xff80, {0x7fc1, 0x7fc1, 0x7fc1}, {0x7fc1, 0x7fc1, 0x7fc1}}, + {0xff80, 0x7fc1, {0xff80, 0xff80, 0xff80}, {0xff80, 0xff80, 0xff80}}, + {0x7c01, 0xff80, {0x7c01, 0x7e01, 0x7e01}, {0x7c01, 0x7e01, 0x7e01}}, + {0x7fc1, 0xfc02, {0x7fc1, 0x7fc1, 0x7fc1}, {0x7fc1, 0xfe02, 0x7fc1}}, + {0x3c00, 0x7c01, {0x7c01, 0x7e01, 0x7e01}, {0x3c00, 0x7e01, 0x3c00}}, + }; + for (auto arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA4}) + for (bool ieee : {false, true}) { + const unsigned profile = arch == ROCJITSU_CODE_ARCH_RDNA4 ? 2 : unsigned(ieee); + for (auto op : + {PackedBinaryOp::ADD, PackedBinaryOp::MUL, PackedBinaryOp::MIN, PackedBinaryOp::MAX}) + for (const auto &test : cases) { + SCOPED_TRACE(::testing::Message() << int(arch) << ':' << ieee << ':' << int(op) << ':' + << test.a << ':' << test.b); + const uint16_t expected = + (op == PackedBinaryOp::ADD || op == PackedBinaryOp::MUL ? test.arithmetic + : test.select)[profile]; + EXPECT_EQ(packed_binary_f16(op, test.a, test.b, 0, 3, false, false, false, arch, ieee), + expected); + EXPECT_EQ(packed_binary_f16(op, test.a, test.b, 0, 3, true, false, true, arch, ieee), + expected > 0x7c00u ? 0 : 0x3c00u); + } + } +} + TEST(ValuFpModeHelpers, RoundingAndSignedZero) { using amdgpu::fp_mode::Arithmetic; for (uint32_t mode = 0; mode < 4; ++mode) { @@ -1204,6 +2127,92 @@ TEST(ValuFpModeHelpers, RoundingAndSignedZero) { } } +TEST(ValuFpModeHelpers, MixedF16FmaExactZeroSigns) { + struct Case { + uint32_t a, b, c; + std::array expected; + }; + constexpr std::array cases{{ + {0, 0x3f800000, 0, {0, 0, 0, 0}}, + {0, 0x3f800000, 0x80000000, {0, 0, 0x8000, 0}}, + {0x80000000, 0x3f800000, 0, {0, 0, 0x8000, 0}}, + {0x80000000, 0x3f800000, 0x80000000, {0x8000, 0x8000, 0x8000, 0x8000}}, + {0, 0xbf800000, 0, {0, 0, 0x8000, 0}}, + {0, 0xbf800000, 0x80000000, {0x8000, 0x8000, 0x8000, 0x8000}}, + {0x80000000, 0xbf800000, 0, {0, 0, 0, 0}}, + {0x80000000, 0xbf800000, 0x80000000, {0, 0, 0x8000, 0}}, + }}; + for (const auto &test : cases) + for (bool zero_second : {false, true}) + for (bool swap : {false, true}) + for (bool clamp : {false, true}) + for (uint32_t mode = 0; mode < 4; ++mode) { + uint32_t a = test.a, b = zero_second ? test.b & 0x80000000u : test.b; + if (swap) + std::swap(a, b); + amdgpu::fp_mode::ScopedEnvironment environment(0); + const uint16_t actual = amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + std::bit_cast(a), std::bit_cast(b), std::bit_cast(test.c), + mode, clamp, false, false); + EXPECT_EQ(actual, clamp ? 0u : test.expected[mode]); + } +} + +TEST(ValuFpModeHelpers, MixedF16FmaCancellationAcrossF32Range) { + constexpr std::array, 5> cases{{ + {0x7f7fffff, 0x3f800000, 0xff7fffff}, + {1, 0x3f800000, 0x80000001}, + {0x00800000, 0x3f000000, 0x80400000}, + {0x5f800000, 0x1f800000, 0xbf800000}, + {0x00800000, 0x34000000, 0x80000001}, + }}; + for (const auto &test : cases) + for (uint32_t mode = 0; mode < 4; ++mode) { + amdgpu::fp_mode::ScopedEnvironment environment(0); + EXPECT_EQ(amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + std::bit_cast(test[0]), std::bit_cast(test[1]), + std::bit_cast(test[2]), mode, false, false, false), + mode == 2 ? 0x8000u : 0u); + } + // These products are far below F16 range, but are nonzero F64 values. A + // sign-only repair must not turn directed underflow into exact cancellation. + for (uint32_t mode = 0; mode < 4; ++mode) { + amdgpu::fp_mode::ScopedEnvironment environment(0); + EXPECT_EQ( + amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + std::bit_cast(1u), std::bit_cast(1u), 0.0f, mode, false, false, false), + mode == 1 ? 1u : 0u); + EXPECT_EQ(amdgpu::fp_mode::detail::fma_f32_to_f16_nearest_environment( + std::bit_cast(0x80000001u), std::bit_cast(1u), 0.0f, mode, false, + false, false), + mode == 2 ? 0x8001u : 0x8000u); + } +} + +TEST(ValuFpModeHelpers, F16FmaRetainsTinyProduct) { + // Both physical cards round 65504 + 2^-48 upward to infinity. A host F64 + // addition alone loses the tiny product and incorrectly returns 65504. + for (uint32_t round = 0; round < 4; ++round) + for (uint32_t denorm = 0; denorm < 4; ++denorm) + for (bool overflow : {false, true}) { + const uint16_t expected = round == 1 && (denorm & 1u) && !overflow ? 0x7c00u : 0x7bffu; + EXPECT_EQ(amdgpu::fp_mode::fma_f16(1, 1, 0x7bff, false, false, false, false, false, false, + round, denorm, 0, false, overflow, false, true), + expected); + } +} + +TEST(ValuFpModeHelpers, F16FmaFlushesBeforePacking) { + // Packing directly to a subnormal half can round these tiny intermediates + // to minimum normal; hardware tests the rounded significand first. + for (uint32_t round = 0; round < 4; ++round) + for (uint16_t sign : {uint16_t{0}, uint16_t{0x8000}}) + EXPECT_EQ(amdgpu::fp_mode::fma_f16(1, sign | 0x03ff, sign | 0x03ff, false, false, false, + false, false, false, round, 1, 0, false, false, false, + true), + sign); +} + TEST(ValuFpModeHelpers, FusedResultAndOutputFlush) { using amdgpu::fp_mode::Arithmetic; EXPECT_EQ(std::bit_cast(amdgpu::fp_mode::arithmetic( @@ -1246,4 +2255,165 @@ TEST(ValuFpModeHelpers, HostFlushControlsDoNotOverrideGpuMode) { #endif } +// Floating compares flush their inputs under MODE.FP_DENORM before the +// relation. Each case runs one decoded RDNA4 compare, built with the generated +// encoders, on lanes whose src0 cycles through +tiny, -tiny, +0 and 1.0 +// with src1 = +0. V_CMP_EQ then selects lane 2 of each group of four when input +// denormals are kept, and lanes 0-2 when they are flushed. F16 sources hold a +// NaN in their high half, so only the .l halves may be read. +struct CompareInputFlushCase { + std::string name; + std::array words; + unsigned width; + bool vop3; + uint32_t mode; + uint32_t expected; +}; + +void PrintTo(const CompareInputFlushCase &test, std::ostream *stream) { *stream << test.name; } + +std::vector compare_input_flush_cases() { + constexpr uint32_t KEEP = 0x44444444u; + constexpr uint32_t FLUSH = 0x77777777u; + // MODE.FP_DENORM is [5:4] for F32 and [7:6] for F16/F64; bit 0 of each field + // allows input denormals. The modes set the two fields differently and vary + // the output bit alone, so reading the other format's field or the output + // bit gives the wrong mask. + constexpr std::array, 8> F32_MODES = {{ + {0xf0u, KEEP}, + {0x00u, FLUSH}, + {0x30u, KEEP}, + {0xc0u, FLUSH}, + {0xe0u, FLUSH}, + {0xd0u, KEEP}, + {0xb0u, KEEP}, + {0x70u, KEEP}, + }}; + constexpr std::array, 8> F16_F64_MODES = {{ + {0xf0u, KEEP}, + {0x00u, FLUSH}, + {0x30u, FLUSH}, + {0xc0u, KEEP}, + {0xe0u, KEEP}, + {0xd0u, KEEP}, + {0xb0u, FLUSH}, + {0x70u, KEEP}, + }}; + struct Form { + const char *name; + std::array words; + unsigned width; + bool vop3; + }; + // Source operands encode VGPR n as 256 + n; VOPC vsrc1 is the VGPR index. + constexpr uint16_t V0 = 256, V1 = 257, V2 = 258; + const auto vopc = [](uint16_t op, uint8_t vsrc1) { + return std::array{rdna4::build_vopc(op, {.src0 = V0, .vsrc1 = vsrc1})[0], 0u}; + }; + const auto vop3 = [](uint16_t op, uint16_t src1) { + return rdna4::build_vop3(op, {.vdst = 6, .src0 = V0, .src1 = src1}); + }; + const std::array forms = {{ + // v_cmp_eq_f32_e32 vcc_lo, v0, v1 + {"EqF32E32", vopc(rdna4::kVCmpEqF32Vopc, 1), 32, false}, + // v_cmp_eq_f32_e64 s6, v0, v1 + {"EqF32E64", vop3(rdna4::kVCmpEqF32Vop3, V1), 32, true}, + // v_cmp_eq_f16_e32 vcc_lo, v0.l, v1.l + {"EqF16E32", vopc(rdna4::kVCmpEqF16Vopc, 1), 16, false}, + // v_cmp_eq_f16_e64 s6, v0.l, v1.l + {"EqF16E64", vop3(rdna4::kVCmpEqF16Vop3, V1), 16, true}, + // v_cmp_eq_f64_e32 vcc_lo, v[0:1], v[2:3] + {"EqF64E32", vopc(rdna4::kVCmpEqF64Vopc, 2), 64, false}, + // v_cmp_eq_f64_e64 s6, v[0:1], v[2:3] + {"EqF64E64", vop3(rdna4::kVCmpEqF64Vop3, V2), 64, true}, + }}; + std::vector cases; + for (const Form &form : forms) + for (const auto &[mode, expected] : form.width == 32 ? F32_MODES : F16_F64_MODES) { + constexpr char HEX[] = "0123456789abcdef"; + const std::string suffix = {HEX[mode >> 4], HEX[mode & 0xfu]}; + cases.push_back({std::string(form.name) + "Mode" + suffix, form.words, form.width, form.vop3, + mode, expected}); + } + return cases; +} + +class ValuCompareInputFlushTest : public testing::TestWithParam {}; + +// Restores the process force-scalar gate if an assertion leaves the test early. +struct ForceScalarGuard { + bool original = util::force_scalar(); + ~ForceScalarGuard() { util::set_force_scalar_for_testing(original); } +}; + +TEST_P(ValuCompareInputFlushTest, HonorsModeOnScalarAndSimdPaths) { + const CompareInputFlushCase &test = GetParam(); + amdgpu::GpuMemory memory("compare_memory"); + amdgpu::L2Cache cache("compare_cache"); + cache.set_backing_memory(&memory); + amdgpu::ComputeUnitCore::Config config{}; + config.arch = ROCJITSU_CODE_ARCH_RDNA4; + config.num_wf_slots = 1; + config.sgprs_per_wf = 106; + config.vgprs_per_wf = 256; + config.lds_size_kb = 64; + std::unique_ptr cu = + amdgpu::ComputeUnitCore::create("valu_compare", config, &memory, &cache); + std::unique_ptr decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4); + amdgpu::Wavefront *wave = cu->dispatch_wf(0, 0, 106, 256); + ASSERT_NE(wave, nullptr); + DecodeResult decoded = decoder->decode(test.words.data()); + ASSERT_FALSE(decoded.failed()); + std::unique_ptr instruction = std::move(decoded).value(); + const uint32_t vgpr = wave->vgpr_alloc().base; + const uint32_t sgpr = wave->sgpr_alloc().base; + const bool wave64 = wave->wf_size() == 64; + const uint64_t full_exec = wave64 ? ~uint64_t{0} : 0xffffffffu; + const uint64_t expected_lanes = test.expected | (uint64_t{test.expected} << 32); + // +tiny, -tiny, +0 and 1.0 in each format. + constexpr std::array F16_SOURCES = {0x7e000001u, 0x7e008001u, 0x7e000000u, + 0x7e003c00u}; + constexpr std::array F32_SOURCES = {0x00000001u, 0x80000001u, 0u, 0x3f800000u}; + constexpr std::array F64_HIGH = {0u, 0x80000000u, 0u, 0x3ff00000u}; + constexpr std::array F64_LOW = {1u, 1u, 0u, 0u}; + ForceScalarGuard guard; + for (const bool scalar : {true, false}) { + util::set_force_scalar_for_testing(scalar); + for (const uint64_t exec : {full_exec, full_exec & uint64_t{0x0f0f0f0f0f0f0f0f}}) { + wave->set_exec(exec); + wave->set_mode_raw(test.mode); + for (uint32_t lane = 0; lane < wave->wf_size(); ++lane) { + if (test.width == 64) { + cu->write_vgpr(vgpr + 0, lane, F64_LOW[lane % 4]); + cu->write_vgpr(vgpr + 1, lane, F64_HIGH[lane % 4]); + cu->write_vgpr(vgpr + 2, lane, 0u); + cu->write_vgpr(vgpr + 3, lane, 0u); + } else { + cu->write_vgpr(vgpr + 0, lane, + test.width == 16 ? F16_SOURCES[lane % 4] : F32_SOURCES[lane % 4]); + cu->write_vgpr(vgpr + 1, lane, test.width == 16 ? 0x7e000000u : 0u); + } + } + // Stale destination bits must be replaced, including inactive lanes. + wave->set_vcc_mask(uint64_t{0xdeadbeefdeadbeef}); + cu->write_sgpr(sgpr + 6, 0xdeadbeefu); + cu->write_sgpr(sgpr + 7, 0xdeadbeefu); + ASSERT_TRUE(cu->execute_instruction(instruction.get(), *wave).succeeded()); + uint64_t result = wave->vcc_mask(); + if (test.vop3) + result = cu->read_sgpr(sgpr + 6) | + (wave64 ? uint64_t{cu->read_sgpr(sgpr + 7)} << 32 : uint64_t{0}); + EXPECT_EQ(result & full_exec, expected_lanes & exec) + << (scalar ? "scalar" : "SIMD") << " exec 0x" << std::hex << exec; + } + } + wave->halt(); +} + +INSTANTIATE_TEST_SUITE_P(CompareInputFlush, ValuCompareInputFlushTest, + testing::ValuesIn(compare_input_flush_cases()), + [](const testing::TestParamInfo &info) { + return info.param.name; + }); + } // namespace diff --git a/emulation/rocjitsu/tests/vop3_conversion_modifiers_test.cpp b/emulation/rocjitsu/tests/vop3_conversion_modifiers_test.cpp index b7670ce51a8b..26d82c06b205 100644 --- a/emulation/rocjitsu/tests/vop3_conversion_modifiers_test.cpp +++ b/emulation/rocjitsu/tests/vop3_conversion_modifiers_test.cpp @@ -9,6 +9,7 @@ #include "rocjitsu/vm/amdgpu/l2_cache.h" #include "rocjitsu/vm/amdgpu/memory_pipeline.h" #include "rocjitsu/vm/amdgpu/wavefront.h" +#include "util/data_types.h" #include "util/simd.h" #include "util/simd_test_hooks.h" @@ -21,6 +22,7 @@ #include #include #include +#include namespace { @@ -271,4 +273,475 @@ TEST_P(Vop3ConversionModifierTest, NearestRoundsTiesUpWithoutRoundingAdjacentInp } } +TEST_P(Vop3ConversionModifierTest, PackedRtzPreservesIndependentSourceModifiers) { + struct Case { + uint32_t input; + uint16_t packed; + }; + // Explicit RTZ results include discarded mantissa bits, signed zero, + // a half subnormal, and finite overflow. The conversion ignores MODE rounding. + constexpr Case cases[] = {{0xbf803fff, 0xbc01}, {0x40490fdb, 0x4248}, {0x80000000, 0x8000}, + {0x477fffff, 0x7bff}, {0xb3ffffff, 0x8001}, {0x3fffffff, 0x3fff}, + {0xc0200000, 0xc100}, {0x3f000000, 0x3800}}; + for (bool force_scalar : {false, true}) { + ForceScalarGuard guard(force_scalar); + amdgpu::GpuMemory memory("packed_rtz_memory"); + amdgpu::L2Cache l2("packed_rtz_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = GetParam(); + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = gfx9() ? 102 : 106; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("packed_rtz", cfg, &memory, &l2); + auto decoder = Decoder::create(GetParam()); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16); + ASSERT_NE(wf, nullptr); + const auto vb = wf->vgpr_alloc().base; + for (uint32_t abs_mask = 0; abs_mask < 4; ++abs_mask) { + for (uint32_t neg_mask = 0; neg_mask < 4; ++neg_mask) { + // GFX9 uses opcode 662; later encodings use opcode 303. + const std::array words{(gfx9() ? 0xd0000002u : 0xd4000002u) | + ((gfx9() ? 662u : 303u) << 16) | (abs_mask << 8), + 256u | (257u << 9) | (neg_mask << 29)}; + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + for (uint32_t mode : {0u, 0xfu}) { + wf->set_mode_raw(mode); + for (uint64_t exec : {wf->wf_size() == 64 ? ~0ull : 0xffffffffull, 0x55ull}) { + SCOPED_TRACE(testing::Message() + << "scalar=" << force_scalar << " abs_mask=" << abs_mask + << " neg_mask=" << neg_mask << " mode=" << mode << " exec=" << exec); + wf->set_exec(exec); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + cu->write_vgpr(vb, lane, cases[lane % std::size(cases)].input); + cu->write_vgpr(vb + 1, lane, cases[(lane + 3) % std::size(cases)].input); + cu->write_vgpr(vb + 2, lane, 0xdeadbeef); + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + uint32_t expected = 0; + for (uint32_t src = 0; src < 2; ++src) { + uint32_t half = cases[(lane + 3 * src) % std::size(cases)].packed; + if (abs_mask & (1u << src)) + half &= 0x7fff; + if (neg_mask & (1u << src)) + half ^= 0x8000; + expected |= half << (16 * src); + } + EXPECT_EQ(cu->read_vgpr(vb + 2, lane), exec & (1ull << lane) ? expected : 0xdeadbeefu) + << "lane=" << lane; + } + } + } + } + } + wf->halt(); + } +} + +TEST_P(Vop3ConversionModifierTest, PackedRtzDppPermutesBeforeSourceModifiers) { + if (gfx9() || GetParam() == ROCJITSU_CODE_ARCH_RDNA1 || GetParam() == ROCJITSU_CODE_ARCH_RDNA2) + GTEST_SKIP() << "This DPP8 encoding is available on RDNA3/3.5/4 and CDNA5."; + for (bool force_scalar : {false, true}) { + ForceScalarGuard guard(force_scalar); + amdgpu::GpuMemory memory("packed_rtz_dpp_memory"); + amdgpu::L2Cache l2("packed_rtz_dpp_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = GetParam(); + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 106; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("packed_rtz_dpp", cfg, &memory, &l2); + auto decoder = Decoder::create(GetParam()); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16); + ASSERT_NE(wf, nullptr); + const auto vb = wf->vgpr_alloc().base; + // LLVM gfx11_asm_vop3_dpp8_from_vop2.s encodes v5, |v1|, -v2 + // with source-0 lanes reversed within each group of eight. + const uint32_t words[] = {0xd52f0105, 0x400204e9, 0x05397701}; + std::unique_ptr inst(decode_valid(*decoder, words)); + ASSERT_NE(inst, nullptr); + wf->set_exec(wf->wf_size() == 64 ? ~0ull : 0xffffffffull); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + cu->write_vgpr(vb + 1, lane, std::bit_cast(-(1.0f + (lane % 8) / 8.0f))); + cu->write_vgpr(vb + 2, lane, std::bit_cast(2.0f + (lane % 8) / 4.0f)); + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const uint32_t lo = 0x3c00 + (7 - lane % 8) * 128; + const uint32_t hi = 0xc000 + (lane % 8) * 128; + EXPECT_EQ(cu->read_vgpr(vb + 5, lane), lo | (hi << 16)) + << "lane=" << lane << " scalar=" << force_scalar; + } + wf->halt(); + } +} + +TEST_P(Vop3ConversionModifierTest, PackedNormalizedRoundsOnceAndSaturatesSymmetrically) { + struct Case { + uint32_t input; + uint16_t unorm, snorm, unorm_negated = 0; + }; + // Physical RDNA3/4 witnesses: false FP32 midpoint ties, signed saturation, + // exact midpoints, infinities and NaN. The normalized conversion ignores MODE. + constexpr Case second_source{0x3f000000, 0x8000, 0x4000}; + constexpr Case f32_only_cases[] = { + {0x37c000c0, 0x0001, 0x0001}, {0x386000e0, 0x0003, 0x0002}, + {0x38b000b0, 0x0005, 0x0003}, {0x38f000f0, 0x0007, 0x0004}, + {0xbf800080, 0x0000, 0x8001, 0xffff}, {0xbf7fff00, 0x0000, 0x8001, 0xfffe}, + {0xbf7ffb00, 0x0000, 0x8003, 0xfffa}, {0xbf7ff700, 0x0000, 0x8005, 0xfff6}}; + // These witnesses are exactly representable in FP16 and exercise both widths. + constexpr Case exact_half_cases[] = {{0x3f002000, 0x801f, 0x400f}, + {0x3f004000, 0x803f, 0x401f}, + {0x3f006000, 0x805f, 0x402f}, + {0x3f008000, 0x807f, 0x403f}, + {0x00000000, 0x0000, 0x0000}, + {0x33800000, 0x0000, 0x0000}, + second_source, + {0x3f800000, 0xffff, 0x7fff}, + {0x40000000, 0xffff, 0x7fff}, + {0x7f800000, 0xffff, 0x7fff}, + {0x7fc00000, 0x0000, 0x0000}, + {0xbf800000, 0x0000, 0x8001, 0xffff}, + {0xc0000000, 0x0000, 0x8001, 0xffff}, + {0xff800000, 0x0000, 0x8001, 0xffff}}; + for (bool force_scalar : {false, true}) { + ForceScalarGuard guard(force_scalar); + amdgpu::GpuMemory memory("packed_normalized_memory"); + amdgpu::L2Cache l2("packed_normalized_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = GetParam(); + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = gfx9() ? 102 : 106; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("packed_normalized", cfg, &memory, &l2); + auto decoder = Decoder::create(GetParam()); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16); + ASSERT_NE(wf, nullptr); + const auto vb = wf->vgpr_alloc().base; + for (bool half : {false, true}) { + const bool early_rdna = + GetParam() == ROCJITSU_CODE_ARCH_RDNA1 || GetParam() == ROCJITSU_CODE_ARCH_RDNA2; + if (half && (gfx9() || early_rdna)) + continue; + std::vector cases(std::begin(exact_half_cases), std::end(exact_half_cases)); + if (!half) + cases.insert(cases.end(), std::begin(f32_only_cases), std::end(f32_only_cases)); + for (bool signed_result : {false, true}) { + // F16 uses opcode 786; F32 uses 660 on GFX9, 872 on RDNA1/2, + // and 801 on later targets. The unsigned opcode follows the signed one. + const uint32_t opcode = (half ? 786 + : gfx9() ? 660 + : early_rdna ? 872 + : 801) + + !signed_result; + for (uint32_t abs_mask = 0; abs_mask < 4; ++abs_mask) { + for (uint32_t neg_mask = 0; neg_mask < 4; ++neg_mask) { + const uint32_t words[] = {(gfx9() ? 0xd0000002u : 0xd4000002u) | (opcode << 16) | + (abs_mask << 8), + 256u | (257u << 9) | (neg_mask << 29)}; + std::unique_ptr inst(decode_valid(*decoder, words)); + ASSERT_NE(inst, nullptr); + for (uint32_t mode : {0u, 0xfu}) { + wf->set_mode_raw(mode); + for (uint64_t exec : {wf->wf_size() == 64 ? ~0ull : 0xffffffffull, 0x55ull}) { + SCOPED_TRACE(testing::Message() + << "scalar=" << force_scalar << " half=" << half + << " signed=" << signed_result << " abs_mask=" << abs_mask + << " neg_mask=" << neg_mask << " mode=" << mode << " exec=" << exec); + wf->set_exec(exec); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const auto &c = cases[lane % cases.size()]; + const uint32_t input = + half ? util::f32_to_f16(std::bit_cast(c.input)) : c.input; + cu->write_vgpr(vb, lane, input); + cu->write_vgpr(vb + 1, lane, + half ? util::f32_to_f16(std::bit_cast(second_source.input)) + : second_source.input); + cu->write_vgpr(vb + 2, lane, 0xdeadbeef); + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const auto &c = cases[lane % cases.size()]; + uint32_t expected = 0; + for (uint32_t source = 0; source < 2; ++source) { + const auto &value = source == 0 ? c : second_source; + const bool negate = + static_cast((abs_mask & (1u << source)) && (value.input >> 31)) ^ + static_cast(neg_mask & (1u << source)); + const uint16_t packed = + signed_result + ? static_cast(negate ? -static_cast(value.snorm) + : value.snorm) + : negate ? value.unorm_negated + : value.unorm; + expected |= static_cast(packed) << (16 * source); + } + EXPECT_EQ(cu->read_vgpr(vb + 2, lane), + exec & (1ull << lane) ? expected : 0xdeadbeefu) + << "lane=" << lane << " input=" << c.input; + } + } + } + } + } + } + } + wf->halt(); + } +} + +TEST_P(Vop3ConversionModifierTest, UnaryNormalizedHalfRoundsOnceAndAppliesModifiers) { + struct Case { + uint16_t input, unorm_magnitude, snorm_magnitude; + }; + // Physical RDNA3/4 witnesses: exact halves, saturation, infinities and NaN. + // For 0x3801, FP32 scaling creates a false midpoint: 32799.49951171875 -> 32799.5. + constexpr Case cases[] = { + {0x0000, 0, 0}, {0x0001, 0, 0}, {0x3800, 0x8000, 0x4000}, + {0x3801, 0x801f, 0x400f}, {0xb801, 0x801f, 0x400f}, {0x3c00, 0xffff, 0x7fff}, + {0xbc00, 0xffff, 0x7fff}, {0x3c01, 0xffff, 0x7fff}, {0xbc01, 0xffff, 0x7fff}, + {0x7c00, 0xffff, 0x7fff}, {0xfc00, 0xffff, 0x7fff}, {0x7e01, 0, 0}}; + const bool early_rdna = + GetParam() == ROCJITSU_CODE_ARCH_RDNA1 || GetParam() == ROCJITSU_CODE_ARCH_RDNA2; + const bool preserve_high = !gfx9() && !early_rdna; + for (bool force_scalar : {false, true}) { + ForceScalarGuard guard(force_scalar); + amdgpu::GpuMemory memory("unary_normalized_memory"); + amdgpu::L2Cache l2("unary_normalized_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = GetParam(); + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = gfx9() ? 102 : 106; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("unary_normalized", cfg, &memory, &l2); + auto decoder = Decoder::create(GetParam()); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16); + ASSERT_NE(wf, nullptr); + const auto vb = wf->vgpr_alloc().base; + for (bool vop3 : {false, true}) { + for (bool signed_result : {false, true}) { + // GFX9 uses VOP1 77 / VOP3 397; later encodings use VOP1 99 / VOP3 483. + // The unsigned opcode follows the signed one. + const uint32_t opcode = + (vop3 ? (gfx9() ? 397u : 483u) : (gfx9() ? 77u : 99u)) + !signed_result; + for (uint32_t modifiers = 0; modifiers < (vop3 ? 4u : 1u); ++modifiers) { + const uint32_t abs_mask = modifiers & 1u; + const uint32_t neg_mask = modifiers >> 1; + const uint32_t words[] = { + vop3 ? ((gfx9() ? 0xd0000002u : 0xd4000002u) | (opcode << 16) | (abs_mask << 8)) + : (0x7e000100u | (2u << 17) | (opcode << 9)), + 256u | (neg_mask << 29)}; + std::unique_ptr inst(decode_valid(*decoder, words)); + ASSERT_NE(inst, nullptr); + for (uint32_t mode : {0u, 0x5u, 0xau, 0xfu}) { + wf->set_mode_raw(mode); + for (uint64_t exec : {wf->wf_size() == 64 ? ~0ull : 0xffffffffull, 0x55ull}) { + SCOPED_TRACE(testing::Message() + << "scalar=" << force_scalar << " vop3=" << vop3 + << " signed=" << signed_result << " abs_mask=" << abs_mask + << " neg_mask=" << neg_mask << " mode=" << mode << " exec=" << exec); + wf->set_exec(exec); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + cu->write_vgpr(vb, lane, 0xdead0000u | cases[lane % std::size(cases)].input); + cu->write_vgpr(vb + 2, lane, 0xabcd1234); + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const auto &c = cases[lane % std::size(cases)]; + const bool negative = ((c.input & 0x8000) != 0 && !abs_mask) ^ (neg_mask != 0); + const uint16_t low = + signed_result + ? static_cast(negative ? -c.snorm_magnitude : c.snorm_magnitude) + : (negative ? 0 : c.unorm_magnitude); + const uint32_t expected = low | (preserve_high ? 0xabcd0000u : 0); + EXPECT_EQ(cu->read_vgpr(vb + 2, lane), + exec & (1ull << lane) ? expected : 0xabcd1234u) + << "lane=" << lane << " input=" << c.input; + } + } + } + } + } + } + wf->halt(); + } +} + +TEST_P(Vop3ConversionModifierTest, HalfToFloatAppliesOutputModifiers) { + struct Case { + uint16_t input; + uint32_t mode, omod; + bool clamp; + uint32_t legacy, modern; + }; + // Captured on gfx1100/gfx1201. OMOD uses the F32 output-denorm control, + // while widening flushes half inputs using the F16 input-denorm control. + constexpr Case cases[] = { + {0x3400, 0, 1, false, 0x3f000000, 0x3f000000}, + {0x3400, 0, 2, false, 0x3f800000, 0x3f800000}, + {0x3400, 0, 3, false, 0x3e000000, 0x3e000000}, + {0x3400, 0x20, 1, false, 0x3e800000, 0x3f000000}, + {0x3400, 0x80, 1, false, 0x3f000000, 0x3f000000}, + {0x3400, 0x200, 1, false, 0x3e800000, 0x3f000000}, + {0x0001, 0x40, 1, false, 0x34000000, 0x34000000}, + {0x0001, 0x10, 1, false, 0, 0}, + {0x4000, 0, 0, true, 0x3f800000, 0x3f800000}, + {0xc000, 0, 0, true, 0, 0}, + {0x3800, 0, 2, true, 0x3f800000, 0x3f800000}, + {0x8000, 0, 1, false, 0, 0}, + {0x7c01, 0, 1, false, 0x7f802000, 0x7fc02000}, + {0x7c01, 0, 0, true, 0x7f802000, 0}, + }; + const bool modern = + GetParam() == ROCJITSU_CODE_ARCH_RDNA4 || GetParam() == ROCJITSU_CODE_ARCH_CDNA5; + amdgpu::GpuMemory memory("half_output_memory"); + amdgpu::L2Cache l2("half_output_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = GetParam(); + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = gfx9() ? 102 : 106; + cfg.vgprs_per_wf = 16; + auto cu = amdgpu::ComputeUnitCore::create("half_output", cfg, &memory, &l2); + auto decoder = Decoder::create(GetParam()); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); + ASSERT_NE(wf, nullptr); + const uint32_t vb = wf->vgpr_alloc().base; + for (bool force_scalar : {false, true}) { + ForceScalarGuard guard(force_scalar); + for (const auto &test : cases) { + const uint32_t words[] = {(gfx9() ? 0xd14b0002u : 0xd58b0002u) | (uint32_t{test.clamp} << 15), + 256u | (test.omod << 27)}; + std::unique_ptr inst(decode_valid(*decoder, words)); + ASSERT_NE(inst, nullptr); + wf->set_mode_raw(test.mode); + for (uint64_t exec : {wf->wf_size() == 64 ? ~0ull : 0xffffffffull, 0x55ull}) { + wf->set_exec(exec); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + cu->write_vgpr(vb, lane, test.input); + cu->write_vgpr(vb + 2, lane, 0xdeadbeef); + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) + EXPECT_EQ(cu->read_vgpr(vb + 2, lane), + exec & (1ull << lane) ? (modern ? test.modern : test.legacy) : 0xdeadbeefu) + << "input=" << test.input << " mode=" << test.mode << " omod=" << test.omod + << " clamp=" << test.clamp << " scalar=" << force_scalar << " lane=" << lane; + } + } + } + wf->halt(); +} + +TEST_P(Vop3ConversionModifierTest, HalfInputConversionsApplyModifiersAndMode) { + struct Case { + uint16_t half; + uint32_t widened; + uint16_t unsigned_value, signed_positive, signed_negative; + }; + // Raw outputs from physical gfx1100/gfx1201, including signaling NaNs. + constexpr Case cases[] = { + {0x0000, 0x00000000, 0, 0, 0}, + {0x0001, 0x33800000, 0, 0, 0}, + {0x03ff, 0x387fc000, 0, 0, 0}, + {0x0400, 0x38800000, 0, 0, 0}, + {0x3e00, 0x3fc00000, 1, 1, 0xffff}, + {0x47c0, 0x40f80000, 7, 7, 0xfff9}, + {0x77ff, 0x46ffe000, 32752, 32752, 0x8010}, + {0x7800, 0x47000000, 32768, 32767, 0x8000}, + {0x7bff, 0x477fe000, 65504, 32767, 0x8000}, + {0x7c00, 0x7f800000, 65535, 32767, 0x8000}, + {0x7c01, 0x7f802000, 0, 0, 0}, + {0x7e01, 0x7fc02000, 0, 0, 0}, + }; + const bool early_rdna = + GetParam() == ROCJITSU_CODE_ARCH_RDNA1 || GetParam() == ROCJITSU_CODE_ARCH_RDNA2; + const bool preserve_half = !early_rdna && !gfx9(); + const bool always_quiet = + GetParam() == ROCJITSU_CODE_ARCH_RDNA4 || GetParam() == ROCJITSU_CODE_ARCH_CDNA5; + for (bool force_scalar : {false, true}) { + ForceScalarGuard guard(force_scalar); + amdgpu::GpuMemory memory("half_input_memory"); + amdgpu::L2Cache l2("half_input_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = GetParam(); + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = gfx9() ? 102 : 106; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("half_input", cfg, &memory, &l2); + auto decoder = Decoder::create(GetParam()); + auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16); + ASSERT_NE(wf, nullptr); + const auto vb = wf->vgpr_alloc().base; + for (bool vop3 : {false, true}) { + for (uint32_t kind = 0; kind < 3; ++kind) { + const bool floating = kind == 2; + // Unsigned/signed VOP1 opcodes 59/60 on GFX9, 82/83 on later targets. + const uint32_t vop1_op = floating ? 11u : (gfx9() ? 59u : 82u) + kind; + const uint32_t opcode = vop1_op + (vop3 ? (gfx9() ? 320u : 384u) : 0u); + for (uint32_t half = 0; + half < (vop3 && !early_rdna && (floating || preserve_half) ? 2u : 1u); ++half) { + for (uint32_t modifiers = 0; modifiers < (vop3 ? 4u : 1u); ++modifiers) { + const uint32_t abs_mask = modifiers & 1u; + const uint32_t neg_mask = modifiers >> 1; + const uint32_t opsel = half | (!floating ? half << 3 : 0u); + const uint32_t words[] = {vop3 ? ((gfx9() ? 0xd0000002u : 0xd4000002u) | + (opcode << 16) | (abs_mask << 8) | (opsel << 11)) + : (0x7e000100u | (2u << 17) | (opcode << 9)), + 256u | (neg_mask << 29)}; + std::unique_ptr inst(decode_valid(*decoder, words)); + ASSERT_NE(inst, nullptr); + for (uint32_t mode : {0u, 0x20fu, 0x40u, 0x24fu, 0x80u, 0x28fu, 0xc0u, 0x2cfu}) { + wf->set_mode_raw(mode); + for (uint64_t exec : {wf->wf_size() == 64 ? ~0ull : 0xffffffffull, 0x55ull}) { + SCOPED_TRACE(testing::Message() + << "scalar=" << force_scalar << " vop3=" << vop3 << " kind=" << kind + << " half=" << half << " abs_mask=" << abs_mask + << " neg_mask=" << neg_mask << " mode=" << mode << " exec=" << exec); + wf->set_exec(exec); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const uint32_t input = + cases[(lane / 2) % std::size(cases)].half | ((lane & 1u) << 15); + cu->write_vgpr(vb, lane, half ? (input << 16) | 0xdead : input | 0xdead0000u); + cu->write_vgpr(vb + 2, lane, 0xabcd1234u); + } + ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded()); + for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) { + const auto &c = cases[(lane / 2) % std::size(cases)]; + const bool negative = ((lane & 1u) && !abs_mask) ^ (neg_mask != 0); + uint32_t expected; + if (floating) { + expected = c.widened; + if (!(mode & 0x40u) && c.half < 0x0400u) + expected = 0; + if ((always_quiet || (mode & 0x200u)) && c.half > 0x7c00u) + expected |= 0x00400000u; + expected |= negative ? 0x80000000u : 0; + } else { + const uint32_t value = kind == 0 + ? (negative ? 0 : c.unsigned_value) + : (negative ? c.signed_negative : c.signed_positive); + expected = preserve_half + ? (half ? (value << 16) | 0x1234u : value | 0xabcd0000u) + : value; + } + EXPECT_EQ(cu->read_vgpr(vb + 2, lane), + exec & (1ull << lane) ? expected : 0xabcd1234u) + << "lane=" << lane << " input=" << c.half; + } + } + } + } + } + } + } + wf->halt(); + } +} + } // namespace diff --git a/emulation/rocjitsu/tests/vskip_test.cpp b/emulation/rocjitsu/tests/vskip_test.cpp new file mode 100644 index 000000000000..a51bf9ca945f --- /dev/null +++ b/emulation/rocjitsu/tests/vskip_test.cpp @@ -0,0 +1,329 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "mma_test_util.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna1/test_encodings.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna2/test_encodings.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/test_encodings.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/test_encodings.h" +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" +#include "rocjitsu/vm/amdgpu/compute_unit.h" +#include "rocjitsu/vm/amdgpu/gpu_memory.h" +#include "rocjitsu/vm/amdgpu/l2_cache.h" +#include "rocjitsu/vm/amdgpu/matrix_coexecution.h" +#include "rocjitsu/vm/plugins/execution_plugin_group.h" + +#include + +#include +#include +#include +#include +#include + +namespace { +using namespace rocjitsu; +using namespace rocjitsu::amdgpu; + +constexpr uint64_t kProgram = 0x100000; +constexpr uint32_t kEnable = 0xBF108081u; // s_setvskip 1, 0 +constexpr uint32_t kDisable = 0xBF108080u; // s_setvskip 0, 0 +constexpr uint32_t kMoveVector = 0x7E000281u; // v_mov_b32 v0, 1 + +class IssueRecorder final : public ExecutionPlugin { +public: + IssueRecorder() : ExecutionPlugin("vskip_recorder") {} + void onAmdgpuBeforeExecuteInstruction(uint64_t, const Instruction &inst, Wavefront &) override { + issued.emplace_back(inst.mnemonic()); + } + std::vector issued; +}; + +class VskipTest : public ::testing::TestWithParam { +protected: + void SetUp() override { + ComputeUnitCore::Config config{}; + config.arch = GetParam(); + config.num_wf_slots = 1; + config.sgprs_per_wf = 104; + config.vgprs_per_wf = 256; + config.lds_size_kb = 64; + config.functional_quantum = 1; + config.memory_wait_diagnostics = MemoryWaitDiagnostics::Warn; + cu = ComputeUnitCore::create("vskip_cu", config, &memory, &l2); + decoder = Decoder::create(GetParam()); + ASSERT_NE(cu, nullptr); + ASSERT_NE(decoder, nullptr); + wf = cu->dispatch_wf(0, kProgram, 104, 256); + ASSERT_NE(wf, nullptr); + wf->set_exec(~uint64_t{0}); + } + + void TearDown() override { + if (wf && !wf->is_halted()) + wf->halt(); + } + + void program(std::span words) { + for (size_t i = 0; i < words.size(); ++i) + memory.write32(kProgram + i * 4, words[i]); + // Supply the complete instruction fetch window at the end of the program. + for (size_t i = 0; i < 4; ++i) + memory.write32(kProgram + (words.size() + i) * 4, 0xBF800000u); + } + + void step(unsigned bytes = 4) { + const uint64_t previous = wf->pc; + (void)cu->step(); + EXPECT_FALSE(wf->is_halted()); + EXPECT_EQ(wf->pc, previous + bytes); + } + + GpuMemory memory{"vskip_memory"}; + L2Cache l2{"vskip_l2"}; + std::unique_ptr cu; + std::unique_ptr decoder; + Wavefront *wf = nullptr; +}; + +TEST_P(VskipTest, SelectedSourceBitUpdatesOnlyVskip) { + // Index 63 aliases bit 31. Literal operands and EXEC=0 still update MODE. + const std::array words{0xBF100100u, kDisable, 0xBF109FFFu, 0x80000000u, 0xBF108081u, kDisable}; + program(words); + constexpr uint32_t mode = 0x000003C5u; + wf->set_mode_raw(mode); + wf->write_scc(1); + wf->set_exec(0); + wf->set_vcc(0x123456789ABCDEF0ull); + cu->write_sgpr(wf->sgpr_alloc().base, 0x80000000u); + cu->write_sgpr(wf->sgpr_alloc().base + 1, 63); + for (unsigned bytes : {4, 4, 8, 4, 4}) { + const bool enabled = wf->pc != kProgram + 4 && wf->pc != kProgram + 20; + step(bytes); + EXPECT_EQ(wf->mode_raw(), mode | (enabled ? Wavefront::VSKIP_BIT : 0)); + EXPECT_EQ(wf->read_scc(), 1u); + EXPECT_EQ(wf->exec(), 0u); + EXPECT_EQ(wf->vcc(), 0x123456789ABCDEF0ull); + } +} + +TEST_P(VskipTest, SkipsVectorEffectsAndResumesAfterScalarDisable) { + const std::array words{kEnable, kMoveVector, + 0x7E080500u, // v_readfirstlane_b32 s4, v0 also skips with EXEC=0. + 0xBE850087u, // s_mov_b32 s5, 7 still executes. + kDisable, kMoveVector}; + program(words); + cu->write_vgpr(wf->vgpr_alloc().base, 0, 99); + cu->write_sgpr(wf->sgpr_alloc().base + 4, 42); + auto plugins = std::make_shared(PluginSinkConfig{}); + auto recorder = std::make_unique(); + auto *record = recorder.get(); + ASSERT_TRUE(plugins->add(std::move(recorder))); + cu->set_plugin_group(plugins); + step(); + step(); + wf->set_exec(0); + step(); + step(); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, 0), 99u); + EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 4), 42u); + EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 5), 7u); + wf->set_exec(~uint64_t{0}); + step(); + step(); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, 0), 1u); + EXPECT_EQ(record->issued, + (std::vector{"s_setvskip", "s_mov_b32", "s_setvskip", "v_mov_b32_e32"})); +} + +TEST_P(VskipTest, SetregAndGetregShareModeStateAndRedispatchClearsIt) { + // MODE writes require two wait states before a vector op or S_GETREG. + constexpr uint32_t kModeWait = 0xBF800001u; // s_nop 1 + const std::array words{0xBA000701u, 1u, // s_setreg_imm32_b32 hwreg(MODE,28,1), 1 + kModeWait, kMoveVector, + 0xB8860701u, // s_getreg_b32 s6, hwreg(MODE,28,1) + 0xB9070701u, // s_setreg_b32 hwreg(MODE,28,1), s7 + kModeWait, kMoveVector, kEnable}; + program(words); + step(8); + step(); + step(); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, 0), 0u); + step(); + EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 6), 1u); + step(); + step(); + step(); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, 0), 1u); + step(); + ASSERT_NE(wf->mode_raw() & Wavefront::VSKIP_BIT, 0u); + wf->halt(); + const uint64_t resumed_vector = kProgram + (words.size() - 2) * sizeof(uint32_t); + wf = cu->dispatch_wf(1, resumed_vector, 104, 256); + ASSERT_NE(wf, nullptr); + EXPECT_EQ(wf->mode_raw() & Wavefront::VSKIP_BIT, 0u); +} + +TEST_P(VskipTest, VectorEncodingFamiliesSkipWithoutIssueOrWaitCounterEffects) { + // Sample the available vector encoding families. Dedicated tests below cover + // scalar-destination VOP3 and matrix forms absent from this table. Unsupported + // vector operations must also skip before reaching their execute callback. + auto check = [&](const auto &encodings) { + std::vector words{kEnable}; + unsigned skipped = 0; + for (const auto &entry : encodings) { + std::string_view name = entry.mnemonic; + if (!(name.starts_with("v_") || name.starts_with("buffer_") || name.starts_with("tbuffer_") || + name.starts_with("image_") || name.starts_with("ds_") || name.starts_with("flat_") || + name.starts_with("global_") || name.starts_with("scratch_") || name == "exp")) + continue; + // The generated sample table has at most two words; wider MFMA extension + // encodings have separate coverage in the async test below. + const std::array padded{entry.words[0], entry.words[1], 0, 0}; + auto inst = decoder->decode(padded.data()); + // Generated samples can violate operand constraints (for example, the + // v_readfirstlane sample selects an SGPR source). Dedicated cases use + // valid encodings and require successful decoding. + if (inst.failed() || inst.value()->size() > 8) + continue; + ASSERT_TRUE(inst.value()->is_vskip_affected()) << name; + words.insert(words.end(), padded.begin(), padded.begin() + inst.value()->size() / 4); + ++skipped; + } + words.push_back(kDisable); + program(words); + auto plugins = std::make_shared(PluginSinkConfig{}); + auto recorder = std::make_unique(); + auto *record = recorder.get(); + ASSERT_TRUE(plugins->add(std::move(recorder))); + cu->set_plugin_group(plugins); + wf->wait_counters().vmcnt = 3; + wf->wait_counters().lgkmcnt = 2; + wf->wait_counters().expcnt = 1; + for (unsigned i = 0; i < skipped + 2; ++i) + (void)cu->step(); + EXPECT_FALSE(wf->is_halted()); + EXPECT_EQ(wf->pc, kProgram + words.size() * 4); + EXPECT_EQ(record->issued, (std::vector{"s_setvskip", "s_setvskip"})); + EXPECT_EQ(wf->wait_counters().vmcnt, 3u); + EXPECT_EQ(wf->wait_counters().lgkmcnt, 2u); + EXPECT_EQ(wf->wait_counters().expcnt, 1u); + wf->wait_counters() = {}; + }; + switch (GetParam()) { + case ROCJITSU_CODE_ARCH_CDNA1: + check(cdna1::test_data::ENCODINGS); + break; + case ROCJITSU_CODE_ARCH_CDNA2: + check(cdna2::test_data::ENCODINGS); + break; + case ROCJITSU_CODE_ARCH_CDNA3: + check(cdna3::test_data::ENCODINGS); + break; + case ROCJITSU_CODE_ARCH_CDNA4: + check(cdna4::test_data::ENCODINGS); + break; + default: + FAIL() << "Unexpected target"; + } +} + +TEST_P(VskipTest, Vop3SubencodingsSkipWithoutIssue) { + // These XML encoding names omit ENC_: VOP3_SDST_ENC and VOP3P_MFMA. + // Neither form is sampled by the generated mnemonic-only encoding table. + const uint32_t matrix = GetParam() == ROCJITSU_CODE_ARCH_CDNA1 ? 0xD3C50000u : 0xD3C58000u; + const std::array, 2> encodings{{ + {0xD1E00806u, 0x040A0300u, 0, 0}, // v_div_scale_f32 v6, s[8:9], v0, v1, v2 + {matrix, 0x04020300u, 0, 0}, // v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[0:3] + }}; + std::vector words{kEnable}; + for (const auto &encoding : encodings) { + auto inst = decoder->decode(encoding.data()); + ASSERT_TRUE(inst.succeeded()); + ASSERT_EQ(inst.value()->size(), 8); + EXPECT_TRUE(inst.value()->is_vskip_affected()) << inst.value()->mnemonic(); + words.insert(words.end(), encoding.begin(), encoding.begin() + 2); + } + words.insert(words.end(), {kDisable, kMoveVector}); + program(words); + cu->write_vgpr(wf->vgpr_alloc().base + 6, 0, 0x12345678u); + cu->write_sgpr(wf->sgpr_alloc().base + 8, 0xA5A5A5A5u); + auto plugins = std::make_shared(PluginSinkConfig{}); + auto recorder = std::make_unique(); + auto *record = recorder.get(); + ASSERT_TRUE(plugins->add(std::move(recorder))); + cu->set_plugin_group(plugins); + for (unsigned bytes : {4, 8, 8, 4, 4}) + step(bytes); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base + 6, 0), 0x12345678u); + EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 8), 0xA5A5A5A5u); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, 0), 1u); + EXPECT_EQ(record->issued, + (std::vector{"s_setvskip", "s_setvskip", "v_mov_b32_e32"})); +} + +INSTANTIATE_TEST_SUITE_P(Cdna, VskipTest, + ::testing::Values(ROCJITSU_CODE_ARCH_CDNA1, ROCJITSU_CODE_ARCH_CDNA2, + ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA4)); + +TEST(VskipAsyncTest, ExtendedMfmaSkipsBeforeAsyncSubmission) { + GpuMemory memory{"vskip_async_memory"}; + L2Cache l2{"vskip_async_l2"}; + ComputeUnitCore::Config config{}; + config.arch = ROCJITSU_CODE_ARCH_CDNA4; + config.num_wf_slots = 1; + config.sgprs_per_wf = 104; + config.vgprs_per_wf = 512; + config.async_resources = std::make_shared(1); + auto cu = ComputeUnitCore::create("vskip_async_cu", config, &memory, &l2); + auto *wf = cu->dispatch_wf(0, kProgram, 104, 256); + ASSERT_NE(wf, nullptr); + wf->set_exec(~uint64_t{0}); + const auto matrix = mma_test::make_cdna4_mfma_scale_words(45, 1, 448, 449); + memory.write32(kProgram, kEnable); + for (size_t i = 0; i < matrix.size(); ++i) + memory.write32(kProgram + 4 + i * 4, matrix[i]); + memory.write32(kProgram + 20, kDisable); + memory.write32(kProgram + 24, kMoveVector); + for (unsigned i = 28; i < 48; i += 4) + memory.write32(kProgram + i, 0xBF800000u); + cu->write_vgpr(wf->vgpr_alloc().base + 64, 0, 0x3F800000); + (void)cu->step(); + (void)cu->step(); + EXPECT_FALSE(wf->is_halted()); + EXPECT_EQ(wf->pc, kProgram + 20); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base + 64, 0), 0x3F800000u); + EXPECT_TRUE(wf->wait_counters().empty()); + (void)cu->step(); + (void)cu->step(); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, 0), 1u); + wf->halt(); +} + +TEST(VskipTargetTest, ModeBitDoesNotSuppressRdnaOrCdna5) { + for (auto arch : {ROCJITSU_CODE_ARCH_RDNA1, ROCJITSU_CODE_ARCH_RDNA2, ROCJITSU_CODE_ARCH_RDNA3, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) { + SCOPED_TRACE(arch); + GpuMemory memory{"vskip_other_memory"}; + L2Cache l2{"vskip_other_l2"}; + ComputeUnitCore::Config config{}; + config.arch = arch; + config.num_wf_slots = 1; + config.sgprs_per_wf = 106; + config.vgprs_per_wf = 256; + auto cu = ComputeUnitCore::create("vskip_other_cu", config, &memory, &l2); + auto *wf = cu->dispatch_wf(0, kProgram, 106, 256); + ASSERT_NE(wf, nullptr); + wf->set_exec(~uint64_t{0}); + wf->set_mode_raw(Wavefront::VSKIP_BIT); + memory.write32(kProgram, kMoveVector); + for (unsigned i = 4; i < 16; i += 4) + memory.write32(kProgram + i, 0xBF800000u); + (void)cu->step(); + EXPECT_EQ(wf->pc, kProgram + 4); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, 0), 1u); + wf->halt(); + } +} +} // namespace diff --git a/emulation/rocjitsu/tests/xcd_distribution_test.cpp b/emulation/rocjitsu/tests/xcd_distribution_test.cpp index 1fb4778627de..d35f04ef3c82 100644 --- a/emulation/rocjitsu/tests/xcd_distribution_test.cpp +++ b/emulation/rocjitsu/tests/xcd_distribution_test.cpp @@ -1426,7 +1426,7 @@ TEST(XcdDistributionTest, ScratchAllocationProbeDoesNotFaultAndReusesBacking) { process.map_pages(kRingVa, ring.data(), sizeof(ring)); process.map_pages(kQueueVa, &queue_desc, sizeof(queue_desc)); uint64_t doorbell = 1; - amdgpu::AqlQueueConfig hw{}; + amdgpu::ComputeQueueConfig hw{}; hw.address_space = address_space; hw.process_id = process.process_id(); hw.queue_id = 1; diff --git a/emulation/rocjitsu/tests/xcd_fanout_kfd_test.cpp b/emulation/rocjitsu/tests/xcd_fanout_kfd_test.cpp index 62f34ae141e0..80f09c080639 100644 --- a/emulation/rocjitsu/tests/xcd_fanout_kfd_test.cpp +++ b/emulation/rocjitsu/tests/xcd_fanout_kfd_test.cpp @@ -8,7 +8,7 @@ /// Every other fan-out regression registers an AqlQueue straight against a /// command processor: vmid 0, no process page table, and a kernel that touches /// no private memory. This brings the driver up instead, creates a compute queue -/// through CREATE_QUEUE -- the path that sets AqlQueueConfig::xcd_fanout -- and +/// through CREATE_QUEUE -- the path that sets ComputeQueueConfig::xcd_fanout -- and /// dispatches a kernel that really does spill, behind a real process page table. /// /// What that reaches, and what it does not: the scratch *addressing* contract is diff --git a/emulation/rocjitsu/tools/rocjitsu/main.cpp b/emulation/rocjitsu/tools/rocjitsu/main.cpp index cf33ad787048..afd1a687531d 100644 --- a/emulation/rocjitsu/tools/rocjitsu/main.cpp +++ b/emulation/rocjitsu/tools/rocjitsu/main.cpp @@ -25,6 +25,7 @@ #include "rocjitsu/kmd/linux/amdgpu_properties.h" #include "rocjitsu/kmd/linux/rpc.h" #include "rocjitsu/vm/amdgpu/partitioning.h" +#include "util/diagnostic.h" #include "embedded_schema.h" #include "launch_preload.h" @@ -506,11 +507,13 @@ int main(int argc, char *argv[]) { // Applying the budget to a copy keeps the override inside this launch: the // config the user named is never rewritten, and simulations built from other // configs in the same process tree keep the budgets their own configs ask for. - auto config_json_for_this_launch = [&]() { + util::StringDiagnostic launch_config_diagnostic; + auto config_json_for_this_launch = [&]() -> rocjitsu::FailureOr { std::string json = rocjitsu::config::read_config_file(abs_config); - if (cpu_thread_budget) - json = rocjitsu::config::json_with_cpu_thread_budget(json, *cpu_thread_budget); - return json; + if (!cpu_thread_budget) + return json; + return rocjitsu::config::json_with_cpu_thread_budget(json, *cpu_thread_budget, + launch_config_diagnostic.emitter()); }; if (thread_budget_table) { @@ -519,8 +522,14 @@ int main(int argc, char *argv[]) { return 1; } try { + rocjitsu::FailureOr json = config_json_for_this_launch(); + if (json.failed()) { + std::cerr << std::format("rocjitsu: thread allocation failed: {}\n", + launch_config_diagnostic.message()); + return 1; + } auto settings = rocjitsu::config::load_execution_thread_settings_from_string( - config_json_for_this_launch(), rocjitsu::kEmbeddedSchema); + json.value(), rocjitsu::kEmbeddedSchema); const uint32_t host = rocjitsu::amdgpu::available_host_threads(); std::cout << "Budget | num_threads | cpu_dispatch_threads per GPU | async_helper_threads | Total\n"; @@ -561,13 +570,15 @@ int main(int argc, char *argv[]) { auto write_config_for_this_launch = [&]() { if (!cpu_thread_budget) return true; - try { - abs_config = rocjitsu::config::write_effective_config(abs_config, *cpu_thread_budget, my_pid); - } catch (const std::exception &e) { - std::cerr << std::format("rocjitsu: {}\n", e.what()); + util::StringDiagnostic diagnostic; + rocjitsu::FailureOr written = rocjitsu::config::write_effective_config( + abs_config, *cpu_thread_budget, my_pid, diagnostic.emitter()); + if (written.failed()) { + std::cerr << std::format("rocjitsu: {}\n", diagnostic.message()); cleanup_runtime_files(my_pid); return false; } + abs_config = std::move(written.value()); return true; }; diff --git a/emulation/rocjitsu/website/README.md b/emulation/rocjitsu/website/README.md index 9527f82f0770..d66894190797 100644 --- a/emulation/rocjitsu/website/README.md +++ b/emulation/rocjitsu/website/README.md @@ -1,60 +1,21 @@ -# Rocjitsu Simulation Performance Dashboard - -React + Vite source for the Rocjitsu simulation-performance dashboard, with MUI components and -ECharts visualizations. This directory contains application source, build configuration, -and tests. Production builds fetch benchmark JSON from `rocjitsu-dashboard/data/` on -the `gh-pages-rocjitsu branch` in the `ROCm/rocm-systems` repository. - -**Live website:** [RocJitsu Performance Dashboard](https://rocm.github.io/rocm-systems/rocjitsu-dashboard/) - -## Quick start: preview with dummy data - -Use npm and a Node.js version matching the `engines` field in [package.json](package.json). -From the `rocm-systems` repository root: - -```bash -cd emulation/rocjitsu/website -npm ci -npm run build -- --mode fixtures -npm run preview -- --mode fixtures -``` - -Open http://localhost:4174 to visualize the website with dummy benchmark data. -Press **Ctrl+C** to stop the preview. The fixture build uses `.test-dist/` and the -fixture preview always uses port 4174; the production preview uses port 4173. - -The production build writes application files to `dist/` without dummy data and loads -JSON from the [`gh-pages-rocjitsu` branch](https://raw.githubusercontent.com/ROCm/rocm-systems/refs/heads/gh-pages-rocjitsu/rocjitsu-dashboard/data/). -The data directory must contain `metadata.json`, `index.json`, `test-catalogs/`, and `runs/`. -Before every data publication, validate the complete staged data directory: - -```bash -npm run validate:data -- /absolute/path/to/staged/data -``` - -Do not publish when this command fails. The browser assumes published input is valid -and fails closed instead of displaying partial history when invalid data bypasses the -publication gate. - -See the [build and test guide](docs/build-and-test.md) for local development with -dummy data or a local data directory (`npm run dev:data -- `), -browser setup, verification commands, and the production build. - -## Source layout - -| Path | Purpose | -| --- | --- | -| `index.html`, `src/main.jsx` | Browser entry points | -| `src/components/`, `src/hooks/` | Views, shared components, interaction state | -| `src/data/` | Data loading, validation, selectors, comparison logic | -| `src/theme/`, `src/utils/`, `src/index.css` | Theme, chart/display helpers, styles | -| `package.json`, `package-lock.json` | Commands and reproducible dependency installation | -| `*.config.js` | Vite, ESLint, Vitest, and Playwright configuration | -| `tests/unit/`, `tests/e2e/`, `tests/fixtures/` | Tests, helpers, and dummy fixtures | -| `docs/` | Build, testing, hosting, and data-contract documentation | - -## Further documentation - -- [Website data contract](docs/website-data-contract.md): JSON schema, comparison - rules, and data publication requirements. -- [Test fixtures](tests/fixtures/README.md): focused test cases and generated history. +# Rocjitsu web applications + +This directory contains the public web applications for Rocjitsu. + +| Directory | Purpose | Published path | +| --- | --- | --- | +| [dashboard/](dashboard/README.md) | Simulation-performance dashboard (React + Vite) | `/rocjitsu-dashboard/` | +| [handbook/](handbook/README.md) | Markdown documentation and blog (MkDocs) | `/rocjitsu/` | + +Each application owns its dependencies, build configuration, and tests. See its +README for local development instructions. The handbook reads its content +directly from [../docs/](../docs/), with [../README.md](../README.md) supplying +the homepage. These are the maintained sources; the website has no content copy. + +The [publishing workflow](../../../.github/workflows/rocjitsu-publish-website.yml) +compares current `develop` with each site's last publication and builds only affected +sites to their separate directories on `gh-pages`. Manual runs and changes to +the shared publishing workflow rebuild both. Benchmark data remains on +`gh-pages-rocjitsu`. See the +[handbook publishing setup](handbook/README.md#verification-and-publishing) +for the shared GitHub App and Pages configuration. diff --git a/emulation/rocjitsu/website/dashboard/.gitignore b/emulation/rocjitsu/website/dashboard/.gitignore new file mode 100644 index 000000000000..1761bc08632f --- /dev/null +++ b/emulation/rocjitsu/website/dashboard/.gitignore @@ -0,0 +1,8 @@ +/node_modules/ +/dist/ +/.test-dist/ +/.test-data/ +/.vite/ +/coverage/ +/test-results/ +/playwright-report/ diff --git a/emulation/rocjitsu/website/dashboard/README.md b/emulation/rocjitsu/website/dashboard/README.md new file mode 100644 index 000000000000..926aa7f914d3 --- /dev/null +++ b/emulation/rocjitsu/website/dashboard/README.md @@ -0,0 +1,73 @@ +# Rocjitsu Simulation Performance Dashboard + +React + Vite source for the Rocjitsu simulation-performance dashboard, with MUI components and +ECharts visualizations. This directory contains application source, build configuration, +and tests. Production builds fetch benchmark JSON from `rocjitsu-dashboard/data/` on +the `gh-pages-rocjitsu branch` in the `ROCm/rocm-systems` repository. + +**Live website:** [RocJitsu Performance Dashboard](https://rocm.github.io/rocm-systems/rocjitsu-dashboard/) + +## Quick start: preview with dummy data + +Use npm and a Node.js version matching the `engines` field in [package.json](package.json). +From the `rocm-systems` repository root: + +```bash +cd emulation/rocjitsu/website/dashboard +npm ci +npm run build -- --mode fixtures +npm run preview -- --mode fixtures +``` + +Open http://localhost:4174 to visualize the website with dummy benchmark data. +Press **Ctrl+C** to stop the preview. The fixture build uses `.test-dist/` and the +fixture preview always uses port 4174; the production preview uses port 4173. + +The production build writes application files to `dist/` without dummy data and loads +JSON from the [`gh-pages-rocjitsu` branch](https://raw.githubusercontent.com/ROCm/rocm-systems/refs/heads/gh-pages-rocjitsu/rocjitsu-dashboard/data/). +The data directory must contain `metadata.json`, `index.json`, `test-catalogs/`, and `runs/`. +Before every data publication, validate the complete staged data directory: + +```bash +npm run validate:data -- /absolute/path/to/staged/data +``` + +Do not publish when this command fails. The browser assumes published input is valid +and fails closed instead of displaying partial history when invalid data bypasses the +publication gate. + +See the [build and test guide](docs/build-and-test.md) for local development with +dummy data or a local data directory (`npm run dev:data -- `), +browser setup, verification commands, and the production build. + +## Publishing to GitHub Pages + +The [rocjitsu-publish-website workflow](../../../../.github/workflows/rocjitsu-publish-website.yml) +compares each site's sources on `develop` with its last published revision and +builds only sites with pending changes. Manual runs and shared workflow changes +rebuild both. Manual publishing also requires `develop`. +The dashboard is published to `gh-pages/rocjitsu-dashboard/`, and the handbook +to `gh-pages/rocjitsu/`; other Pages files are preserved. Benchmark data +continues to come from `gh-pages-rocjitsu`. + +See the [handbook publishing setup](../handbook/README.md#verification-and-publishing) +for GitHub App secrets, permissions, and Pages configuration. + +## Source layout + +| Path | Purpose | +| --- | --- | +| `index.html`, `src/main.jsx` | Browser entry points | +| `src/components/`, `src/hooks/` | Views, shared components, interaction state | +| `src/data/` | Data loading, validation, selectors, comparison logic | +| `src/theme/`, `src/utils/`, `src/index.css` | Theme, chart/display helpers, styles | +| `package.json`, `package-lock.json` | Commands and reproducible dependency installation | +| `*.config.js` | Vite, ESLint, Vitest, and Playwright configuration | +| `tests/unit/`, `tests/e2e/`, `tests/fixtures/` | Tests, helpers, and dummy fixtures | +| `docs/` | Build, testing, hosting, and data-contract documentation | + +## Further documentation + +- [Website data contract](docs/website-data-contract.md): JSON schema, comparison + rules, and data publication requirements. +- [Test fixtures](tests/fixtures/README.md): focused test cases and generated history. diff --git a/emulation/rocjitsu/website/docs/build-and-test.md b/emulation/rocjitsu/website/dashboard/docs/build-and-test.md similarity index 84% rename from emulation/rocjitsu/website/docs/build-and-test.md rename to emulation/rocjitsu/website/dashboard/docs/build-and-test.md index 14263c0011dd..29e0f0a117a0 100644 --- a/emulation/rocjitsu/website/docs/build-and-test.md +++ b/emulation/rocjitsu/website/dashboard/docs/build-and-test.md @@ -1,6 +1,6 @@ # Website build and test -The Rocjitsu simulation-performance dashboard lives in `emulation/rocjitsu/website`. It is a standalone +The Rocjitsu simulation-performance dashboard lives in `emulation/rocjitsu/website/dashboard`. It is a standalone React + Vite source package. Real benchmark data belongs on the `gh-pages-rocjitsu` branch under `rocjitsu-dashboard/data/`; dummy data is retained only as test fixtures. @@ -9,7 +9,7 @@ installation, GPU, or benchmark service. ## Prerequisites -- Node.js matching `website/package.json`: `^20.19.0 || ^22.13.0 || >=24.0.0`. +- Node.js matching `website/dashboard/package.json`: `^20.19.0 || ^22.13.0 || >=24.0.0`. - npm and network access to install the locked dependencies. - Chromium and its system libraries for Playwright browser tests. The install command below downloads Chromium and may need administrator privileges to @@ -20,7 +20,7 @@ installation, GPU, or benchmark service. Run from the `rocm-systems` repository root: ```bash -cd emulation/rocjitsu/website +cd emulation/rocjitsu/website/dashboard node --version npm --version npm ci @@ -29,13 +29,14 @@ npm run verify ``` `npm run verify` runs ESLint, a production build, Vitest unit tests, Playwright -desktop and mobile browser tests, and the chart interaction race test ten times -sequentially. A successful run exits with status 0. Playwright builds the current +desktop and mobile browser tests, the chart interaction race test ten times +sequentially, and the production-hosting smoke test. A successful run exits with +status 0. Playwright builds the current source with dummy fixtures into `.test-dist/`, starts its own preview server at `http://127.0.0.1:4174`, and stops it when done. Keep port 4174 free. Browser tests leave the data-free production build in `dist/` untouched. -`npm run test:e2e:production` separately rebuilds and previews the production +`npm run test:e2e:production` (also included in `verify`) rebuilds and previews the production artifact on port 4175, loads the published GitHub Raw dataset, and verifies real browser-cache reuse without request interception. CI runs this live production smoke test after the fixture suite; it requires network access and valid @@ -44,7 +45,7 @@ published data. If port 4174 is occupied, select a free port without stopping other servers: `PLAYWRIGHT_PORT=4176 npm run verify` (or use the same variable with `npm run test:e2e`). -Individual commands, all run from `website/`: +Individual commands, all run from `website/dashboard/`: | Command | Purpose | | --- | --- | @@ -58,7 +59,7 @@ Individual commands, all run from `website/`: | `npm run test:e2e:production` | Smoke test the production build, GitHub Raw hosting, and browser caching | | `npm run test:e2e:chart-race` | Run the chart interaction race test ten times sequentially | | `npm test` | Run the unit tests, the browser suites, and the ten-repeat chart race test | -| `npm run verify` | Run lint, build, and everything in `npm test` | +| `npm run verify` | Run lint, build, everything in `npm test`, and the production-hosting smoke test | Run the two browser commands one at a time. Both rebuild the shared `.test-dist/` fixture output before starting their server, so a concurrent run deletes files the other @@ -73,7 +74,7 @@ after updating Playwright if its required browser version changes. ## Run locally ```bash -# From emulation/rocjitsu/website: +# From emulation/rocjitsu/website/dashboard: npm run dev:fixtures -- --host 127.0.0.1 npm run dev:data -- /absolute/path/to/staged/data --host 127.0.0.1 ``` @@ -121,9 +122,12 @@ validated JSON independently under that directory. It contains `metadata.json`, `index.json`, `test-catalogs/`, and `runs/`; a data-only update does not require rebuilding the application. -This source package does not provide a deployment workflow. The hosting owner -chooses how to publish the contents of `dist/` through its existing release -process. +The [rocjitsu-publish-website workflow](../../../../../.github/workflows/rocjitsu-publish-website.yml) +verifies the dashboard and publishes `dist/` from `develop` to +`gh-pages/rocjitsu-dashboard/`. It runs the same `npm run verify` suite as website +CI (lint, build, unit, browser, chart-race, and production-hosting tests) before +creating a publishing token. Benchmark data remains independently published on +`gh-pages-rocjitsu`. ```bash npm run validate:data -- /absolute/path/to/staged/data @@ -144,5 +148,5 @@ npm run preview -- --host 127.0.0.1 Dependencies, generated fixture data, production/test build output, coverage, and Playwright reports/results are ignored by Git. Keep the source, lockfile, tests, and test fixtures under version -control. Parent-repository automation should use `emulation/rocjitsu/website` as its +control. Parent-repository automation should use `emulation/rocjitsu/website/dashboard` as its working directory and its `package-lock.json` as the npm cache key. diff --git a/emulation/rocjitsu/website/docs/website-data-contract.md b/emulation/rocjitsu/website/dashboard/docs/website-data-contract.md similarity index 100% rename from emulation/rocjitsu/website/docs/website-data-contract.md rename to emulation/rocjitsu/website/dashboard/docs/website-data-contract.md diff --git a/emulation/rocjitsu/website/eslint.config.js b/emulation/rocjitsu/website/dashboard/eslint.config.js similarity index 100% rename from emulation/rocjitsu/website/eslint.config.js rename to emulation/rocjitsu/website/dashboard/eslint.config.js diff --git a/emulation/rocjitsu/website/index.html b/emulation/rocjitsu/website/dashboard/index.html similarity index 100% rename from emulation/rocjitsu/website/index.html rename to emulation/rocjitsu/website/dashboard/index.html diff --git a/emulation/rocjitsu/website/package-lock.json b/emulation/rocjitsu/website/dashboard/package-lock.json similarity index 100% rename from emulation/rocjitsu/website/package-lock.json rename to emulation/rocjitsu/website/dashboard/package-lock.json diff --git a/emulation/rocjitsu/website/package.json b/emulation/rocjitsu/website/dashboard/package.json similarity index 94% rename from emulation/rocjitsu/website/package.json rename to emulation/rocjitsu/website/dashboard/package.json index 877b01b5c41f..3a88927144bd 100644 --- a/emulation/rocjitsu/website/package.json +++ b/emulation/rocjitsu/website/dashboard/package.json @@ -21,7 +21,7 @@ "test:e2e:production": "playwright test --config=playwright.production.config.js", "test:e2e:chart-race": "playwright test --project=desktop tests/e2e/benchmarks.e2e.js --grep \"benchmark explorer switches among single, grid, and aggregate modes\" --repeat-each=10 --workers=1", "test": "npm run test:unit && npm run test:e2e && npm run test:e2e:chart-race", - "verify": "npm run lint && npm run build && npm test" + "verify": "npm run lint && npm run build && npm test && npm run test:e2e:production" }, "dependencies": { "@emotion/react": "^11.14.0", diff --git a/emulation/rocjitsu/website/playwright.config.js b/emulation/rocjitsu/website/dashboard/playwright.config.js similarity index 100% rename from emulation/rocjitsu/website/playwright.config.js rename to emulation/rocjitsu/website/dashboard/playwright.config.js diff --git a/emulation/rocjitsu/website/playwright.production.config.js b/emulation/rocjitsu/website/dashboard/playwright.production.config.js similarity index 100% rename from emulation/rocjitsu/website/playwright.production.config.js rename to emulation/rocjitsu/website/dashboard/playwright.production.config.js diff --git a/emulation/rocjitsu/website/scripts/dashboard-data-dir.mjs b/emulation/rocjitsu/website/dashboard/scripts/dashboard-data-dir.mjs similarity index 100% rename from emulation/rocjitsu/website/scripts/dashboard-data-dir.mjs rename to emulation/rocjitsu/website/dashboard/scripts/dashboard-data-dir.mjs diff --git a/emulation/rocjitsu/website/scripts/dashboard-data-source.mjs b/emulation/rocjitsu/website/dashboard/scripts/dashboard-data-source.mjs similarity index 100% rename from emulation/rocjitsu/website/scripts/dashboard-data-source.mjs rename to emulation/rocjitsu/website/dashboard/scripts/dashboard-data-source.mjs diff --git a/emulation/rocjitsu/website/scripts/dev-with-data.mjs b/emulation/rocjitsu/website/dashboard/scripts/dev-with-data.mjs similarity index 100% rename from emulation/rocjitsu/website/scripts/dev-with-data.mjs rename to emulation/rocjitsu/website/dashboard/scripts/dev-with-data.mjs diff --git a/emulation/rocjitsu/website/scripts/validate-dashboard-data.mjs b/emulation/rocjitsu/website/dashboard/scripts/validate-dashboard-data.mjs similarity index 100% rename from emulation/rocjitsu/website/scripts/validate-dashboard-data.mjs rename to emulation/rocjitsu/website/dashboard/scripts/validate-dashboard-data.mjs diff --git a/emulation/rocjitsu/website/src/App.jsx b/emulation/rocjitsu/website/dashboard/src/App.jsx similarity index 100% rename from emulation/rocjitsu/website/src/App.jsx rename to emulation/rocjitsu/website/dashboard/src/App.jsx diff --git a/emulation/rocjitsu/website/src/components/benchmarks/AggregatePerformanceChart.jsx b/emulation/rocjitsu/website/dashboard/src/components/benchmarks/AggregatePerformanceChart.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/benchmarks/AggregatePerformanceChart.jsx rename to emulation/rocjitsu/website/dashboard/src/components/benchmarks/AggregatePerformanceChart.jsx diff --git a/emulation/rocjitsu/website/src/components/benchmarks/BenchmarkHistoryChart.jsx b/emulation/rocjitsu/website/dashboard/src/components/benchmarks/BenchmarkHistoryChart.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/benchmarks/BenchmarkHistoryChart.jsx rename to emulation/rocjitsu/website/dashboard/src/components/benchmarks/BenchmarkHistoryChart.jsx diff --git a/emulation/rocjitsu/website/src/components/benchmarks/BenchmarkPicker.jsx b/emulation/rocjitsu/website/dashboard/src/components/benchmarks/BenchmarkPicker.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/benchmarks/BenchmarkPicker.jsx rename to emulation/rocjitsu/website/dashboard/src/components/benchmarks/BenchmarkPicker.jsx diff --git a/emulation/rocjitsu/website/src/components/benchmarks/BenchmarkResultDialog.jsx b/emulation/rocjitsu/website/dashboard/src/components/benchmarks/BenchmarkResultDialog.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/benchmarks/BenchmarkResultDialog.jsx rename to emulation/rocjitsu/website/dashboard/src/components/benchmarks/BenchmarkResultDialog.jsx diff --git a/emulation/rocjitsu/website/src/components/benchmarks/ChartPointSelector.jsx b/emulation/rocjitsu/website/dashboard/src/components/benchmarks/ChartPointSelector.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/benchmarks/ChartPointSelector.jsx rename to emulation/rocjitsu/website/dashboard/src/components/benchmarks/ChartPointSelector.jsx diff --git a/emulation/rocjitsu/website/src/components/benchmarks/HistoricalRecords.jsx b/emulation/rocjitsu/website/dashboard/src/components/benchmarks/HistoricalRecords.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/benchmarks/HistoricalRecords.jsx rename to emulation/rocjitsu/website/dashboard/src/components/benchmarks/HistoricalRecords.jsx diff --git a/emulation/rocjitsu/website/src/components/benchmarks/RunDetailsDialog.jsx b/emulation/rocjitsu/website/dashboard/src/components/benchmarks/RunDetailsDialog.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/benchmarks/RunDetailsDialog.jsx rename to emulation/rocjitsu/website/dashboard/src/components/benchmarks/RunDetailsDialog.jsx diff --git a/emulation/rocjitsu/website/src/components/compare/RunSelector.jsx b/emulation/rocjitsu/website/dashboard/src/components/compare/RunSelector.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/compare/RunSelector.jsx rename to emulation/rocjitsu/website/dashboard/src/components/compare/RunSelector.jsx diff --git a/emulation/rocjitsu/website/src/components/failures/RunReliability.jsx b/emulation/rocjitsu/website/dashboard/src/components/failures/RunReliability.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/failures/RunReliability.jsx rename to emulation/rocjitsu/website/dashboard/src/components/failures/RunReliability.jsx diff --git a/emulation/rocjitsu/website/src/components/layout/DashboardHeader.jsx b/emulation/rocjitsu/website/dashboard/src/components/layout/DashboardHeader.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/layout/DashboardHeader.jsx rename to emulation/rocjitsu/website/dashboard/src/components/layout/DashboardHeader.jsx diff --git a/emulation/rocjitsu/website/src/components/layout/FiltersBar.jsx b/emulation/rocjitsu/website/dashboard/src/components/layout/FiltersBar.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/layout/FiltersBar.jsx rename to emulation/rocjitsu/website/dashboard/src/components/layout/FiltersBar.jsx diff --git a/emulation/rocjitsu/website/src/components/overview/DurationHistory.jsx b/emulation/rocjitsu/website/dashboard/src/components/overview/DurationHistory.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/overview/DurationHistory.jsx rename to emulation/rocjitsu/website/dashboard/src/components/overview/DurationHistory.jsx diff --git a/emulation/rocjitsu/website/src/components/overview/LargestChanges.jsx b/emulation/rocjitsu/website/dashboard/src/components/overview/LargestChanges.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/overview/LargestChanges.jsx rename to emulation/rocjitsu/website/dashboard/src/components/overview/LargestChanges.jsx diff --git a/emulation/rocjitsu/website/src/components/overview/MetricsGrid.jsx b/emulation/rocjitsu/website/dashboard/src/components/overview/MetricsGrid.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/overview/MetricsGrid.jsx rename to emulation/rocjitsu/website/dashboard/src/components/overview/MetricsGrid.jsx diff --git a/emulation/rocjitsu/website/src/components/overview/OverviewView.jsx b/emulation/rocjitsu/website/dashboard/src/components/overview/OverviewView.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/overview/OverviewView.jsx rename to emulation/rocjitsu/website/dashboard/src/components/overview/OverviewView.jsx diff --git a/emulation/rocjitsu/website/src/components/overview/RecentRuns.jsx b/emulation/rocjitsu/website/dashboard/src/components/overview/RecentRuns.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/overview/RecentRuns.jsx rename to emulation/rocjitsu/website/dashboard/src/components/overview/RecentRuns.jsx diff --git a/emulation/rocjitsu/website/src/components/overview/ResultsTable.jsx b/emulation/rocjitsu/website/dashboard/src/components/overview/ResultsTable.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/overview/ResultsTable.jsx rename to emulation/rocjitsu/website/dashboard/src/components/overview/ResultsTable.jsx diff --git a/emulation/rocjitsu/website/src/components/shared/Chart.jsx b/emulation/rocjitsu/website/dashboard/src/components/shared/Chart.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/shared/Chart.jsx rename to emulation/rocjitsu/website/dashboard/src/components/shared/Chart.jsx diff --git a/emulation/rocjitsu/website/src/components/shared/CommitComparison.jsx b/emulation/rocjitsu/website/dashboard/src/components/shared/CommitComparison.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/shared/CommitComparison.jsx rename to emulation/rocjitsu/website/dashboard/src/components/shared/CommitComparison.jsx diff --git a/emulation/rocjitsu/website/src/components/shared/DetailItem.jsx b/emulation/rocjitsu/website/dashboard/src/components/shared/DetailItem.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/shared/DetailItem.jsx rename to emulation/rocjitsu/website/dashboard/src/components/shared/DetailItem.jsx diff --git a/emulation/rocjitsu/website/src/components/shared/DetailLayout.jsx b/emulation/rocjitsu/website/dashboard/src/components/shared/DetailLayout.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/shared/DetailLayout.jsx rename to emulation/rocjitsu/website/dashboard/src/components/shared/DetailLayout.jsx diff --git a/emulation/rocjitsu/website/src/components/shared/SectionCard.jsx b/emulation/rocjitsu/website/dashboard/src/components/shared/SectionCard.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/shared/SectionCard.jsx rename to emulation/rocjitsu/website/dashboard/src/components/shared/SectionCard.jsx diff --git a/emulation/rocjitsu/website/src/components/shared/StatusChip.jsx b/emulation/rocjitsu/website/dashboard/src/components/shared/StatusChip.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/shared/StatusChip.jsx rename to emulation/rocjitsu/website/dashboard/src/components/shared/StatusChip.jsx diff --git a/emulation/rocjitsu/website/src/components/views/BenchmarksView.jsx b/emulation/rocjitsu/website/dashboard/src/components/views/BenchmarksView.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/views/BenchmarksView.jsx rename to emulation/rocjitsu/website/dashboard/src/components/views/BenchmarksView.jsx diff --git a/emulation/rocjitsu/website/src/components/views/CompareRunsView.jsx b/emulation/rocjitsu/website/dashboard/src/components/views/CompareRunsView.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/views/CompareRunsView.jsx rename to emulation/rocjitsu/website/dashboard/src/components/views/CompareRunsView.jsx diff --git a/emulation/rocjitsu/website/src/components/views/FailuresView.jsx b/emulation/rocjitsu/website/dashboard/src/components/views/FailuresView.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/views/FailuresView.jsx rename to emulation/rocjitsu/website/dashboard/src/components/views/FailuresView.jsx diff --git a/emulation/rocjitsu/website/src/components/views/PluginComparisonView.jsx b/emulation/rocjitsu/website/dashboard/src/components/views/PluginComparisonView.jsx similarity index 100% rename from emulation/rocjitsu/website/src/components/views/PluginComparisonView.jsx rename to emulation/rocjitsu/website/dashboard/src/components/views/PluginComparisonView.jsx diff --git a/emulation/rocjitsu/website/src/data/dashboardData.js b/emulation/rocjitsu/website/dashboard/src/data/dashboardData.js similarity index 100% rename from emulation/rocjitsu/website/src/data/dashboardData.js rename to emulation/rocjitsu/website/dashboard/src/data/dashboardData.js diff --git a/emulation/rocjitsu/website/src/data/dashboardDataError.js b/emulation/rocjitsu/website/dashboard/src/data/dashboardDataError.js similarity index 100% rename from emulation/rocjitsu/website/src/data/dashboardDataError.js rename to emulation/rocjitsu/website/dashboard/src/data/dashboardDataError.js diff --git a/emulation/rocjitsu/website/src/data/dashboardValidation.js b/emulation/rocjitsu/website/dashboard/src/data/dashboardValidation.js similarity index 100% rename from emulation/rocjitsu/website/src/data/dashboardValidation.js rename to emulation/rocjitsu/website/dashboard/src/data/dashboardValidation.js diff --git a/emulation/rocjitsu/website/src/data/pluginComparison.js b/emulation/rocjitsu/website/dashboard/src/data/pluginComparison.js similarity index 100% rename from emulation/rocjitsu/website/src/data/pluginComparison.js rename to emulation/rocjitsu/website/dashboard/src/data/pluginComparison.js diff --git a/emulation/rocjitsu/website/src/data/problemDetails.js b/emulation/rocjitsu/website/dashboard/src/data/problemDetails.js similarity index 100% rename from emulation/rocjitsu/website/src/data/problemDetails.js rename to emulation/rocjitsu/website/dashboard/src/data/problemDetails.js diff --git a/emulation/rocjitsu/website/src/data/provenance.js b/emulation/rocjitsu/website/dashboard/src/data/provenance.js similarity index 100% rename from emulation/rocjitsu/website/src/data/provenance.js rename to emulation/rocjitsu/website/dashboard/src/data/provenance.js diff --git a/emulation/rocjitsu/website/src/data/publishedDataUrls.js b/emulation/rocjitsu/website/dashboard/src/data/publishedDataUrls.js similarity index 100% rename from emulation/rocjitsu/website/src/data/publishedDataUrls.js rename to emulation/rocjitsu/website/dashboard/src/data/publishedDataUrls.js diff --git a/emulation/rocjitsu/website/src/data/runOrdering.js b/emulation/rocjitsu/website/dashboard/src/data/runOrdering.js similarity index 100% rename from emulation/rocjitsu/website/src/data/runOrdering.js rename to emulation/rocjitsu/website/dashboard/src/data/runOrdering.js diff --git a/emulation/rocjitsu/website/src/data/selectors.js b/emulation/rocjitsu/website/dashboard/src/data/selectors.js similarity index 100% rename from emulation/rocjitsu/website/src/data/selectors.js rename to emulation/rocjitsu/website/dashboard/src/data/selectors.js diff --git a/emulation/rocjitsu/website/src/hooks/useDashboardState.js b/emulation/rocjitsu/website/dashboard/src/hooks/useDashboardState.js similarity index 100% rename from emulation/rocjitsu/website/src/hooks/useDashboardState.js rename to emulation/rocjitsu/website/dashboard/src/hooks/useDashboardState.js diff --git a/emulation/rocjitsu/website/src/index.css b/emulation/rocjitsu/website/dashboard/src/index.css similarity index 100% rename from emulation/rocjitsu/website/src/index.css rename to emulation/rocjitsu/website/dashboard/src/index.css diff --git a/emulation/rocjitsu/website/src/main.jsx b/emulation/rocjitsu/website/dashboard/src/main.jsx similarity index 100% rename from emulation/rocjitsu/website/src/main.jsx rename to emulation/rocjitsu/website/dashboard/src/main.jsx diff --git a/emulation/rocjitsu/website/src/theme/styles.js b/emulation/rocjitsu/website/dashboard/src/theme/styles.js similarity index 100% rename from emulation/rocjitsu/website/src/theme/styles.js rename to emulation/rocjitsu/website/dashboard/src/theme/styles.js diff --git a/emulation/rocjitsu/website/src/theme/theme.js b/emulation/rocjitsu/website/dashboard/src/theme/theme.js similarity index 100% rename from emulation/rocjitsu/website/src/theme/theme.js rename to emulation/rocjitsu/website/dashboard/src/theme/theme.js diff --git a/emulation/rocjitsu/website/src/utils/chartColors.js b/emulation/rocjitsu/website/dashboard/src/utils/chartColors.js similarity index 100% rename from emulation/rocjitsu/website/src/utils/chartColors.js rename to emulation/rocjitsu/website/dashboard/src/utils/chartColors.js diff --git a/emulation/rocjitsu/website/src/utils/chartGaps.js b/emulation/rocjitsu/website/dashboard/src/utils/chartGaps.js similarity index 100% rename from emulation/rocjitsu/website/src/utils/chartGaps.js rename to emulation/rocjitsu/website/dashboard/src/utils/chartGaps.js diff --git a/emulation/rocjitsu/website/src/utils/chartStyles.js b/emulation/rocjitsu/website/dashboard/src/utils/chartStyles.js similarity index 100% rename from emulation/rocjitsu/website/src/utils/chartStyles.js rename to emulation/rocjitsu/website/dashboard/src/utils/chartStyles.js diff --git a/emulation/rocjitsu/website/src/utils/chartZoom.js b/emulation/rocjitsu/website/dashboard/src/utils/chartZoom.js similarity index 100% rename from emulation/rocjitsu/website/src/utils/chartZoom.js rename to emulation/rocjitsu/website/dashboard/src/utils/chartZoom.js diff --git a/emulation/rocjitsu/website/src/utils/durationAxis.js b/emulation/rocjitsu/website/dashboard/src/utils/durationAxis.js similarity index 100% rename from emulation/rocjitsu/website/src/utils/durationAxis.js rename to emulation/rocjitsu/website/dashboard/src/utils/durationAxis.js diff --git a/emulation/rocjitsu/website/src/utils/formatters.js b/emulation/rocjitsu/website/dashboard/src/utils/formatters.js similarity index 100% rename from emulation/rocjitsu/website/src/utils/formatters.js rename to emulation/rocjitsu/website/dashboard/src/utils/formatters.js diff --git a/emulation/rocjitsu/website/src/utils/performance.js b/emulation/rocjitsu/website/dashboard/src/utils/performance.js similarity index 100% rename from emulation/rocjitsu/website/src/utils/performance.js rename to emulation/rocjitsu/website/dashboard/src/utils/performance.js diff --git a/emulation/rocjitsu/website/src/utils/values.js b/emulation/rocjitsu/website/dashboard/src/utils/values.js similarity index 100% rename from emulation/rocjitsu/website/src/utils/values.js rename to emulation/rocjitsu/website/dashboard/src/utils/values.js diff --git a/emulation/rocjitsu/website/tests/e2e/benchmarks.e2e.js b/emulation/rocjitsu/website/dashboard/tests/e2e/benchmarks.e2e.js similarity index 100% rename from emulation/rocjitsu/website/tests/e2e/benchmarks.e2e.js rename to emulation/rocjitsu/website/dashboard/tests/e2e/benchmarks.e2e.js diff --git a/emulation/rocjitsu/website/tests/e2e/comparisons.e2e.js b/emulation/rocjitsu/website/dashboard/tests/e2e/comparisons.e2e.js similarity index 100% rename from emulation/rocjitsu/website/tests/e2e/comparisons.e2e.js rename to emulation/rocjitsu/website/dashboard/tests/e2e/comparisons.e2e.js diff --git a/emulation/rocjitsu/website/tests/e2e/failures.e2e.js b/emulation/rocjitsu/website/dashboard/tests/e2e/failures.e2e.js similarity index 100% rename from emulation/rocjitsu/website/tests/e2e/failures.e2e.js rename to emulation/rocjitsu/website/dashboard/tests/e2e/failures.e2e.js diff --git a/emulation/rocjitsu/website/tests/e2e/helpers/chart.js b/emulation/rocjitsu/website/dashboard/tests/e2e/helpers/chart.js similarity index 100% rename from emulation/rocjitsu/website/tests/e2e/helpers/chart.js rename to emulation/rocjitsu/website/dashboard/tests/e2e/helpers/chart.js diff --git a/emulation/rocjitsu/website/tests/e2e/helpers/dialog.js b/emulation/rocjitsu/website/dashboard/tests/e2e/helpers/dialog.js similarity index 100% rename from emulation/rocjitsu/website/tests/e2e/helpers/dialog.js rename to emulation/rocjitsu/website/dashboard/tests/e2e/helpers/dialog.js diff --git a/emulation/rocjitsu/website/tests/e2e/loading.e2e.js b/emulation/rocjitsu/website/dashboard/tests/e2e/loading.e2e.js similarity index 100% rename from emulation/rocjitsu/website/tests/e2e/loading.e2e.js rename to emulation/rocjitsu/website/dashboard/tests/e2e/loading.e2e.js diff --git a/emulation/rocjitsu/website/tests/e2e/mobile.e2e.js b/emulation/rocjitsu/website/dashboard/tests/e2e/mobile.e2e.js similarity index 100% rename from emulation/rocjitsu/website/tests/e2e/mobile.e2e.js rename to emulation/rocjitsu/website/dashboard/tests/e2e/mobile.e2e.js diff --git a/emulation/rocjitsu/website/tests/e2e/overview.e2e.js b/emulation/rocjitsu/website/dashboard/tests/e2e/overview.e2e.js similarity index 100% rename from emulation/rocjitsu/website/tests/e2e/overview.e2e.js rename to emulation/rocjitsu/website/dashboard/tests/e2e/overview.e2e.js diff --git a/emulation/rocjitsu/website/tests/fixtures/README.md b/emulation/rocjitsu/website/dashboard/tests/fixtures/README.md similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/README.md rename to emulation/rocjitsu/website/dashboard/tests/fixtures/README.md diff --git a/emulation/rocjitsu/website/tests/fixtures/data/index.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/index.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/index.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/index.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/metadata.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/metadata.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/metadata.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/metadata.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606010530-86b362ea.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606010530-86b362ea.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606010530-86b362ea.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606010530-86b362ea.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606020530-187a7544.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606020530-187a7544.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606020530-187a7544.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606020530-187a7544.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606140530-6fb48d1c.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606140530-6fb48d1c.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606140530-6fb48d1c.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606140530-6fb48d1c.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606150530-43f35f88.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606150530-43f35f88.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606150530-43f35f88.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606150530-43f35f88.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606160530-dac48bd5.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606160530-dac48bd5.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606160530-dac48bd5.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606160530-dac48bd5.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606170530-968f242c.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606170530-968f242c.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606170530-968f242c.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606170530-968f242c.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606180530-ff16d949.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606180530-ff16d949.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606180530-ff16d949.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606180530-ff16d949.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606190530-66b63aa1.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606190530-66b63aa1.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606190530-66b63aa1.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606190530-66b63aa1.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606200530-bf5a4b49.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606200530-bf5a4b49.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606200530-bf5a4b49.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606200530-bf5a4b49.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606210530-ad8f25c6.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606210530-ad8f25c6.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606210530-ad8f25c6.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606210530-ad8f25c6.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606220530-4be8e6da.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606220530-4be8e6da.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606220530-4be8e6da.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606220530-4be8e6da.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606230530-163b6c23.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606230530-163b6c23.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606230530-163b6c23.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606230530-163b6c23.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606240530-851ac25e.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606240530-851ac25e.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606240530-851ac25e.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606240530-851ac25e.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606250530-8c54b2c0.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606250530-8c54b2c0.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606250530-8c54b2c0.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606250530-8c54b2c0.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606260530-5ddf25ed.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606260530-5ddf25ed.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606260530-5ddf25ed.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606260530-5ddf25ed.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606270530-46c26daf.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606270530-46c26daf.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606270530-46c26daf.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606270530-46c26daf.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606280530-aef13790.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606280530-aef13790.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606280530-aef13790.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606280530-aef13790.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606290530-de3d5289.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606290530-de3d5289.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606290530-de3d5289.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606290530-de3d5289.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606300530-fb70f1ea.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606300530-fb70f1ea.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202606300530-fb70f1ea.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202606300530-fb70f1ea.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607010530-a841295b.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607010530-a841295b.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607010530-a841295b.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607010530-a841295b.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607020530-598cf7d9.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607020530-598cf7d9.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607020530-598cf7d9.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607020530-598cf7d9.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607030530-bd535aa7.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607030530-bd535aa7.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607030530-bd535aa7.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607030530-bd535aa7.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607040530-350aca20.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607040530-350aca20.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607040530-350aca20.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607040530-350aca20.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607050530-162d2d6e.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607050530-162d2d6e.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607050530-162d2d6e.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607050530-162d2d6e.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607060530-ea8723a9.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607060530-ea8723a9.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607060530-ea8723a9.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607060530-ea8723a9.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607070530-2e24042a.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607070530-2e24042a.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607070530-2e24042a.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607070530-2e24042a.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607080530-1b0e137f.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607080530-1b0e137f.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607080530-1b0e137f.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607080530-1b0e137f.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607250530-8e0c5183-asan.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607250530-8e0c5183-asan.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607250530-8e0c5183-asan.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607250530-8e0c5183-asan.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607250530-8e0c5183.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607250530-8e0c5183.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202607250530-8e0c5183.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202607250530-8e0c5183.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608020530-68c7dece.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608020530-68c7dece.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608020530-68c7dece.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608020530-68c7dece.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608130530-784750dd.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608130530-784750dd.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608130530-784750dd.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608130530-784750dd.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608140530-f25f5a48.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608140530-f25f5a48.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608140530-f25f5a48.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608140530-f25f5a48.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608150530-8418072e.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608150530-8418072e.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608150530-8418072e.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608150530-8418072e.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608260530-daaf4bae.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608260530-daaf4bae.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608260530-daaf4bae.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608260530-daaf4bae.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608270530-0db03af1.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608270530-0db03af1.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608270530-0db03af1.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608270530-0db03af1.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608280530-87c0b32c.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608280530-87c0b32c.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608280530-87c0b32c.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608280530-87c0b32c.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608290530-19872076.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608290530-19872076.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608290530-19872076.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608290530-19872076.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608300530-9398bd3f.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608300530-9398bd3f.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608300530-9398bd3f.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608300530-9398bd3f.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608310530-255eabe3.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608310530-255eabe3.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608310530-255eabe3.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608310530-255eabe3.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311310-9f774d29.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311310-9f774d29.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311310-9f774d29.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311310-9f774d29.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-asan.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-asan.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-asan.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-asan.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-tsan.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-tsan.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-tsan.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-tsan.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-ubsan.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-ubsan.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-ubsan.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311945-31369c4d-ubsan.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311945-31369c4d.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311945-31369c4d.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202608311945-31369c4d.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202608311945-31369c4d.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202609010115-8418072e-manual.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202609010115-8418072e-manual.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/benchmark-202609010115-8418072e-manual.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/benchmark-202609010115-8418072e-manual.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-commit-20260826T103000000Z.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-commit-20260826T103000000Z.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-commit-20260826T103000000Z.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-commit-20260826T103000000Z.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-commit-20260826T153000000Z.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-commit-20260826T153000000Z.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-commit-20260826T153000000Z.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-commit-20260826T153000000Z.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-commit-20260830T153000000Z.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-commit-20260830T153000000Z.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-commit-20260830T153000000Z.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-commit-20260830T153000000Z.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-17.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-17.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-17.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-17.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-18.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-18.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-18.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-18.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-19.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-19.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-19.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-19.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-20.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-20.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-20.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-20.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-21.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-21.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-21.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-21.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-22.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-22.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-22.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-22.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-23.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-23.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-23.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-23.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-24.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-24.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-24.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-24.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-26.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-26.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-26.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-26.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-27.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-27.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-27.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-27.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-28.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-28.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-28.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-28.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-29.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-29.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-29.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-29.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-30.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-30.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-30.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-30.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-31.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-31.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-07-31.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-07-31.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-01.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-01.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-01.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-01.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-03.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-03.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-03.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-03.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-04.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-04.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-04.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-04.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-05.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-05.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-05.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-05.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-06.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-06.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-06.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-06.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-07.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-07.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-07.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-07.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-08.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-08.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-08.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-08.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-09.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-09.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-09.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-09.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-10.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-10.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-10.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-10.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-11.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-11.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-11.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-11.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-12.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-12.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-12.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-12.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-16.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-16.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-16.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-16.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-17.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-17.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-17.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-17.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-18.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-18.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-18.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-18.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-19.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-19.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-19.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-19.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-20.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-20.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-20.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-20.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-21.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-21.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-21.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-21.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-22.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-22.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-22.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-22.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-23.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-23.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-23.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-23.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-24.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-24.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-24.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-24.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-25.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-25.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/runs/generated-history-2026-08-25.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/runs/generated-history-2026-08-25.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/test-catalogs/rocjitsu-core-v1.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/test-catalogs/rocjitsu-core-v1.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/test-catalogs/rocjitsu-core-v1.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/test-catalogs/rocjitsu-core-v1.json diff --git a/emulation/rocjitsu/website/tests/fixtures/data/test-catalogs/rocjitsu-core-v2.json b/emulation/rocjitsu/website/dashboard/tests/fixtures/data/test-catalogs/rocjitsu-core-v2.json similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/data/test-catalogs/rocjitsu-core-v2.json rename to emulation/rocjitsu/website/dashboard/tests/fixtures/data/test-catalogs/rocjitsu-core-v2.json diff --git a/emulation/rocjitsu/website/tests/fixtures/publishedData.js b/emulation/rocjitsu/website/dashboard/tests/fixtures/publishedData.js similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/publishedData.js rename to emulation/rocjitsu/website/dashboard/tests/fixtures/publishedData.js diff --git a/emulation/rocjitsu/website/tests/fixtures/syntheticDataset.js b/emulation/rocjitsu/website/dashboard/tests/fixtures/syntheticDataset.js similarity index 100% rename from emulation/rocjitsu/website/tests/fixtures/syntheticDataset.js rename to emulation/rocjitsu/website/dashboard/tests/fixtures/syntheticDataset.js diff --git a/emulation/rocjitsu/website/tests/production/production.e2e.js b/emulation/rocjitsu/website/dashboard/tests/production/production.e2e.js similarity index 100% rename from emulation/rocjitsu/website/tests/production/production.e2e.js rename to emulation/rocjitsu/website/dashboard/tests/production/production.e2e.js diff --git a/emulation/rocjitsu/website/tests/unit/compareRuns.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/compareRuns.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/compareRuns.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/compareRuns.test.js diff --git a/emulation/rocjitsu/website/tests/unit/dashboardDataDir.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/dashboardDataDir.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/dashboardDataDir.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/dashboardDataDir.test.js diff --git a/emulation/rocjitsu/website/tests/unit/dashboardDataError.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/dashboardDataError.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/dashboardDataError.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/dashboardDataError.test.js diff --git a/emulation/rocjitsu/website/tests/unit/dashboardDataSource.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/dashboardDataSource.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/dashboardDataSource.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/dashboardDataSource.test.js diff --git a/emulation/rocjitsu/website/tests/unit/loader.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/loader.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/loader.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/loader.test.js diff --git a/emulation/rocjitsu/website/tests/unit/publishedData.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/publishedData.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/publishedData.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/publishedData.test.js diff --git a/emulation/rocjitsu/website/tests/unit/publishedDataUrls.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/publishedDataUrls.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/publishedDataUrls.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/publishedDataUrls.test.js diff --git a/emulation/rocjitsu/website/tests/unit/selectors.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/selectors.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/selectors.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/selectors.test.js diff --git a/emulation/rocjitsu/website/tests/unit/utils.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/utils.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/utils.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/utils.test.js diff --git a/emulation/rocjitsu/website/tests/unit/validationCli.test.js b/emulation/rocjitsu/website/dashboard/tests/unit/validationCli.test.js similarity index 100% rename from emulation/rocjitsu/website/tests/unit/validationCli.test.js rename to emulation/rocjitsu/website/dashboard/tests/unit/validationCli.test.js diff --git a/emulation/rocjitsu/website/vite.config.js b/emulation/rocjitsu/website/dashboard/vite.config.js similarity index 100% rename from emulation/rocjitsu/website/vite.config.js rename to emulation/rocjitsu/website/dashboard/vite.config.js diff --git a/emulation/rocjitsu/website/vitest.config.js b/emulation/rocjitsu/website/dashboard/vitest.config.js similarity index 100% rename from emulation/rocjitsu/website/vitest.config.js rename to emulation/rocjitsu/website/dashboard/vitest.config.js diff --git a/emulation/rocjitsu/website/handbook/.gitignore b/emulation/rocjitsu/website/handbook/.gitignore new file mode 100644 index 000000000000..67584006e180 --- /dev/null +++ b/emulation/rocjitsu/website/handbook/.gitignore @@ -0,0 +1,2 @@ +/.venv/ +/site/ diff --git a/emulation/rocjitsu/website/handbook/README.md b/emulation/rocjitsu/website/handbook/README.md new file mode 100644 index 000000000000..1f20f01c6274 --- /dev/null +++ b/emulation/rocjitsu/website/handbook/README.md @@ -0,0 +1,174 @@ +# Rocjitsu handbook + +Publishing configuration for the Rocjitsu documentation and blog at +. The engineering-handbook +layout uses Material for MkDocs, system fonts, light/dark themes, search, and +Mermaid diagrams. The site uses a plain-text name without a logo. + +Guide and blog Markdown lives in [../../docs/](../../docs/), the single source of +truth. MkDocs reads that directory directly: no copied guides, generated Markdown +mirror, or synchronization step. The dashboard introduction and blog +live there. The `docs/sphinx/` subtree is excluded from this publication. + +This directory contains only publishing configuration, theme assets, and tests. +The homepage renders the existing [../../README.md](../../README.md) through an +in-memory MkDocs file. There is no separate `docs/index.md` or second overview. +Links between the README and published guides resolve to handbook pages, +including section anchors. README edits trigger CI and live reload. +The [simulation-performance dashboard](../dashboard/README.md) is a +separate application. + +## Build and preview + +Use Python 3.12. From this directory: + +```bash +python3.12 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt +.venv/bin/mkdocs build --strict +.venv/bin/python -m unittest discover -s tests +node --test tests/repository-stats.test.cjs +.venv/bin/mkdocs serve --dev-addr 127.0.0.1:8000 +``` + +The browser-script tests use Node.js 22.13 or newer with no npm dependencies. +Open the local address printed by MkDocs (including `/rocm-systems/rocjitsu/`). +The production build writes `site/`; generated files and the virtual environment +are ignored by Git. The pinned MkDocs 1.x and Material versions match the tested +theme and hook APIs; review compatibility before changing them. + +## Adding a guide + +1. Add the Markdown file under `emulation/rocjitsu/docs/`. +2. Register its path, relative to `docs/`, in the `nav` section of + [mkdocs.yml](mkdocs.yml). Choose **User guide** for building, running, + configuring, or troubleshooting Rocjitsu; **Developer guide** for understanding + or extending its internals; or **Project** for project information. + For example, add `- My guide: my-guide.md` under the appropriate existing group. +3. From `emulation/rocjitsu/website/handbook/`, using the environment configured + in [Build and preview](#build-and-preview), run: + + ```bash + .venv/bin/python -m mkdocs build --strict -f mkdocs.yml + ``` + +MkDocs discovers Markdown files automatically, but sidebar placement is explicit. +A regular guide missing from `nav` fails the strict build in both CI and +publication. Preview the site and check the guide's sidebar placement and links +before submitting the change. + +Use Markdown that renders consistently on GitHub and in MkDocs: + +- Leave a blank line between introductory prose and a list. +- Indent nested lists by four spaces per level, including bullets inside a + numbered step. Align continuation lines with the nested item's text. +- Indent paragraphs and fenced code blocks within a list item by four spaces + after a blank line so they remain part of that item. +- Put C++ template names in backticks, such as `Clocked`, instead of + escaping angle brackets. + +Check the rendered page to confirm that list nesting, numbering, and template +names appear as intended. + +Blog posts under `docs/blog/posts/` are registered by the blog plugin and do not +need individual `nav` entries. `docs/sphinx/` is excluded from this handbook. +If a page should deliberately be published without a sidebar entry, document that +choice in `mkdocs.yml` using `not_in_nav`; use `exclude_docs` for content that +should not be published. Keep navigation validation enabled. + +## Editing + +| Path | Purpose | +| --- | --- | +| `mkdocs.yml` | Navigation, theme, plugins, and production URL | +| `../../docs/` | Canonical guide pages, dashboard introduction, and blog | +| `../../README.md` | Homepage and project overview | +| `assets/handbook.css` | Layout, colors, and focus styles | +| `hooks.py` | Source-link resolution, Markdown renderers, and presentation icons | +| `overrides/partials/` | Repository link and counts | + +Edit technical instructions in `../../docs/`; never add a second copy here. +The handbook presents complete documentation pages, not a directory of links to +raw Markdown. Keep layout, typography, and reading controls in this theme; +author page-specific structure once in the canonical Markdown. Material cards, +callouts, tabs, and expandable details are available where they aid reading. +They render on the website; GitHub's Markdown viewer may display their syntax +differently. Use tables for comparisons, not for homepage navigation. +The build adds guide-title icons without editing the canonical Markdown or its +heading anchors. Use plain text on subsection headings. +Sidebar icons belong to groups; individual navigation links remain text-only. +Keep explicit heading IDs stable when changing titles. The repository header +link points directly to Rocjitsu; its stars/forks describe the containing +`ROCm/rocm-systems` repository. A best-effort GitHub request at build time embeds +the counts in the HTML, so readers do not need API access. Browser requests +refresh them with a 15-minute session cache; failed refreshes retain existing +counts. Tooltips identify the initial build snapshot. If neither build nor +browser can fetch counts, the icons display an em dash. Set +`ROCJITSU_DOCS_OFFLINE=1` to skip the build request for offline builds. +The standard Material footer contains +the project copyright, license, GitHub and issue links, and theme attribution. +The README's GitHub CI badge is omitted during rendering, without changing its +Markdown source. + +Relative links between documentation pages stay within the handbook. Links to +existing source files outside `docs/` open the corresponding GitHub location. +Missing targets still fail the strict build. Source files are never rewritten. + +Blog posts live in `../../docs/blog/posts/` and use `date`, `title`, and `slug` +front matter. Keep unfinished articles marked `draft: true`; the production +build excludes them. To preview drafts locally: + +```bash +.venv/bin/mkdocs serve --dev-addr 127.0.0.1:8000 +``` + +Check technical claims against the corresponding document in `../../docs/` before +publishing a post. Diagrams must identify whether they show component organization +or a specific execution path. + +## Verification and publishing + +The `rocjitsu-handbook` CI workflow runs a strict build and regression tests +on Rocjitsu changes in pull requests and `develop` pushes, including source files +linked from the canonical docs. +Sphinx-only changes do not trigger it. Tests verify that canonical edits reach +the site, source links resolve, Sphinx is excluded, and builds do not write back +into `docs/`. Before submitting a change, run these checks and inspect affected +pages at desktop and mobile widths in both color themes. + +The `rocjitsu-publish-website` workflow compares current `develop` with each site's +last published source revision, then builds and publishes only affected sites: + +- Handbook: `rocjitsu/` +- Dashboard: `rocjitsu-dashboard/` + +Changes anywhere in `emulation/rocjitsu/` select the handbook, since canonical +documentation can link to source files outside `docs/` and link resolution checks +whether those targets exist. Changes in `website/dashboard/` also select the +dashboard. Sphinx-only changes are excluded. A publishing-workflow change or a +manual run selects both. +Handbook-only runs do not install dashboard dependencies or update its Pages files. +Each published directory contains a `.source-revision` marker. Missing markers +trigger an initial build. Comparing against publication rather than the latest +push also catches changes from failed or superseded pending runs. Delayed runs +check out current `develop` so they cannot roll a site back to an older event. +If a history rewrite makes a recorded revision unavailable, a manual run rebuilds +both sites and refreshes their markers without using the old revisions. + +Each selected site passes its verification suite before a publishing token is +created. The dashboard runs `npm run verify`, including production-hosting tests; +the handbook runs its strict build and Python/Node regression tests against the +same checkout that is published. + +Both destinations use the same publishing workflow and concurrency group. Each +destination is replaced independently; files elsewhere on `gh-pages` are preserved. +Dashboard benchmark data remains on `gh-pages-rocjitsu`. + +Publishing uses the existing `APP_ID` and `APP_PRIVATE_KEY` secrets to obtain a +GitHub App token with repository Contents write permission. The App must be +installed and allowed to push to `gh-pages`; Pages must use `gh-pages` at its +root. Manual publishing is restricted to `develop`. See the +[publishing workflow](../../../../.github/workflows/rocjitsu-publish-website.yml). + +The root Pages index is maintained separately. This workflow does not overwrite +it. diff --git a/emulation/rocjitsu/website/handbook/assets/handbook.css b/emulation/rocjitsu/website/handbook/assets/handbook.css new file mode 100644 index 000000000000..16fbb4c501ed --- /dev/null +++ b/emulation/rocjitsu/website/handbook/assets/handbook.css @@ -0,0 +1,182 @@ +/* Presentation for canonical documentation and site navigation. */ +:root { + --md-text-font: "Segoe UI", Arial, sans-serif; + --md-code-font: "SFMono-Regular", Consolas, monospace; + --md-primary-fg-color: #234b66; + --md-primary-fg-color--dark: #1b3b50; +} +[data-md-color-scheme="default"] { + --handbook-heading: #213c50; + --handbook-link: #21639b; + --handbook-teal: #087c78; + --handbook-border: #d9e4eb; + --handbook-surface: #f0f6f9; + --handbook-active: #e4f1f7; + --handbook-muted: #586f80; +} +[data-md-color-scheme="slate"] { + --md-default-bg-color: #18232e; + --md-primary-fg-color: #203f54; + --md-primary-fg-color--dark: #162e40; + --handbook-heading: #e0edf6; + --handbook-link: #86c9ff; + --handbook-teal: #63d5c7; + --handbook-border: #364957; + --handbook-surface: #223340; + --handbook-active: #284557; + --handbook-muted: #a9bdcb; +} +.md-grid { max-width: 1320px; } +.md-header { box-shadow: none; border-bottom: 2px solid #248e99; } +.md-header__inner { height: 52px; } +.md-header__title { font-size: 18px; } +.md-typeset { font-size: 15px; line-height: 1.7; } +.md-typeset h1 { color: var(--handbook-heading); font-size: 30px; font-weight: 600; line-height: 1.3; margin-bottom: 24px; } +.md-typeset h2 { color: var(--handbook-heading); font-size: 23px; font-weight: 600; line-height: 1.4; margin: 2em 0 0.8em; } +.md-typeset h3 { color: var(--handbook-heading); font-size: 18px; font-weight: 600; line-height: 1.45; } +.md-typeset h4, .md-typeset h5, .md-typeset h6 { color: var(--handbook-heading); font-size: 15px; font-weight: 600; } +.md-typeset :is(h1, h2, h3, h4, h5, h6) { overflow-wrap: anywhere; } +.md-typeset h1 .twemoji { color: var(--handbook-teal); margin-right: 4px; } +.md-typeset a { color: var(--handbook-link); } +.md-typeset p a:is(:hover, :focus-visible), .md-typeset li a:is(:hover, :focus-visible) { text-decoration: underline; text-underline-offset: 3px; } +.md-typeset :not(pre) > code { overflow-wrap: anywhere; } +.md-typeset pre > code { font-size: 13px; line-height: 1.65; border-radius: 4px; padding: 14px 16px; scrollbar-width: thin; } +/* Keep the copy control beside the code, outside its horizontal scroll area. */ +.md-typeset .highlight:has(.md-code__button) pre { + padding-right: 40px; + background: var(--md-code-bg-color); + border-radius: 4px; +} +.md-typeset table:not([class]) { font-size: 13px; line-height: 1.6; box-shadow: none; border: 1px solid var(--handbook-border); border-radius: 4px; } +.md-typeset table:not([class]) th { color: var(--handbook-heading); background: var(--handbook-surface); vertical-align: top; } +.md-typeset table:not([class]) td { border-top-color: var(--handbook-border); } +.md-typeset .admonition, .md-typeset details { font-size: 14px; box-shadow: none; } +/* Native Markdown cards stay in the documentation source and stack on phones. */ +.md-typeset .grid.cards { gap: 16px; margin: 24px 0; } +.md-typeset .grid.cards > ul > li { + border: 1px solid var(--handbook-border); + border-radius: 6px; + padding: 20px; + box-shadow: none; +} +.md-typeset .grid.cards > ul > li:hover { + border-color: var(--handbook-teal); + box-shadow: none; +} +.md-typeset .grid.cards > ul > li > :first-child { + color: var(--handbook-heading); + font-size: 17px; +} +.md-typeset .grid.cards .twemoji { color: var(--handbook-teal); } +.md-typeset .grid.cards hr { border-bottom-color: var(--handbook-border); margin: 16px 0; } +@media (max-width: 40em) { + .md-typeset { font-size: 16px; } + .md-typeset h1 { font-size: 28px; } + .md-typeset h2 { font-size: 22px; } + .md-typeset .highlight { margin-inline: 0; } + .md-typeset .grid.cards { grid-template-columns: minmax(0, 1fr); } + .md-typeset .grid.cards > ul > li { padding: 18px; } +} +.md-nav { font-size: 13px; } +.md-sidebar--primary { border-right: 1px solid var(--handbook-border); } +.md-post { padding-bottom: 20px; border-bottom: 1px solid var(--handbook-border); } +@media (max-width: 76.234375em) { + .md-sidebar--primary { border-right: 0; } + .md-nav--primary .md-nav__link { min-height: 36px; align-items: center; } +} +/* Use the site name without a logo. */ +.md-header .md-logo, .md-nav--primary .md-nav__title .md-logo { display: none; } +.md-nav__link--active { + color: var(--handbook-link); + font-weight: 700; + background: var(--handbook-active); + border-left: 2px solid var(--handbook-teal); + padding: 5px 8px; +} +a:focus-visible, button:focus-visible, input:focus-visible { + outline: 2px solid var(--handbook-link); outline-offset: 3px; +} +.md-typeset .md-code__button { color: var(--handbook-muted); } +.md-typeset .md-code__button:hover { color: var(--handbook-link); } +.md-typeset .mermaid { overflow-x: auto; padding: 16px; border: 1px solid var(--handbook-border); border-radius: 4px; background: var(--handbook-surface); } +/* Desktop hierarchy: site title, spaced section headings, indented pages. */ +@media (min-width: 76.25em) { + .md-nav--primary > .md-nav__title { + color: var(--handbook-heading); + font-size: 17px; + font-weight: 750; + padding-bottom: 14px; + border-bottom: 1px solid var(--handbook-border); + margin-bottom: 12px; + } + .md-nav--primary .md-nav__item--section > .md-nav__link { + color: var(--handbook-heading); + font-size: 13px; + font-weight: 750; + margin-top: 24px; + margin-bottom: 9px; + } + .md-nav--primary .md-nav__item--section > .md-nav > .md-nav__list { + border-left: 1px solid var(--handbook-border); + margin-left: 3px; + padding-left: 12px; + } + .md-nav--secondary > .md-nav__title { + color: var(--handbook-heading); + font-weight: 700; + } +} +.md-nav__item .md-nav__link--active:not(.md-nav__link--index) { + color: var(--handbook-link); +} +.md-nav__link:hover, .md-nav__link:focus-visible { color: var(--handbook-link); } +.md-nav--primary .md-nav__item--section > label.md-nav__link[for] { color: var(--handbook-heading); } +/* The reading position should not change the table of contents geometry. */ +.md-nav--secondary .md-nav__link { line-height: 1.5; } +.md-nav--secondary .md-nav__link--active { + padding: 0; + border: 0; + background: none; +} +/* Keep the site name stable when Material activates its scrolled header. */ +.md-header__title .md-header__topic:first-child { + opacity: 1; + transform: none; + transition: none; + pointer-events: auto; + z-index: 0; +} +.md-header__title .md-header__topic + .md-header__topic { display: none; } +/* A simple text launch link, with an external-link arrow. */ +.md-typeset a.dashboard-launch { + display: inline-flex; + align-items: center; + gap: 12px; + max-width: 100%; + margin: 12px 0; + padding: 6px 0; + color: var(--handbook-link); + line-height: 1.6; + border-bottom: 1px solid var(--handbook-border); +} +.md-typeset a.dashboard-launch strong { font-weight: 600; } +.md-typeset a.dashboard-launch .twemoji { flex-shrink: 0; } +.md-typeset a.dashboard-launch:hover, +.md-typeset a.dashboard-launch:focus-visible { + border-bottom-color: var(--handbook-link); +} +/* Keep repository identity and counts compact beside the GitHub icon. */ +.md-header__source { width: auto; } +.md-source__repository { max-width: none; } +.md-source__repository--active .md-source__facts, +.md-source__repository--active .md-source__fact { animation: none; } +.md-nav__source .md-source__repository { + white-space: normal; + overflow-wrap: anywhere; +} +.md-nav__source .md-source { display: flex; align-items: center; } +.md-nav__source .md-source__repository { min-width: 0; } +.source-links { display: flex; align-items: center; } +.source-links .md-source { padding-right: 8px; } +/* Keep the source link's full focus ring visible inside the header and drawer. */ +.source-links .md-source:focus-visible { outline: 2px solid #fff; outline-offset: -3px; } diff --git a/emulation/rocjitsu/website/handbook/assets/repository-stats.js b/emulation/rocjitsu/website/handbook/assets/repository-stats.js new file mode 100644 index 000000000000..e3993b8a32de --- /dev/null +++ b/emulation/rocjitsu/website/handbook/assets/repository-stats.js @@ -0,0 +1,56 @@ +// Show GitHub repository counts in the header and mobile navigation. +// Loaded by MkDocs extra_javascript; failures preserve the build snapshot. +(() => { + const cacheKey = 'rocjitsu-repository-stats'; + const maxAge = 15 * 60 * 1000; + const labels = { stargazers_count: 'Stars', forks_count: 'Forks' }; + + function validCounts(data) { + return data && Object.keys(labels).every( + key => Number.isSafeInteger(data[key]) && data[key] >= 0, + ); + } + + function render(data) { + document.querySelectorAll('[data-repository-stat]').forEach(element => { + const key = element.dataset.repositoryStat; + const count = data[key].toLocaleString('en-US'); + element.textContent = count; + element.title = `${labels[key]}: ${count} · ROCm/rocm-systems`; + element.setAttribute('aria-label', element.title); + }); + } + + async function load() { + try { + const cached = JSON.parse(sessionStorage.getItem(cacheKey)); + if (cached && validCounts(cached.data) && Number.isFinite(cached.time) + && Date.now() >= cached.time) { + render(cached.data); + if (Date.now() - cached.time < maxAge) return; + } + } catch { + // Storage may be unavailable; fetching counts does not require it. + } + + try { + const response = await fetch('https://api.github.com/repos/ROCm/rocm-systems'); + if (!response.ok) return; + const data = await response.json(); + if (!validCounts(data)) return; + render(data); + try { + sessionStorage.setItem(cacheKey, JSON.stringify({ time: Date.now(), data: { + stargazers_count: data.stargazers_count, + forks_count: data.forks_count, + } })); + } catch { + // Keep the displayed counts when storage is unavailable. + } + } catch { + // Keep the build snapshot or cached counts when GitHub cannot be reached. + } + } + + return load(); +})(); diff --git a/emulation/rocjitsu/website/handbook/hooks.py b/emulation/rocjitsu/website/handbook/hooks.py new file mode 100644 index 000000000000..6a69e6398304 --- /dev/null +++ b/emulation/rocjitsu/website/handbook/hooks.py @@ -0,0 +1,221 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# Publish canonical documentation with source links and presentation-only icons. +# Usage: mkdocs build --strict -f emulation/rocjitsu/website/handbook/mkdocs.yml + +from functools import lru_cache +from http.client import HTTPException +import json +import os +from pathlib import Path +import posixpath +import re +from urllib.parse import quote, unquote, urlsplit, urlunsplit +from urllib.request import Request, urlopen + +import material +from markdown.extensions import Extension +from markdown.treeprocessors import Treeprocessor +from material.extensions import emoji +from mkdocs.structure.files import File +from pymdownx.superfences import fence_code_format + +SECTION_ICONS = { + "User guide": "material/book-open-variant", + "Developer guide": "material/code-braces", + "Project": "material/folder-outline", +} + +PAGE_ICONS = { + 'building.md': 'hammer-wrench', + 'rocjitsu-cli.md': 'console', + 'configuration.md': 'cog-outline', + 'architecture.md': 'sitemap', + 'style.md': 'source-branch', + 'race-detector.md': 'bug-outline', + 'rocgdb-debugging.md': 'bug-outline', + 'benchmarking.md': 'chart-line', + 'simdojo.md': 'cog-outline', + 'vm-design.md': 'memory', + 'dbt-design.md': 'swap-horizontal', + 'dbi-design.md': 'magnify', + 'plugins.md': 'puzzle-outline', + 'performance-dashboard.md': 'chart-line', + 'blog/index.md': 'newspaper-variant-outline', +} + + +@lru_cache(maxsize=1) +def repository_stats(): + # Fetch once per build/preview process; an offline build remains usable. + try: + request = Request( + 'https://api.github.com/repos/ROCm/rocm-systems', + headers={'User-Agent': 'rocjitsu-handbook'}, + ) + with urlopen(request, timeout=3) as response: + data = json.load(response) + keys = ('stargazers_count', 'forks_count') + if isinstance(data, dict) and all( + type(data.get(key)) is int and data[key] >= 0 for key in keys + ): + return {key: data[key] for key in keys} + except (OSError, ValueError, HTTPException): + pass + return {} + + +class SourceLinksProcessor(Treeprocessor): + def __init__(self, md, extension): + super().__init__(md) + self.extension = extension + # Material retains excerpt renderers until after other pages render. + self.source_path = extension.source_path + self.page_uri = extension.page_uri + + def run(self, root): + extension = self.extension + # The README keeps its CI badge on GitHub, but the handbook omits it. + if self.source_path == extension.docs_dir.parent / 'README.md': + for paragraph in list(root): + if paragraph.tag != 'p' or len(paragraph) != 1 or paragraph.text: + continue + link = paragraph[0] + if link.tag != 'a' or len(link) != 1 or link.text or link.tail: + continue + image = link[0] + source = urlsplit(image.get('src', '')) + if ( + image.tag == 'img' + and not image.tail + and source.netloc == 'github.com' + and '/actions/workflows/' in source.path + and source.path.endswith('/badge.svg') + ): + root.remove(paragraph) + for link in root.iter('a'): + url = urlsplit(link.get('href', '')) + if url.scheme or url.netloc or not url.path or url.path.startswith('/'): + continue + target = (self.source_path.parent / unquote(url.path)).resolve() + if target in extension.published_sources: + destination = posixpath.relpath( + extension.published_sources[target], + posixpath.dirname(self.page_uri) or '.', + ) + link.set('href', urlunsplit(url._replace(path=destination))) + continue + if target.is_relative_to(extension.docs_dir): + continue + if ( + not target.is_relative_to(extension.docs_dir.parent) + or not target.exists() + ): + continue + relative = target.relative_to(extension.docs_dir.parent).as_posix() + source_url = extension.source_url + if target.is_file(): + source_url = source_url.replace('/tree/', '/blob/', 1) + destination = urlsplit(f'{source_url}/{quote(relative)}') + link.set( + 'href', + urlunsplit( + destination._replace(query=url.query, fragment=url.fragment) + ), + ) + + +class SourceLinksExtension(Extension): + def __init__(self, docs_dir, source_url): + super().__init__() + self.docs_dir = Path(docs_dir).resolve() + self.source_url = source_url.rstrip('/') + self.source_path = self.docs_dir / 'index.md' + self.page_uri = 'index.md' + self.published_sources = {} + + def extendMarkdown(self, md): + # Run before MkDocs (priority 0) and Material excerpts (priority 1) + # resolve local links into output URLs. + md.treeprocessors.register(SourceLinksProcessor(md, self), 'source_links', 2) + + +def on_config(config): + config.extra['repository_stats'] = ( + {} if os.environ.get('ROCJITSU_DOCS_OFFLINE') == '1' else repository_stats() + ) + # Keep Python callables here so mkdocs.yml remains standard, lintable YAML. + config.mdx_configs['pymdownx.emoji'] = { + 'emoji_index': emoji.twemoji, + 'emoji_generator': emoji.to_svg, + } + config.mdx_configs['pymdownx.superfences'] = { + 'custom_fences': [ + {'name': 'mermaid', 'class': 'mermaid', 'format': fence_code_format}, + ], + } + config.markdown_extensions.append( + SourceLinksExtension(config.docs_dir, config.repo_url) + ) + return config + + +def on_files(files, config): + readme = Path(config.docs_dir).parent / 'README.md' + if readme.exists(): + files.append(File.generated(config, 'index.md', abs_src_path=str(readme))) + for extension in config.markdown_extensions: + if isinstance(extension, SourceLinksExtension): + extension.published_sources = { + Path(file.abs_src_path).resolve(): file.src_uri + for file in files + if file.is_documentation_page() and file.abs_src_path + } + for name in ('handbook.css', 'repository-stats.js'): + asset = Path(__file__).parent / 'assets' / name + files.append(File.generated(config, f'assets/{name}', abs_src_path=str(asset))) + return files + + +def on_page_markdown(markdown, page, config, files): + for extension in config.markdown_extensions: + if isinstance(extension, SourceLinksExtension): + extension.page_uri = page.file.src_uri + extension.source_path = ( + Path(page.file.abs_src_path) + if page.file.abs_src_path + else Path(config.docs_dir) / page.file.src_uri + ) + return markdown + + +def on_page_content(html, page, config, files): + icon = PAGE_ICONS.get(page.file.src_uri) + if icon: + icon_path = ( + Path(material.__file__).parent + / 'templates' + / '.icons' + / 'material' + / f'{icon}.svg' + ) + decoration = ( + f' ' + ) + html = re.sub( + r'(]*>)', lambda match: match[1] + decoration, html, count=1 + ) + return html + + +def on_serve(server, config, builder): + server.watch(str(Path(__file__).parent / 'assets')) + server.watch(str(Path(config.docs_dir).parent / 'README.md')) + return server + + +def on_nav(nav, **kwargs): + for section in nav.items: + if section.is_section and section.title in SECTION_ICONS: + section.meta = {"icon": SECTION_ICONS[section.title]} + return nav diff --git a/emulation/rocjitsu/website/handbook/mkdocs.yml b/emulation/rocjitsu/website/handbook/mkdocs.yml new file mode 100644 index 000000000000..6bd417d0dc42 --- /dev/null +++ b/emulation/rocjitsu/website/handbook/mkdocs.yml @@ -0,0 +1,116 @@ +site_name: Rocjitsu +site_description: User and developer documentation for the ROCm Just-in-Time Suite. +site_url: https://rocm.github.io/rocm-systems/rocjitsu/ +repo_url: https://github.com/ROCm/rocm-systems/tree/develop/emulation/rocjitsu +repo_name: ROCm/rocm-systems +docs_dir: ../../docs +exclude_docs: | + /sphinx/ +validation: + nav: + omitted_files: warn + links: + anchors: warn +site_dir: site +hooks: + - hooks.py +theme: + name: material + custom_dir: overrides + icon: + repo: fontawesome/brands/github + favicon: "data:," + font: false + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: custom + accent: teal + toggle: + icon: material/weather-night + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: custom + accent: teal + toggle: + icon: material/weather-sunny + name: Switch to light mode + features: + - navigation.sections + - navigation.expand + - navigation.top + - navigation.tracking + - toc.follow + - content.code.copy + - search.suggest + - search.highlight +plugins: + - search + - blog: + post_url_format: '{slug}' + archive: false + categories: false + authors: false +markdown_extensions: + - admonition + - tables + - attr_list + - md_in_html + - pymdownx.details + - pymdownx.tabbed: + alternate_style: true + - pymdownx.emoji + - pymdownx.superfences + - pymdownx.highlight + - toc: + permalink: true +extra_css: + - assets/handbook.css +extra_javascript: + - assets/repository-stats.js +nav: + - Overview: index.md + - User guide: + - Build from source: building.md + - Command-line interface: rocjitsu-cli.md + - Configuration: configuration.md + - Race detection: race-detector.md + - GPU debugging: rocgdb-debugging.md + - Benchmarking: benchmarking.md + - QEMU VFIO-user compute: qemu-vfio.md + - Vulkan compute: vulkan-compute.md + - Memory wait diagnostics: memory-wait-diagnostics.md + - Developer guide: + - Architecture: architecture.md + - Code style: style.md + - Simulation engine: simdojo.md + - GPU hardware model: vm-design.md + - Binary translation: dbt-design.md + - Translation tool: rj_dbt_translate.md + - DBT guest mode: rocjitsu_dbt_guest.md + - Binary instrumentation: dbi-design.md + - Instrumentation overview: dbi-overview.md + - Execution plugins: plugins.md + - ISA code generation: codegen.md + - ISA target providers: isa-target-providers.md + - ISA gap audit: isa-gap-audit.md + - Adding a GPU: npi.md + - Asynchronous MMA execution: async-instructions.md + - MMA lookahead admission: mma-admission.md + - Concurrent dispatch: concurrent-dispatch.md + - Memory wait coverage: memory-wait-counter-coverage.md + - Project: + - Blog: blog/index.md + - Performance dashboard: performance-dashboard.md +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/ROCm/rocm-systems/tree/develop/emulation/rocjitsu + name: Rocjitsu on GitHub + - icon: material/bug-outline + link: https://github.com/ROCm/rocm-systems/issues + name: Report an issue +copyright: >- + Copyright © 2025–2026 Advanced Micro Devices, Inc.
+ MIT License diff --git a/emulation/rocjitsu/website/handbook/overrides/partials/source.html b/emulation/rocjitsu/website/handbook/overrides/partials/source.html new file mode 100644 index 000000000000..3f7268b0fa98 --- /dev/null +++ b/emulation/rocjitsu/website/handbook/overrides/partials/source.html @@ -0,0 +1,16 @@ +{# Keep repository facts visible even if GitHub is temporarily unavailable. #} +{% set stats = config.extra.repository_stats %} +
+ +
+ {% include ".icons/" ~ config.theme.icon.repo ~ ".svg" %} +
+
+ {{ config.repo_name }} +
    +
  • {{ stats.get('stargazers_count', '—') }}
    • +
  • {{ stats.get('forks_count', '—') }}
    • +
+
+ +
diff --git a/emulation/rocjitsu/website/handbook/requirements.txt b/emulation/rocjitsu/website/handbook/requirements.txt new file mode 100644 index 000000000000..0c525b8a62c3 --- /dev/null +++ b/emulation/rocjitsu/website/handbook/requirements.txt @@ -0,0 +1,2 @@ +mkdocs==1.6.1 +mkdocs-material==9.7.7 diff --git a/emulation/rocjitsu/website/handbook/tests/repository-stats.test.cjs b/emulation/rocjitsu/website/handbook/tests/repository-stats.test.cjs new file mode 100644 index 000000000000..c2d36ef33061 --- /dev/null +++ b/emulation/rocjitsu/website/handbook/tests/repository-stats.test.cjs @@ -0,0 +1,143 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT +// Verify repository counts, cache expiry, and graceful network/storage failures. +// Usage: node --test tests/repository-stats.test.cjs + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const vm = require('node:vm'); + +const source = fs.readFileSync( + path.join(__dirname, '../assets/repository-stats.js'), 'utf8', +); +const now = 1_800_000_000_000; +const cacheKey = 'rocjitsu-repository-stats'; +const counts = { stargazers_count: 1234, forks_count: 0 }; + +async function run({ cache = null, response, storageThrows = false, initial = {} } = {}) { + const elements = ['header', 'mobile'].flatMap(context => + ['stargazers_count', 'forks_count'].map(key => ({ + context, + dataset: { repositoryStat: key }, + textContent: initial[key] ?? '—', + title: 'Count unavailable', + attributes: {}, + setAttribute(name, value) { this.attributes[name] = value; }, + })), + ); + const requests = []; + const writes = []; + const result = vm.runInNewContext(source, { + Date: { now: () => now }, + document: { + querySelectorAll(selector) { + assert.equal(selector, '[data-repository-stat]'); + return elements; + }, + }, + sessionStorage: { + getItem(key) { + if (storageThrows) throw new Error('Storage disabled'); + assert.equal(key, cacheKey); + return cache; + }, + setItem(key, value) { + if (storageThrows) throw new Error('Storage disabled'); + writes.push({ key, value: JSON.parse(value) }); + }, + }, + async fetch(url) { + requests.push(url); + if (response instanceof Error) throw response; + return response || { ok: true, json: async () => counts }; + }, + }); + assert.equal(typeof result?.then, 'function', 'The loader must be awaitable'); + await result; + return { elements, requests, writes }; +} + +function assertRendered(elements) { + assert.deepEqual(elements.map(element => element.textContent), [ + '1,234', '0', '1,234', '0', + ]); + for (const element of elements) { + const expected = element.dataset.repositoryStat === 'stargazers_count' + ? 'Stars: 1,234 · ROCm/rocm-systems' + : 'Forks: 0 · ROCm/rocm-systems'; + assert.equal(element.title, expected); + assert.equal(element.attributes['aria-label'], expected); + } +} + +test('one request updates desktop/mobile counts, including zero, and caches them', async () => { + const { elements, requests, writes } = await run(); + assertRendered(elements); + assert.deepEqual(requests, ['https://api.github.com/repos/ROCm/rocm-systems']); + assert.deepEqual(writes, [{ key: cacheKey, value: { time: now, data: counts } }]); +}); + +for (const [name, response] of [ + ['network rejection', new Error('Offline')], + ['HTTP failure', { ok: false, json: async () => counts }], + ['malformed JSON', { ok: true, json: async () => { throw new Error('Invalid JSON'); } }], + ['partial counts', { ok: true, json: async () => ({ stargazers_count: 1234 }) }], + ['invalid counts', { ok: true, json: async () => ({ stargazers_count: -1, forks_count: '0' }) }], +]) { + test(`${name} preserves both placeholders without caching`, async () => { + const { elements, requests, writes } = await run({ response }); + assert.equal(requests.length, 1); + assert.deepEqual(writes, []); + for (const element of elements) { + assert.equal(element.textContent, '—'); + assert.equal(element.title, 'Count unavailable'); + assert.deepEqual(element.attributes, {}); + } + }); +} + +test('fresh cache renders both contexts without a request', async () => { + const cache = JSON.stringify({ time: now - 60_000, data: counts }); + const { elements, requests, writes } = await run({ cache }); + assertRendered(elements); + assert.deepEqual(requests, []); + assert.deepEqual(writes, []); +}); + +test('expired counts remain visible when the refresh fails', async () => { + const cache = JSON.stringify({ time: now - 16 * 60 * 1000, data: counts }); + const { elements, requests, writes } = await run({ cache, response: new Error('Offline') }); + assertRendered(elements); + assert.equal(requests.length, 1); + assert.deepEqual(writes, []); +}); + +test('server-rendered counts survive a failed browser request without a cache', async () => { + const initial = { stargazers_count: '516', forks_count: '425' }; + const { elements, requests, writes } = await run({ initial, response: new Error('Blocked') }); + assert.deepEqual(elements.map(element => element.textContent), ['516', '425', '516', '425']); + assert.equal(requests.length, 1); + assert.deepEqual(writes, []); +}); + +for (const [name, cache] of [ + ['expired', JSON.stringify({ time: now - 15 * 60 * 1000, data: counts })], + ['future timestamp', JSON.stringify({ time: now + 1000, data: counts })], + ['malformed', '{broken'], + ['partial', JSON.stringify({ time: now, data: { forks_count: 0 } })], +]) { + test(`${name} cache is replaced from the API`, async () => { + const { elements, requests, writes } = await run({ cache }); + assertRendered(elements); + assert.equal(requests.length, 1); + assert.deepEqual(writes, [{ key: cacheKey, value: { time: now, data: counts } }]); + }); +} + +test('unavailable storage does not prevent fetching or rendering', async () => { + const { elements, requests } = await run({ storageThrows: true }); + assertRendered(elements); + assert.equal(requests.length, 1); +}); diff --git a/emulation/rocjitsu/website/handbook/tests/test_canonical_docs.py b/emulation/rocjitsu/website/handbook/tests/test_canonical_docs.py new file mode 100644 index 000000000000..385d5696419e --- /dev/null +++ b/emulation/rocjitsu/website/handbook/tests/test_canonical_docs.py @@ -0,0 +1,249 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# Verify canonical documentation publication and source-link rendering. +# Usage: python -m unittest discover -s tests + +import importlib.util +from http.client import IncompleteRead +import io +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from markdown import Markdown +import yaml + +HANDBOOK = Path(__file__).resolve().parents[1] +SOURCE_URL = 'https://github.com/ROCm/rocm-systems/tree/develop/emulation/rocjitsu' +spec = importlib.util.spec_from_file_location('handbook_hooks', HANDBOOK / 'hooks.py') +hooks = importlib.util.module_from_spec(spec) +spec.loader.exec_module(hooks) + + +class RepositoryStatsTests(unittest.TestCase): + def tearDown(self): + hooks.repository_stats.cache_clear() + + def test_valid_counts_are_fetched_once_including_zero(self): + with patch.object( + hooks, + 'urlopen', + return_value=io.StringIO('{"stargazers_count":0,"forks_count":42}'), + ) as fetch: + expected = {'stargazers_count': 0, 'forks_count': 42} + self.assertEqual(hooks.repository_stats(), expected) + self.assertEqual(hooks.repository_stats(), expected) + fetch.assert_called_once() + + def test_invalid_responses_leave_an_offline_build_usable(self): + for response in ( + 'not json', + 'null', + '[]', + '{}', + '{"stargazers_count":true,"forks_count":2}', + '{"stargazers_count":-1,"forks_count":2}', + ): + with self.subTest(response=response): + hooks.repository_stats.cache_clear() + with patch.object(hooks, 'urlopen', return_value=io.StringIO(response)): + self.assertEqual(hooks.repository_stats(), {}) + + def test_network_failure_leaves_an_offline_build_usable(self): + for failure in (OSError('offline'), IncompleteRead(b'{')): + with self.subTest(failure=failure): + hooks.repository_stats.cache_clear() + with patch.object(hooks, 'urlopen', side_effect=failure): + self.assertEqual(hooks.repository_stats(), {}) + + +class SourceLinksTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.docs = self.root / 'rocjitsu' / 'docs' + self.docs.mkdir(parents=True) + (self.docs / 'architecture.md').write_text('# Architecture\n') + (self.docs.parent / 'tools').mkdir() + (self.docs.parent / 'tools' / 'run.py').write_text('# Source\n') + (self.root / 'outside.md').write_text('# Outside Rocjitsu\n') + extension = hooks.SourceLinksExtension(self.docs, SOURCE_URL) + extension.source_path = self.docs / 'index.md' + self.markdown = Markdown(extensions=[extension, 'fenced_code']) + + def test_existing_source_file_and_directory_links(self): + html = self.markdown.convert( + '[Source](../tools/run.py?plain=1#L2) [Directory](../tools/)' + ) + self.assertIn( + 'href="https://github.com/ROCm/rocm-systems/blob/develop/' + 'emulation/rocjitsu/tools/run.py?plain=1#L2"', + html, + ) + self.assertIn(f'href="{SOURCE_URL}/tools"', html) + + def test_document_missing_and_outside_links_remain_unchanged(self): + links = [ + 'architecture.md#layers', + '../missing.py', + '../../outside.md', + '#local-heading', + 'https://example.com/source.py', + ] + for link in links: + with self.subTest(link=link): + html = self.markdown.reset().convert(f'[Link]({link})') + self.assertIn(f'href="{link}"', html) + + def test_nested_page_resolves_source_relative_to_page(self): + nested = self.docs / 'guides' + nested.mkdir() + extension = hooks.SourceLinksExtension(self.docs, SOURCE_URL) + extension.source_path = nested / 'guide.md' + html = Markdown(extensions=[extension]).convert('[Source](../../tools/run.py)') + self.assertIn( + 'href="https://github.com/ROCm/rocm-systems/blob/develop/' + 'emulation/rocjitsu/tools/run.py"', + html, + ) + + def test_code_examples_are_not_rewritten(self): + example = '[Source](../tools/run.py)' + html = self.markdown.convert(f'`{example}`\n\n```markdown\n{example}\n```') + self.assertIn(f'{example}', html) + self.assertIn(f'{example}\n', html) + self.assertNotIn('github.com', html) + + +class CanonicalBuildTests(unittest.TestCase): + def test_build_reads_canonical_docs_without_modifying_them(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / 'rocjitsu' + docs = root / 'docs' + handbook = root / 'website' / 'handbook' + handbook.mkdir(parents=True) + (docs / 'sphinx').mkdir(parents=True) + readme = root / 'README.md' + (root / 'CMakeLists.txt').write_text('# Build configuration\n') + building = docs / 'building.md' + building.write_text('# Building\n\nCANONICAL_FIRST_REVISION\n') + (docs / 'sphinx' / 'private.md').write_text( + '# Excluded\n\nSPHINX_ONLY_MARKER\n' + ) + for directory in ('assets', 'overrides'): + shutil.copytree(HANDBOOK / directory, handbook / directory) + shutil.copy2(HANDBOOK / 'hooks.py', handbook / 'hooks.py') + config = yaml.safe_load((HANDBOOK / 'mkdocs.yml').read_text()) + config['nav'] = [ + {'Overview': 'index.md'}, + {'Building': 'building.md'}, + {'Blog': 'blog/index.md'}, + ] + (docs / 'blog' / 'posts' / 'nested').mkdir(parents=True) + (docs / 'blog' / 'index.md').write_text('# Blog\n') + for name, relative in ( + ('first', '../../../'), + ('nested/second', '../../../../'), + ): + (docs / 'blog' / 'posts' / f'{name}.md').write_text( + f'---\ndate: 2025-01-01\n---\n\n# {Path(name).name}\n\n' + f'[Source]({relative}CMakeLists.txt?plain=1#L1)\n\n' + f'[Home]({relative}README.md#supported-architectures)\n\n' + '\n\nFull post body.\n' + ) + (handbook / 'mkdocs.yml').write_text(yaml.safe_dump(config)) + + for marker in ('CANONICAL_FIRST_REVISION', 'CANONICAL_SECOND_REVISION'): + with self.subTest(marker=marker): + readme.write_text( + f'# Project\n\n## Supported architectures\n\n{marker}\n\n' + '[![CI](https://github.com/ROCm/rocm-systems/actions/' + 'workflows/rocjitsu-ci.yml/badge.svg)]' + '(https://github.com/ROCm/rocm-systems/actions)\n\n' + '[Build](docs/building.md#setup)\n' + ) + readme_before = readme.read_bytes() + building.write_text( + f'# Building\n\n{marker}\n\n' + '[Source](../CMakeLists.txt)\n\n## Setup\n\n' + '[Support](../README.md#supported-architectures)\n' + ) + before = { + path.relative_to(docs): path.read_bytes() + for path in docs.rglob('*') + if path.is_file() + } + result = subprocess.run( + [sys.executable, '-m', 'mkdocs', 'build', '--strict'], + cwd=handbook, + env={**os.environ, 'ROCJITSU_DOCS_OFFLINE': '1'}, + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, result.stdout + result.stderr + ) + site = handbook / 'site' + overview = (site / 'index.html').read_text() + self.assertIn(marker, overview) + self.assertNotIn('/badge.svg', overview) + self.assertIn('Material for MkDocs', overview) + self.assertIn('href="building/#setup"', overview) + self.assertFalse((site / 'project-overview').exists()) + self.assertEqual(readme_before, readme.read_bytes()) + page = (site / 'building' / 'index.html').read_text() + self.assertIn('href="../#supported-architectures"', page) + self.assertIn(marker, page) + self.assertIn('

Setup', page) + self.assertIn( + 'href="https://github.com/ROCm/rocm-systems/blob/develop/' + 'emulation/rocjitsu/CMakeLists.txt"', + page, + ) + if marker == 'CANONICAL_SECOND_REVISION': + self.assertNotIn('CANONICAL_FIRST_REVISION', page) + search = (site / 'search' / 'search_index.json').read_text() + self.assertIn(marker, search) + self.assertNotIn('SPHINX_ONLY_MARKER', search) + for output, home, count in ( + ('blog/index.html', '../#supported-architectures', 2), + ('blog/first/index.html', '../../#supported-architectures', 1), + ('blog/second/index.html', '../../#supported-architectures', 1), + ): + rendered = (site / output).read_text() + self.assertEqual( + rendered.count( + 'href="https://github.com/ROCm/rocm-systems/blob/develop/' + 'emulation/rocjitsu/CMakeLists.txt?plain=1#L1"' + ), + count, + output, + ) + self.assertEqual( + rendered.count(f'href="{home}">Home'), count, output + ) + self.assertNotIn('href="../../../', rendered, output) + self.assertFalse((site / 'sphinx').exists()) + self.assertEqual( + (site / 'assets' / 'handbook.css').read_bytes(), + (handbook / 'assets' / 'handbook.css').read_bytes(), + ) + self.assertEqual( + before, + { + path.relative_to(docs): path.read_bytes() + for path in docs.rglob('*') + if path.is_file() + }, + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/emulation/rocjitsu/website/handbook/tests/test_guide_rendering.py b/emulation/rocjitsu/website/handbook/tests/test_guide_rendering.py new file mode 100644 index 000000000000..ca1749d89c57 --- /dev/null +++ b/emulation/rocjitsu/website/handbook/tests/test_guide_rendering.py @@ -0,0 +1,173 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# Check that publishing existing guides preserves their structural meaning. +# Usage: python -m unittest discover -s tests + +from html.parser import HTMLParser +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from xml.etree.ElementTree import Element, SubElement + +HANDBOOK = Path(__file__).resolve().parents[1] + + +class RenderedLists(HTMLParser): + """Keep list hierarchy and each item's complete visible text.""" + + def __init__(self, html): + super().__init__() + self.root = Element('root') + self.stack = [self.root] + self.feed(html) + + def handle_starttag(self, tag, attrs): + if tag in ('ol', 'ul', 'li'): + self.stack.append(SubElement(self.stack[-1], tag)) + + def handle_endtag(self, tag): + if tag in ('ol', 'ul', 'li'): + self.stack.pop() + + def handle_data(self, data): + for node in self.stack[1:]: + node.text = (node.text or '') + data + + +class GuideRenderingTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + temporary = tempfile.TemporaryDirectory() + cls.addClassCleanup(temporary.cleanup) + cls.site = Path(temporary.name) / 'site' + result = subprocess.run( + [ + sys.executable, + '-m', + 'mkdocs', + 'build', + '--strict', + '--site-dir', + str(cls.site), + ], + cwd=HANDBOOK, + env={**os.environ, 'ROCJITSU_DOCS_OFFLINE': '1'}, + capture_output=True, + text=True, + ) + if result.returncode: + raise AssertionError(result.stdout + result.stderr) + + def page(self, name): + return (self.site / name / 'index.html').read_text() + + def test_race_checks_belong_to_step_five_of_six(self): + lists = RenderedLists(self.page('race-detector')) + steps = [ + node + for node in lists.root.iter('ol') + if 'Wave 0 executes ds_write_b32' in (node.text or '') + ] + self.assertEqual(len(steps), 1) + items = steps[0].findall('li') + self.assertEqual(len(items), 6) + self.assertIn('Wave 0 executes ds_read_b32', items[4].text) + checks = items[4].findall('./ul/li') + self.assertEqual(len(checks), 3) + for item, label in zip(checks, ('Fast path', 'Slow path', 'Race reported.')): + self.assertIn(label, item.text) + self.assertIn('What s_barrier would fix', items[5].text) + + def test_lists_after_introductory_prose_are_list_items(self): + for page, marker in ( + ('vm-design', 'kernel_entry_pc'), + ('vm-design', 's_gl1_inv'), + ('dbi-design', 'VGPRs — a direct'), + ('dbi-design', 'anchor_offset is dword aligned.'), + ('dbi-design', 'build_scratch_store_dword / build_scratch_load_dword'), + ('rocjitsu-cli', 'RPC_IOCTL (ALLOC_MEMORY response)'), + ('rocjitsu-cli', 'UnixTransport::listen(endpoint)'), + ('rocjitsu-cli', 'Unix: Unix domain socket + SCM_RIGHTS'), + ): + with self.subTest(page=page, marker=marker): + lists = RenderedLists(self.page(page)) + self.assertTrue( + any(marker in (li.text or '') for li in lists.root.iter('li')), + f'{marker} must be inside a list item', + ) + + def test_continuation_paragraphs_stay_in_their_list_item(self): + for page, first, continuation, next_item in ( + ( + 'dbi-overview', + 'Sampling', + 'Finer granularity', + 'Filtering in the probe.', + ), + ( + 'dbi-overview', + 'How long it must live.', + 'Completion is observable', + 'Read from the dispatch packet.', + ), + ( + 'rocjitsu_dbt_guest', + 'HSA_TOOLS_LIB', + 'The supported DBT launch path', + 'HSA_TOOLS_DISABLE_REGISTER=1', + ), + ): + with self.subTest(page=page): + lists = RenderedLists(self.page(page)) + items = [ + li + for li in lists.root.iter('li') + if continuation in (li.text or '') + ] + self.assertEqual(len(items), 1) + self.assertIn(first, items[0].text) + self.assertNotIn(next_item, items[0].text) + + def test_numbered_principles_remain_single_lists(self): + for page, marker, count, item_index, continuation in ( + ('dbi-overview', 'Transparency.', 7, 0, 'This extends to state'), + ( + 'rocjitsu_dbt_guest', + 'ROCR must internally discover', + 5, + 1, + 'Calls that applications use', + ), + ): + with self.subTest(page=page): + lists = RenderedLists(self.page(page)) + ordered = [ + node + for node in lists.root.iter('ol') + if marker in (node.text or '') + ] + self.assertEqual(len(ordered), 1) + items = ordered[0].findall('li') + self.assertEqual(len(items), count) + self.assertIn(continuation, items[item_index].text) + + def test_template_names_are_literal_code(self): + html = self.page('simdojo') + for name, parameters in ( + ('Clocked', 'Base'), + ('Functional', 'Base'), + ('ExecBase', 'Mode, Base'), + ('Cache', 'NumSets, Assoc, LineSizeBits'), + ('VectorReg', 'N, T'), + ): + with self.subTest(name=name): + expected = f'{name}<{parameters}>' + self.assertTrue(expected in html, expected) + self.assertFalse(f'{name}\\' in html, f'Escaped template: {name}') + + +if __name__ == '__main__': + unittest.main() diff --git a/profilers/profiler-hub/CMakeLists.txt b/profilers/profiler-hub/CMakeLists.txt index d4971eabf2f0..24157d468029 100644 --- a/profilers/profiler-hub/CMakeLists.txt +++ b/profilers/profiler-hub/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -cmake_minimum_required(VERSION 3.21 FATAL_ERROR) +cmake_minimum_required(VERSION 3.25 FATAL_ERROR) project(profiler-hub VERSION 0.2.0) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -18,9 +18,14 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) # ------------------------------------------------------------ # option(PROFILER_HUB_BUILD_TESTS "Build unit tests" ON) -option(PROFILER_HUB_BUILD_BENCHMARKS "Build performance benchmarks" ON) +option(PROFILER_HUB_BUILD_BENCHMARKS "Build performance benchmarks" OFF) option(PROFILER_HUB_ENABLE_LOGGING "Enable logging" OFF) option(PROFILER_HUB_ENABLE_COVERAGE "Enable code coverage" OFF) +option( + PROFILER_HUB_FETCH_DEPENDENCIES + "Download missing or unsuitable dependencies at configure time" + OFF +) if(PROFILER_HUB_BUILD_TESTS) enable_testing() @@ -196,10 +201,9 @@ add_library(profiler-hub SHARED $) add_library(profiler-hub-static STATIC $) set_target_properties(profiler-hub-static PROPERTIES OUTPUT_NAME profiler-hub) -target_link_options( - profiler-hub - PRIVATE -Wl,--exclude-libs,$ -) +if(TARGET profiler-hub-sqlite3-shared) + set_target_properties(profiler-hub PROPERTIES INSTALL_RPATH "$ORIGIN") +endif() # Embed semantic version + ABI version on the shared library so that # libprofiler-hub.so. is the SONAME and libprofiler-hub.so... @@ -247,9 +251,10 @@ endforeach() # target_link_libraries() above (that only supplies the $ # object library), so consumers of the *installed* static package would # otherwise get no transitive link requirements at all. Explicitly restore -# the ones that are genuinely resolvable by real consumers; queries, -# sqlite3, and profiler-hub-objects itself are always in-tree-only and -# never belong here. nlohmann_json is deliberately excluded too: it is a +# the ones that are genuinely resolvable by real consumers; queries and +# profiler-hub-objects itself are always in-tree-only and never belong here. +# sqlite3 is omitted, as it has always been: listing it would change the +# installed static contract. nlohmann_json is deliberately excluded too: it is a # PRIVATE, header-only dependency of profiler-hub-objects that never appears # in the installed public headers, so it carries no link symbols a static # consumer needs. profiler-hub (SHARED) needs no such list: its dependencies diff --git a/profilers/profiler-hub/cmake/benchmark.cmake b/profilers/profiler-hub/cmake/benchmark.cmake index 49d3c26371be..94184498e145 100644 --- a/profilers/profiler-hub/cmake/benchmark.cmake +++ b/profilers/profiler-hub/cmake/benchmark.cmake @@ -3,20 +3,24 @@ include_guard(DIRECTORY) -set(BENCHMARK_VERSION "1.8.3" CACHE STRING "Google Benchmark version") +set(BENCHMARK_VERSION "1.8.3" CACHE STRING "Minimum Google Benchmark version") -find_package(benchmark QUIET) +# Fetching is Off by default: a missing or old package errors out. +if(NOT PROFILER_HUB_FETCH_DEPENDENCIES) + find_package(benchmark ${BENCHMARK_VERSION}) + + if(NOT benchmark_FOUND) + message( + FATAL_ERROR + "profiler-hub requires Google Benchmark ${BENCHMARK_VERSION} or newer on CMAKE_PREFIX_PATH. Configure with -DPROFILER_HUB_BUILD_BENCHMARKS=OFF to skip the benchmarks, or with -DPROFILER_HUB_FETCH_DEPENDENCIES=ON to download it instead." + ) + endif() -if(benchmark_FOUND) message( STATUS "Using system Google Benchmark (version ${benchmark_VERSION})" ) else() - message( - STATUS - "System Google Benchmark not found, fetching version ${BENCHMARK_VERSION}" - ) include(FetchContent) FetchContent_Declare( @@ -24,6 +28,10 @@ else() GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v${BENCHMARK_VERSION} GIT_SHALLOW TRUE + # Without this, the MakeAvailable below always fetches. + # FetchContent derives the find_package call from the content name, which + # here is not the name Google Benchmark installs itself under. + FIND_PACKAGE_ARGS ${BENCHMARK_VERSION} NAMES benchmark ) set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) @@ -31,6 +39,7 @@ else() set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "" FORCE) set(BENCHMARK_USE_BUNDLED_GTEST OFF CACHE BOOL "" FORCE) + # Tries find_package() first, fetches only if that fails. FetchContent_MakeAvailable(googlebenchmark) if(NOT TARGET benchmark::benchmark) diff --git a/profilers/profiler-hub/cmake/fetch_sqlite3.cmake b/profilers/profiler-hub/cmake/fetch_sqlite3.cmake new file mode 100644 index 000000000000..5d75793638b6 --- /dev/null +++ b/profilers/profiler-hub/cmake/fetch_sqlite3.cmake @@ -0,0 +1,79 @@ +# Copyright (c) Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Run by `cmake -P` from the add_custom_command in sqlite3.cmake. A COMMAND +# list cannot branch on a result, so the clone's full-clone retry lives here. + +if(EXISTS "${SQLITE3_SOURCE_DIR}") + file(REMOVE_RECURSE "${SQLITE3_SOURCE_DIR}") +endif() + +message( + STATUS + "[profiler-hub] Cloning SQLite3 from ${SQLITE3_GIT_URL} @ ${SQLITE3_GIT_TAG}" +) + +execute_process( + COMMAND + ${GIT_EXECUTABLE} clone --depth 1 --filter=blob:none --branch + ${SQLITE3_GIT_TAG} ${SQLITE3_GIT_URL} ${SQLITE3_SOURCE_DIR} + RESULT_VARIABLE _sqlite3_clone_rc +) + +if(NOT _sqlite3_clone_rc EQUAL 0) + message(STATUS "[profiler-hub] Optimized clone failed; retrying full clone") + if(EXISTS "${SQLITE3_SOURCE_DIR}") + file(REMOVE_RECURSE "${SQLITE3_SOURCE_DIR}") + endif() + execute_process( + COMMAND + ${GIT_EXECUTABLE} clone --branch ${SQLITE3_GIT_TAG} + ${SQLITE3_GIT_URL} ${SQLITE3_SOURCE_DIR} + RESULT_VARIABLE _sqlite3_clone_rc + ) +endif() + +if(NOT _sqlite3_clone_rc EQUAL 0) + message( + FATAL_ERROR + "[profiler-hub] git clone of SQLite3 failed (rc=${_sqlite3_clone_rc})" + ) +endif() + +message(STATUS "[profiler-hub] Generating SQLite3 amalgamation") + +execute_process( + COMMAND ./configure --disable-tcl + WORKING_DIRECTORY ${SQLITE3_SOURCE_DIR} + RESULT_VARIABLE _sqlite3_configure_rc +) + +if(NOT _sqlite3_configure_rc EQUAL 0) + message( + FATAL_ERROR + "[profiler-hub] SQLite3 ./configure failed (rc=${_sqlite3_configure_rc})" + ) +endif() + +execute_process( + COMMAND ${MAKE_COMMAND} sqlite3.c + WORKING_DIRECTORY ${SQLITE3_SOURCE_DIR} + RESULT_VARIABLE _sqlite3_make_rc +) + +if(NOT _sqlite3_make_rc EQUAL 0) + message( + FATAL_ERROR + "[profiler-hub] SQLite3 amalgamation generation failed (rc=${_sqlite3_make_rc})" + ) +endif() + +if( + NOT EXISTS "${SQLITE3_SOURCE_DIR}/sqlite3.c" + OR NOT EXISTS "${SQLITE3_SOURCE_DIR}/sqlite3.h" +) + message( + FATAL_ERROR + "[profiler-hub] SQLite3 amalgamation files not found after build" + ) +endif() diff --git a/profilers/profiler-hub/cmake/fmt.cmake b/profilers/profiler-hub/cmake/fmt.cmake index c0fe0773ffff..a722c9505a9a 100644 --- a/profilers/profiler-hub/cmake/fmt.cmake +++ b/profilers/profiler-hub/cmake/fmt.cmake @@ -3,14 +3,21 @@ include_guard(DIRECTORY) -set(FMT_VERSION "11.2.0" CACHE STRING "fmt version") +set(FMT_VERSION "11.1.3" CACHE STRING "Minimum fmt version") -find_package(fmt ${FMT_VERSION} QUIET) +# Fetching is Off by default: a missing or old package errors out. +if(NOT PROFILER_HUB_FETCH_DEPENDENCIES) + find_package(fmt ${FMT_VERSION}) + + if(NOT fmt_FOUND) + message( + FATAL_ERROR + "profiler-hub requires fmt ${FMT_VERSION} or newer on CMAKE_PREFIX_PATH. Configure with -DPROFILER_HUB_FETCH_DEPENDENCIES=ON to download it instead." + ) + endif() -if(fmt_FOUND) message(STATUS "Using system fmt (version ${fmt_VERSION})") else() - message(STATUS "System fmt not found, fetching version ${FMT_VERSION}") include(FetchContent) FetchContent_Declare( @@ -18,6 +25,8 @@ else() GIT_REPOSITORY https://github.com/fmtlib/fmt.git GIT_TAG ${FMT_VERSION} GIT_SHALLOW TRUE + # Without this, the MakeAvailable below always fetches. + FIND_PACKAGE_ARGS ${FMT_VERSION} ) set(FMT_INSTALL OFF CACHE BOOL "" FORCE) @@ -31,6 +40,7 @@ else() set(_PROFILER_HUB_BUILD_SHARED_LIBS_BACKUP ${BUILD_SHARED_LIBS}) set(BUILD_SHARED_LIBS OFF) + # Tries find_package() first, fetches only if that fails. FetchContent_MakeAvailable(fmt) set(BUILD_SHARED_LIBS ${_PROFILER_HUB_BUILD_SHARED_LIBS_BACKUP}) diff --git a/profilers/profiler-hub/cmake/gtest.cmake b/profilers/profiler-hub/cmake/gtest.cmake index 5d6d020091fc..210e2c1d7998 100644 --- a/profilers/profiler-hub/cmake/gtest.cmake +++ b/profilers/profiler-hub/cmake/gtest.cmake @@ -3,17 +3,21 @@ include_guard(DIRECTORY) -set(GTEST_VERSION "1.14.0" CACHE STRING "Google Test version") +set(GTEST_VERSION "1.14.0" CACHE STRING "Minimum Google Test version") -find_package(GTest QUIET) +# Fetching is Off by default: a missing or old package errors out. +if(NOT PROFILER_HUB_FETCH_DEPENDENCIES) + find_package(GTest ${GTEST_VERSION}) + + if(NOT GTest_FOUND) + message( + FATAL_ERROR + "profiler-hub requires GoogleTest ${GTEST_VERSION} or newer on CMAKE_PREFIX_PATH. Configure with -DPROFILER_HUB_BUILD_TESTS=OFF to skip the unit tests, or with -DPROFILER_HUB_FETCH_DEPENDENCIES=ON to download it instead." + ) + endif() -if(GTest_FOUND) message(STATUS "Using system GoogleTest (version ${GTest_VERSION})") else() - message( - STATUS - "System GoogleTest not found, fetching version ${GTEST_VERSION}" - ) include(FetchContent) FetchContent_Declare( @@ -21,12 +25,17 @@ else() GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG v${GTEST_VERSION} GIT_SHALLOW TRUE + # Without this, the MakeAvailable below always fetches. + # FetchContent derives the find_package call from the content name, which + # here is not the name GoogleTest installs itself under. + FIND_PACKAGE_ARGS ${GTEST_VERSION} NAMES GTest ) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) set(BUILD_GMOCK ON CACHE BOOL "" FORCE) set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) + # Tries find_package() first, fetches only if that fails. FetchContent_MakeAvailable(googletest) if(NOT TARGET GTest::gtest) diff --git a/profilers/profiler-hub/cmake/nlohmann_json.cmake b/profilers/profiler-hub/cmake/nlohmann_json.cmake index 1f707b2deb81..718b6f43e33c 100644 --- a/profilers/profiler-hub/cmake/nlohmann_json.cmake +++ b/profilers/profiler-hub/cmake/nlohmann_json.cmake @@ -3,23 +3,38 @@ include_guard(DIRECTORY) -set(NLOHMANN_JSON_VERSION "3.11.3" CACHE STRING "nlohmann_json version") - -# Always fetch rather than find_package(): some super-project build orchestrators -# (e.g. TheRock) intercept find_package() calls and reject any package not -# explicitly declared as a dependency of this subproject. Vendoring -# unconditionally avoids that dependency-declaration requirement entirely. -message(STATUS "Fetching nlohmann_json version ${NLOHMANN_JSON_VERSION}") -include(FetchContent) - -FetchContent_Declare( - nlohmann_json - GIT_REPOSITORY https://github.com/nlohmann/json.git - GIT_TAG v${NLOHMANN_JSON_VERSION} - GIT_SHALLOW TRUE -) - -set(JSON_BuildTests OFF CACHE BOOL "" FORCE) -set(JSON_Install OFF CACHE BOOL "" FORCE) - -FetchContent_MakeAvailable(nlohmann_json) +set(NLOHMANN_JSON_VERSION "3.11.3" CACHE STRING "Minimum nlohmann_json version") + +# Fetching is Off by default: a missing or old package errors out. +if(NOT PROFILER_HUB_FETCH_DEPENDENCIES) + find_package(nlohmann_json ${NLOHMANN_JSON_VERSION}) + + if(NOT nlohmann_json_FOUND) + message( + FATAL_ERROR + "profiler-hub requires nlohmann_json ${NLOHMANN_JSON_VERSION} or newer on CMAKE_PREFIX_PATH. Configure with -DPROFILER_HUB_FETCH_DEPENDENCIES=ON to download it instead." + ) + endif() + + message( + STATUS + "Using system nlohmann_json (version ${nlohmann_json_VERSION})" + ) +else() + include(FetchContent) + + FetchContent_Declare( + nlohmann_json + GIT_REPOSITORY https://github.com/nlohmann/json.git + GIT_TAG v${NLOHMANN_JSON_VERSION} + GIT_SHALLOW TRUE + # Without this, the MakeAvailable below always fetches. + FIND_PACKAGE_ARGS ${NLOHMANN_JSON_VERSION} + ) + + set(JSON_BuildTests OFF CACHE BOOL "" FORCE) + set(JSON_Install OFF CACHE BOOL "" FORCE) + + # Tries find_package() first, fetches only if that fails. + FetchContent_MakeAvailable(nlohmann_json) +endif() diff --git a/profilers/profiler-hub/cmake/spdlog.cmake b/profilers/profiler-hub/cmake/spdlog.cmake index 972a1c33ea2e..34eb0375bdca 100644 --- a/profilers/profiler-hub/cmake/spdlog.cmake +++ b/profilers/profiler-hub/cmake/spdlog.cmake @@ -3,38 +3,21 @@ include_guard(DIRECTORY) -set(SPDLOG_VERSION "1.15.3" CACHE STRING "spdlog version") +set(SPDLOG_VERSION "1.15.3" CACHE STRING "Minimum spdlog version") -find_package(spdlog ${SPDLOG_VERSION} QUIET) +# Fetching is Off by default: a missing or old package errors out. +if(NOT PROFILER_HUB_FETCH_DEPENDENCIES) + find_package(spdlog ${SPDLOG_VERSION}) -# A system spdlog is only safe to reuse if it was built with external fmt. -# If it was built against its bundled fmt, its public headers pull in -# and libspdlog exports bundled-fmt symbols, which -# would coexist with the external fmt that profiler-hub links - two fmt -# copies in one binary. Detect this via the interface compile definition -# that spdlog's exported target carries when SPDLOG_FMT_EXTERNAL was set. -if(spdlog_FOUND) - get_target_property( - _spdlog_iface_defs - spdlog::spdlog - INTERFACE_COMPILE_DEFINITIONS - ) - if(NOT _spdlog_iface_defs MATCHES "SPDLOG_FMT_EXTERNAL") + if(NOT spdlog_FOUND) message( - STATUS - "System spdlog uses bundled fmt; falling back to FetchContent with external fmt" + FATAL_ERROR + "profiler-hub requires spdlog ${SPDLOG_VERSION} or newer on CMAKE_PREFIX_PATH. Configure with -DPROFILER_HUB_FETCH_DEPENDENCIES=ON to download it instead." ) - set(spdlog_FOUND FALSE) endif() -endif() -if(spdlog_FOUND) message(STATUS "Using system spdlog (version ${spdlog_VERSION})") else() - message( - STATUS - "System spdlog not found, fetching version ${SPDLOG_VERSION}" - ) include(FetchContent) FetchContent_Declare( @@ -42,6 +25,8 @@ else() GIT_REPOSITORY https://github.com/gabime/spdlog.git GIT_TAG v${SPDLOG_VERSION} GIT_SHALLOW TRUE + # Without this, the MakeAvailable below always fetches. + FIND_PACKAGE_ARGS ${SPDLOG_VERSION} ) set(SPDLOG_BUILD_SHARED OFF CACHE BOOL "" FORCE) @@ -53,6 +38,7 @@ else() set(_PROFILER_HUB_BUILD_SHARED_LIBS_BACKUP ${BUILD_SHARED_LIBS}) set(BUILD_SHARED_LIBS OFF) + # Tries find_package() first, fetches only if that fails. FetchContent_MakeAvailable(spdlog) set(BUILD_SHARED_LIBS ${_PROFILER_HUB_BUILD_SHARED_LIBS_BACKUP}) @@ -66,3 +52,20 @@ else() add_library(spdlog::spdlog ALIAS spdlog) endif() endif() + +# The exported target's interface definitions are the only reliable record +# of which fmt a prebuilt spdlog was compiled against. +get_target_property( + _spdlog_iface_defs + spdlog::spdlog + INTERFACE_COMPILE_DEFINITIONS +) + +if(NOT _spdlog_iface_defs MATCHES "SPDLOG_FMT_EXTERNAL") + message( + FATAL_ERROR + "profiler-hub requires an spdlog built with SPDLOG_FMT_EXTERNAL. The one provided is built against its bundled fmt, which would put two fmt copies in one binary. Provide a suitable spdlog on CMAKE_PREFIX_PATH, or configure with -DPROFILER_HUB_FETCH_DEPENDENCIES=ON -DCMAKE_DISABLE_FIND_PACKAGE_spdlog=ON to bypass it and download one." + ) +endif() + +unset(_spdlog_iface_defs) diff --git a/profilers/profiler-hub/cmake/sqlite3.cmake b/profilers/profiler-hub/cmake/sqlite3.cmake index 706154fb2c1d..e3bb872a79a5 100644 --- a/profilers/profiler-hub/cmake/sqlite3.cmake +++ b/profilers/profiler-hub/cmake/sqlite3.cmake @@ -1,146 +1,118 @@ # Copyright (c) Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -# ----------------------------------------------------------------------------------------# -# -# SQLite3 - cloned from upstream and built from the amalgamation -# -# Mirrors the pattern used by sibling rocprofiler-systems -# (projects/rocprofiler-systems/cmake/SQLite3.cmake): fetch the upstream -# git repository at a pinned tag, then build locally. Avoids vendoring -# any binary blobs in the source tree. -# -# ----------------------------------------------------------------------------------------# - -set(SQLITE3_GIT_URL - "https://github.com/sqlite/sqlite.git" - CACHE STRING - "Upstream SQLite3 git repository URL" -) -set(SQLITE3_GIT_TAG - "version-3.45.3" - CACHE STRING - "Upstream SQLite3 git tag to check out" -) - -message( - STATUS - "[profiler-hub] Cloning SQLite3 from ${SQLITE3_GIT_URL} @ ${SQLITE3_GIT_TAG}" -) - -find_package(Git REQUIRED) -find_program(MAKE_COMMAND NAMES make gmake REQUIRED) - -set(SQLITE3_SOURCE_DIR "${PROJECT_BINARY_DIR}/external/sqlite3") -set(SQLITE3_AMALG_C "${SQLITE3_SOURCE_DIR}/sqlite3.c") -set(SQLITE3_AMALG_H "${SQLITE3_SOURCE_DIR}/sqlite3.h") - -# checkout: shallow + partial first, retry full on failure -if(NOT EXISTS "${SQLITE3_SOURCE_DIR}/configure") - if(EXISTS "${SQLITE3_SOURCE_DIR}") - file(REMOVE_RECURSE "${SQLITE3_SOURCE_DIR}") - endif() - execute_process( - COMMAND - ${GIT_EXECUTABLE} clone --depth 1 --filter=blob:none --branch - ${SQLITE3_GIT_TAG} ${SQLITE3_GIT_URL} ${SQLITE3_SOURCE_DIR} - RESULT_VARIABLE _sqlite3_clone_rc +include_guard(DIRECTORY) + +# The rocpd schema profiler-hub executes uses JSON_EXTRACT, which only became an +# unconditional part of the SQLite core in 3.38.0 (before that it was the opt-in +# SQLITE_ENABLE_JSON1 extension). Everything else profiler-hub calls is far older. +set(SQLITE3_VERSION "3.38.0" CACHE STRING "Minimum SQLite3 version") + +find_package(SQLite3 ${SQLITE3_VERSION}) + +# Fetching is Off by default: a missing or old package errors out. +if(NOT SQLite3_FOUND AND NOT PROFILER_HUB_FETCH_DEPENDENCIES) + message( + FATAL_ERROR + "profiler-hub requires SQLite3 ${SQLITE3_VERSION} or newer on CMAKE_PREFIX_PATH. Configure with -DPROFILER_HUB_FETCH_DEPENDENCIES=ON to download it instead." ) - if(NOT _sqlite3_clone_rc EQUAL 0) - message( - STATUS - "[profiler-hub] Optimized clone failed; retrying full clone" - ) - if(EXISTS "${SQLITE3_SOURCE_DIR}") - file(REMOVE_RECURSE "${SQLITE3_SOURCE_DIR}") - endif() - execute_process( - COMMAND - ${GIT_EXECUTABLE} clone --branch ${SQLITE3_GIT_TAG} - ${SQLITE3_GIT_URL} ${SQLITE3_SOURCE_DIR} - RESULT_VARIABLE _sqlite3_clone_rc - ) - endif() - if(NOT _sqlite3_clone_rc EQUAL 0) - message( - FATAL_ERROR - "[profiler-hub] git clone of SQLite3 failed (rc=${_sqlite3_clone_rc})" - ) - endif() endif() -# generate amalgamation (sqlite3.c + sqlite3.h) via upstream autotools -if(NOT EXISTS "${SQLITE3_AMALG_C}" OR NOT EXISTS "${SQLITE3_AMALG_H}") - message(STATUS "[profiler-hub] Generating SQLite3 amalgamation") - execute_process( - COMMAND ./configure --disable-tcl - WORKING_DIRECTORY ${SQLITE3_SOURCE_DIR} - RESULT_VARIABLE _sqlite3_configure_rc +if(SQLite3_FOUND) + message(STATUS "Using system SQLite3 (version ${SQLite3_VERSION})") + + # SQLite::SQLite3 is the target name both CMake's own FindSQLite3 module and + # TheRock's bundled sqlite3-config.cmake define, so one line covers both. + add_library(profiler-hub-sqlite3 INTERFACE) + target_link_libraries( + profiler-hub-sqlite3 + INTERFACE SQLite::SQLite3 ${CMAKE_DL_LIBS} ) - if(NOT _sqlite3_configure_rc EQUAL 0) - message( - FATAL_ERROR - "[profiler-hub] SQLite3 ./configure failed (rc=${_sqlite3_configure_rc})" - ) - endif() - execute_process( - COMMAND ${MAKE_COMMAND} sqlite3.c - WORKING_DIRECTORY ${SQLITE3_SOURCE_DIR} - RESULT_VARIABLE _sqlite3_make_rc +else() + # SQLite3 is not vendored into this tree. rocprofiler-sdk takes the same + # position for its own SQLite3, sourcing it from upstream as a git submodule + # (projects/rocprofiler-sdk/external/). + set(SQLITE3_GIT_URL + "https://github.com/sqlite/sqlite.git" + CACHE STRING + "Upstream SQLite3 git repository URL" + ) + set(SQLITE3_GIT_TAG + "version-3.45.3" + CACHE STRING + "Upstream SQLite3 git tag to check out" + ) + + find_package(Git REQUIRED) + find_program(MAKE_COMMAND NAMES make gmake REQUIRED) + + set(SQLITE3_SOURCE_DIR "${PROJECT_BINARY_DIR}/external/sqlite3") + set(SQLITE3_AMALG_C "${SQLITE3_SOURCE_DIR}/sqlite3.c") + set(SQLITE3_AMALG_H "${SQLITE3_SOURCE_DIR}/sqlite3.h") + + # The retry lives in the -P script because a COMMAND list cannot branch on + # a result. No USES_TERMINAL: it would put this in ninja's console pool and + # serialise the very edge this moved out of configure. + add_custom_command( + OUTPUT ${SQLITE3_AMALG_C} ${SQLITE3_AMALG_H} + COMMAND + ${CMAKE_COMMAND} -DGIT_EXECUTABLE=${GIT_EXECUTABLE} + -DMAKE_COMMAND=${MAKE_COMMAND} -DSQLITE3_GIT_URL=${SQLITE3_GIT_URL} + -DSQLITE3_GIT_TAG=${SQLITE3_GIT_TAG} + -DSQLITE3_SOURCE_DIR=${SQLITE3_SOURCE_DIR} -P + ${CMAKE_CURRENT_LIST_DIR}/fetch_sqlite3.cmake + DEPENDS ${CMAKE_CURRENT_LIST_DIR}/fetch_sqlite3.cmake + COMMENT "[profiler-hub] Fetching SQLite3 ${SQLITE3_GIT_TAG}" + VERBATIM + ) + + add_library( + profiler-hub-sqlite3-shared + SHARED + ${SQLITE3_AMALG_C} + ${SQLITE3_AMALG_H} ) - if(NOT _sqlite3_make_rc EQUAL 0) - message( - FATAL_ERROR - "[profiler-hub] SQLite3 amalgamation generation failed (rc=${_sqlite3_make_rc})" - ) - endif() - if(NOT EXISTS "${SQLITE3_AMALG_C}" OR NOT EXISTS "${SQLITE3_AMALG_H}") - message( - FATAL_ERROR - "[profiler-hub] SQLite3 amalgamation files not found after build" - ) - endif() -endif() -add_library(profiler-hub-sqlite3-static STATIC ${SQLITE3_AMALG_C}) - -target_include_directories( - profiler-hub-sqlite3-static - PUBLIC $ -) - -target_compile_definitions( - profiler-hub-sqlite3-static - PRIVATE - SQLITE_DEFAULT_MEMSTATUS=0 - SQLITE_THREADSAFE=1 - SQLITE_DEFAULT_WAL_SYNCHRONOUS=1 - SQLITE_LIKE_DOESNT_MATCH_BLOBS=1 - SQLITE_OMIT_DEPRECATED=1 - SQLITE_OMIT_PROGRESS_CALLBACK=1 - SQLITE_OMIT_SHARED_CACHE=1 -) - -# Seal the bundled SQLite symbols so they are not exported from -# libprofiler-hub.{so,a} and cannot collide with (or be interposed by) -# other sqlite3 versions bundled by sibling components on TheRock. -target_compile_options( - profiler-hub-sqlite3-static - PRIVATE -O2 -fPIC -fvisibility=hidden -) - -set_target_properties( - profiler-hub-sqlite3-static - PROPERTIES POSITION_INDEPENDENT_CODE ON C_STANDARD 11 -) - -add_library(profiler-hub-sqlite3 INTERFACE) -target_link_libraries( - profiler-hub-sqlite3 - INTERFACE profiler-hub-sqlite3-static ${CMAKE_DL_LIBS} -) - -message( - STATUS - "[profiler-hub] SQLite3 amalgamation source: ${SQLITE3_SOURCE_DIR}" -) + target_include_directories( + profiler-hub-sqlite3-shared + PUBLIC $ + ) + + target_compile_definitions( + profiler-hub-sqlite3-shared + PRIVATE + SQLITE_DEFAULT_MEMSTATUS=0 + SQLITE_THREADSAFE=1 + SQLITE_DEFAULT_WAL_SYNCHRONOUS=1 + SQLITE_LIKE_DOESNT_MATCH_BLOBS=1 + SQLITE_OMIT_DEPRECATED=1 + SQLITE_OMIT_PROGRESS_CALLBACK=1 + SQLITE_OMIT_SHARED_CACHE=1 + ) + + target_compile_options(profiler-hub-sqlite3-shared PRIVATE -O2 -fPIC) + + set_target_properties( + profiler-hub-sqlite3-shared + PROPERTIES + POSITION_INDEPENDENT_CODE ON + C_STANDARD 11 + OUTPUT_NAME profiler-hub-sqlite3 + ) + + install( + TARGETS profiler-hub-sqlite3-shared + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT profiler-hub + ) + + add_library(profiler-hub-sqlite3 INTERFACE) + target_link_libraries( + profiler-hub-sqlite3 + INTERFACE profiler-hub-sqlite3-shared ${CMAKE_DL_LIBS} + ) + + message( + STATUS + "[profiler-hub] SQLite3 amalgamation source: ${SQLITE3_SOURCE_DIR}" + ) +endif() diff --git a/profilers/profiler-hub/tests/unit/CMakeLists.txt b/profilers/profiler-hub/tests/unit/CMakeLists.txt index 238b2b4c3d82..f108cdd85e39 100644 --- a/profilers/profiler-hub/tests/unit/CMakeLists.txt +++ b/profilers/profiler-hub/tests/unit/CMakeLists.txt @@ -9,7 +9,6 @@ set(UNIT_TEST_SOURCES table_select_query_test.cpp entity_utility_test.cpp json_serializer_test.cpp - symbol_seal_test.cpp writer_test.cpp reader_test.cpp sqlite_backend_impl_test.cpp @@ -43,10 +42,6 @@ target_compile_definitions( PUBLIC ROCPD_DB_PATH="${ROCPD_DB_PATH}" ) -target_compile_definitions( - profiler-hub_unit_tests - PRIVATE PROFILER_HUB_SHARED_LIB="$" -) add_dependencies(profiler-hub_unit_tests profiler-hub) target_include_directories( @@ -67,17 +62,9 @@ if(PROFILER_HUB_ENABLE_COVERAGE) profiler_hub_add_coverage_flags(profiler-hub_unit_tests) endif() -# ASAN: TheRock forces -shared-libsan fleet-wide but amd-llvm ships only the static asan -# runtime, so link it statically here. -if( - CMAKE_CXX_COMPILER_ID MATCHES "Clang" - AND CMAKE_CXX_FLAGS MATCHES "-fsanitize=address" -) - target_link_options(profiler-hub_unit_tests PRIVATE -static-libsan) -endif() - gtest_discover_tests( profiler-hub_unit_tests + DISCOVERY_MODE PRE_TEST WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTIES LABELS "unit" ) diff --git a/profilers/profiler-hub/tests/unit/symbol_seal_test.cpp b/profilers/profiler-hub/tests/unit/symbol_seal_test.cpp deleted file mode 100644 index 77f75c36483c..000000000000 --- a/profilers/profiler-hub/tests/unit/symbol_seal_test.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright (c) Advanced Micro Devices, Inc. -// SPDX-License-Identifier: MIT - -// Catch regressions where bundled sqlite3_* symbols leak out of libprofiler-hub.so. -// -// SQLite is compiled with -fvisibility=hidden and the shared library uses -// --exclude-libs on ELF so we do not step on another sqlite3 in the same process -// (TheRock has one). We parse nm/dumpbin output on the built artifact, not at -// runtime. PROFILER_HUB_SHARED_LIB comes from CMake. Skip if the host has no -// suitable tool. - -#include - -#if !defined(_WIN32) -# include -#endif - -#include -#include -#include -#include -#include -#include -#include - -namespace -{ - -// Set by CMake (absolute path to libprofiler-hub.so). -constexpr const char* k_shared_library = PROFILER_HUB_SHARED_LIB; - -#if defined(_WIN32) -# define PHUB_POPEN _popen -# define PHUB_PCLOSE _pclose -#else -# define PHUB_POPEN popen -# define PHUB_PCLOSE pclose -#endif - -struct file_op -{ - void operator()(FILE* pipe) const noexcept - { - if(pipe != nullptr) - { - PHUB_PCLOSE(pipe); - } - } -}; - -// Run command, collect stdout. Returns the child exit status via `status` -// (-1 if the shell could not run it). False only when the pipe could not open. -bool -run_capture(const std::string& command, std::string& output, int& status) -{ - output.clear(); - status = -1; - std::unique_ptr pipe(PHUB_POPEN(command.c_str(), "r")); - if(!pipe) - { - return false; - } - - std::array buffer{}; - while(std::fgets(buffer.data(), static_cast(buffer.size()), pipe.get()) != - nullptr) - { - output += buffer.data(); - } - - status = PHUB_PCLOSE(pipe.release()); - return true; -} - -// Convenience overload: success means the command ran and exited zero. -bool -run_capture(const std::string& command, std::string& output) -{ - int status = -1; - if(!run_capture(command, output, status)) - { - return false; - } -#if defined(_WIN32) - // _pclose gives the child exit code directly (-1 on error). - return status == 0; -#else - return status != -1 && WIFEXITED(status) && WEXITSTATUS(status) == 0; -#endif -} - -struct symbol_holder -{ - std::string command; - std::regex pattern; -}; - -// A probe only needs to prove the tool exists and runs. Some tools (notably -// dumpbin /?) print their banner to stdout but exit non-zero, so presence is -// decided by captured output, not by the exit status. -bool -tool_available(const std::string& probe) -{ - std::string out; - int status = -1; - return run_capture(probe, out, status) && !out.empty(); -} - -// First nm/llvm-nm/dumpbin that works on this platform, or false. -bool -pick_tool(const std::string& library, symbol_holder& out) -{ - const std::string quoted = "\"" + library + "\""; - -#if defined(_WIN32) - // dumpbin export table: ordinal, hint, RVA, name. - if(tool_available("dumpbin /? 2>NUL")) - { - out = { - "dumpbin /nologo /exports " + quoted + " 2>NUL", - std::regex{ - R"(^\s*[0-9]+\s+[0-9A-Fa-f]+\s+[0-9A-Fa-f]+\s+(sqlite3_[A-Za-z0-9_]+))" } - }; - return true; - } - if(tool_available("llvm-nm --version 2>NUL")) - { - out = { "llvm-nm --defined-only " + quoted + " 2>NUL", - std::regex{ R"(\b[A-Za-z] (sqlite3_[A-Za-z0-9_]+)$)" } }; - return true; - } -#else - for(const char* candidate : { "nm", "llvm-nm" }) - { - if(tool_available(std::string{ candidate } + " --version 2>/dev/null")) - { - out = { - std::string{ candidate } + " -D --defined-only " + quoted + - " 2>/dev/null", - std::regex{ - R"(^[0-9a-fA-F]+ [A-Za-z] (sqlite3_[A-Za-z0-9_]+(?:@{1,2}[A-Za-z0-9_.-]+)?)$)" } - }; - return true; - } - } -#endif - - return false; -} - -TEST(sqlite3_symbol_seal, shared_library_exports_no_sqlite3_symbols) -{ - symbol_holder lister; - if(!pick_tool(k_shared_library, lister)) - { - GTEST_SKIP() << "need nm, llvm-nm, or dumpbin to inspect " << k_shared_library; - } - - std::string output; - ASSERT_TRUE(run_capture(lister.command, output)) - << "failed to inspect exported symbols of " << k_shared_library; - - std::vector leaked; - std::istringstream stream(output); - std::string line; - while(std::getline(stream, line)) - { - if(!line.empty() && line.back() == '\r') - { - line.pop_back(); - } - - std::smatch match; - if(std::regex_search(line, match, lister.pattern)) - { - leaked.push_back(match[1].str()); - } - } - - std::string detail; - for(const auto& symbol : leaked) - { - detail += "\n " + symbol; - } - - EXPECT_TRUE(leaked.empty()) - << k_shared_library << " exports bundled sqlite3 symbol(s):" << detail - << " (expected none, heck -fvisibility=hidden on profiler-hub-sqlite3-static " - "and --exclude-libs on the profiler-hub shared link)"; -} - -} // namespace diff --git a/projects/amdsmi/.pre-commit-config.yaml b/projects/amdsmi/.pre-commit-config.yaml index 3b79b0c3e336..5bb78496baef 100644 --- a/projects/amdsmi/.pre-commit-config.yaml +++ b/projects/amdsmi/.pre-commit-config.yaml @@ -28,6 +28,8 @@ exclude: | | esmi_ib_library/ | ualoe_lib/ | third_party/ + # Verbatim upstream license text; the whitespace is theirs, not ours. + | py-interface/THIRD_PARTY_LICENSES/ fail_fast: false repos: diff --git a/projects/amdsmi/CHANGELOG.md b/projects/amdsmi/CHANGELOG.md index 3a14692c8f04..08a06bdf0109 100644 --- a/projects/amdsmi/CHANGELOG.md +++ b/projects/amdsmi/CHANGELOG.md @@ -4,6 +4,26 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr ***All information listed below is for reference and subject to change.*** +## amd_smi_lib for ROCm 10.2.0 + +### Added + +- **Added amdgpu version details to `amdsmi_get_gpu_driver_info()`**. + - `driver_kernel_version` and `amdgpu_driver_version` split `/sys/module/amdgpu/version`, such as `6.19.14` and `31400000`. A 3-part version such as `6.19.4` fills only `driver_kernel_version`. + - `driver_build_version` reports the build number of the active DKMS package when its version matches the loaded module. + - `driver_full_version` is `driver_version` plus `-build` when the build is known, otherwise `driver_version`. `driver_version` is unchanged. + - On the WSL backend, `driver_full_version` is a copy of the WDDM `driver_version`. + +### Changed + +- **`amd-smi`, `amd-smi version`, and `amd-smi static --driver` include the DKMS build in the amdgpu version**. + - For example `6.19.14.31400000-2370381`. Labels and JSON/CSV keys are unchanged. + +### Resolved Issues + +- **Fixed runtime fatal CPERs reporting no AFIDs**. + - `amd-smi ras --cper` showed an empty `list afids` column for fatal records, `amd-smi ras --afid --cper-file` printed `-`, and `amdsmi_get_afids_from_cper()` returned no AFIDs. amdgpu writes fatal crashdump sections 32 bytes shorter than `sizeof(cper_sec_crashdump)`, and the section bounds check required the full struct, so every such section was skipped. The check now requires only the dump member the record type uses. + ## amd_smi_lib for ROCm 10.1.0 ### Added @@ -28,9 +48,23 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - Reports the amdgpu `chip_rev` and `external_rev` values from the `AMDGPU_INFO_DEV_INFO` DRM query, both distinct from `rev_id`, which is the PCI config-space revision. `external_rev_id` is family-scoped, so interpret it alongside `device_id`. - Exposed under the same names in the Python `amdsmi_get_gpu_asic_info()` dictionary and in `amd-smi static --asic`. The C fields report `0xFFFFFFFF` when unsupported; Python and the CLI render that as `N/A`. - ABI-preserving: the fields take two `uint32_t` slots from `amdsmi_asic_info_t.reserved`, which shrinks from 17 to 15 entries. The structure size and all other field offsets are unchanged. +- **Added NPM (Node Power Management) power limit setting**. + - New `amd-smi set --node-power-limit`/`-n` to set the node-level power limit in watts (node-wide, not per-GPU). + - New API: `amdsmi_set_npm_limit()`. + - New `current_node_power` field in `amdsmi_npm_info_t` (returned by `amdsmi_get_npm_info()`), the current (instantaneous) node power in watts, queried once per node rather than once per GPU; `amd-smi node -p` now displays it when available. + - New `max_node_power_limit` field in `amdsmi_npm_info_t` (returned by `amdsmi_get_npm_info()`), the platform max bound for `amdsmi_set_npm_limit()` requests. + - `amdsmi_set_npm_limit()` rejects the request with `AMDSMI_STATUS_INVAL` when NPM is disabled on the node (`amdsmi_npm_info_t::status == AMDSMI_NPM_STATUS_DISABLED`), instead of writing a limit that would have no defined effect. `amd-smi set --node-power-limit` performs the same check up front to fail fast with a friendlier message. + +- **Added `AMDSMI_VRAM_TYPE_HBM4` to `amdsmi_vram_type_t`**. + - Identifies HBM Generation 4 VRAM, reported by `amdsmi_get_gpu_vram_info()`. + - Also added the pre-existing `HBM3E` value to the Python `AmdSmiVramType` enum, which had been missing it. ### Changed +- **`amdsmi_get_npm_info()` and `amdsmi_set_npm_limit()` now reject `amdsmi_node_handle` values not vended by `amdsmi_get_node_handle()`**. + - Previously any non-null handle was dereferenced directly; an unregistered/garbage handle now returns `AMDSMI_STATUS_INVAL` instead. +- **NPM sysfs numeric reads (e.g. `cur_node_power_limit`, `max_node_power_limit`) now reject negative or malformed content**. + - Previously a leading `-` (e.g. `"-1"`) parsed successfully as `UINT64_MAX`; such content now fails with `RSMI_STATUS_UNEXPECTED_DATA`. - **`amdsmi_get_clock_info()` now returns `AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS` for clock values that exceed `INT_MAX`**. - Such values were previously narrowed to a negative number and returned as data. - The `UINT_MAX` "unavailable" sentinel is exempt: a domain with no minimum dpm level or no deep-sleep state keeps reporting the clock as unavailable instead of failing the call. @@ -56,10 +90,94 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - Both functions the header declares are C++ symbols, which the version script's `amdsmi_*` export glob does not match, so including the header only ever led to a link error. It joins the `_test` and WSL impl headers that are already build-only. - The example is the one shipped file that included that header, so it went with it rather than being left unbuildable against an install tree. +- **Reworked `amd-smi` CLI process exit codes.** + How exit codes are chosen: + - Library failures exit with the underlying `AMDSMI_STATUS_*` value (0-56). + - CLI-only errors use dedicated codes in the reserved 193-253 band. + - When more than one failure with different codes is recorded in a single run — across devices, or across sub-steps of one command (e.g. `reset --clocks`) — the process exits `204`. + + Multi-device `set`/`reset` behavior: + - A per-device GPU failure is now recorded, and the command continues to the remaining devices. + - The exit code is decided once, after every device has been attempted. Previously a single device failure could abort the loop early. + + Notable value changes (many cases previously aliased onto `1`/`2`; each now has a distinct code): + + | Case | Was | Now | + | --- | --- | --- | + | Import failure | `1` | `193` | + | Invalid command | `1` | `194` | + | Invalid parameter | `2` | `195` | + | Device not found (bad device index, or no devices present) | `3` / `255` | `196` | + | Invalid file path | `4` | `197` | + | Invalid parameter value | `5` | `198` | + | Missing parameter value | `6` | `199` | + | Command not supported (CLI, parse-time) | `7` | `200` (distinct from library `NOT_SUPPORTED` = `2`) | + | Device interface unavailable (CLI, runtime) | — | `201` | + | Required target/argument missing | `9` | `202` | + | Invalid subcommand | `10` | `203` | + | Permission denied | `11` | `10` (real `AMDSMI_STATUS_NO_PERM`) | + | Mixed device/field failures (differing codes) | — | `204` | + | `amdsmi_init()` watchdog timeout | `2` | `205` | + | Drivers not loaded | `255` | `206` | + | Interactive confirmation declined (answering no, or closing stdin) | `1` | `207` | + | Library (device) failure | `(1000 + status)` wrapped to a byte | underlying `AMDSMI_STATUS_*` (`0`-`56`) | + | Unknown/unmapped library error | `231` | `255` (library `UNKNOWN_ERROR` reported as its low byte) | + | Library status too large to report as an exit code | — | `253` (not reachable with current library versions) | + + The `Was`/`Now` values are process exit codes (`$?`). Previously the printed `code` field showed the *negative* of these (e.g. `-7`) while the process exited with the absolute value (`7`); the rework makes the printed `code` and `$?` agree (both `200`). + +- **`amd-smi set --power-cap` now applies per GPU instead of aborting on the first out-of-range device.** + - Each GPU is validated against its own reported range. + - An out-of-range value on one GPU is reported for that GPU, while in-range GPUs still apply. + - The process exit code reflects the per-device failure. + +- **Clearer, more consistent `amd-smi` error messages.** + - Errors now read `[AMDSMI_STATUS_] `, so the underlying status is obvious at a glance. The internal class name is no longer prepended, and `--json`/`--csv` errors are now valid JSON/CSV. + - Before: `amdsmi_cli_exceptions.AmdSmiLibraryErrorException: AMDSMI has returned error '-1002' - 'Command not supported' Error code: -1002` + - After (human): `[AMDSMI_STATUS_NOT_SUPPORTED] Command not supported Error code: 2 [AMDSMI_STATUS_NOT_SUPPORTED]` + - After (`--json`): `{"error": "[AMDSMI_STATUS_NOT_SUPPORTED] Command not supported", "code": 2, "error_type": "AMDSMI_STATUS_NOT_SUPPORTED"}` + - `--json` and `--csv` errors gain an `error_type` field (e.g. `INVALID_PARAMETER_VALUE` or an `AMDSMI_STATUS_*` name) alongside the numeric `code`. `--csv` gains a third column (`error,code,error_type`); a reader that unpacks exactly two fields per row will need updating. + - `amd-smi ras` reports a readable error when a CPER file can't be read or decoded, instead of printing a Python traceback. + +- **`amd-smi ras` now exits non-zero when a CPER file fails to decode.** + - Affects `--afid --folder`, `--afid --cper-file`, and `--cper`, which previously exited `0`. + - The exit code is the underlying `AMDSMI_STATUS_*`, for example `43`. When files fail with different codes, the command exits `205`. + - The failing file shows `[AMDSMI_STATUS_] ` in place of its AFIDs. + - A record that decodes successfully but carries no AFIDs (e.g. a corrected entry) shows `N/A` in place of its AFIDs. + - `--json` and `--csv` gain `status`, `message`, and `code` fields. Existing fields are unchanged. + - `--afid --csv` now emits CSV instead of the human-readable table. + +- **`amd-smi set --compute-partition` / `-C` now attempts each GPU individually.** + - An unsupported GPU reports `NOT_SUPPORTED` on its own, instead of the whole command aborting up front. + - Input is validated against the static partition type names when profiles cannot be enumerated. + - A numeric profile INDEX (e.g. `-C 0`) is also accepted, matched against an estimated `SPX, DPX, TPX, QPX, CPX` ordering. + - The `-C` help now lists `SPX, DPX, TPX, QPX, CPX` and `0-4` instead of `N/A`. + +### Removed + +- **Removed the internal `amd-smi` CLI exception classes `AmdSmiParameterNotSupportedException` and `AmdSmiUnknownErrorException`**. + These were CLI-internal (not part of the public `amdsmi` Python library), their exit-code behavior is superseded by the reworked CLI process exit codes described above. + +- **`amd-smi metric` no longer prints fields the GPU's `gpu_metrics` table version cannot carry** (human-readable output only). + - These fields previously printed as `N/A` and are now omitted from human-readable output. On a metrics v1.3 GPU this removes 88 of the 110 `N/A` values in `amd-smi metric -g 0` (80%), taking the command from 140 lines to 95. + - **Fields that stop appearing on current accelerators.** On the metrics v1.6 and later tables, which cover MI300X, MI325X and MI355X, the `POWER` section loses `THROTTLE_STATUS` and the `USAGE` section loses `MM_ACTIVITY`, `VCN_ACTIVITY` and `JPEG_ACTIVITY`. Naming the section does not bring them back, so `amd-smi metric --power` no longer prints `THROTTLE_STATUS` and `amd-smi metric --usage` no longer prints the three activity rows. On a metrics v1.3 GPU plain `amd-smi metric` instead drops the whole `THROTTLE` section, and `THROTTLE_STATUS` is still printed. **`--show-unsupported` restores all of them**, and `--json` and `--csv` never stopped emitting them. + - **`--json` and `--csv` are unaffected and still emit every field, `N/A` ones included.** They are the machine-readable contract, so their key sets and column sets stay complete and stable: any key or column a script indexed before still resolves. Anything parsing the human-readable output does see fewer fields, and `--show-unsupported` gives it back the previous output. + - **Pass `--show-unsupported` to restore the full human-readable output.** The flag is accepted with `--json` and `--csv` and has no effect there, since those formats are never filtered. + - Scoped to the metrics table version, not to ASIC support: fields the CLI can also source from hwmon or sysfs are always printed, namely the `edge`, `hotspot` and `mem` temperature sensors, the fan section, the voltages, and the `gfx_N`, `vclk_N`, `dclk_N`, `mem_N` and `socclk_N` clock slots. Entries sourced only from the metrics blobs are eligible for omission: the `hbm_stacks`, `mid`, `aid` and `xcd` temperature arrays, and the `uclk_aid` and `socclks_mid` clock arrays. + - A section named on the command line is never emptied. Plain `amd-smi metric` drops a section the version can populate nothing of, which on a metrics v1.3 GPU removes the whole `throttle` section; `amd-smi metric --throttle` prints all 31 rows instead. A named section that is only partly suppressed is still filtered. `--partition` scopes the data rather than naming a section, so `amd-smi metric --partition` protects nothing and filters like plain `amd-smi metric`. + - A field the version does carry but the ASIC or driver leaves unpopulated still prints `N/A`, a field reporting a real value is never removed, and an unrecognized metrics version or an unreadable header suppresses nothing. + ### Optimized ### Resolved Issues +- **Fixed `amd-smi` printing a Python traceback when an unknown NIC or switch is selected**. + - `amd-smi static --nic 999` and `--switch 999` failed while building the "device not found" error, so the command exited `1` with a traceback and no readable message. `--json` and `--csv` produced no parseable output. + - Both now report `Can not find a device: NIC '999'` (or `SWITCH`) and exit `196`, matching `--gpu`, `--cpu`, and `--core`. + +- **Fixed xGMI read and write data counters reading as unavailable on MI450**. + - On MI450 the GPU connects to the CPU over xGMI, and the driver reports that link's traffic as a single counter. It was dropped instead of being stored as the first link, so `xgmi_read_data_acc` and `xgmi_write_data_acc` from `amdsmi_get_gpu_metrics_info()`, and the link `read`/`write` from `amdsmi_get_link_metrics()`, read `UINT64_MAX` (`N/A`). + - **Fixed `rsmi_dev_reg_table_get()` failing on register-state images that contain no SMN entries**. - The loop-back test ran before the SMN and instance counters reached zero, so an image with no SMN entries re-entered the loop and read past the end of the image; the call then returned an error for a well-formed file. @@ -107,6 +225,10 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - **Fixed `amd-smi metric --clock` reporting FCLK `MAX_CLK` as 0 MHz**. - The FCLK range came from `pp_od_clk_voltage`, which on some GPUs (e.g. MI45x) has no FCLK section, leaving the parsed maximum at 0. It now falls back to `pp_dpm_fclk` (and likewise `pp_dpm_sclk`/`pp_dpm_mclk`) when the overdrive file omits a domain. +- **Fixed `amd-smi set`/`reset` on a GPU silently exiting `0` after a per-device failure.** + - A device error during GPU `set`/`reset` is now recorded, so the process exits with a non-zero code instead of reporting success. + - Example: `set --memory-partition` on a GPU that does not support it. + - **Fixed `amd-smi set -L/--clk-limit max ` not enforcing caps that fall between clock levels**. - For `mclk` and `fclk` ONLY, which expose a discrete DPM table, the requested `max` is now rounded down to the nearest selectable clock level, so the enforced limit never exceeds the requested value. - `sclk` supports a continuous frequency range, so its requested `max` is honored exactly (e.g. `600` enforces a limit of 600MHz) and is not snapped. diff --git a/projects/amdsmi/CMakeLists.txt b/projects/amdsmi/CMakeLists.txt index e93620776b89..2a6e11db41c8 100644 --- a/projects/amdsmi/CMakeLists.txt +++ b/projects/amdsmi/CMakeLists.txt @@ -167,7 +167,7 @@ cmake_dependent_option( AUTO_BUILD_STATIC_LIBS "auto-build static library (tests/examples link to static)" ON - "NOT BUILD_SHARED_LIBS OR BUILD_TESTS" + "NOT BUILD_SHARED_LIBS OR BUILD_TESTS OR BUILD_EXAMPLES" OFF ) cmake_dependent_option( diff --git a/projects/amdsmi/amdsmi_cli/CMakeLists.txt b/projects/amdsmi/amdsmi_cli/CMakeLists.txt index 1a8da2d56934..b2b8eab8f036 100644 --- a/projects/amdsmi/amdsmi_cli/CMakeLists.txt +++ b/projects/amdsmi/amdsmi_cli/CMakeLists.txt @@ -19,6 +19,7 @@ set(CLI_FILES amdsmi_helpers.py amdsmi_init.py amdsmi_logger.py + amdsmi_metrics_field_support.py amdsmi_parser.py amdsmi_cli_exceptions.py amdsmi_rocm_smi_compat.py diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_cli.py b/projects/amdsmi/amdsmi_cli/amdsmi_cli.py index a6235893639f..069bf1904d90 100755 --- a/projects/amdsmi/amdsmi_cli/amdsmi_cli.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_cli.py @@ -53,7 +53,9 @@ except ImportError as e: print(f"Unhandled import error: {e}") print(f"Unable to import amdsmi_cli files. Check {cli_files_path} if they are present.") - sys.exit(1) + from amdsmi_cli_exceptions import AmdSmiExitCode + + sys.exit(int(AmdSmiExitCode.IMPORT_ERROR)) def _print_error(e, destination): @@ -129,7 +131,15 @@ def configure_logging_and_execute(args, amd_smi_commands): sys.tracebacklimit = -1 amd_smi_helpers = AMDSMIHelpers() - amd_smi_commands = AMDSMICommands(helpers=amd_smi_helpers) + # Device init below can fail before argv is parsed, and it still has to + # report in the format the user asked for. Neither flag has a short form. + if "--json" in sys.argv: + init_format = "json" + elif "--csv" in sys.argv: + init_format = "csv" + else: + init_format = "human_readable" + amd_smi_commands = AMDSMICommands(format=init_format, helpers=amd_smi_helpers) amd_smi_parser = AMDSMIParser( amd_smi_commands.version, amd_smi_commands.list_devices, @@ -219,8 +229,9 @@ def configure_logging_and_execute(args, amd_smi_commands): elif sys.argv[1] in valid_commands: args = amd_smi_parser.parse_args(args=None) else: + # Raised before args are parsed, so the format comes from sys.argv. raise amdsmi_cli_exceptions.AmdSmiInvalidSubcommandException( - sys.argv[1], amd_smi_commands.logger.destination + sys.argv[1], amd_smi_helpers.get_output_format() ) # Handle command modifiers before subcommand execution @@ -232,27 +243,24 @@ def configure_logging_and_execute(args, amd_smi_commands): if hasattr(args, "file") and args.file: amd_smi_commands.logger.destination = args.file configure_logging_and_execute(args, amd_smi_commands) + + sys.exit(amd_smi_helpers.error_collector.resolve_exit_code()) except amdsmi_cli_exceptions.AmdSmiException as e: - _print_error( - f"{type(e).__module__}.{type(e).__name__}: {str(e)}", - amd_smi_commands.logger.destination, - ) + _print_error(str(e), amd_smi_commands.logger.destination) sys.exit(abs(e.value)) except amdsmi_exception.AmdSmiLibraryException as e: + # A library error that escaped the single-device path. Print it, record + # it as a device failure, and finalize so the exit code matches the + # multi-device path (the underlying AMDSMI_STATUS_* code). exc = amdsmi_cli_exceptions.AmdSmiLibraryErrorException( amd_smi_commands.logger.format, e.get_error_code() ) - _print_error( - f"{type(exc).__module__}.{type(exc).__name__}: {str(exc)}", - amd_smi_commands.logger.destination, - ) - sys.exit(abs(exc.value)) + _print_error(str(exc), amd_smi_commands.logger.destination) + amd_smi_helpers.error_collector.record_library_error(e.get_error_code()) + sys.exit(amd_smi_helpers.error_collector.resolve_exit_code()) except PermissionError as e: command = sys.argv[1] if len(sys.argv) > 1 else "" outputformat = amd_smi_commands.logger.format exc = amdsmi_cli_exceptions.AmdSmiPermissionDeniedException(command, outputformat) - _print_error( - f"{type(exc).__module__}.{type(exc).__name__}: {str(exc)}", - amd_smi_commands.logger.destination, - ) + _print_error(str(exc), amd_smi_commands.logger.destination) sys.exit(abs(exc.value)) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_cli_exceptions.py b/projects/amdsmi/amdsmi_cli/amdsmi_cli_exceptions.py index 7eda0aa5f74e..10008194471e 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_cli_exceptions.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_cli_exceptions.py @@ -2,8 +2,127 @@ # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT +import csv +import enum +import io import json +# NOTE: intentionally no module-level ``from amdsmi import amdsmi_wrapper`` here. +# This module (and its AmdSmiExitCode enum) must stay importable even when the +# amdsmi package failed to load, so the import-error paths can exit with +# AmdSmiExitCode.IMPORT_ERROR instead of a hardcoded number. The wrapper is +# imported lazily where it is actually needed (see AmdSmiPermissionDeniedException). + + +@enum.unique +class AmdSmiExitCode(enum.IntEnum): + """CLI-invented process exit codes, in the reserved 193-253 band. + + The single byte a process can return is split four ways:: + + 0 success + 1-128 library AMDSMI_STATUS_* values (currently 0-56) + 129-192 POSIX signal codes (128 + signal number, SIGRTMAX = 64). + Owned by the OS, which reports them for any process + 193-253 the CLI-invented codes below + 254/255 library sentinels: AMDSMI_STATUS_MAP_ERROR (0xFFFFFFFE) -> 254, + AMDSMI_STATUS_UNKNOWN_ERROR (0xFFFFFFFF) -> 255 + + Library statuses are NOT redefined here: a library failure surfaces as its + real AMDSMI_STATUS_* value via library_code_to_exit_code. 254/255 are + *library* errors even though they sit above the CLI band (enforced by + test_cli_codes_never_collide_with_library_exit_codes). + Each member carries a human-readable ``.note``. + + Note on "not supported": a *library* AMDSMI_STATUS_NOT_SUPPORTED (2) is a + runtime result from a device call and is honored as-is (exit code 2). The + CLI's own COMMAND_NOT_SUPPORTED (200) is a parse-time decision (the command + isn't available on this system, so no library call is made), so callers can + tell the two apart from the exit code alone. DEVICE_INTERFACE_UNAVAILABLE + (201) covers the remaining case: the command is valid and the device is + present, but an interface the CLI reads directly yielded no usable data, so + there is no library status to fold. + + Next free code is 208. Assigning one is free only until it ships: a released + code keeps its meaning forever, because scripts branch on the number. + """ + + def __new__(cls, value, note=""): + # Per-code doc string, queryable for a future --help/exit-code table (no runtime consumer today). + obj = int.__new__(cls, value) + obj._value_ = value + obj.note = note + return obj + + SUCCESS = (0, "all recorded operations succeeded") + IMPORT_ERROR = (193, "Python import failure") + INVALID_COMMAND = (194, "unrecognized command") + INVALID_PARAMETER = (195, "invalid parameter") + # No devices present = a live driver enumerating zero handles. A missing + # driver never reaches here, it exits DRIVERS_NOT_LOADED. + DEVICE_NOT_FOUND = (196, "requested device not found, or no devices present") + INVALID_FILE_PATH = (197, "invalid file path") + INVALID_PARAMETER_VALUE = (198, "invalid parameter value") + MISSING_PARAMETER_VALUE = (199, "missing parameter value") + COMMAND_NOT_SUPPORTED = (200, "command not available on this system (parse-time)") + # TEMPORARY. Sole user is the gpu_od fan OD_RANGE read, which parses sysfs + # directly because no API reports the range minimum. Retire this code once + # amdsmi_get_gpu_fan_speed_range() lands and that branch can record the + # library status instead (see the TODO in tests/python/common/common.py). + DEVICE_INTERFACE_UNAVAILABLE = (201, "required device interface unavailable to the CLI") + REQUIRED_COMMAND = (202, "required command/target missing") + INVALID_SUBCOMMAND = (203, "invalid subcommand") + MIXED_DEVICE_ERRORS = (204, "aggregated: >1 recorded failure with DIFFERING codes") + INIT_TIMEOUT = (205, "amdsmi_init() watchdog fired (library call hung)") + DRIVERS_NOT_LOADED = (206, "no usable AMD drivers / modules not loaded") + USER_ABORTED = (207, "user declined an interactive confirmation prompt") + # Sits at the top of the band as a sentinel, mirroring the library's 254/255. + UNREPRESENTABLE_LIBRARY_STATUS = (253, "library status outside 0-128 and not a known sentinel") + + +# Reserved band for the CLI-invented codes above. Starts one past the highest +# POSIX signal code (128 + SIGRTMAX = 192) and stops below the library's 254/255 +# sentinels, so a CLI code can collide with neither. +CLI_EXIT_CODE_BAND_START = 193 +CLI_EXIT_CODE_BAND_END = 253 + +# The library's half of the byte. It may not grow past this into signal space. +LIBRARY_STATUS_MAX = 128 + + +class AmdSmiErrorSeverity(enum.Enum): + """Whether an error should stop the whole command or just this device. + + FATAL - abort immediately (import / init / driver-not-loaded / bad command + syntax / missing required target / permission). Continuing makes no + sense because nothing device-specific succeeded. + DEVICE - record the failure and keep iterating the remaining devices; the + process exit code is decided once at the end by + ``AmdSmiErrorCollector.resolve_exit_code()``. + """ + + FATAL = "fatal" + DEVICE = "device" + + +class AmdSmiDeviceKind(enum.Enum): + """Device kinds the CLI can report as missing. The value is the label used + in the message; NIC covers both AI-NIC and Broadcom, which ``-N`` selects + together. + """ + + GPU = "GPU" + CPU = "CPU" + CPU_CORE = "CPU CORE" + NIC = "NIC" + SWITCH = "SWITCH" + + +# The two 32-bit library sentinels (see amdsmi_wrapper.amdsmi_status_t), named here +# so this module stays importable without the amdsmi package and avoids bare hex. +AMDSMI_STATUS_MAP_ERROR = 0xFFFFFFFE +AMDSMI_STATUS_UNKNOWN_ERROR = 0xFFFFFFFF + AMDSMI_ERROR_MESSAGES = { 0: "Success", @@ -35,6 +154,7 @@ 33: "No more free slot", 34: "Driver not loaded", # Reserved for future error messages + 39: "More data available than the provided buffer", 40: "No data was found for given input", 41: "Insufficient size for operation", 42: "Unexpected size of data was read", @@ -51,18 +171,72 @@ 53: "Parsed argument is invalid", 54: "AMDGPU restart error", 55: "Setting is not available", - 0xFFFFFFFE: "AMD-SMI Library error did not map to a status code", - 0xFFFFFFFF: "Unknown error", + 56: "EEPROM is corrupted", + AMDSMI_STATUS_MAP_ERROR: "AMD-SMI Library error did not map to a status code", + AMDSMI_STATUS_UNKNOWN_ERROR: "Unknown error", } def _get_error_message(error_code): - if abs(error_code) in AMDSMI_ERROR_MESSAGES: - return AMDSMI_ERROR_MESSAGES[abs(error_code)] - return "Generic error" + # test_every_library_status_has_a_friendly_message guarantees a mapping for + # every known status, so this fallback only fires for an unrecognized value + # (e.g. a mismatched library version). Include the raw code so it stays + # distinct from AMDSMI_STATUS_UNKNOWN_ERROR's own "Unknown error". + code = abs(error_code) + if code in AMDSMI_ERROR_MESSAGES: + return AMDSMI_ERROR_MESSAGES[code] + return f"Unrecognized error code {code}" + + +def _get_error_name(error_code): + """Return the AMDSMI_STATUS_* name for a status code. + + e.g. 43 -> 'AMDSMI_STATUS_UNEXPECTED_DATA'. Imports the wrapper lazily so + this module stays importable without the amdsmi package, and falls back to + a synthetic 'AMDSMI_STATUS_' if the code is not recognized (e.g. a + mismatched library version). + """ + code = abs(int(error_code)) + try: + from amdsmi import amdsmi_wrapper + + name = amdsmi_wrapper.amdsmi_status_t__enumvalues.get(code) + if name: + return name + except Exception: + pass + return f"AMDSMI_STATUS_{code}" + + +def _get_error_type_name(value): + """Return the name for an exit code, whichever space it's from. + + Tries the CLI-invented ``AmdSmiExitCode`` band first (e.g. 198 -> + 'INVALID_PARAMETER_VALUE'), since those values fall outside the library's + status range and ``_get_error_name`` would otherwise synthesize a bogus + 'AMDSMI_STATUS_' for them. Falls back to ``_get_error_name`` for a + real library AMDSMI_STATUS_* value. + """ + try: + return AmdSmiExitCode(value).name + except ValueError: + return _get_error_name(value) + + +def _csv_error(message, code, error_type): + # Messages carry commas and newlines (option lists, multi-line hints). + buf = io.StringIO() + csv.writer(buf, lineterminator="\n").writerows( + [["error", "code", "error_type"], [message, code, error_type]] + ) + return buf.getvalue().rstrip("\n") class AmdSmiException(Exception): + # Default: an unclassified error stops everything. Per-device failures + # (see AmdSmiLibraryErrorException) override this to DEVICE. + severity = AmdSmiErrorSeverity.FATAL + def __init__(self): self.json_message = {} self.csv_message = "" @@ -72,6 +246,24 @@ def __init__(self): self.device_type = "" self.value = 0 + def _build_output_messages(self, common_message, error_type_name=None): + """Render *common_message* into each output format __str__ selects from. + + *error_type_name* lets a subclass supply an already-resolved name (see + ``AmdSmiLibraryErrorException``, whose ``status_name`` is derived from + the raw library code, not the already-mapped ``self.value``). + """ + # json/csv are read a line at a time, so they carry the message without + # the layout whitespace the human-readable hints use. + flat = " ".join(common_message.split()) + if error_type_name is None: + error_type_name = _get_error_type_name(self.value) + self.json_message["error"] = flat + self.json_message["code"] = self.value + self.json_message["error_type"] = error_type_name + self.csv_message = _csv_error(flat, self.value, error_type_name) + self.stdout_message = f"{common_message} Error code: {self.value} [{error_type_name}]" + def __str__(self): # Return message according to the current output format if self.output_format == "json": @@ -87,7 +279,7 @@ def __str__(self): class AmdSmiInvalidCommandException(AmdSmiException): def __init__(self, command, outputformat: str, message=None): super().__init__() - self.value = -1 + self.value = int(AmdSmiExitCode.INVALID_COMMAND) self.command = command self.output_format = outputformat @@ -96,16 +288,13 @@ def __init__(self, command, outputformat: str, message=None): if message: common_message = message - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiInvalidParameterException(AmdSmiException): - def __init__(self, command, arg, outputformat: str): + def __init__(self, command, arg, outputformat: str, message=None, hint=None): super().__init__() - self.value = -2 + self.value = int(AmdSmiExitCode.INVALID_PARAMETER) self.command = command self.arg = arg self.output_format = outputformat @@ -113,41 +302,31 @@ def __init__(self, command, arg, outputformat: str): common_message = ( f"Parameter '{self.arg}' is invalid. Run 'amd-smi {self.command} -h' for more info." ) + if message: + common_message = message + if hint: + common_message += f" {hint}" - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiDeviceNotFoundException(AmdSmiException): - def __init__(self, command, outputformat: str, gpu: bool, cpu: bool, core: bool): + def __init__(self, command, outputformat: str, device_kind: AmdSmiDeviceKind): super().__init__() - self.value = -3 + self.value = int(AmdSmiExitCode.DEVICE_NOT_FOUND) self.command = command self.output_format = outputformat - - # Handle different devices - self.device_type = "" - if gpu: - self.device_type = "GPU" - elif cpu: - self.device_type = "CPU" - elif core: - self.device_type = "CPU CORE" + self.device_type = device_kind.value common_message = f"Can not find a device: {self.device_type} '{self.command}'" - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiInvalidFilePathException(AmdSmiException): def __init__(self, command, outputformat: str, message=None): super().__init__() - self.value = -4 + self.value = int(AmdSmiExitCode.INVALID_FILE_PATH) self.command = command self.output_format = outputformat @@ -156,16 +335,13 @@ def __init__(self, command, outputformat: str, message=None): if message: common_message = message - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiInvalidParameterValueException(AmdSmiException): def __init__(self, command, arg, outputformat: str, hint: str = None): super().__init__() - self.value = -5 + self.value = int(AmdSmiExitCode.INVALID_PARAMETER_VALUE) self.command = command self.arg = arg self.output_format = outputformat @@ -174,31 +350,28 @@ def __init__(self, command, arg, outputformat: str, hint: str = None): if hint: common_message += f" {hint}" - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiMissingParameterValueException(AmdSmiException): def __init__(self, command, outputformat: str): super().__init__() - self.value = -6 + self.value = int(AmdSmiExitCode.MISSING_PARAMETER_VALUE) self.command = command self.output_format = outputformat common_message = f"Parameter '{self.command}' requires a value. Run '--help' for more info." - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiCommandNotSupportedException(AmdSmiException): def __init__(self, command, outputformat: str): super().__init__() - self.value = -7 + # CLI-level (parse-time) decision: the command name is not available on + # this system, so no library call was made. Distinct from a library + # AMDSMI_STATUS_NOT_SUPPORTED (2), which is honored as-is elsewhere. + self.value = int(AmdSmiExitCode.COMMAND_NOT_SUPPORTED) self.command = command self.output_format = outputformat @@ -206,61 +379,41 @@ def __init__(self, command, outputformat: str): f"Command '{self.command}' is not supported on the system. Run '--help' for more info." ) - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" - - -class AmdSmiParameterNotSupportedException(AmdSmiException): - def __init__(self, command, outputformat: str): - super().__init__() - self.value = -8 - self.command = command - self.output_format = outputformat - - common_message = f"Parameter '{self.command}' is not supported on the system. Run '--help' for more info." - - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiRequiredCommandException(AmdSmiException): def __init__(self, command, outputformat: str): super().__init__() - self.value = -9 + self.value = int(AmdSmiExitCode.REQUIRED_COMMAND) self.command = command self.output_format = outputformat common_message = f"Command '{self.command}' requires a target argument. Run 'amd-smi {self.command} -h' for more info." - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiInvalidSubcommandException(AmdSmiException): def __init__(self, command, outputformat: str): super().__init__() - self.value = -10 + self.value = int(AmdSmiExitCode.INVALID_SUBCOMMAND) self.command = command self.output_format = outputformat common_message = f"AMD-SMI Command '{self.command}' is invalid. Must receive valid AMD-SMI Command first. Run 'amd-smi -h' for more info." - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiPermissionDeniedException(AmdSmiException): def __init__(self, command, outputformat: str): super().__init__() - self.value = -11 + # Only raised after a live library call returned NO_PERM, so amdsmi is + # loaded here. Import lazily so this module stays importable without it. + from amdsmi import amdsmi_wrapper + + self.value = int(amdsmi_wrapper.AMDSMI_STATUS_NO_PERM) self.command = command self.output_format = outputformat @@ -268,37 +421,104 @@ def __init__(self, command, outputformat: str): f"AMD-SMI Command '{self.command}' requires elevation (sudo privileges required)" ) - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" - - -class AmdSmiUnknownErrorException(AmdSmiException): - def __init__(self, command, outputformat: str): - super().__init__() - self.value = -100 - self.command = command - self.output_format = outputformat - - common_message = "An unknown error has occurred. Run 'help' for more info." - - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + self._build_output_messages(common_message) class AmdSmiLibraryErrorException(AmdSmiException): - def __init__(self, outputformat: str, error_code): + # A library error is (almost always) specific to one device, so it is + # recorded and the command keeps going to the next device. + severity = AmdSmiErrorSeverity.DEVICE + + def __init__(self, outputformat: str, error_code, detail=None): super().__init__() - self.value = -1000 - abs(error_code) - self.smilibcode = error_code + # Surface the underlying AMDSMI_STATUS_* value as the exit code. + self.value = library_code_to_exit_code(error_code) + self.amdsmi_lib_code = error_code self.output_format = outputformat - common_message = f"AMDSMI has returned error '{self.value}' - '{AMDSMI_ERROR_MESSAGES[abs(self.smilibcode)]}'" + # Format: "[AMDSMI_STATUS_] ". ``detail`` overrides the + # generic library message with caller-specific context when provided. + # status_name / status_message are exposed so callers can render the + # pieces separately (e.g. structured JSON/CSV rows). + self.status_name = _get_error_name(self.amdsmi_lib_code) + self.status_message = detail if detail else _get_error_message(self.amdsmi_lib_code) + common_message = f"[{self.status_name}] {self.status_message}" + + self._build_output_messages(common_message, error_type_name=self.status_name) + + +def library_code_to_exit_code(error_code): + """Map an AMDSMI_STATUS_* value to a POSIX process exit code (0-255). + + The library owns 0-128 plus its two 32-bit sentinels, which report as their + low byte (``AMDSMI_STATUS_MAP_ERROR`` = 0xFFFFFFFE -> 254, + ``AMDSMI_STATUS_UNKNOWN_ERROR`` = 0xFFFFFFFF -> 255). Statuses currently run + 0-56, so real values pass through and the exit code *is* the status. + + Anything else cannot be reported faithfully. 129-192 is signal territory, + and a value above 255 loses its identity in a byte -- status 300 would + arrive as 44, a real but unrelated status, and status 256 as 0, reporting + success. Those report UNREPRESENTABLE_LIBRARY_STATUS instead of a plausible + wrong answer. + """ + status = abs(int(error_code)) + if status in (AMDSMI_STATUS_MAP_ERROR, AMDSMI_STATUS_UNKNOWN_ERROR): + return status & 0xFF + if status > LIBRARY_STATUS_MAX: + return int(AmdSmiExitCode.UNREPRESENTABLE_LIBRARY_STATUS) + return status + + +class AmdSmiErrorCollector: + """Collects per-device failures across a command and decides one exit code. + + This is the heart of the "record-then-finalize" model: instead of a device + error raising and unwinding past the device loop (which aborts the remaining + devices and drops their buffered output), each device's failure is recorded + here and the loop keeps going. ``resolve_exit_code()`` is consulted once, at + the very end, to pick the single process exit code. + """ - self.json_message["error"] = common_message - self.json_message["code"] = self.value - self.csv_message = f"error,code\n{common_message}, {self.value}" - self.stdout_message = f"{common_message} Error code: {self.value}" + def __init__(self): + self._codes = [] + + def reset(self): + """Clear recorded codes (e.g. between watch-mode iterations). + + Note: this is a future-proofing measure. Set/reset + are not watch enabled commands, but we may want to add for + future watch mode commands. Today, this is not a requirement. + """ + self._codes = [] + + def record(self, exit_code): + """Record one device failure by its exit code.""" + self._codes.append(int(abs(exit_code))) + + def record_library_error(self, error_code): + """Record a device failure from a raw AMDSMI_STATUS_* value.""" + self._codes.append(library_code_to_exit_code(error_code)) + + @property + def has_errors(self): + return bool(self._codes) + + def resolve_exit_code(self): + """Pick the process exit code from everything recorded. + + none recorded -> SUCCESS (0) + all recorded are equal -> that code + mixed different codes -> MIXED_DEVICE_ERRORS + + "Mixed" is any set of recorded failures with differing codes in a single + run: across multiple devices, OR across multiple sub-steps/fields of one + command (e.g. reset --clocks). So a single GPU can resolve to + MIXED_DEVICE_ERRORS if its sub-steps fail with different codes -- the + name keeps DEVICE for registry compatibility, but the semantics are + per-run, not strictly per-device. + """ + if not self._codes: + return int(AmdSmiExitCode.SUCCESS) + if len(set(self._codes)) == 1: + return self._codes[0] + return int(AmdSmiExitCode.MIXED_DEVICE_ERRORS) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py index 1bd82ab796b4..e2f7354feb81 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py @@ -9,6 +9,7 @@ from amdsmi_helpers import AMDSMIHelpers from amdsmi_logger import AMDSMILogger from amdsmi import amdsmi_exception, amdsmi_interface +from amdsmi_cli_exceptions import AmdSmiDeviceKind, AmdSmiExitCode, AmdSmiLibraryErrorException from subcommands import ( BadPagesCommands, @@ -32,6 +33,19 @@ ) +def _exit_on_init_error(output_format, error, device_kind: AmdSmiDeviceKind): + """Report a device-init library failure and exit with its status code. + + Device init runs before argv is parsed, so this cannot reach the top-level + handler; ``output_format`` comes from the raw argv scan in amdsmi_cli.main. + The traceback stays at debug level -- the user gets the formatted error. + """ + exc = AmdSmiLibraryErrorException(output_format, error.err_code) + logging.debug(f"Unexpected library error during {device_kind.value} device init", exc_info=True) + print(exc) + sys.exit(exc.value) + + class AMDSMICommands( BadPagesCommands, DefaultCommands, @@ -97,7 +111,7 @@ def __init__(self, format="human_readable", destination="stdout", helpers=None) "Unable to get devices, driver not initialized (amdgpu not found in modules)" ) else: - raise e + _exit_on_init_error(self.logger.format, e, AmdSmiDeviceKind.GPU) if len(self.device_handles) == 0: # No GPU's found post amdgpu driver initialization @@ -126,7 +140,7 @@ def __init__(self, format="human_readable", destination="stdout", helpers=None) "Unable to get devices, driver not initialized (BRCMNIC not found in modules)" ) else: - raise e + _exit_on_init_error(self.logger.format, e, AmdSmiDeviceKind.NIC) # Resolve the node handle (independent of AINIC init; needed for amd-smi node). for dev in self.device_handles: @@ -153,7 +167,7 @@ def __init__(self, format="human_readable", destination="stdout", helpers=None) "Unable to detect any CPU devices, check amd_hsmp (or) hsmp_acpi version and module status (sudo modprobe amd_hsmp (or) sudo modprobe hsmp_acpi)" ) else: - raise e + _exit_on_init_error(self.logger.format, e, AmdSmiDeviceKind.CPU) # core handles try: @@ -167,7 +181,7 @@ def __init__(self, format="human_readable", destination="stdout", helpers=None) "Unable to get CORE devices, amd_hsmp driver not loaded (sudo modprobe amd_hsmp)" ) else: - raise e + _exit_on_init_error(self.logger.format, e, AmdSmiDeviceKind.CPU_CORE) if len(self.cpu_handles) == 0 and len(self.core_handles) == 0: # No CPU's found post amd_hsmp driver initialization @@ -182,7 +196,7 @@ def __init__(self, format="human_readable", destination="stdout", helpers=None) version_args.cpu_version = False version_args.nic_version = False self.version(version_args) - sys.exit(-1) + sys.exit(int(AmdSmiExitCode.DEVICE_NOT_FOUND)) def profile(self, args): """Not applicable to linux baremetal""" diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_helpers.py b/projects/amdsmi/amdsmi_cli/amdsmi_helpers.py index 4e0a10303df6..a759e7a9d62c 100755 --- a/projects/amdsmi/amdsmi_cli/amdsmi_helpers.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_helpers.py @@ -57,6 +57,10 @@ def __init__(self) -> None: amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_UNKNOWN_ERROR ) + # Per-device failure aggregator for the record-then-finalize exit-code + # model. See amdsmi_cli_exceptions.AmdSmiErrorCollector. + self.error_collector = amdsmi_cli_exceptions.AmdSmiErrorCollector() + # Check if the system is a virtual OS if self.operating_system.startswith("Linux"): self._is_linux = True @@ -874,6 +878,123 @@ def get_oam_0_device_handle(self): amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_FOUND ) + def run_device_subcommand(self, subcommand, args, **device_kwarg): + """Run one device's subcommand as part of a multi-device loop. + + Safety net for the record-then-finalize model: a DEVICE-severity failure + (or a raw library error that a handler forgot to convert) is recorded and + we keep going to the next device. A FATAL failure propagates and aborts + the whole command, as intended. + """ + try: + subcommand(args, multiple_devices=True, **device_kwarg) + except amdsmi_cli_exceptions.AmdSmiException as e: + if e.severity != amdsmi_cli_exceptions.AmdSmiErrorSeverity.DEVICE: + raise + # A device-scoped CLI error; prefer its underlying library status + # code when present so it matches the record-and-return path. + if hasattr(e, "amdsmi_lib_code"): + self.error_collector.record_library_error(e.amdsmi_lib_code) + else: + self.error_collector.record(e.value) + except amdsmi_exception.AmdSmiLibraryException as e: + # Backstops any handler that let a library error escape; record_or_raise + # owns the NO_PERM-is-command-wide rule the handlers apply themselves. + self.record_or_raise(e) + + # CPER-specific detail text for the statuses amdsmi_get_afids_from_cper can + # return. Anything else falls back to the generic library message. + CPER_DECODE_MESSAGES = { + amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_INVAL: "Invalid CPER file input", + amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_SIZE: "Unexpected CPER file data size", + amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_DATA: "Unexpected data in the CPER file", + amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED: "AFID decoding is not supported", + } + + def build_afid_record(self, cper_file, output_format, afids=None, code=None): + """Build one AFID result record, shared by ``ras --afid`` and ``ras --cper``. + + ``code=None`` means a successful decode; otherwise it is the failing + AMDSMI_STATUS_*. ``afids`` stays a list so JSON consumers keep indexing + and len()-ing it; ``decode_failed`` stays alongside the newer + status/message/code fields so pre-existing consumers keep working. + """ + if code is None: + return { + "cper_file": str(cper_file), + "afids": list(afids) if afids else [], + "decode_failed": False, + "status": "AMDSMI_STATUS_SUCCESS", + "message": "Success", + "code": 0, + } + exc = amdsmi_cli_exceptions.AmdSmiLibraryErrorException( + output_format, code, detail=self.CPER_DECODE_MESSAGES.get(code) + ) + return { + "cper_file": str(cper_file), + "afids": [], + "decode_failed": True, + "status": exc.status_name, + "message": exc.status_message, + "code": exc.value, + } + + @staticmethod + def afid_cell(record): + """Render one record's ``afids`` column for the human/CSV tables.""" + if record["decode_failed"]: + return f"[{record['status']}] {record['message']}" + return " ".join(map(str, record["afids"])) if record["afids"] else "N/A" + + def record_or_raise(self, exception, context=None): + """Record a per-device library failure, or raise when it is command-wide. + + NO_PERM means the user needs elevation, which affects every device, so it + aborts instead of being recorded per device and retried. Every other status + is device-scoped: record it and let the caller continue. + + Returns the per-device message when *context* names the device (e.g. + ``"CPU 3"``), so a caller can store it wherever it collects output. + """ + if exception.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError("Command requires elevation") from exception + self.error_collector.record_library_error(exception.get_error_code()) + if context is None: + return None + return f"Error occurred for {context} - {exception.get_error_info()}" + + def store_device_error(self, logger, device, key, message, exception=None, code=None): + """Show a per-device error AND record it for the final exit code. + + Record-then-finalize helper: fuses "show" + "count" so a per-device + branch can't display an error but forget to record it (which silently + leaves the exit code at 0). Pass ``exception=`` for a library failure + (surfaces its AMDSMI_STATUS_*) or ``code=`` for a CLI-level failure. + Iteration continues. + + When to use what: + - success -> logger.store_output(...); do NOT record. + - per-device failure -> store_device_error(exception=e | code=X) <- prefer this + - system-wide (gtt) -> logger.output[key]=... + error_collector.record_library_error(...) + - aggregated dict -> record per failing sub-step, one store_output at the end + (e.g. reset --clocks) + + exception= (record_library_error) = a caught AmdSmiLibraryException; + code= (record) = a CLI decision with no library call. + """ + if exception is None and code is None: + raise TypeError( + "store_device_error requires exception= (library failure) or " + "code= (CLI failure); showing an error without recording it " + "would leave the exit code at 0" + ) + logger.store_output(device, key, message) + if exception is not None: + self.error_collector.record_library_error(exception.get_error_code()) + else: + self.error_collector.record(code) + def handle_gpus(self, args, logger, subcommand): """This function will run execute the subcommands based on the number of gpus passed in via args. @@ -894,8 +1015,8 @@ def handle_gpus(self, args, logger, subcommand): if isinstance(args.gpu, list): if len(args.gpu) > 1: for device_handle in args.gpu: - # Handle multiple_devices to print all output at once - subcommand(args, multiple_devices=True, gpu=device_handle) + # Record per-device failures and keep going. + self.run_device_subcommand(subcommand, args, gpu=device_handle) logger.print_output(multiple_device_enabled=True) return True, args.gpu elif len(args.gpu) == 1: @@ -935,7 +1056,7 @@ def handle_switchs(self, args, logger, subcommand): amdsmi_interface.amdsmi_wrapper.AMDSMI_PROCESSOR_TYPE_BRCM_SWITCH ).name ): - subcommand(args, multiple_devices=True, switch=device_handle) + self.run_device_subcommand(subcommand, args, switch=device_handle) logger.print_output(multiple_device_enabled=True) return True, args.switch @@ -976,7 +1097,7 @@ def handle_brcm_nics(self, args, logger, subcommand): amdsmi_interface.amdsmi_wrapper.AMDSMI_PROCESSOR_TYPE_BRCM_NIC ).name ): - subcommand(args, multiple_devices=True, nic=device_handle) + self.run_device_subcommand(subcommand, args, nic=device_handle) logger.print_output(multiple_device_enabled=True) return True, args.nic @@ -1017,7 +1138,7 @@ def handle_ainics(self, args, logger, subcommand): amdsmi_interface.amdsmi_wrapper.AMDSMI_PROCESSOR_TYPE_AMD_NIC ).name ): - subcommand(args, multiple_devices=True, nic=device_handle) + self.run_device_subcommand(subcommand, args, nic=device_handle) logger.print_output(multiple_device_enabled=True) return True, args.nic @@ -1048,8 +1169,8 @@ def handle_cpus(self, args, logger, subcommand): if isinstance(args.cpu, list): if len(args.cpu) > 1: for device_handle in args.cpu: - # Handle multiple_devices to print all output at once - subcommand(args, multiple_devices=True, cpu=device_handle) + # Record per-device failures and keep going. + self.run_device_subcommand(subcommand, args, cpu=device_handle) logger.print_output(multiple_device_enabled=True) return True, args.cpu elif len(args.cpu) == 1: @@ -1057,6 +1178,8 @@ def handle_cpus(self, args, logger, subcommand): return False, args.cpu else: logging.debug("args.cpu has empty list") + # Callers ignore the 2nd value whenever the 1st is True; matches handle_gpus. + return True, args.cpu else: return False, args.cpu @@ -1078,8 +1201,8 @@ def handle_cores(self, args, logger, subcommand): if isinstance(args.core, list): if len(args.core) > 1: for device_handle in args.core: - # Handle multiple_devices to print all output at once - subcommand(args, multiple_devices=True, core=device_handle) + # Record per-device failures and keep going. + self.run_device_subcommand(subcommand, args, core=device_handle) logger.print_output(multiple_device_enabled=True) return True, args.core elif len(args.core) == 1: @@ -1087,6 +1210,8 @@ def handle_cores(self, args, logger, subcommand): return False, args.core else: logging.debug("args.core has empty list") + # Callers ignore the 2nd value whenever the 1st is True; matches handle_gpus. + return True, args.core else: return False, args.core @@ -1399,26 +1524,54 @@ def get_accelerator_partition_profile_config(self): break return accelerator_partition_profiles + def estimate_accelerator_partition_profile_indicies(self): + """Best-effort profile_type/profile_index pairing, used only when real + profiles can't be enumerated: not running as root (enumeration itself + needs sudo), or a device that doesn't support accelerator partitioning + at all. Assumes the driver's fixed SPX,DPX,TPX,QPX,CPX order, which the + CLI can't verify without querying the device -- an actual set attempt + is still validated by the amdsmi library/driver against the real + hardware either way. This estimate also feeds the CLI's hint text for + an invalid -C value. + """ + estimated_profiles = {"profile_indices": [], "profile_types": []} + i = 0 + for name in amdsmi_interface.AmdSmiComputePartitionType.__members__: + if name != "INVALID": + estimated_profiles["profile_types"].append(name) + estimated_profiles["profile_indices"].append(str(i)) + i += 1 + return estimated_profiles + def get_accelerator_choices_types_indices(self): - return_val = ("N/A", {"profile_indices": [], "profile_types": []}) + # (choices, accelerator_profiles_dict) if os.geteuid() != 0: + # Not root: profile INDEX values need sudo to enumerate, so estimate them. logging.debug( - "AMDSMIHelpers.get_accelerator_choices_types_indices - Not root, unable to get accelerator partition profiles" + "AMDSMIHelpers.get_accelerator_choices_types_indices - Not root, estimating partition profiles" ) - # If not root, we can't get the accelerator partition profiles - return return_val - else: - logging.debug( - "AMDSMIHelpers.get_accelerator_choices_types_indices - Root, getting accelerator partition profiles" + estimated_profiles = self.estimate_accelerator_partition_profile_indicies() + compute_partitions_list = ( + estimated_profiles["profile_types"] + estimated_profiles["profile_indices"] ) + return (compute_partitions_list, estimated_profiles) + + logging.debug( + "AMDSMIHelpers.get_accelerator_choices_types_indices - Root, getting accelerator partition profiles" + ) accelerator_partition_profiles = self.get_accelerator_partition_profile_config() if len(accelerator_partition_profiles["profile_types"]) != 0: compute_partitions_list = ( accelerator_partition_profiles["profile_types"] + accelerator_partition_profiles["profile_indices"] ) - return_val = (compute_partitions_list, accelerator_partition_profiles) - return return_val + return (compute_partitions_list, accelerator_partition_profiles) + # Device doesn't support partitioning; fall back to the same estimate. + estimated_profiles = self.estimate_accelerator_partition_profile_indicies() + compute_partitions_list = ( + estimated_profiles["profile_types"] + estimated_profiles["profile_indices"] + ) + return (compute_partitions_list, estimated_profiles) def get_memory_partition_types(self): memory_partitions_str = [ @@ -1528,6 +1681,28 @@ def format_power_cap(self, value): return f"{converted} W" return value + def get_max_node_power_limit(self): + """Get the platform max NPM node power limit (Watts), for help-text bounds. + + Mirrors get_power_caps(): queried once at parser-build time so + --node-power-limit's help text can show its valid upper bound. NPM is + node-wide, so the first device that yields a reading is sufficient. + """ + for dev in self.get_gpu_handles(): + try: + node_handle = amdsmi_interface.amdsmi_get_node_handle(dev) + max_node_power_limit = amdsmi_interface.amdsmi_get_npm_info(node_handle)[ + "max_node_power_limit" + ] + if max_node_power_limit != "N/A": + return f"{max_node_power_limit} W" + except (amdsmi_interface.AmdSmiLibraryException, KeyError) as e: + logging.debug( + f"AMDSMIHelpers.get_max_node_power_limit - Unable to get NPM info for device {dev}: {str(e)}" + ) + continue + return "N/A" + @staticmethod def detect_gpu_od(bdf): """Detect if a GPU has the gpu_od sysfs interface. @@ -1802,6 +1977,24 @@ def read_pending_gtt_pages(self): continue return None + def _read_confirmation(self, question, auto_respond=False): + """Read an answer to a confirmation prompt. + + A closed stdin returns "", which matches no caller's accept-list and so + takes the same decline branch as answering "no" -- exiting USER_ABORTED + rather than raising EOFError at the prompt. Callers must abort in that + branch; this returns an answer, it does not exit. + + @param question: Prompt to display when there is no automatic response + @param auto_respond: Response to automatically provide for all prompts + """ + if auto_respond: + return auto_respond + try: + return input(question) + except EOFError: + return "" + def confirm_out_of_spec_warning(self, auto_respond=False): """Print the warning for running outside of specification and prompt user to accept the terms. @@ -1820,14 +2013,12 @@ def confirm_out_of_spec_warning(self, auto_respond=False): MAY NOT BE COVERED BY YOUR BOARD OR SYSTEM MANUFACTURER'S WARRANTY. Please use this utility with caution. """) - if not auto_respond: - user_input = input("Do you accept these terms? [y/n] ") - else: - user_input = auto_respond + user_input = self._read_confirmation("Do you accept these terms? [y/n] ", auto_respond) if user_input in ["y", "Y", "yes", "Yes", "YES"]: return else: - sys.exit("Confirmation not given. Exiting without setting value") + print("Confirmation not given. Exiting without setting value", file=sys.stderr) + sys.exit(int(amdsmi_cli_exceptions.AmdSmiExitCode.USER_ABORTED)) def confirm_changing_memory_partition_gpu_reload_warning(self, auto_respond=False): """Print the warning for running outside of specification and prompt user to accept the terms. @@ -1858,16 +2049,13 @@ def confirm_changing_memory_partition_gpu_reload_warning(self, auto_respond=Fals workloads across all devices. """) - if not auto_respond: - user_input = input("Do you accept these terms? [Y/N] ") - else: - user_input = auto_respond + user_input = self._read_confirmation("Do you accept these terms? [Y/N] ", auto_respond) if user_input in ["Yes", "yes", "y", "Y", "YES"]: print("") return else: print("Confirmation not given. Exiting without setting value") - sys.exit(1) + sys.exit(int(amdsmi_cli_exceptions.AmdSmiExitCode.USER_ABORTED)) def is_valid_profile(self, profile): profile_presets = ( @@ -2455,6 +2643,8 @@ def dump_cper_entries( list: JSON row dicts when JSON format is active, otherwise ``[]``. """ json_output = logger is not None and logger.is_json_format() + # Only the AFID message text varies by format. json_output picks the branch. + output_format = logger.format if logger is not None else "human" if cper_counter is None: cper_counter = [0] @@ -2481,10 +2671,17 @@ def dump_cper_entries( if isinstance(raw_bytes, list): # Handle signed bytes (convert negative values to unsigned) raw_bytes = bytes(x & 0xFF for x in raw_bytes) - afids = self.cper_dump_afids(raw_bytes) - except Exception as e: - afids = [] + afid_record = self.build_afid_record( + cper_path, output_format, afids=self.cper_dump_afids(raw_bytes) + ) + except amdsmi_exception.AmdSmiLibraryException as e: logging.debug(f"Failed to fetch AFIDs for {cper_path}: {e}") + # Records the status so the decode failure reaches the exit + # code. NO_PERM aborts instead, since it is not per-file. + self.record_or_raise(e) + afid_record = self.build_afid_record( + cper_path, output_format, code=e.get_error_code() + ) json_rows.append( { "timestamp": timestamp, @@ -2492,7 +2689,11 @@ def dump_cper_entries( "severity": severity, "cper_file": cper_path_str, "metadata_file": json_path_str, - "afids": afids, + "afids": afid_record["afids"], + "decode_failed": afid_record["decode_failed"], + "status": afid_record["status"], + "message": afid_record["message"], + "code": afid_record["code"], } ) if emit_inline: @@ -2506,11 +2707,18 @@ def dump_cper_entries( if isinstance(raw_bytes, list): # Handle signed bytes (convert negative values to unsigned) raw_bytes = bytes(x & 0xFF for x in raw_bytes) - afids = self.cper_dump_afids(raw_bytes) - afids_str = " ".join(map(str, afids)) - except Exception as e: - afids_str = "Error fetching AFIDs" + afid_record = self.build_afid_record( + cper_path, output_format, afids=self.cper_dump_afids(raw_bytes) + ) + except amdsmi_exception.AmdSmiLibraryException as e: logging.debug(f"Failed to fetch AFIDs for {cper_path}: {e}") + # Records the status so the decode failure reaches the exit + # code. NO_PERM aborts instead, since it is not per-file. + self.record_or_raise(e) + afid_record = self.build_afid_record( + cper_path, output_format, code=e.get_error_code() + ) + afids_str = self.afid_cell(afid_record) self.cper_print( f"{timestamp:<20} {gpu_id:<7} {severity:<20} {fname:<17} {afids_str}", logger, @@ -2583,24 +2791,11 @@ def cper_dump_afids(self, cper_file: Union[bytes, Path]): raw = cper_file.read_bytes() else: raw = cper_file - try: - afids, num_afids = amdsmi_interface.amdsmi_get_afids_from_cper(raw) - return afids - except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_INVAL: - raise ValueError("Invalid CPER file inputs") from e - elif ( - e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_SIZE - ): - raise ValueError("Invalid CPER file data size") from e - elif ( - e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_DATA - ): - raise ValueError("Unexpected data in CPER file") from e - elif e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED: - raise NotImplementedError("AFID decoding not supported") from e - else: - raise ValueError("Unexpected Error getting afids from CPER file") from e + # A decode failure raises AmdSmiLibraryException carrying the real + # AMDSMI_STATUS_*; callers catch it (or let it reach the top-level + # handler) to surface a truthful exit code. No translation here. + afids, num_afids = amdsmi_interface.amdsmi_get_afids_from_cper(raw) + return afids def get_partition_id(self, device_handle, gpu_id=None) -> int: partition_id = -1 @@ -2741,11 +2936,17 @@ def ras_cper(self, args, device_handle, logger, gpu_idx, emit_inline=True, cper_ or e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_FILE_NOT_FOUND ): - raise FileNotFoundError( - "Error accessing CPER files. This command requires CPER to be enabled." + raise amdsmi_cli_exceptions.AmdSmiLibraryErrorException( + logger.format, + e.get_error_code(), + detail="Error accessing CPER files. This command requires CPER to be enabled.", ) from e if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_FILE_ERROR: - raise OSError("Error opening CPER file. Unable to read CPER File") from e + raise amdsmi_cli_exceptions.AmdSmiLibraryErrorException( + logger.format, + e.get_error_code(), + detail="Error opening CPER file. Unable to read CPER File", + ) from e else: logging.debug(f"Cannot retrieve CPER entries: {e}") break @@ -3176,6 +3377,9 @@ def validate_and_set_power_cap( } return f"{power_type_key} power cap is already set to {requested_power_cap}W" elif current_power_cap == 0: + self.error_collector.record( + int(amdsmi_cli_exceptions.AmdSmiExitCode.INVALID_PARAMETER_VALUE) + ) if logger.is_json_format() or logger.is_csv_format(): return { "status": "error", @@ -3186,13 +3390,25 @@ def validate_and_set_power_cap( } return f"Unable to set {power_type_key} power cap to {requested_power_cap}W, current value is {current_power_cap}W" elif not min_power_cap <= requested_power_cap <= max_power_cap: - # Raise so the caller exits with a non-zero return code - raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException( - sys.argv[1] if len(sys.argv) > 1 else "unknown", - f"{requested_power_cap}W", - self.get_output_format(), - hint=f"Power cap must be between {min_power_cap}W and {max_power_cap}W", + # Record this device's out-of-range failure and keep going: + # another device may have a different valid range (e.g. a higher + # max), so it could still succeed. Raising here would abort the + # whole loop before the other GPUs are tried (record-then-finalize). + self.error_collector.record( + int(amdsmi_cli_exceptions.AmdSmiExitCode.INVALID_PARAMETER_VALUE) ) + message = ( + f"Unable to set {power_type_key} power cap to {requested_power_cap}W. " + f"Power cap must be between {min_power_cap}W and {max_power_cap}W" + ) + if logger.is_json_format() or logger.is_csv_format(): + return { + "status": "error", + "sensor": power_type_key, + "requested_power_cap": self.unit_format(logger, requested_power_cap, "W"), + "message": message, + } + return message # Set the power cap new_power_cap = self.convert_SI_unit( requested_power_cap, AMDSMIHelpers.SI_Unit.BASE, AMDSMIHelpers.SI_Unit.MICRO @@ -3210,6 +3426,7 @@ def validate_and_set_power_cap( if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError("Command requires elevation") from e error_msg = f"[{e.get_error_info(detailed=False)}] Unable to set {power_type_key} power cap to {requested_power_cap}W" + self.error_collector.record_library_error(e.get_error_code()) if logger.is_json_format() or logger.is_csv_format(): return { "status": "error", @@ -3220,6 +3437,105 @@ def validate_and_set_power_cap( } return error_msg + def validate_and_set_node_power_limit(self, node_handle, requested_limit, logger): + """Validate and set the NPM (Node Power Management) power limit for a node. + + Mirrors validate_and_set_power_cap(): fetches the platform max bound via + amdsmi_get_npm_info() (amdsmi_npm_info_t::max_node_power_limit, sourced from + board/max_node_power_limit) and rejects out-of-range requests before ever + issuing the write. This is an early, CLI-friendly pre-check only: + amdsmi_set_npm_limit() itself already validates `limit` against the same + bound internally and fails closed if that bound can't be read (see its + docstring); this CLI-side check exists purely to fail fast and surface a + clean, user-facing hint here rather than relying solely on the library + call's exception. + + There is no driver-exposed minimum bound beyond positivity (no sysfs file + or design-doc source documents a real minimum), so the lower bound enforced + here is simply "> 0". + + Also rejects the request up front if NPM itself is disabled on this node + (amdsmi_npm_info_t::status == AMDSMI_NPM_STATUS_DISABLED) -- there is no + max bound to validate against in that case, and writing + board/cur_node_power_limit while NPM is disabled has no defined effect. + Same fail-fast pattern as the max-bound check above: amdsmi_set_npm_limit() + itself already enforces this too, so this is a CLI-friendly early exit, + not the sole enforcement. + + Fails closed when the platform max is unavailable ("N/A"): rather than + allowing any positive value through in that degraded-driver scenario, the + request is rejected outright here too, matching amdsmi_set_npm_limit()'s + own fail-closed behavior for the same condition. + + Args: + node_handle: Node handle obtained via amdsmi_get_node_handle() + requested_limit: Requested node power limit value in watts + logger: AMDSMILogger instance for format-aware output + + Returns: + dict or str: Structured data for JSON/CSV or formatted string for human-readable output + """ + try: + npm_info = amdsmi_interface.amdsmi_get_npm_info(node_handle) + max_node_power_limit = npm_info["max_node_power_limit"] + + if npm_info["status"] == amdsmi_interface.amdsmi_wrapper.AMDSMI_NPM_STATUS_DISABLED: + raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException( + sys.argv[1] if len(sys.argv) > 1 else "unknown", + f"{requested_limit}", + self.get_output_format(), + hint="Node power limit cannot be set: NPM is disabled on this node", + ) + elif max_node_power_limit == "N/A": + # Fail closed: the platform max is unreadable (degraded driver / + # unavailable sysfs), so there is no bound to validate the + # request against. Reject rather than let an unbounded value + # through. + raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException( + sys.argv[1] if len(sys.argv) > 1 else "unknown", + f"{requested_limit}", + self.get_output_format(), + hint="Node power limit cannot be validated: platform maximum is unavailable", + ) + elif not 0 < requested_limit <= max_node_power_limit: + # Raise so the caller exits with a non-zero return code + raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException( + sys.argv[1] if len(sys.argv) > 1 else "unknown", + f"{requested_limit}", + self.get_output_format(), + hint=f"Node power limit must be between 1W and {max_node_power_limit}W", + ) + + amdsmi_interface.amdsmi_set_npm_limit(node_handle, requested_limit) + if logger.is_json_format() or logger.is_csv_format(): + return { + "status": "success", + "requested_limit": f"{requested_limit}", + "message": f"Successfully set node power limit to {requested_limit} W", + } + return f"Successfully set node power limit to {requested_limit} W." + except amdsmi_exception.AmdSmiLibraryException as e: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError("Command requires elevation") from e + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_INVAL: + # Raise so the caller exits with a non-zero return code, same as + # the CLI-side bound checks above. + raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException( + sys.argv[1] if len(sys.argv) > 1 else "unknown", + f"{requested_limit}", + self.get_output_format(), + hint=e.get_error_info(detailed=False), + ) from e + error_msg = f"[{e.get_error_info(detailed=False)}] Unable to set node power limit to {requested_limit} W" + if logger.is_json_format() or logger.is_csv_format(): + return { + "status": "error", + "requested_limit": f"{requested_limit}", + "error": e.get_error_info(detailed=False), + "message": error_msg, + } + return error_msg + def prompt_reboot(self): """Prompt user to reboot and execute if confirmed diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_init.py b/projects/amdsmi/amdsmi_cli/amdsmi_init.py index 9b8374a67ea7..0469d5ad55d8 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_init.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_init.py @@ -12,6 +12,8 @@ import threading from pathlib import Path +from amdsmi_cli_exceptions import AmdSmiExitCode + # CLI module resolution order (distinct from `import amdsmi` in a user script): # 1. this installation's share/amd_smi copy -- the modules the CLI shipped # with, always preferred so `amd-smi` uses its own version even on a host @@ -41,7 +43,7 @@ print( "Failed to import the amdsmi Python library. Install amd-smi-lib (rpm/deb) or pip install the amdsmi wheel." ) - sys.exit(1) + sys.exit(int(AmdSmiExitCode.IMPORT_ERROR)) # Using basic python logging for user errors and development logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.ERROR) # User level logging @@ -181,18 +183,25 @@ def _run_init(): init_thread.join(timeout=_INIT_TIMEOUT_SEC) if init_thread.is_alive(): + # The library hung with no status, so exit with a CLI code (not a borrowed + # library status) -- the exit code unambiguously means the CLI watchdog gave up. logging.error( "amdsmi_init() timed out after %ds. The GPU driver may be unresponsive.", _INIT_TIMEOUT_SEC, ) - sys.exit(2) + sys.exit(int(AmdSmiExitCode.INIT_TIMEOUT)) if isinstance( init_result["exception"], (amdsmi_interface.AmdSmiLibraryException, amdsmi_interface.AmdSmiParameterException), ): e = init_result["exception"] - # parameter exception thrown if init_flag is 0, but err_code will be set to 0 in that case, so must check if init_flag is 0 too + # Normalize "drivers not loaded" to one CLI code, reached two ways: + # 1) Library returns NOT_INIT/DRIVER_NOT_LOADED + # 2) No drivers detected up front -> init_flag == 0. (amdsmi_init raises + # AmdSmiParameterException with err_code=None, which won't match (1)'s + # tuple, so the init_flag == 0 check catches it.) + # We don't want to clash with library error codes, so we use a CLI code (AmdSmiExitCode.DRIVERS_NOT_LOADED) for this case. if ( e.err_code in ( @@ -204,7 +213,7 @@ def _run_init(): logging.error( "Drivers not loaded (amdgpu, amd_hsmp, ionic, bnxt_en drivers not found in modules)" ) - sys.exit(-1) + sys.exit(int(AmdSmiExitCode.DRIVERS_NOT_LOADED)) else: raise e elif init_result["exception"] is not None: diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py index f474c6668d95..bfb1ba69c311 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py @@ -1249,21 +1249,11 @@ def print_default_output(self, output: Dict): rocm_version = "N/A" if output["version_info"]["rocm version"][0]: rocm_version = str(output["version_info"]["rocm version"][1]).ljust(8) - driver_version = output["version_info"]["amdgpu version"] - if driver_version == "N/A": - amdgpu_version = "N/A".ljust(8) + driver_info = output["version_info"]["amdgpu version"] + if driver_info == "N/A": + amdgpu_version = "N/A" else: - # Example driver version string for amdgpu: 6.8.0-60 : 'Linuxversion6.8.0-60-generic(buildd@lcy02-amd64-098)(x86_64-linux-gnu-gcc-12(Ubuntu12.3.0-1ubuntu1~22.04)12.3.0,GNUld(GNUBinutilsforUbuntu)2.38)#63~22.04.1-UbuntuSMPPREEMPT_DYNAMICTueApr2219:00:15UTC2' - # Extract version before "-generic" if it exists - if "-generic" in driver_version["driver_version"]: - # Extract version using regex to find pattern like "6.8.0-60" - match = re.search(r"(\d+\.\d+\.\d+-\d+)", driver_version["driver_version"]) - if match: - amdgpu_version = match.group(1)[:80] - else: - amdgpu_version = "N/A" - else: - amdgpu_version = str(driver_version["driver_version"])[:80] + amdgpu_version = str(driver_info["driver_full_version"]) fw_pldm_version = str(output["version_info"]["fw pldm version"]) vbios_version = str(output["version_info"]["vbios version"]) kernel_version = str(output["version_info"]["kernel version"]) @@ -1290,10 +1280,9 @@ def _trunc(s): print(default_line_1) print("| AMD-SMI {0:<{w}s} |".format(amd_smi_version, w=_COL_WIDTH)) - # Print amdgpu or kernel version based on availability, if neither then don't print - if amdgpu_version.strip() != "N/A": + if amdgpu_version != "N/A": print("| amdgpu Version: {0:<{w}s} |".format(amdgpu_version, w=_COL_WIDTH)) - elif kernel_version.strip() != "N/A": + elif kernel_version != "N/A": print("| OS kernel Version: {0:<{w}s} |".format(kernel_version, w=_COL_WIDTH)) if rocm_version != "N/A": diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_metrics_field_support.py b/projects/amdsmi/amdsmi_cli/amdsmi_metrics_field_support.py new file mode 100644 index 000000000000..adf3964833eb --- /dev/null +++ b/projects/amdsmi/amdsmi_cli/amdsmi_metrics_field_support.py @@ -0,0 +1,402 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Which ``amd-smi metric`` fields the detected gpu_metrics version can populate. + +Backs the default field filtering of human-readable ``amd-smi metric`` output, +which ``--show-unsupported`` turns off. ``--json`` and ``--csv`` are consumed by +scripts and never filter, so ``metric.py`` does not call this module for them. + +The field set carried by ``gpu_metrics`` is determined by its header version and +is not cumulative: v1.4 drops 18 of the fields v1.3 carried, v1.6 drops +jpeg_activity/vcn_activity/throttle_status. Each version is therefore an explicit +set, mirroring the per-version ``copy_internal_to_external_metrics()`` in +``rocm_smi_gpu_metrics.cc``. + +Three rules keep the feature from ever losing data: + +* an unmapped version yields ``None``, and callers then suppress nothing, so a + future metrics version degrades to today's output; +* :func:`filter_unsupported` drops a path only when every leaf under it already + reads "N/A", so a mis-mapped path can under-report but can never hide a real + value; +* a section the user named on the command line is passed as protected and is + left whole rather than emptied, so an explicit ``--throttle`` never answers + with silence. + +The all-N/A gate is what makes the partition blob safe to approximate. It is the +second eligible source and is versioned independently, by +``amdgpu_partition_metric_version_translation_table`` in the same C++ file, which +this module does not read: every partition-backed path (``xcp_stats.*``, +``temperature_mid``, ``current_socclks_mid``) is judged by the GPU metrics +version instead. A wrong approximation can then only change whether an already +empty column is printed, which on a mixed-version fleet shows up as a key-set +difference between GPUs rather than as a lost value. +""" + +from functools import lru_cache +from typing import Any, Dict, FrozenSet, Optional, Tuple + +NA = "N/A" + +# (format_revision, content_revision) as reported by the gpu_metrics header. +VersionKey = Tuple[int, int] +# A (section, key) coordinate in the ``metric`` command's output dict. +CliPath = Tuple[str, str] +# Public ``amdsmi_gpu_metrics_t`` member names. +FieldSet = FrozenSet[str] + + +def _fields(names: str) -> FieldSet: + """Field sets are written as whitespace-separated text to keep the table readable.""" + return frozenset(names.split()) + + +# Public ``amdsmi_gpu_metrics_t`` members each header version can populate. +_VERSION_FIELDS: Dict[VersionKey, FieldSet] = { + (1, 0): _fields( + """ + average_dclk0_frequency average_dclk1_frequency average_gfx_activity + average_gfxclk_frequency average_mm_activity average_socclk_frequency average_socket_power + average_uclk_frequency average_umc_activity average_vclk0_frequency average_vclk1_frequency + current_dclk0 current_dclk1 current_fan_speed current_gfxclk current_socclk current_uclk + current_vclk0 current_vclk1 energy_accumulator pcie_link_speed pcie_link_width + system_clock_counter temperature_edge temperature_hotspot temperature_mem temperature_vrgfx + temperature_vrmem temperature_vrsoc throttle_status + """ + ), + (1, 1): _fields( + """ + average_dclk0_frequency average_dclk1_frequency average_gfx_activity + average_gfxclk_frequency average_mm_activity average_socclk_frequency average_socket_power + average_uclk_frequency average_umc_activity average_vclk0_frequency average_vclk1_frequency + current_dclk0 current_dclk1 current_fan_speed current_gfxclk current_socclk current_uclk + current_vclk0 current_vclk1 energy_accumulator gfx_activity_acc mem_activity_acc + pcie_link_speed pcie_link_width system_clock_counter temperature_edge temperature_hbm + temperature_hotspot temperature_mem temperature_vrgfx temperature_vrmem temperature_vrsoc + throttle_status + """ + ), + (1, 2): _fields( + """ + average_dclk0_frequency average_dclk1_frequency average_gfx_activity + average_gfxclk_frequency average_mm_activity average_socclk_frequency average_socket_power + average_uclk_frequency average_umc_activity average_vclk0_frequency average_vclk1_frequency + current_dclk0 current_dclk1 current_fan_speed current_gfxclk current_socclk current_uclk + current_vclk0 current_vclk1 energy_accumulator firmware_timestamp gfx_activity_acc + mem_activity_acc pcie_link_speed pcie_link_width system_clock_counter temperature_edge + temperature_hbm temperature_hotspot temperature_mem temperature_vrgfx temperature_vrmem + temperature_vrsoc throttle_status + """ + ), + (1, 3): _fields( + """ + average_dclk0_frequency average_dclk1_frequency average_gfx_activity + average_gfxclk_frequency average_mm_activity average_socclk_frequency average_socket_power + average_uclk_frequency average_umc_activity average_vclk0_frequency average_vclk1_frequency + current_dclk0 current_dclk0s current_dclk1 current_fan_speed current_gfxclk current_gfxclks + current_socclk current_socclks current_uclk current_vclk0 current_vclk0s current_vclk1 + energy_accumulator firmware_timestamp gfx_activity_acc indep_throttle_status + mem_activity_acc pcie_link_speed pcie_link_width system_clock_counter temperature_edge + temperature_hbm temperature_hotspot temperature_mem temperature_vrgfx temperature_vrmem + temperature_vrsoc throttle_status vcn_activity voltage_gfx voltage_mem voltage_soc + """ + ), + (1, 4): _fields( + """ + average_gfx_activity average_umc_activity current_dclk0 current_dclk0s current_dclk1 + current_gfxclk current_gfxclks current_socclk current_socclks current_socket_power + current_uclk current_vclk0 current_vclk0s current_vclk1 energy_accumulator + firmware_timestamp gfx_activity_acc gfxclk_lock_status mem_activity_acc pcie_bandwidth_acc + pcie_bandwidth_inst pcie_l0_to_recov_count_acc pcie_link_speed pcie_link_width + pcie_replay_count_acc pcie_replay_rover_count_acc system_clock_counter temperature_hotspot + temperature_mem temperature_vrsoc throttle_status vcn_activity xgmi_link_speed + xgmi_link_width xgmi_read_data_acc xgmi_write_data_acc + """ + ), + (1, 5): _fields( + """ + average_gfx_activity average_umc_activity current_dclk0 current_dclk0s current_dclk1 + current_gfxclk current_gfxclks current_socclk current_socclks current_socket_power + current_uclk current_vclk0 current_vclk0s current_vclk1 energy_accumulator + firmware_timestamp gfx_activity_acc gfxclk_lock_status jpeg_activity mem_activity_acc + pcie_bandwidth_acc pcie_bandwidth_inst pcie_l0_to_recov_count_acc pcie_link_speed + pcie_link_width pcie_nak_rcvd_count_acc pcie_nak_sent_count_acc pcie_replay_count_acc + pcie_replay_rover_count_acc system_clock_counter temperature_hotspot temperature_mem + temperature_vrsoc throttle_status vcn_activity xgmi_link_speed xgmi_link_width + xgmi_read_data_acc xgmi_write_data_acc + """ + ), + (1, 6): _fields( + """ + accumulation_counter average_gfx_activity average_umc_activity current_dclk0 current_dclk0s + current_dclk1 current_gfxclk current_gfxclks current_socclk current_socclks + current_socket_power current_uclk current_vclk0 current_vclk0s current_vclk1 + energy_accumulator firmware_timestamp gfx_activity_acc gfxclk_lock_status + hbm_thm_residency_acc mem_activity_acc num_partition pcie_bandwidth_acc pcie_bandwidth_inst + pcie_l0_to_recov_count_acc pcie_lc_perf_other_end_recovery pcie_link_speed pcie_link_width + pcie_nak_rcvd_count_acc pcie_nak_sent_count_acc pcie_replay_count_acc + pcie_replay_rover_count_acc ppt_residency_acc prochot_residency_acc socket_thm_residency_acc + system_clock_counter temperature_hotspot temperature_mem temperature_vrsoc + vr_thm_residency_acc xcp_stats.gfx_busy_acc xcp_stats.gfx_busy_inst xcp_stats.jpeg_busy + xcp_stats.vcn_busy xgmi_link_speed xgmi_link_width xgmi_read_data_acc xgmi_write_data_acc + """ + ), + (1, 7): _fields( + """ + accumulation_counter average_gfx_activity average_umc_activity current_dclk0 current_dclk0s + current_dclk1 current_gfxclk current_gfxclks current_socclk current_socclks + current_socket_power current_uclk current_vclk0 current_vclk0s current_vclk1 + energy_accumulator firmware_timestamp gfx_activity_acc gfxclk_lock_status + hbm_thm_residency_acc mem_activity_acc num_partition pcie_bandwidth_acc pcie_bandwidth_inst + pcie_l0_to_recov_count_acc pcie_lc_perf_other_end_recovery pcie_link_speed pcie_link_width + pcie_nak_rcvd_count_acc pcie_nak_sent_count_acc pcie_replay_count_acc + pcie_replay_rover_count_acc ppt_residency_acc prochot_residency_acc socket_thm_residency_acc + system_clock_counter temperature_hotspot temperature_mem temperature_vrsoc + vr_thm_residency_acc vram_max_bandwidth xcp_stats.gfx_below_host_limit_acc + xcp_stats.gfx_busy_acc xcp_stats.gfx_busy_inst xcp_stats.jpeg_busy xcp_stats.vcn_busy + xgmi_link_speed xgmi_link_status xgmi_link_width xgmi_read_data_acc xgmi_write_data_acc + """ + ), + (1, 8): _fields( + """ + accumulation_counter average_gfx_activity average_umc_activity current_dclk0 current_dclk0s + current_dclk1 current_gfxclk current_gfxclks current_socclk current_socclks + current_socket_power current_uclk current_vclk0 current_vclk0s current_vclk1 + energy_accumulator firmware_timestamp gfx_activity_acc gfxclk_lock_status + hbm_thm_residency_acc mem_activity_acc num_partition pcie_bandwidth_acc pcie_bandwidth_inst + pcie_l0_to_recov_count_acc pcie_lc_perf_other_end_recovery pcie_link_speed pcie_link_width + pcie_nak_rcvd_count_acc pcie_nak_sent_count_acc pcie_replay_count_acc + pcie_replay_rover_count_acc ppt_residency_acc prochot_residency_acc socket_thm_residency_acc + system_clock_counter temperature_hotspot temperature_mem temperature_vrsoc + vr_thm_residency_acc vram_max_bandwidth xcp_stats.gfx_below_host_limit_ppt_acc + xcp_stats.gfx_below_host_limit_thm_acc xcp_stats.gfx_below_host_limit_total_acc + xcp_stats.gfx_busy_acc xcp_stats.gfx_busy_inst xcp_stats.gfx_low_utilization_acc + xcp_stats.jpeg_busy xcp_stats.temperature_xcd xcp_stats.vcn_busy xgmi_link_speed + xgmi_link_status xgmi_link_width xgmi_read_data_acc xgmi_write_data_acc + """ + ), + # APU v2.4 and v3.0 share ApuMetricsBase_v30_t but fill different public members. + (2, 4): _fields( + """ + average_dclk0_frequency average_gfx_activity average_gfxclk_frequency average_mm_activity + average_socclk_frequency average_socket_power average_uclk_frequency average_vclk0_frequency + current_dclk0 current_gfxclk current_socclk current_uclk current_vclk0 indep_throttle_status + system_clock_counter throttle_status + """ + ), + (3, 0): _fields( + """ + average_gfx_activity average_gfxclk_frequency average_socclk_frequency average_socket_power + average_uclk_frequency average_vclk0_frequency system_clock_counter + """ + ), +} + +# (1, >=9) is the kernel-schema-driven dynamic path, so this is a static upper +# bound: the kernel may advertise fewer attributes than the library can emit. +_DYNAMIC_FIELDS: FieldSet = _fields( + """ + accumulation_counter average_gfx_activity average_umc_activity current_dclk0 current_dclk0s + current_dclk1 current_gfxclk current_gfxclks current_socclk current_socclks + current_socclks_mid current_socket_power current_uclk current_uclk_aid current_vclk0 + current_vclk0s current_vclk1 energy_accumulator firmware_timestamp gfx_activity_acc + gfxclk_lock_status hbm_thm_residency_acc mem_activity_acc pcie_bandwidth_acc + pcie_bandwidth_inst pcie_l0_to_recov_count_acc pcie_lc_perf_other_end_recovery + pcie_link_speed pcie_link_width pcie_nak_rcvd_count_acc pcie_nak_sent_count_acc + pcie_replay_count_acc pcie_replay_rover_count_acc ppt_residency_acc prochot_residency_acc + socket_thm_residency_acc system_clock_counter temperature_aid temperature_hbm_stacks + temperature_hotspot temperature_mem temperature_mid temperature_vrsoc vr_thm_residency_acc + vram_max_bandwidth xcp_stats.gfx_below_host_limit_ppt_acc + xcp_stats.gfx_below_host_limit_thm_acc xcp_stats.gfx_below_host_limit_total_acc + xcp_stats.gfx_busy_acc xcp_stats.gfx_busy_inst xcp_stats.gfx_low_utilization_acc + xcp_stats.jpeg_busy xcp_stats.temperature_xcd xcp_stats.vcn_busy xgmi_link_speed + xgmi_link_status xgmi_link_width xgmi_read_data_acc xgmi_write_data_acc + """ +) + +_DYNAMIC_FORMAT_REVISION = 1 +_DYNAMIC_MIN_CONTENT_REVISION = 9 + +# (section, key) -> every gpu_metrics field the path needs. All of them are +# required, so a missing one makes the path suppressible. An allow-list: a CLI +# path absent here is never suppressed. +# +# A path may be mapped only when the metrics blobs are its only source. If +# metric.py can also fill it from hwmon, sysfs or another API, its absence is not +# determined by a metrics version, and suppression would follow live GPU state +# rather than a header. Notably absent for that reason: temperature +# edge/hotspot/mem and the whole fan section (hwmon), the voltages +# (amdsmi_get_power_info), pcie replay_count and the per-engine clock slots +# (sysfs), and usage.vcn_busy (amdsmi_get_vcn_busy_percent on Navi). +_CLI_PATH_REQUIRED_SOURCES: Dict[CliPath, Tuple[str, ...]] = { + ("usage", "mm_activity"): ("average_mm_activity",), + ("usage", "vcn_activity"): ("vcn_activity",), + ("usage", "jpeg_activity"): ("jpeg_activity",), + ("usage", "gfx_busy_inst"): ("xcp_stats.gfx_busy_inst",), + ("usage", "jpeg_busy"): ("xcp_stats.jpeg_busy",), + ("power", "throttle_status"): ("throttle_status",), + ("clock", "uclk_aid"): ("current_uclk_aid",), + ("clock", "socclks_mid"): ("current_socclks_mid",), + ("temperature", "hbm_stacks"): ("temperature_hbm_stacks",), + ("temperature", "mid"): ("temperature_mid",), + ("temperature", "aid"): ("temperature_aid",), + ("temperature", "xcd"): ("xcp_stats.temperature_xcd",), + ("pcie", "bandwidth"): ("pcie_bandwidth_inst",), + ("pcie", "l0_to_recovery_count"): ("pcie_l0_to_recov_count_acc",), + ("pcie", "replay_roll_over_count"): ("pcie_replay_rover_count_acc",), + ("pcie", "nak_sent_count"): ("pcie_nak_sent_count_acc",), + ("pcie", "nak_received_count"): ("pcie_nak_rcvd_count_acc",), + ("pcie", "lc_perf_other_end_recovery_count"): ("pcie_lc_perf_other_end_recovery",), + ("throttle", "accumulation_counter"): ("accumulation_counter",), +} + +# Throttle rows all come from the residency accumulators, exposed through +# amdsmi_get_violation_status(); each row has accumulated/status/activity forms. +_THROTTLE_SOURCES: Dict[str, str] = { + "prochot": "prochot_residency_acc", + "ppt": "ppt_residency_acc", + "socket_thermal": "socket_thm_residency_acc", + "vr_thermal": "vr_thm_residency_acc", + "hbm_thermal": "hbm_thm_residency_acc", + "gfx_clk_below_host_limit": "xcp_stats.gfx_below_host_limit_acc", + "gfx_clk_below_host_limit_power": "xcp_stats.gfx_below_host_limit_ppt_acc", + "gfx_clk_below_host_limit_thermal": "xcp_stats.gfx_below_host_limit_thm_acc", + "total_gfx_clk_below_host_limit": "xcp_stats.gfx_below_host_limit_total_acc", + "low_utilization": "xcp_stats.gfx_low_utilization_acc", +} + +for _row, _source in _THROTTLE_SOURCES.items(): + _CLI_PATH_REQUIRED_SOURCES[("throttle", f"{_row}_accumulated")] = (_source,) + # amdsmi_get_violation_status() computes the activity percentage as the + # accumulator over the counter and derives the status from it, leaving both + # unset without the counter, so both forms require both fields. + _CLI_PATH_REQUIRED_SOURCES[("throttle", f"{_row}_violation_status")] = ( + _source, + "accumulation_counter", + ) + _CLI_PATH_REQUIRED_SOURCES[("throttle", f"{_row}_violation_activity")] = ( + _source, + "accumulation_counter", + ) + + +def known_versions() -> FrozenSet[VersionKey]: + """The explicitly mapped ``(format_revision, content_revision)`` pairs. + + No CLI caller: this is the seam the table's tests and the C++ drift guard + enumerate versions through. + """ + return frozenset(_VERSION_FIELDS) + + +def _normalize_version(format_revision: Any, content_revision: Any) -> Optional[VersionKey]: + try: + return (int(format_revision), int(content_revision)) + except (TypeError, ValueError): + return None + + +def populatable_fields(format_revision: Any, content_revision: Any) -> Optional[FieldSet]: + """Public gpu_metrics fields this version can carry, or None if unmapped. + + Explicit entries win over the dynamic bound, matching + ``translate_header_to_flag_version()``, which consults + ``amdgpu_metric_version_translation_table`` first and only falls back to the + (1, >=9) dynamic reader on a miss. + """ + return _populatable_fields(_normalize_version(format_revision, content_revision)) + + +def _populatable_fields(version: Optional[VersionKey]) -> Optional[FieldSet]: + if version is None: + return None + fields = _VERSION_FIELDS.get(version) + if fields is not None: + return fields + if version[0] == _DYNAMIC_FORMAT_REVISION and version[1] >= _DYNAMIC_MIN_CONTENT_REVISION: + return _DYNAMIC_FIELDS + return None + + +def build_suppression_set(header: Any) -> FrozenSet[CliPath]: + """CLI paths the version in ``header`` can never populate. + + ``header`` is the dict from ``amdsmi_get_gpu_metrics_header_info()``. Anything + unreadable or unmapped yields an empty set, which suppresses nothing. + """ + if not isinstance(header, dict): + return frozenset() + return _suppression_set( + _normalize_version(header.get("format_revision"), header.get("content_revision")) + ) + + +# Depends only on the version, so `metric --watch` across GPUs and iterations +# walks the source map once per distinct version rather than once per call. +@lru_cache(maxsize=128) +def _suppression_set(version: Optional[VersionKey]) -> FrozenSet[CliPath]: + fields = _populatable_fields(version) + if fields is None: + return frozenset() + return frozenset( + path + for path, sources in _CLI_PATH_REQUIRED_SOURCES.items() + if any(source not in fields for source in sources) + ) + + +def _is_all_na(value: Any) -> bool: + """True when nothing under ``value`` carries data.""" + if isinstance(value, str): + if value == NA: + return True + # Human-readable output renders a list as "[N/A, 42 %]". + if value.startswith("[") and value.endswith("]"): + # A naive split mis-splits an element containing a comma, which then + # compares unequal to "N/A" and keeps the field. Fails safe. + items = value[1:-1].split(",") + return bool(value[1:-1].strip()) and all(item.strip() == NA for item in items) + return False + if isinstance(value, dict): + return bool(value) and all(_is_all_na(item) for item in value.values()) + if isinstance(value, (list, tuple)): + return bool(value) and all(_is_all_na(item) for item in value) + return False + + +def filter_unsupported( + values_dict: Dict[str, Any], + suppressed: FrozenSet[CliPath], + protected_sections: FrozenSet[str] = frozenset(), +) -> Dict[str, Any]: + """``values_dict`` without the suppressed all-N/A paths. + + An empty ``suppressed`` set is handed straight back as ``values_dict`` + itself. Otherwise the top level is a new dict, and the sections it did not + have to change are the objects it was given. + + Sections emptied by the filter are dropped, except those in + ``protected_sections``: the user asked for those by name, so they are handed + back unfiltered rather than as nothing. A section that arrived empty is kept + so filtering never changes anything but suppressed fields. + """ + if not suppressed: + return values_dict + + filtered = {} + for section, section_value in values_dict.items(): + if not isinstance(section_value, dict): + filtered[section] = section_value + continue + kept = { + key: value + for key, value in section_value.items() + if (section, key) not in suppressed or not _is_all_na(value) + } + if not kept and section_value: + if section in protected_sections: + filtered[section] = section_value + continue + filtered[section] = kept + return filtered diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py index 8f9ba66f669f..dc60e7042507 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py @@ -349,14 +349,10 @@ def _is_valid_string(self, string_value, sub_arg=None): sys.argv[1], string_value, outputformat ) - def _is_command_supported(self, user_input, acceptable_values, command_name): - if acceptable_values == "N/A": - outputformat = self.helpers.get_output_format() - raise amdsmi_cli_exceptions.AmdSmiPermissionDeniedException(command_name, outputformat) - elif str(user_input).upper() not in acceptable_values: - print(f"Valid inputs are {acceptable_values}") + def _is_command_supported(self, user_input, acceptable_values, hint=None): + if str(user_input).upper() not in acceptable_values: raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException( - sys.argv[1], str(user_input).upper(), self.helpers.get_output_format() + sys.argv[1], str(user_input).upper(), self.helpers.get_output_format(), hint=hint ) else: return str(user_input).upper() @@ -381,11 +377,17 @@ def __call__( # Check if the sclk and mclk parameters are valid if clk_type not in valid_clk_types: raise amdsmi_cli_exceptions.AmdSmiInvalidParameterException( - sys.argv[1], clk_type, output_format + sys.argv[1], + clk_type, + output_format, + hint=f"Valid options are: {', '.join(valid_clk_types)}.", ) if lim_type not in valid_lim_types: raise amdsmi_cli_exceptions.AmdSmiInvalidParameterException( - sys.argv[1], lim_type, output_format + sys.argv[1], + lim_type, + output_format, + hint=f"Valid options are: {', '.join(valid_lim_types)}.", ) # Check if the val is a valid integer value @@ -425,7 +427,10 @@ def __call__( # Check if the sclk and mclk parameters are valid if clk_type not in valid_clk_types: raise amdsmi_cli_exceptions.AmdSmiInvalidParameterException( - sys.argv[1], clk_type, output_format + sys.argv[1], + clk_type, + output_format, + hint=f"Valid options are: {', '.join(valid_clk_types)}.", ) if not perf_levels_str: @@ -708,7 +713,10 @@ def __call__(self, parser, args, values, option_string=None): .lower() ) except Exception: - sys.exit("Confirmation not given. Exiting without setting value") + print( + "Confirmation not given. Exiting without setting value", file=sys.stderr + ) + sys.exit(int(amdsmi_cli_exceptions.AmdSmiExitCode.USER_ABORTED)) if resp in ("a", "append"): setattr(args, self.dest, path) return @@ -717,12 +725,11 @@ def __call__(self, parser, args, values, option_string=None): setattr(args, self.dest, path) return else: - # User declined to overwrite - raise amdsmi_cli_exceptions.AmdSmiInvalidFilePathException( - path, - CheckOutputFilePath.outputformat, - "User declined to overwrite or append existing file.", + # Declining is not a bad path: the file was fine, the user said no. + print( + "User declined to overwrite or append existing file.", file=sys.stderr ) + sys.exit(int(amdsmi_cli_exceptions.AmdSmiExitCode.USER_ABORTED)) else: raise amdsmi_cli_exceptions.AmdSmiInvalidFilePathException( path, CheckOutputFilePath.outputformat @@ -819,9 +826,7 @@ def __call__(self, parser, args, values, option_string=None): raise amdsmi_cli_exceptions.AmdSmiDeviceNotFoundException( selected_device_handles, _GPUSelectAction.outputformat, - True, - False, - False, + amdsmi_cli_exceptions.AmdSmiDeviceKind.GPU, ) return _GPUSelectAction @@ -855,7 +860,9 @@ def __call__(self, parser, args, values, option_string=None): ) else: raise amdsmi_cli_exceptions.AmdSmiDeviceNotFoundException( - selected_device_handles, _NICSelectAction.output_format + selected_device_handles, + _NICSelectAction.output_format, + amdsmi_cli_exceptions.AmdSmiDeviceKind.NIC, ) return _NICSelectAction @@ -889,7 +896,9 @@ def __call__(self, parser, args, values, option_string=None): ) else: raise amdsmi_cli_exceptions.AmdSmiDeviceNotFoundException( - selected_device_handles, _SwitchSelectAction.output_format + selected_device_handles, + _SwitchSelectAction.output_format, + amdsmi_cli_exceptions.AmdSmiDeviceKind.SWITCH, ) return _SwitchSelectAction @@ -928,9 +937,7 @@ def __call__(self, parser, args, values, option_string=None): raise amdsmi_cli_exceptions.AmdSmiDeviceNotFoundException( selected_device_handles, _CPUSelectAction.outputformat, - False, - True, - False, + amdsmi_cli_exceptions.AmdSmiDeviceKind.CPU, ) return _CPUSelectAction @@ -969,9 +976,7 @@ def __call__(self, parser, args, values, option_string=None): raise amdsmi_cli_exceptions.AmdSmiDeviceNotFoundException( selected_device_handles, _CoreSelectAction.outputformat, - False, - False, - True, + amdsmi_cli_exceptions.AmdSmiDeviceKind.CPU_CORE, ) return _CoreSelectAction @@ -1272,18 +1277,26 @@ def __call__(self, parser, args, values, option_string=None): ### Building parsers ### @staticmethod - def _guard_gtt_gpu_conflict(parser, gtt_flags=("--gtt", "-G")): - """Override *parser*.error() so that combining any GTT flag with + def _guard_gtt_gpu_conflict( + parser, gtt_flags=("--gtt", "-G"), reason="--gtt is a system-wide setting, not per-GPU" + ): + """Override *parser*.error() so that combining any of *gtt_flags* with --gpu / -g produces a clear mutual-exclusion message instead of - the confusing "expected at least one argument" from --gpu.""" + the confusing "expected at least one argument" from --gpu. Only + argparse's missing-value diagnostic is replaced.""" _original_error = parser.error def _intercept(message): - if set(gtt_flags).intersection(sys.argv) and {"--gpu", "-g"}.intersection(sys.argv): + error_prefix = message.split(":", 1)[0] + if ( + set(gtt_flags).intersection(sys.argv) + and {"--gpu", "-g"}.intersection(sys.argv) + and error_prefix in {"argument -g/--gpu", "argument --gpu/-g"} + and "expected at least one argument" in message + ): flag_str = "/".join(gtt_flags) _original_error( - f"argument {flag_str}: not allowed with argument --gpu/-g " - "(--gtt is a system-wide setting, not per-GPU)" + f"argument {flag_str}: not allowed with argument --gpu/-g ({reason})" ) _original_error(message) @@ -1783,6 +1796,12 @@ def _add_metric_parser(self, subparsers: argparse._SubParsersAction, func): " (XCP/AID/MID) data sources; combine with those flags to scope it;" "\n Only available for MI300 or newer ASICs" ) + show_unsupported_help = ( + "Print every field, including the ones the GPU's gpu_metrics\n" + " table version cannot carry and which are omitted by default;\n" + " affects human-readable output only, since --json and --csv\n" + " always print every field" + ) # Help text for Arguments only on Hypervisors schedule_help = "All scheduling information" @@ -2182,7 +2201,10 @@ def _add_metric_parser(self, subparsers: argparse._SubParsersAction, func): # Add Universal Arguments & watch Args self._add_watch_arguments(metric_parser) self._add_device_arguments(metric_parser, required=False) - self._add_command_modifiers(metric_parser) + command_modifier_group = self._add_command_modifiers(metric_parser) + command_modifier_group.add_argument( + "--show-unsupported", action="store_true", required=False, help=show_unsupported_help + ) def _add_process_parser(self, subparsers: argparse._SubParsersAction, func): if self.helpers.is_hypervisor(): @@ -2411,6 +2433,7 @@ def _add_set_value_parser(self, subparsers: argparse._SubParsersAction, func): memory_partition_choices_str = ", ".join(self.helpers.get_memory_partition_types()) accelerator_set_choices_str = ", ".join(accelerator_set_choices) set_compute_partition_help = f"Set one of the following accelerator TYPE or profile INDEX:\n\t{accelerator_set_choices_str}.\n\tUse `sudo amd-smi partition --accelerator` to find acceptable values." + set_compute_partition_hint = f"\nValid inputs are: {accelerator_set_choices_str}.\nUse `sudo amd-smi partition --accelerator` to find acceptable values." set_memory_partition_help = f"Set one of the following the memory partition modes:\n\t{memory_partition_choices_str}" soc_pstate_help_info = ", ".join(self.helpers.get_soc_pstates()) set_soc_pstate_help = f"Set the GPU soc pstate policy using policy id, an integer. Valid id's include:\n\t{soc_pstate_help_info}" @@ -2525,7 +2548,7 @@ def _add_set_value_parser(self, subparsers: argparse._SubParsersAction, func): action="store", choices=accelerator_set_choices, type=lambda value: self._is_command_supported( - value, accelerator_set_choices, "--compute-partition" + value, accelerator_set_choices, hint=set_compute_partition_hint ), required=False, help=set_compute_partition_help, @@ -2655,6 +2678,19 @@ def _add_set_value_parser(self, subparsers: argparse._SubParsersAction, func): metavar="GB", ) + # Node power limit is enabled on guest (1VF), maintain order + max_node_power_limit = self.helpers.get_max_node_power_limit() + set_node_power_limit_help = f"Set the node-level (NPM) power limit in watts.\n\tThis is a node-wide setting, not per-GPU.\n\tMax node power limit: {max_node_power_limit}" + set_value_exclusive_group.add_argument( + "-n", + "--node-power-limit", + action="store", + type=lambda value: self._positive_int(value, "--node-power-limit"), + required=False, + help=set_node_power_limit_help, + metavar="WATTS", + ) + if self.helpers.is_amd_hsmp_initialized(): if self.helpers.is_baremetal(): # Optional CPU Args @@ -2863,6 +2899,15 @@ def _add_set_value_parser(self, subparsers: argparse._SubParsersAction, func): # Reject --gtt combined with --gpu at the argparse level self._guard_gtt_gpu_conflict(set_value_parser, gtt_flags=("--gtt", "-G")) + # Improve the --gpu error message if combined with --node-power-limit; + # actual rejection happens at runtime in set_value.py, since these two + # flags sit in separate argparse groups and argparse itself never + # raises for this combination. + self._guard_gtt_gpu_conflict( + set_value_parser, + gtt_flags=("--node-power-limit", "-n"), + reason="--node-power-limit is a node-wide setting, not per-GPU", + ) # Set accepts default devices of all self._add_device_arguments(set_value_parser, required=False) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_rocm_smi_compat.py b/projects/amdsmi/amdsmi_cli/amdsmi_rocm_smi_compat.py index 6d83e3fc0026..cda05b0aade0 100755 --- a/projects/amdsmi/amdsmi_cli/amdsmi_rocm_smi_compat.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_rocm_smi_compat.py @@ -15,6 +15,7 @@ from __future__ import print_function import sys import os +import enum import logging # Version information @@ -24,8 +25,20 @@ SMI_HASH = "amdsmi" __version__ = "%s.%s.%s+%s" % (SMI_MAJ, SMI_MIN, SMI_PAT, SMI_HASH) -# Set to 1 if an error occurs -RETCODE = 0 + +# `amd-smi --rocm-smi` mirrors legacy rocm-smi's default output, so it keeps +# rocm-smi's binary exit convention (0 = success, 1 = error) instead of amd-smi's +# exit-code redesign (the 193-253 AmdSmiExitCode band + library status codes). +# Adopting the new codes here could break scripts that rely on rocm-smi's existing +# $? contract. +@enum.unique +class RocmSmiCompatExitCode(enum.IntEnum): + SUCCESS = 0 + ERROR = 1 + + +# Set to RocmSmiCompatExitCode.ERROR if an error occurs +RETCODE = int(RocmSmiCompatExitCode.SUCCESS) # If we want JSON format output instead PRINT_JSON = False @@ -814,7 +827,7 @@ def showAllConcise(deviceList): if PRINT_JSON: print("NOT_SUPPORTED: Cannot print JSON/CSV output for concise output") - sys.exit(1) + sys.exit(int(RocmSmiCompatExitCode.ERROR)) silent = True @@ -992,17 +1005,17 @@ def main(): except ImportError: print("ERROR: Could not import amdsmi module") print("Install with: pip install amdsmi") - return 1 + return int(RocmSmiCompatExitCode.ERROR) # Check if driver is initialized if not driverInitialized(): logging.error("Unable to detect amdgpu driver. Exiting...") - return 1 + return int(RocmSmiCompatExitCode.ERROR) # Initialize AMD SMI (equivalent to rocm_smi.initializeRsmi()) if not initializeRsmi(): logging.error("Failed to initialize AMD SMI") - return 1 + return int(RocmSmiCompatExitCode.ERROR) try: # Get processor handles (equivalent to rocm_smi.listDevices()) @@ -1010,7 +1023,7 @@ def main(): if not deviceList: logging.error("No AMD GPU devices found") - return 1 + return int(RocmSmiCompatExitCode.ERROR) # Check AMD GPUs (equivalent to rocm_smi.checkAmdGpus()) if not checkAmdGpus(deviceList): diff --git a/projects/amdsmi/amdsmi_cli/subcommands/metric.py b/projects/amdsmi/amdsmi_cli/subcommands/metric.py index 9fcc4111efda..9b1fddbfbcac 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/metric.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/metric.py @@ -6,9 +6,16 @@ import logging import time +import amdsmi_metrics_field_support + from amdsmi import amdsmi_exception, amdsmi_interface from amdsmi.amdsmi_interface import AMDSMI_MAX_RAIL_INDEX +# metric args whose values_dict section keys are not just the arg name. Only +# --overdrive differs; --partition and the hypervisor-only args fill no section +# of their own. +_SECTION_KEYS_BY_METRIC_ARG = {"overdrive": ("overdrive", "mem_overdrive")} + class MetricCommands: def metric_gpu( @@ -204,6 +211,19 @@ def metric_gpu( args.xgmi, ] + # Sections the user named, which unsupported-field filtering may narrow + # but must never empty. Captured here because the block further down + # turns every section on when none was named, and memoized because that + # block mutates the args object shared by the per-GPU recursion and by + # every watch iteration. + if not hasattr(args, "requested_metric_sections"): + args.requested_metric_sections = frozenset( + section + for arg in current_platform_args + if getattr(args, arg, False) + for section in _SECTION_KEYS_BY_METRIC_ARG.get(arg, (arg,)) + ) + # Handle No GPU passed if args.gpu == None: args.gpu = self.device_handles @@ -256,23 +276,24 @@ def metric_gpu( # Get gpu_id for logging gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu) - if args.loglevel == "DEBUG": - try: - # Get GPU Metrics table version - gpu_metric_version_info = amdsmi_interface.amdsmi_get_gpu_metrics_header_info( - args.gpu - ) - gpu_metric_version_str = json.dumps(gpu_metric_version_info, indent=4) - logging.debug( - "GPU Metrics table Version for GPU %s | %s", gpu_id, gpu_metric_version_str - ) - except amdsmi_exception.AmdSmiLibraryException as e: + # Fields this metrics version cannot populate are filtered out of + # human-readable output unless --show-unsupported asks for them. JSON and + # CSV are consumed by scripts, so they keep every field and every key. + filter_human_fields = self.logger.is_human_readable_format() and not getattr( + args, "show_unsupported", False + ) + + gpu_metric_version_info = None + if args.loglevel == "DEBUG" or filter_human_fields: + gpu_metric_version_info = self._gpu_metrics_header(args.gpu, gpu_id) + if args.loglevel == "DEBUG" and gpu_metric_version_info is not None: logging.debug( - "#1 - Unable to load GPU Metrics table version for %s | %s", + "GPU Metrics table Version for GPU %s | %s", gpu_id, - e.get_error_info(), + json.dumps(gpu_metric_version_info, indent=4), ) + if args.loglevel == "DEBUG": try: # Get GPU Metrics table gpu_metric_debug_info = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu) @@ -2352,6 +2373,21 @@ def metric_gpu( del values_dict[section_key] # else: keep the N/A for explicitly named sections + if filter_human_fields: + suppressed = amdsmi_metrics_field_support.build_suppression_set(gpu_metric_version_info) + if not suppressed: + header = gpu_metric_version_info or {} + logging.debug( + "unsupported-field filtering suppresses nothing on gpu %s: " + "gpu_metrics version (%s, %s) is unmapped or its header was unreadable", + gpu_id, + header.get("format_revision", "N/A"), + header.get("content_revision", "N/A"), + ) + values_dict = amdsmi_metrics_field_support.filter_unsupported( + values_dict, suppressed, args.requested_metric_sections + ) + # Store timestamp first if watching_output is enabled if watching_output: self.logger.store_output(args.gpu, "timestamp", int(time.time())) @@ -2368,6 +2404,40 @@ def metric_gpu( if watching_output: # End of single gpu add to watch_output self.logger.store_watch_output(multiple_device_enabled=False) + def _gpu_metrics_header(self, device_handle, gpu_id): + """The gpu_metrics header for ``device_handle``, read at most once. + + The header is fixed for the life of the handle while the read pulls a + whole metrics blob, so `metric --watch` must not pay for it once per GPU + per iteration. An unreadable header caches as None, which suppresses + nothing. + """ + if not hasattr(self, "_gpu_metrics_header_cache"): + self._gpu_metrics_header_cache = {} + # Handles are unhashable ctypes pointers, so key on identity and keep the + # handle alive in the entry so its id cannot be recycled. + entry = self._gpu_metrics_header_cache.get(id(device_handle)) + if entry is not None: + return entry[1] + + header = None + try: + header = amdsmi_interface.amdsmi_get_gpu_metrics_header_info(device_handle) + except amdsmi_exception.AmdSmiException as e: + # AmdSmiParameterException is a sibling of AmdSmiLibraryException, so it + # reaches here on a bad handle but carries no get_error_info(). + logging.debug( + "#1 - Unable to load GPU Metrics table version for %s | %s", + gpu_id, + ( + e.get_error_info() + if isinstance(e, amdsmi_exception.AmdSmiLibraryException) + else e + ), + ) + self._gpu_metrics_header_cache[id(device_handle)] = (device_handle, header) + return header + def metric_cpu( self, args, diff --git a/projects/amdsmi/amdsmi_cli/subcommands/node.py b/projects/amdsmi/amdsmi_cli/subcommands/node.py index 57fed1f5b3d9..bcfe33b5afb7 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/node.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/node.py @@ -66,7 +66,12 @@ def node( self.group_check_printed = True # Initialize variables for both power management and base board temps - npm_dict = {"limit": "N/A", "status": "N/A", "threshold": "N/A"} + npm_dict = { + "limit": "N/A", + "status": "N/A", + "threshold": "N/A", + "current_node_power": "N/A", + } power_unit = "W" limit = "N/A" base_board_temp_dict = {} @@ -89,6 +94,7 @@ def node( limit = npm_info.get("limit", "N/A") status = npm_info.get("status", npm_info.get("current", "N/A")) ubb_power_threshold = npm_info.get("ubb_power_threshold", "N/A") + current_node_power = npm_info.get("current_node_power", "N/A") if limit != "N/A": npm_dict["limit"] = limit @@ -101,6 +107,9 @@ def node( # Add UBB power threshold if available if ubb_power_threshold != "N/A": npm_dict["threshold"] = ubb_power_threshold + # Add current node power if available + if current_node_power != "N/A": + npm_dict["current_node_power"] = current_node_power # Get base board temperatures using node_handle if args.base_board_temps: @@ -156,6 +165,8 @@ def node( node_output.append(f" STATUS: {npm_dict.get('status', 'N/A')}") threshold = npm_dict.get("threshold", "N/A") node_output.append(f" THRESHOLD: {threshold} {power_unit}") + current_node_power = npm_dict.get("current_node_power", "N/A") + node_output.append(f" CURRENT_NODE_POWER: {current_node_power} {power_unit}") if args.base_board_temps and base_board_temp_dict: node_output.append(" BASEBOARD:") node_output.append(" TEMPERATURE:") @@ -181,11 +192,14 @@ def node( print("\n".join(node_output)) else: if self.logger.is_csv_format(): - csv_dict = {} + # Only node 0 is currently supported; mirror the gpu/cpu/nic + # CSV surfaces, which always lead with a device identifier. + csv_dict = {"node": 0} if args.power_management: csv_dict["limit"] = npm_dict.get("limit", "N/A") csv_dict["status"] = npm_dict.get("status", "N/A") csv_dict["threshold"] = npm_dict.get("threshold", "N/A") + csv_dict["current_node_power"] = npm_dict.get("current_node_power", "N/A") if args.base_board_temps and base_board_temp_dict: csv_dict.update(base_board_temp_dict) if args.gtt and gtt_dict: @@ -208,6 +222,11 @@ def node( npm_dict["threshold"] = self.helpers.unit_format( self.logger, threshold, power_unit ) + current_node_power_value = npm_dict.get("current_node_power", "N/A") + if current_node_power_value != "N/A": + npm_dict["current_node_power"] = self.helpers.unit_format( + self.logger, current_node_power_value, power_unit + ) node_output["power_management"] = npm_dict if args.base_board_temps and base_board_temp_dict: node_output["base_board"] = {"temperature": base_board_temp_dict} diff --git a/projects/amdsmi/amdsmi_cli/subcommands/ras.py b/projects/amdsmi/amdsmi_cli/subcommands/ras.py index 15590ae6eaeb..a573b67d27ce 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/ras.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/ras.py @@ -10,8 +10,38 @@ from amdsmi_cli_exceptions import AmdSmiInvalidCommandException, AmdSmiInvalidFilePathException +from amdsmi import amdsmi_exception + class RasCommands: + def _build_afid_record(self, cper_file, afids=None, code=None): + """Shared with ``ras --cper``; see AMDSMIHelpers.build_afid_record.""" + return self.helpers.build_afid_record(cper_file, self.logger.format, afids=afids, code=code) + + def _emit_afid_records(self, records): + """Render decoded --afid records. + + Human output is a compact ``file_name | list of afids`` table (the + logger's tabular renderer smushes these wide rows). json/csv keep the + structured schema (cper_file, afids, status, code, message) so machine + consumers still get the real AMDSMI_STATUS per file. + """ + if self.logger.is_human_readable_format(): + name_w = max([len("file_name")] + [len(Path(r["cper_file"]).name) for r in records]) + print(f"{'file_name':<{name_w}} list of afids") + for r in records: + name = Path(r["cper_file"]).name + print(f"{name:<{name_w}} {self.helpers.afid_cell(r)}") + else: + rows = records + if self.logger.is_csv_format(): + # csv.DictWriter would render the list as "[1, 2, 3]". JSON keeps the list. + rows = [{**r, "afids": self.helpers.afid_cell(r)} for r in rows] + self.logger.multiple_device_output = rows + # emit_empty keeps a folder whose files were all skipped rendering as + # `[]` rather than nothing, so stdout stays parseable. + self.logger.print_output(multiple_device_enabled=True, emit_empty=True) + def _validate_ras_args(self, args): """Validate ``ras`` arguments up front, before any driver call or file I/O. @@ -104,31 +134,17 @@ def _decode_afid_folder(self, args): finally: os.close(fd) - decode_failed = False - afids = [] try: afids = self.helpers.cper_dump_afids(raw) - except Exception as e: + results.append(self._build_afid_record(cper_path, afids=afids)) + except amdsmi_exception.AmdSmiLibraryException as e: logging.debug("Failed to decode AFIDs from %s: %s", cper_path, e) - decode_failed = True - results.append( - {"cper_file": str(cper_path), "afids": afids, "decode_failed": decode_failed} - ) + # Records the real library status so the decode failure reaches + # the exit code. NO_PERM aborts instead, since it is not per-file. + self.helpers.record_or_raise(e) + results.append(self._build_afid_record(cper_path, code=e.get_error_code())) - if self.logger.is_json_format(): - self.logger.multiple_device_output = results - self.logger.print_output(multiple_device_enabled=True, emit_empty=True) - else: - print(f"{'file_name':<32} list of afids") - for entry in results: - if entry["decode_failed"]: - afids_str = "decode failed" - elif entry["afids"]: - afids_str = " ".join(map(str, entry["afids"])) - else: - afids_str = "-" - fname = Path(entry["cper_file"]).name - print(f"{fname:<32} {afids_str}") + self._emit_afid_records(results) def ras( self, @@ -179,13 +195,22 @@ def ras( if args.afid: if args.cper_file: - afids = self.helpers.cper_dump_afids(args.cper_file) - if self.logger.is_json_format(): - afid_output = {"cper_file": str(args.cper_file), "afids": afids} - self.logger.output = afid_output - self.logger.print_output() - else: - print(" ".join(map(str, afids)) if afids else "-") + # Read failure -> clean file-path error. Decode failure -> a + # unified per-file record with its AMDSMI_STATUS (same schema as + # --folder), and a non-zero exit via the error collector. + try: + afids = self.helpers.cper_dump_afids(args.cper_file) + record = self._build_afid_record(args.cper_file, afids=afids) + except OSError as e: + raise AmdSmiInvalidFilePathException( + args.cper_file, + self.logger.format, + f"Unable to read CPER file '{args.cper_file}': {e}", + ) from e + except amdsmi_exception.AmdSmiLibraryException as e: + self.helpers.record_or_raise(e) + record = self._build_afid_record(args.cper_file, code=e.get_error_code()) + self._emit_afid_records([record]) return # --afid --folder: read-only decode of a pre-existing folder. diff --git a/projects/amdsmi/amdsmi_cli/subcommands/reset.py b/projects/amdsmi/amdsmi_cli/subcommands/reset.py index 0d415cbe0d23..bad21cc0c0f9 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/reset.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/reset.py @@ -6,6 +6,8 @@ import sys from amdsmi_cli_exceptions import AmdSmiRequiredCommandException +from amdsmi_cli_exceptions import AmdSmiInvalidParameterException +from amdsmi_cli_exceptions import AmdSmiExitCode from amdsmi_helpers import AMDSMIHelpers from amdsmi import amdsmi_exception, amdsmi_interface @@ -44,8 +46,9 @@ def reset( clean_local_data (bool, optional): Value override for args.run_cleaner_shader. Defaults to None. Raises: - ValueError: Value error if no gpu value is provided - IndexError: Index error if gpu list is empty + AmdSmiInvalidParameterException: If an invalid parameter combination is provided (e.g. --gtt with --gpu) + AmdSmiRequiredCommandException: If no reset subcommand argument is provided + PermissionError: If a reset operation requires elevation (AMDSMI_STATUS_NO_PERM) Return: Nothing @@ -76,12 +79,13 @@ def reset( # Special GTT handling (system-wide, not per-GPU) — handle before device dispatch if hasattr(args, "gtt") and args.gtt: if hasattr(args, "gpu") and args.gpu is not None: - print( + msg = ( "amd-smi reset: error: argument --gtt: not allowed with argument --gpu/-g " - "(--gtt is a system-wide setting, not per-GPU)", - file=sys.stderr, + "(--gtt is a system-wide setting, not per-GPU)" + ) + raise AmdSmiInvalidParameterException( + "reset", "--gtt", self.helpers.get_output_format(), msg ) - sys.exit(2) try: amdsmi_interface.amdsmi_reset_ttm_pages_limit() self.logger.output["reset_gtt"] = ( @@ -98,6 +102,7 @@ def reset( self.logger.output["reset_gtt"] = ( f"[{e.get_error_info(detailed=False)}] Unable to reset GTT" ) + self.helpers.error_collector.record_library_error(e.get_error_code()) self.logger.print_output() return @@ -197,11 +202,13 @@ def reset( raise PermissionError("Command requires elevation") from e result = f"[{e.get_error_info(detailed=False)}] Unable to reset GPU" self.logger.store_output(args.gpu, "gpu_reset", result) + self.helpers.error_collector.record_library_error(e.get_error_code()) self.logger.print_output() self.logger.clear_multiple_devices_output() return else: result = "Unable to reset non-amd GPU" + self.helpers.error_collector.record(AmdSmiExitCode.DEVICE_NOT_FOUND) self.logger.store_output(args.gpu, "gpu_reset", result) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -220,6 +227,7 @@ def reset( reset_clocks_results["overdrive"] = ( f"[{e.get_error_info(detailed=False)}] Unable to reset overdrive to 0" ) + self.helpers.error_collector.record_library_error(e.get_error_code()) # continue to reset clocks and performance level try: level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO @@ -231,6 +239,7 @@ def reset( reset_clocks_results["clocks"] = ( f"[{e.get_error_info(detailed=False)}] Unable to reset performance level to auto" ) + self.helpers.error_collector.record_library_error(e.get_error_code()) logging.debug( "Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info() ) @@ -248,6 +257,7 @@ def reset( reset_clocks_results["performance"] = ( f"[{e.get_error_info(detailed=False)}] Unable to reset performance level to auto" ) + self.helpers.error_collector.record_library_error(e.get_error_code()) logging.debug( "Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info() ) @@ -265,7 +275,9 @@ def reset( raise PermissionError("Command requires elevation") from e result = f"[{e.get_error_info(detailed=False)}] Unable to reset fan speed to driver control" logging.debug("Failed to reset fans on gpu %s | %s", gpu_id, e.get_error_info()) - self.logger.store_output(args.gpu, "reset_fans", result) + self.helpers.store_device_error( + self.logger, args.gpu, "reset_fans", result, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -289,6 +301,7 @@ def reset( reset_profile_results["power_profile"] = ( f"[{e.get_error_info(detailed=False)}] Unable to reset Power Profile to default (bootup default)" ) + self.helpers.error_collector.record_library_error(e.get_error_code()) logging.debug( "Failed to reset power profile on gpu %s | %s", gpu_id, e.get_error_info() ) @@ -312,7 +325,9 @@ def reset( result = ( f"[{e.get_error_info(detailed=False)}] Unable to reset XGMI Error count" ) - self.logger.store_output(args.gpu, "reset_xgmi_err", result) + self.helpers.store_device_error( + self.logger, args.gpu, "reset_xgmi_err", result, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -332,7 +347,9 @@ def reset( "Failed to set perf level on gpu %s | %s", gpu_id, e.get_error_info() ) result = f"[{e.get_error_info(detailed=False)}] Unable to reset Performance Level to default (auto)" - self.logger.store_output(args.gpu, "reset_perf_determinism", result) + self.helpers.store_device_error( + self.logger, args.gpu, "reset_perf_determinism", result, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -386,6 +403,7 @@ def reset( final_output[f"ppt{current_sensor_num}"] = ( f"[{e.get_error_info(detailed=False)}] Unable to reset cap to default power cap" ) + self.helpers.error_collector.record_library_error(e.get_error_code()) self.logger.store_output(args.gpu, "powercap", final_output) if multiple_devices: self.logger.store_multiple_device_output() @@ -405,7 +423,9 @@ def reset( if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError("Command requires elevation") from e result = f"[{e.get_error_info(detailed=False)}] Unable to clean local data" - self.logger.store_output(args.gpu, "clean_local_data", result) + self.helpers.store_device_error( + self.logger, args.gpu, "clean_local_data", result, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return diff --git a/projects/amdsmi/amdsmi_cli/subcommands/set_value.py b/projects/amdsmi/amdsmi_cli/subcommands/set_value.py index d0bc1fa7dc53..a9258f8c58f4 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/set_value.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/set_value.py @@ -8,6 +8,8 @@ import sys from amdsmi_cli_exceptions import AmdSmiRequiredCommandException +from amdsmi_cli_exceptions import AmdSmiInvalidParameterException +from amdsmi_cli_exceptions import AmdSmiExitCode from amdsmi import amdsmi_exception, amdsmi_interface from amdsmi.amdsmi_interface import AMDSMI_MAX_PPT_LIMIT, AMDSMI_MAX_UTIL @@ -34,8 +36,8 @@ def set_core( core_msr_floor_limit (list, optional): Value override for args.core_msr_floor_limit. Defaults to None. Raises: - ValueError: Value error if no core value is provided - IndexError: Index error if core list is empty + AmdSmiRequiredCommandException: If no core target or subcommand argument is provided + PermissionError: If a set operation requires elevation (AMDSMI_STATUS_NO_PERM) Return: Nothing @@ -50,7 +52,8 @@ def set_core( args.core_msr_floor_limit = core_msr_floor_limit if args.core == None: - raise ValueError("No Core provided, specific Core targets(S) are needed") + command = " ".join(sys.argv[1:]) + raise AmdSmiRequiredCommandException(command, self.logger.format) # Handle multiple cores handled_multiple_cores, device_handle = self.helpers.handle_cores( @@ -98,8 +101,8 @@ def set_core( else: static_dict["set_core_boost_limit"]["Response"] = f"{boost_limit} MHz" except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["set_core_boost_limit"]["Response"] = ( - f"Error occurred for Core {core_id} - {e.get_error_info()}" + static_dict["set_core_boost_limit"]["Response"] = self.helpers.record_or_raise( + e, f"Core {core_id}" ) logging.debug( "Failed to set core boost limit for core %s | %s", core_id, e.get_error_info() @@ -139,8 +142,8 @@ def set_core( else: static_dict["floor_limit"]["Response"] = f"Set, VALUE: {flimit} MHz, successful" except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["floor_limit"]["Response"] = ( - f"Error occurred for Core {core_id} - {e.get_error_info()}" + static_dict["floor_limit"]["Response"] = self.helpers.record_or_raise( + e, f"Core {core_id}" ) logging.debug( "Failed to set core floor limit for core %s | %s", core_id, e.get_error_info() @@ -185,8 +188,8 @@ def set_core( f"Set, VALUE: {effflimit} MHz, successful" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["msr_floor_limit"]["Response"] = ( - f"Error occurred for Core {core_id} - {e.get_error_info()}" + static_dict["msr_floor_limit"]["Response"] = self.helpers.record_or_raise( + e, f"Core {core_id}" ) logging.debug( "Failed to set core MSR floor limit for core %s | %s", @@ -253,8 +256,8 @@ def set_cpu( cpu_sdps_limit (int, optional): Value override for args.cpu_sdps_limit. Defaults to None. Raises: - ValueError: Value error if no cpu value is provided - IndexError: Index error if cpu list is empty + AmdSmiRequiredCommandException: If no cpu target or subcommand argument is provided + PermissionError: If a set operation requires elevation (AMDSMI_STATUS_NO_PERM) Return: Nothing @@ -301,7 +304,8 @@ def set_cpu( args.cpu_sdps_limit = cpu_sdps_limit if args.cpu == None: - raise ValueError("No CPU provided, specific CPU targets(S) are needed") + command = " ".join(sys.argv[1:]) + raise AmdSmiRequiredCommandException(command, self.logger.format) # Handle multiple CPU's handled_multiple_cpus, device_handle = self.helpers.handle_cpus( @@ -362,8 +366,8 @@ def set_cpu( f"{args.cpu_pwr_limit[0][0] / 1000:.3f} W" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["set_pwr_limit"]["Response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["set_pwr_limit"]["Response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set power limit for cpu %s | %s", cpu_id, e.get_error_info() @@ -379,8 +383,8 @@ def set_cpu( f"{args.cpu_xgmi_link_width[0][0]} - {args.cpu_xgmi_link_width[0][1]}" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["set_xgmi_link_width"]["Response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["set_xgmi_link_width"]["Response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set xgmi link width for cpu %s | %s", cpu_id, e.get_error_info() @@ -399,8 +403,8 @@ def set_cpu( f"NBIO[{args.cpu_lclk_dpm_level[0][0]}]" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["set_lclk_dpm_level"]["Response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["set_lclk_dpm_level"]["Response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set lclk dpm level for cpu %s | %s", cpu_id, e.get_error_info() @@ -441,8 +445,8 @@ def set_cpu( "Set power efficiency mode operation successful" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["pwr_eff_mode"]["response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["pwr_eff_mode"]["response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set power efficiency mode for cpu %s | %s", @@ -460,8 +464,8 @@ def set_cpu( f"{args.cpu_gmi3_link_width[0][0]} - {args.cpu_gmi3_link_width[0][1]}" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["set_gmi3_link_width"]["response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["set_gmi3_link_width"]["response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set gmi3 link width for cpu %s | %s", cpu_id, e.get_error_info() @@ -475,8 +479,8 @@ def set_cpu( ) static_dict["set_pcie_link_rate"]["prev_mode"] = resp except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["set_pcie_link_rate"]["prev_mode"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["set_pcie_link_rate"]["prev_mode"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set pcie link rate for cpu %s | %s", cpu_id, e.get_error_info() @@ -490,8 +494,8 @@ def set_cpu( ) static_dict["set_df_pstate_range"]["response"] = "Set Operation successful" except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["set_df_pstate_range"]["response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["set_df_pstate_range"]["response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set df pstate range for cpu %s | %s", cpu_id, e.get_error_info() @@ -505,9 +509,7 @@ def set_cpu( "Enabled DF - Pstate performance boost algorithm" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["apbenable"]["state"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" - ) + static_dict["apbenable"]["state"] = self.helpers.record_or_raise(e, f"CPU {cpu_id}") logging.debug("Failed to enable APB for cpu %s | %s", cpu_id, e.get_error_info()) if args.cpu_disable_apb: @@ -518,8 +520,8 @@ def set_cpu( "Disabled DF - Pstate performance boost algorithm" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["apbdisable"]["state"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["apbdisable"]["state"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug("Failed to enable APB for cpu %s | %s", cpu_id, e.get_error_info()) @@ -531,9 +533,8 @@ def set_cpu( ) static_dict["set_soc_boost_limit"]["Response"] = "Set Operation successful" except amdsmi_exception.AmdSmiLibraryException as e: - # static_dict["set_soc_boost_limit"]["Response"] = "N/A" - static_dict["set_soc_boost_limit"]["Response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["set_soc_boost_limit"]["Response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set socket boost limit for cpu %s | %s", cpu_id, e.get_error_info() @@ -549,8 +550,8 @@ def set_cpu( f"Set, MIN_PSTATE: {args.cpu_xgmi_pstate_range[0][0]}, MAX_PSTATE: {args.cpu_xgmi_pstate_range[0][1]}, successful" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["xgmi_pstate_range"]["response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["xgmi_pstate_range"]["response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set xgmi pstate range for cpu %s | %s", cpu_id, e.get_error_info() @@ -564,8 +565,8 @@ def set_cpu( ) static_dict["railisofreq_policy"]["response"] = f"Set, VALUE: {resp}, successful" except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["railisofreq_policy"]["response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["railisofreq_policy"]["response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set ISO frequency policy for cpu %s | %s", cpu_id, e.get_error_info() @@ -579,8 +580,8 @@ def set_cpu( ) static_dict["dfcstate_ctrl"]["response"] = f"Set, VALUE: {resp}, successful" except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["dfcstate_ctrl"]["response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["dfcstate_ctrl"]["response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set dfcstate control for cpu %s | %s", cpu_id, e.get_error_info() @@ -594,8 +595,8 @@ def set_cpu( f"Set, VALUE: {args.cpu_pc6_enable[0][0]}, successful" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["pc6_enable"]["response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["pc6_enable"]["response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set PC6 enable for cpu %s | %s", cpu_id, e.get_error_info() @@ -609,8 +610,8 @@ def set_cpu( f"Set, VALUE: {args.cpu_cc6_enable[0][0]}, successful" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["cc6_enable"]["response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["cc6_enable"]["response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set CC6 enable for cpu %s | %s", cpu_id, e.get_error_info() @@ -649,8 +650,8 @@ def set_cpu( else: static_dict["floor_limit"]["Response"] = f"Set, VALUE: {flimit} MHz, successful" except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["floor_limit"]["Response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["floor_limit"]["Response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set socket floor limit for cpu %s | %s", cpu_id, e.get_error_info() @@ -693,8 +694,8 @@ def set_cpu( f"Set, VALUE: {effflimit} MHz, successful" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["msr_floor_limit"]["Response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["msr_floor_limit"]["Response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set CPU MSR floor limit for cpu %s | %s", cpu_id, e.get_error_info() @@ -720,8 +721,8 @@ def set_cpu( "Set DIMM sideband register write operation successful" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["dimm_sb_reg"]["Response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["dimm_sb_reg"]["Response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to write DIMM sideband register for cpu %s | %s", @@ -738,8 +739,8 @@ def set_cpu( f"Set, VALUE: {sdps_limit_watts:.3f} W, successful" ) except amdsmi_exception.AmdSmiLibraryException as e: - static_dict["sdps_limit"]["Response"] = ( - f"Error occurred for CPU {cpu_id} - {e.get_error_info()}" + static_dict["sdps_limit"]["Response"] = self.helpers.record_or_raise( + e, f"CPU {cpu_id}" ) logging.debug( "Failed to set socket SDPS limit for cpu %s | %s", cpu_id, e.get_error_info() @@ -870,8 +871,9 @@ def set_gpu( ptl_format(string, optional): Value override for args.ptl_format. Defaults to None. compute_partition_mem_alloc_mode (str, optional): Value override for args.compute_partition_mem_alloc_mode. Defaults to None. Raises: - ValueError: Value error if no gpu value is provided - IndexError: Index error if gpu list is empty + AmdSmiRequiredCommandException: If no gpu subcommand argument is provided + AmdSmiInvalidParameterException: If mutually exclusive/invalid parameters are supplied + PermissionError: If a set operation requires elevation (AMDSMI_STATUS_NO_PERM) Return: Nothing @@ -1007,7 +1009,16 @@ def format_fan_error(message, include_driver_note=False): f"Unable to read gpu_od OD_RANGE from {gpu_od_path}. Cannot set fan speed.", include_driver_note=True, ) - self.logger.store_output(args.gpu, "fan", result) + # No library status to fold: parse_gpu_od_fan_range() reads sysfs + # directly because no API reports the OD_RANGE minimum. Switch to + # record_library_error() when amdsmi_get_gpu_fan_speed_range() lands. + self.helpers.store_device_error( + self.logger, + args.gpu, + "fan", + result, + code=AmdSmiExitCode.DEVICE_INTERFACE_UNAVAILABLE, + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1031,7 +1042,13 @@ def format_fan_error(message, include_driver_note=False): f"Invalid fan speed value {input_value} for gpu_od interface. Valid range: {od_min}-{od_max} or use percentage (0-100%)", include_driver_note=True, ) - self.logger.store_output(args.gpu, "fan", result) + self.helpers.store_device_error( + self.logger, + args.gpu, + "fan", + result, + code=AmdSmiExitCode.INVALID_PARAMETER_VALUE, + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1050,7 +1067,13 @@ def format_fan_error(message, include_driver_note=False): ) # round down (aka floor) to nearest whole number else: result = f"Invalid fan speed value {input_value}. Valid range: 0-255 or use percentage (0-100%)" - self.logger.store_output(args.gpu, "fan", result) + self.helpers.store_device_error( + self.logger, + args.gpu, + "fan", + result, + code=AmdSmiExitCode.INVALID_PARAMETER_VALUE, + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1065,6 +1088,7 @@ def format_fan_error(message, include_driver_note=False): include_driver_note=has_gpu_od, ) self.logger.store_output(args.gpu, "fan", result) + self.helpers.error_collector.record_library_error(e.get_error_code()) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1089,6 +1113,7 @@ def format_fan_error(message, include_driver_note=False): "perflevel", f"[{e.get_error_info(detailed=False)}] Unable to set performance level to {args.perf_level}", ) + self.helpers.error_collector.record_library_error(e.get_error_code()) perf_options = ( str(self.helpers.get_perf_levels()[0][0:-1]) .replace("[", "") @@ -1131,10 +1156,12 @@ def format_fan_error(message, include_driver_note=False): f"Failed to fetch available profiles: {e.get_error_info()}" ) - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "profile", f"Invalid profile: {args.profile}\n\nAvailable profiles: {available_str}", + code=int(AmdSmiExitCode.INVALID_PARAMETER_VALUE), ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1166,7 +1193,9 @@ def format_fan_error(message, include_driver_note=False): ) error_msg = f"[{e.get_error_info(detailed=False)}] Unable to set power profile to {args.profile}" - self.logger.store_output(args.gpu, "profile", error_msg) + self.helpers.store_device_error( + self.logger, args.gpu, "profile", error_msg, exception=e + ) print(f"\nAvailable Power Profiles:\n\t{available_str}\n") self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1183,10 +1212,12 @@ def format_fan_error(message, include_driver_note=False): except amdsmi_exception.AmdSmiLibraryException as e: if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError("Command requires elevation") from e - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "perfdeterminism", f"[{e.get_error_info(detailed=False)}] Unable to enable performance determinism and set GFX clock frequency to {args.perf_determinism} MHz", + exception=e, ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1201,10 +1232,8 @@ def format_fan_error(message, include_driver_note=False): self.logger.clear_multiple_devices_output() return if args.compute_partition: - current_set_count = self.helpers.get_set_count() - future_set_count = 0 - attempted_to_set = "N/A" user_requested_partition_args = "N/A" + accelerator_set_choices = [] try: (accelerator_set_choices, accelerator_profiles) = ( self.helpers.get_accelerator_choices_types_indices() @@ -1216,7 +1245,6 @@ def format_fan_error(message, include_driver_note=False): ) if args.compute_partition in accelerator_profiles["profile_types"]: index = accelerator_profiles["profile_types"].index(args.compute_partition) - attempted_to_set = f"Attempted to set accelerator partition to {args.compute_partition} (profile #{accelerator_profiles['profile_indices'][int(index)]}) on {gpu_string}" user_requested_partition_args = f"{args.compute_partition} (profile #{accelerator_profiles['profile_indices'][int(index)]})" amdsmi_interface.amdsmi_set_gpu_accelerator_partition_profile( args.gpu, int(accelerator_profiles["profile_indices"][index]) @@ -1226,23 +1254,43 @@ def format_fan_error(message, include_driver_note=False): index = accelerator_profiles["profile_indices"].index( args.compute_partition ) - attempted_to_set = f"Attempted to set accelerator partition to {accelerator_profiles['profile_types'][int(index)]} (profile #{args.compute_partition}) on {gpu_string}" user_requested_partition_args = f"{accelerator_profiles['profile_types'][int(index)]} (profile #{args.compute_partition})" amdsmi_interface.amdsmi_set_gpu_accelerator_partition_profile( args.gpu, accelerator_partition ) - else: - raise ValueError( - f"Invalid accelerator configuration {args.compute_partition} on {gpu_string}" + elif ( + args.compute_partition + in amdsmi_interface.AmdSmiComputePartitionType.__members__ + ): + # Profiles couldn't be enumerated (device without accelerator + # partitions); attempt the set anyway so the driver reports + # this device's real status. + compute_partition = amdsmi_interface.AmdSmiComputePartitionType[ + args.compute_partition + ] + user_requested_partition_args = args.compute_partition + amdsmi_interface.amdsmi_set_gpu_compute_partition( + args.gpu, compute_partition ) - self.helpers.increment_set_count() - future_set_count = self.helpers.get_set_count() - if current_set_count == future_set_count - 1: - self.logger.store_output( + else: + # Not a valid accelerator type or profile index -- a CLI + # parameter error. Record and keep going. + self.helpers.store_device_error( + self.logger, args.gpu, "accelerator_partition", - f"Successfully set accelerator partition to {user_requested_partition_args}", + f"Invalid accelerator configuration {args.compute_partition}", + code=int(AmdSmiExitCode.INVALID_PARAMETER_VALUE), ) + self.logger.print_output() + self.logger.clear_multiple_devices_output() + return + + self.logger.store_output( + args.gpu, + "accelerator_partition", + f"Successfully set accelerator partition to {user_requested_partition_args}", + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1250,31 +1298,24 @@ def format_fan_error(message, include_driver_note=False): except amdsmi_exception.AmdSmiLibraryException as e: if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError("Command requires elevation") from e - elif ( - e.get_error_code() - == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED - ): - self.helpers.increment_set_count() - future_set_count = self.helpers.get_set_count() - if current_set_count == future_set_count - 1: - out = f"[AMDSMI_STATUS_NOT_SUPPORTED] Unable to set compute partition to {user_requested_partition_args}" - self.logger.store_output(args.gpu, "accelerator_partition", out) - elif ( + out = ( + f"[{e.get_error_info(detailed=False)}] Unable to set accelerator partition " + f"to {user_requested_partition_args}" + ) + # SETTING_UNAVAILABLE means the value is valid but not allowed + # in the current state (e.g. wrong memory partition mode), so + # provide hint on how to resolve the issue. + if ( e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_SETTING_UNAVAILABLE ): - print( - f"\n{attempted_to_set}\n" - f"\n[AMDSMI_STATUS_SETTING_UNAVAILABLE] Please check amd-smi partition --memory --accelerator for available profiles.\n" - "Users may need to switch memory partition to another mode in order to enable the desired accelerator partition.\n" + out += ( + ".\nPlease check `sudo amd-smi partition -am` for available profiles.\n" ) - raise ValueError( - f"[AMDSMI_STATUS_SETTING_UNAVAILABLE] Unable to set accelerator partition to {args.compute_partition} on {gpu_string}" - ) from e - else: - raise ValueError( - f"Unable to set accelerator partition to {args.compute_partition} on {gpu_string}" - ) from e + out += "Some modes may require a memory partition change to enable." + self.helpers.store_device_error( + self.logger, args.gpu, "accelerator_partition", out, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1324,12 +1365,16 @@ def format_fan_error(message, include_driver_note=False): raise PermissionError("Command requires elevation") from e elif e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_INVAL: print(f"Valid Memory partition Modes: {memory_dict['caps']}\n") - self.logger.store_output(args.gpu, "memory_partition", out) + self.helpers.store_device_error( + self.logger, args.gpu, "memory_partition", out, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return else: - self.logger.store_output(args.gpu, "memory_partition", out) + self.helpers.store_device_error( + self.logger, args.gpu, "memory_partition", out, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1357,10 +1402,12 @@ def format_fan_error(message, include_driver_note=False): ", " ) # Remove trailing comma and space print(f"Valid SOC P-State Policies: [{policy_string}]\n") - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "socpstate", f"[{e.get_error_info(detailed=False)}] Unable to set soc pstate dpm policy to {args.soc_pstate}", + exception=e, ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1393,10 +1440,12 @@ def format_fan_error(message, include_driver_note=False): ", " ) # Remove trailing comma and space print(f"Valid XGMI PLPD Policies: [{policy_string}]\n") - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "xgmiplpd", f"[{e.get_error_info(detailed=False)}] Unable to set XGMI per-link power down policy to {args.xgmi_plpd}", + exception=e, ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1425,10 +1474,6 @@ def format_fan_error(message, include_driver_note=False): "get_clock_freq": f"Unable to retrieve {clk_type} frequency levels", "set_clock": f"Unable to set {clk_type} perf level(s) to {perf_levels_str}", } - if clk_type not in smi_clk_type_mapping: - raise ValueError( - f"Invalid clock type {clk_type}. Valid options are: {', '.join(smi_clk_type_mapping.keys())}" - ) # Set perf level to manual if not already set try: @@ -1442,7 +1487,9 @@ def format_fan_error(message, include_driver_note=False): results_clk_lvl["perf_level"] = ( f"[{e.get_error_info(detailed=False)}] Unable to set performance level to MANUAL" ) - self.logger.store_output(args.gpu, "clk_level", results_clk_lvl) + self.helpers.store_device_error( + self.logger, args.gpu, "clk_level", results_clk_lvl, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1461,7 +1508,9 @@ def format_fan_error(message, include_driver_note=False): results_clk_lvl["get_clock_freq"] = ( f"[{e.get_error_info(detailed=False)}] Unable to retrieve {clk_type} frequency levels" ) - self.logger.store_output(args.gpu, "clk_level", results_clk_lvl) + self.helpers.store_device_error( + self.logger, args.gpu, "clk_level", results_clk_lvl, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1479,7 +1528,9 @@ def format_fan_error(message, include_driver_note=False): results_clk_lvl["get_clock_freq"] = ( f"[{e.get_error_info(detailed=False)}] Unable to retrieve {clk_type} frequency levels" ) - self.logger.store_output(args.gpu, "clk_level", results_clk_lvl) + self.helpers.store_device_error( + self.logger, args.gpu, "clk_level", results_clk_lvl, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1503,7 +1554,13 @@ def format_fan_error(message, include_driver_note=False): results_clk_lvl["set_clock"] = ( f"Invalid level(s) {invalid_levels_str} are not within the range of supported levels for {clk_type}" ) - self.logger.store_output(args.gpu, "clk_level", results_clk_lvl) + self.helpers.store_device_error( + self.logger, + args.gpu, + "clk_level", + results_clk_lvl, + code=int(AmdSmiExitCode.INVALID_PARAMETER_VALUE), + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1527,7 +1584,9 @@ def format_fan_error(message, include_driver_note=False): results_clk_lvl["set_clock"] = ( f"[{e.get_error_info(detailed=False)}] Unable to set {clk_type} perf level(s) to {perf_levels_str}" ) - self.logger.store_output(args.gpu, "clk_level", results_clk_lvl) + self.helpers.store_device_error( + self.logger, args.gpu, "clk_level", results_clk_lvl, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1552,7 +1611,9 @@ def format_fan_error(message, include_driver_note=False): results_clk_lvl["set_clock"] = ( f"[{e.get_error_info(detailed=False)}] Unable to set {clk_type} perf level(s) to {perf_levels_str}" ) - self.logger.store_output(args.gpu, "clk_level", results_clk_lvl) + self.helpers.store_device_error( + self.logger, args.gpu, "clk_level", results_clk_lvl, exception=e + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1569,10 +1630,12 @@ def format_fan_error(message, include_driver_note=False): except amdsmi_exception.AmdSmiLibraryException as e: if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError("Command requires elevation") from e - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "ptlstatus", f"[{e.get_error_info(detailed=False)}] Unable to set ptl status to {args.ptl_status}", + exception=e, ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1602,10 +1665,12 @@ def format_fan_error(message, include_driver_note=False): except amdsmi_exception.AmdSmiLibraryException as e: if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError("Command requires elevation") from e - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "ptlformat", f"[{e.get_error_info(detailed=False)}] Unable to set PTL format to {requested_str}", + exception=e, ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1656,8 +1721,12 @@ def format_fan_error(message, include_driver_note=False): amdsmi_clk_type = amdsmi_interface.AmdSmiClkType.DF else: print("Valid clock types are: sclk, mclk, fclk\n") - self.logger.store_output( - args.gpu, "clk_limit", f"Invalid clock type {args.clk_limit.clk_type}" + self.helpers.store_device_error( + self.logger, + args.gpu, + "clk_limit", + f"Invalid clock type {args.clk_limit.clk_type}", + code=int(AmdSmiExitCode.INVALID_PARAMETER_VALUE), ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1670,7 +1739,13 @@ def format_fan_error(message, include_driver_note=False): # stays settable. bound_error = self._check_opposing_bound(clk_tuple, lim_type, clk_type, val) if bound_error is not None: - self.logger.store_output(args.gpu, "clk_limit", bound_error) + self.helpers.store_device_error( + self.logger, + args.gpu, + "clk_limit", + bound_error, + code=int(AmdSmiExitCode.INVALID_PARAMETER_VALUE), + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1715,8 +1790,12 @@ def format_fan_error(message, include_driver_note=False): and clk_type == "mclk" ): logging.debug("Setting mclk min is not supported") - self.logger.store_output( - args.gpu, "clk_limit", "Setting mclk min is not supported" + self.helpers.store_device_error( + self.logger, + args.gpu, + "clk_limit", + f"[{e.get_error_info(detailed=False)}] Unable to change {args.clk_limit.lim_type} of {args.clk_limit.clk_type} to {args.clk_limit.val}MHz", + exception=e, ) else: logging.debug( @@ -1724,10 +1803,12 @@ def format_fan_error(message, include_driver_note=False): gpu_id, e.get_error_info(), ) - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "clk_limit", f"[{e.get_error_info(detailed=False)}] Unable to change {args.clk_limit.lim_type} of {args.clk_limit.clk_type} to {args.clk_limit.val}MHz", + exception=e, ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1747,14 +1828,20 @@ def format_fan_error(message, include_driver_note=False): and clk_type == "mclk" ): logging.debug("Setting mclk min is not supported") - self.logger.store_output( - args.gpu, "clk_limit", "Setting mclk min is not supported" + self.helpers.store_device_error( + self.logger, + args.gpu, + "clk_limit", + f"[{e.get_error_info(detailed=False)}] Unable to change {args.clk_limit.lim_type} of {args.clk_limit.clk_type} to {args.clk_limit.val}MHz", + exception=e, ) else: - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "clk_limit", f"[{e.get_error_info(detailed=False)}] Unable to set {args.clk_limit.lim_type} of {args.clk_limit.clk_type} to {args.clk_limit.val}MHz", + exception=e, ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1797,10 +1884,12 @@ def format_fan_error(message, include_driver_note=False): except amdsmi_exception.AmdSmiLibraryException as e: if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError("Command requires elevation") from e - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "process_isolation", f"[{e.get_error_info(detailed=False)}] Unable to set process isolation to {status_string}", + exception=e, ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1814,8 +1903,11 @@ def format_fan_error(message, include_driver_note=False): if args.mem_carveout is not None: # Validate single GPU (VRAM is per-GPU) if isinstance(args.gpu, list) and len(args.gpu) > 1: - raise ValueError( - "VRAM carveout can only be set for a single GPU. Please specify --gpu " + raise AmdSmiInvalidParameterException( + "set", + "--mem-carveout", + self.logger.format, + "VRAM carveout can only be set for a single GPU. Please specify --gpu ", ) try: @@ -1825,10 +1917,12 @@ def format_fan_error(message, include_driver_note=False): # Validate index if args.mem_carveout >= len(options): - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "mem_carveout", f"Invalid index {args.mem_carveout}. Valid range: 0-{len(options) - 1}", + code=int(AmdSmiExitCode.INVALID_PARAMETER_VALUE), ) self.logger.print_output() self.logger.clear_multiple_devices_output() @@ -1869,17 +1963,21 @@ def format_fan_error(message, include_driver_note=False): # Surface an actionable message instead of a raw error code. # Avoid naming specific products here so the message does not # age as new ASICs add or drop UMA carveout support. - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "mem_carveout", "Not supported: UMA carveout is only available on APUs whose" ' VBIOS exposes the ATCS "Set UMA Allocation Size" function.', + exception=e, ) else: - self.logger.store_output( + self.helpers.store_device_error( + self.logger, args.gpu, "mem_carveout", f"[{e.get_error_info(detailed=False)}] Unable to set VRAM carveout to index {args.mem_carveout}", + exception=e, ) self.logger.print_output() @@ -1898,7 +1996,13 @@ def format_fan_error(message, include_driver_note=False): "Invalid accelerator partition memory allocation mode " f"{args.compute_partition_mem_alloc_mode}" ) - self.logger.store_output(args.gpu, "accelerator_partition_mem_alloc_mode", out) + self.helpers.store_device_error( + self.logger, + args.gpu, + "accelerator_partition_mem_alloc_mode", + out, + code=int(AmdSmiExitCode.INVALID_PARAMETER_VALUE), + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -1911,7 +2015,13 @@ def format_fan_error(message, include_driver_note=False): self.logger.clear_multiple_devices_output() raise PermissionError("Command requires elevation") from e else: - self.logger.store_output(args.gpu, "accelerator_partition_mem_alloc_mode", out) + self.helpers.store_device_error( + self.logger, + args.gpu, + "accelerator_partition_mem_alloc_mode", + out, + exception=e, + ) self.logger.print_output() self.logger.clear_multiple_devices_output() return @@ -2009,8 +2119,9 @@ def set_value( xgmi_plpd (int, optional): Value override for args.xgmi_plpd. Defaults to None. process_isolation (int, optional): Value override for args.process_isolation. Defaults to None. Raises: - ValueError: Value error if no gpu value is provided - IndexError: Index error if gpu list is empty + AmdSmiRequiredCommandException: If no device target or argument is provided + AmdSmiInvalidParameterException: If GPU/CPU/CORE arguments are combined or --gtt is misused + PermissionError: If a set operation requires elevation (AMDSMI_STATUS_NO_PERM) Return: Nothing @@ -2027,12 +2138,13 @@ def set_value( # Special GTT handling (system-wide, not per-GPU) — handle before device dispatch if hasattr(args, "gtt") and args.gtt is not None: if hasattr(args, "gpu") and args.gpu is not None: - print( + msg = ( "amd-smi set: error: argument --gtt/-G: not allowed with argument --gpu/-g " - "(--gtt is a system-wide setting, not per-GPU)", - file=sys.stderr, + "(--gtt is a system-wide setting, not per-GPU)" + ) + raise AmdSmiInvalidParameterException( + "set", "--gtt", self.helpers.get_output_format(), msg ) - sys.exit(2) gb_value = args.gtt pages = self.helpers.gb_to_pages(gb_value) try: @@ -2051,8 +2163,35 @@ def set_value( self.logger.output["set_gtt"] = ( f"[{e.get_error_info(detailed=False)}] Unable to set GTT to {gb_value:.2f} GB" ) + # record-then-finalize: GTT is system-wide, so record the library + # error (not per-device) so the exit code reflects the failure. + self.helpers.error_collector.record_library_error(e.get_error_code()) + self.logger.print_output() + return + + # Special node power limit handling (node-wide, not per-GPU) — handle before device dispatch + if hasattr(args, "node_power_limit") and args.node_power_limit is not None: + if hasattr(args, "gpu") and args.gpu is not None: + print( + "amd-smi set: error: argument --node-power-limit/-n: not allowed with " + "argument --gpu/-g (--node-power-limit is a node-wide setting, not per-GPU)", + file=sys.stderr, + ) + sys.exit(2) + requested_limit = args.node_power_limit + if self.node_handle is None: + self.logger.output["set_node_power_limit"] = ( + "[AMDSMI_STATUS_NOT_SUPPORTED - Feature not supported] " + "Unable to set node power limit: no NPM-capable node found" + ) self.logger.print_output() return + result = self.helpers.validate_and_set_node_power_limit( + self.node_handle, requested_limit, self.logger + ) + self.logger.output["set_node_power_limit"] = result + self.logger.print_output() + return # Check if a GPU argument has been set gpu_args_enabled = False @@ -2203,14 +2342,23 @@ def set_value( raise AmdSmiRequiredCommandException(command, self.logger.format) # Only allow one device's arguments to be set at a time + command = " ".join(sys.argv[1:]) if not any([gpu_args_enabled, cpu_args_enabled, core_args_enabled]): - raise ValueError( - "No GPU, CPU, or CORE arguments provided, specific arguments are needed" - ) + raise AmdSmiRequiredCommandException(command, self.logger.format) elif all([gpu_args_enabled, cpu_args_enabled, core_args_enabled]): - raise ValueError("Cannot set GPU, CPU, and CORE arguments at the same time") + raise AmdSmiInvalidParameterException( + "set", + "--gpu/--cpu/--core", + self.logger.format, + "Cannot set GPU, CPU, and CORE arguments at the same time", + ) elif not (gpu_args_enabled ^ cpu_args_enabled ^ core_args_enabled): - raise ValueError("Cannot set GPU, CPU, or CORE arguments at the same time") + raise AmdSmiInvalidParameterException( + "set", + "--gpu/--cpu/--core", + self.logger.format, + "Cannot set GPU, CPU, or CORE arguments at the same time", + ) if self.helpers.is_amdgpu_initialized() and gpu_args_enabled: if args.gpu == None: @@ -2229,7 +2377,8 @@ def set_value( # Print out all CPU and all GPU static info only if no device was specified. # If a GPU or CPU argument is provided only print out the specified device. if args.cpu == None and args.gpu == None and args.core == None: - raise ValueError("No GPU, CPU, or CORE provided, specific target(s) are needed") + command = " ".join(sys.argv[1:]) + raise AmdSmiRequiredCommandException(command, self.logger.format) if args.cpu: self.set_cpu( @@ -2292,7 +2441,8 @@ def set_value( ) elif self.helpers.is_amd_hsmp_initialized(): # Only CPU is initialized if args.cpu == None and args.core == None: - raise ValueError("No CPU or CORE provided, specific target(s) are needed") + command = " ".join(sys.argv[1:]) + raise AmdSmiRequiredCommandException(command, self.logger.format) if args.cpu: self.set_cpu( args, diff --git a/projects/amdsmi/amdsmi_cli/subcommands/static.py b/projects/amdsmi/amdsmi_cli/subcommands/static.py index efde5e511823..d8470bd1cb94 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/static.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/static.py @@ -680,7 +680,7 @@ def static_gpu( try: driver_info = amdsmi_interface.amdsmi_get_gpu_driver_info(args.gpu) driver_info_dict["name"] = driver_info["driver_name"] - driver_info_dict["version"] = driver_info["driver_version"] + driver_info_dict["version"] = driver_info["driver_full_version"] except amdsmi_exception.AmdSmiLibraryException as e: logging.debug( "Failed to get driver info for gpu %s | %s", gpu_id, e.get_error_info() diff --git a/projects/amdsmi/amdsmi_cli/subcommands/version.py b/projects/amdsmi/amdsmi_cli/subcommands/version.py index c045b5b9b60f..9c407e130393 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/version.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/version.py @@ -60,15 +60,10 @@ def version(self, args, gpu_version=None, cpu_version=None, nic_version=None): try: gpus = amdsmi_interface.amdsmi_get_processor_handles() if isinstance(gpus, list) and len(gpus) > 0: - gpu_version_info = amdsmi_interface.amdsmi_get_gpu_driver_info(gpus[0]) - gpu_version_str = gpu_version_info["driver_version"] - else: - gpu_version_str = "N/A" + driver_info = amdsmi_interface.amdsmi_get_gpu_driver_info(gpus[0]) + self.logger.output["amdgpu_version"] = driver_info["driver_full_version"] except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get amdgpu version | %s", e.get_error_info()) - gpu_version_str = "N/A" - - self.logger.output["amdgpu_version"] = gpu_version_str + logging.debug("Failed to get amdgpu driver versions | %s", e.get_error_info()) if args.cpu_version: try: ret = amdsmi_interface.amdsmi_get_cpu_handles() @@ -110,7 +105,8 @@ def version(self, args, gpu_version=None, cpu_version=None, nic_version=None): ) if args.gpu_version: human_readable_output = ( - human_readable_output + f" | amdgpu version: {gpu_version_str}" + human_readable_output + + f" | amdgpu version: {self.logger.output['amdgpu_version']}" ) if args.cpu_version: human_readable_output = ( diff --git a/projects/amdsmi/docs/conceptual/ras.md b/projects/amdsmi/docs/conceptual/ras.md index 5ff9db678241..9432bba89f25 100644 --- a/projects/amdsmi/docs/conceptual/ras.md +++ b/projects/amdsmi/docs/conceptual/ras.md @@ -246,32 +246,42 @@ must be supplied. Unlike the `--cper --folder` write path, the AFID `--folder` must already exist and contain at least one `.cper` file — it is not auto-created. +Human output is a `file_name | list of afids` table. `--json` and `--csv` emit a +structured per-file record instead (`cper_file, afids, decode_failed, status, +message, code`). In JSON `afids` is a list. In CSV it is a space-joined string. +Either way the command exits with the underlying `AMDSMI_STATUS_*` on a decode +failure (or `205` when files fail with differing codes), so a scripted caller +sees the failure. + :::::{tab-set} ::::{tab-item} Single file -Parse one CPER record and print the space-separated list of AFIDs encoded in -it. A single `-` is printed when the record contains no AFID payload (e.g. -corrected entries). +Parse one CPER record and print a one-row `file_name | list of afids` table. +`N/A` is shown when the record contains no AFID payload (e.g. corrected +entries). A file that cannot be decoded shows `[AMDSMI_STATUS_] ` +in place of its AFIDs and the command exits with that status. ```shell-session ~$ amd-smi ras --afid --cper-file /tmp/cper_dump/fatal-1.cper -30 31 32 33 +file_name list of afids +fatal-1.cper 30 31 32 33 ``` :::: ::::{tab-item} Folder Parse every `*.cper` in a pre-existing directory and print a `file_name | list of afids` table, one row per record. A record with no AFID -payload shows `-`, and a file that cannot be parsed shows `decode failed`. +payload shows `N/A`, and a file that cannot be parsed shows +`[AMDSMI_STATUS_] ` in place of its AFIDs. Symlinks in the folder are skipped (they are never followed). ```shell-session ~$ amd-smi ras --afid --folder /tmp/cper_dump/ -file_name list of afids -fatal-1.cper 30 31 32 33 -fatal-2.cper 30 31 32 33 -nonfatal-1.cper - -truncated.cper decode failed +file_name list of afids +fatal-1.cper 30 31 32 33 +fatal-2.cper 30 31 32 33 +nonfatal-1.cper N/A +truncated.cper [AMDSMI_STATUS_UNEXPECTED_DATA] Unexpected data in the CPER file ``` :::: diff --git a/projects/amdsmi/docs/how-to/amdsmi-cli-tool.md b/projects/amdsmi/docs/how-to/amdsmi-cli-tool.md index 6075ea46e3cd..48fe55ef9696 100644 --- a/projects/amdsmi/docs/how-to/amdsmi-cli-tool.md +++ b/projects/amdsmi/docs/how-to/amdsmi-cli-tool.md @@ -329,7 +329,7 @@ usage: amd-smi metric [-h] [-g GPU [GPU ...] | -U CPU [CPU ...] | -O CORE [CORE [--cpu-dimm-pow-consumption DIMM_ADDR] [--cpu-dimm-thermal-sensor DIMM_ADDR] [--core-boost-limit] [--core-curr-active-freq-core-limit] [--core-energy] - [--json | --csv] [--file FILE] [--loglevel LEVEL] + [--json | --csv] [--file FILE] [--loglevel LEVEL] [--show-unsupported] If no GPU is specified, returns metric information for all GPUs on the system. If no metric argument is provided, all metric information will be displayed. @@ -431,8 +431,46 @@ Command Modifiers: --file FILE Saves output into a file on the provided path (stdout by default). --loglevel LEVEL Set the logging level from the possible choices: DEBUG, INFO, WARNING, ERROR, CRITICAL + --show-unsupported Print every field, including the ones the GPU's gpu_metrics + table version cannot carry and which are omitted by default; + affects human-readable output only, since --json and --csv + always print every field ``` +The `gpu_metrics` table the driver exposes has a version, and each version +carries a different set of fields. Fields the detected version cannot carry are +omitted from human-readable output. Pass `--show-unsupported` to print them as +`N/A` instead, which restores the output of earlier releases. + +`--json` and `--csv` are never filtered. They are consumed by scripts, so they +keep emitting every field, the `N/A` ones included, and their key and column +sets are unchanged from earlier releases. `--show-unsupported` is accepted +alongside them and has no effect. + +This is scoped to the metrics table version, not to what the ASIC supports. Only +fields whose sole sources are the metrics blobs are eligible, such as the +`hbm_stacks`, `mid`, `aid` and `xcd` temperature arrays and the `uclk_aid` and +`socclks_mid` clock arrays. Anything the CLI can also read from hwmon or sysfs is +always printed: the `edge`, `hotspot` and `mem` temperature sensors, the fan +section, the voltages, and the `gfx_N`, `vclk_N`, `dclk_N`, `mem_N` and +`socclk_N` clock slots. A field the version *does* carry but the ASIC or driver +leaves unpopulated also still prints `N/A`. + +Filtering never removes a field that reports a value, and it suppresses nothing +at all when the metrics version is unrecognized or its header cannot be read. + +A section named on the command line is never emptied by filtering. Plain +`amd-smi metric` prints every section, so a section the version can populate +nothing of is dropped entirely; on a metrics v1.3 GPU that removes the whole +`throttle` section. Asking for that section by name instead, as in `amd-smi +metric --throttle`, prints it in full rather than answering with silence. A named +section that is only partly suppressed is still filtered, so `amd-smi metric +--usage` on a v1.9 GPU still omits `jpeg_activity`. + +`--partition` scopes the data rather than naming a section, so it protects +nothing and `amd-smi metric --partition` filters exactly like plain `amd-smi +metric`. + (cmd-process)= ### amd-smi process @@ -618,7 +656,7 @@ Set options for specified devices. ~$ amd-smi set --help usage: amd-smi set [-h] (-g GPU [GPU ...] | -U CPU [CPU ...] | -O CORE [CORE ...]) [-f %] [-l LEVEL] [-P SETPROFILE] [-d SCLKMAX] [-C PARTITION] [-M PARTITION] - [-a MODE] [-o WATTS] [-p POLICY_ID] [-x POLICY_ID] [-R STATUS] + [-a MODE] [-o WATTS] [-p POLICY_ID] [-x POLICY_ID] [-R STATUS] [-n WATTS] [--cpu-pwr-limit PWR_LIMIT] [--cpu-xgmi-link-width MIN_WIDTH MAX_WIDTH] [--cpu-lclk-dpm-level NBIOID MIN_DPM MAX_DPM] [--cpu-pwr-eff-mode MODE [UTIL PPT_LIMIT]] [--cpu-gmi3-link-width MIN_LW MAX_LW] [--cpu-pcie-link-rate LINK_RATE] @@ -668,6 +706,9 @@ Set Arguments: -R, --process-isolation STATUS Enable or disable the GPU process isolation on a per partition basis: 0 for disable and 1 for enable. --ptl-status STATUS Enable or disable the PTL on a GPU processor: 0 for disable and 1 for enable --ptl-format FRMT1,FRMT2 Set the PTL format on a GPU processor. For example, --ptl-format I8,F32 + -n, --node-power-limit WATTS Set the node-level (NPM) power limit in watts. + This is a node-wide setting, not per-GPU. + Max node power limit: 6000 W CPU Arguments: --cpu-pwr-limit PWR_LIMIT Set power limit for the given socket. Input parameter is power limit value. @@ -1086,7 +1127,7 @@ information, GPU status, and running processes. ~$ amd-smi +------------------------------------------------------------------------------+ | AMD-SMI 27.0.0 | -| amdgpu Version: 6.19.4 | +| amdgpu Version: 6.19.14.31400000-2370381 | | ROCm Version: 7.14.0 | | Platform: Linux Baremetal | |-------------------------------------+----------------------------------------| @@ -1261,7 +1302,7 @@ GPU: 0 PTL_FORMAT: N/A DRIVER: NAME: amdgpu - VERSION: 6.19.4 + VERSION: 6.19.14.31400000-2370381 OS_KERNEL_VERSION: 5.15.0-generic BOARD: MODEL_NUMBER: N/A @@ -1661,6 +1702,23 @@ users inspect and tune the BIOS VRAM carveout and the TTM `pages_limit` (shared GTT) respectively. Both features talk directly to kernel UAPI interfaces (sysfs / modprobe.d) and do **not** require libdrm. +`amd-smi node -p` / `amd-smi node --power-management` also reports a +`CURRENT_NODE_POWER` line alongside the existing `LIMIT`/`STATUS`/`THRESHOLD` +fields: the current (instantaneous) node power draw in watts, read once per +node rather than once per GPU. Use `amd-smi set -n WATTS` / +`amd-smi set --node-power-limit WATTS` to change the node-level power limit +(also a node-wide, not per-GPU, setting): + +```shell-session +~$ amd-smi node -p +NODE: + POWER_MANAGEMENT: + LIMIT: 6000 W + STATUS: ENABLED + THRESHOLD: N/A W + CURRENT_NODE_POWER: 5800 W +``` + ### Supported ASICs | Feature | Hardware | Status | diff --git a/projects/amdsmi/docs/reference/amdsmi-py-api.md b/projects/amdsmi/docs/reference/amdsmi-py-api.md index 9510da991507..7e655ee1e4d0 100644 --- a/projects/amdsmi/docs/reference/amdsmi-py-api.md +++ b/projects/amdsmi/docs/reference/amdsmi-py-api.md @@ -568,9 +568,13 @@ Output: Dictionary with fields Field | Content ---|--- -`driver_name` | driver name -`driver_version` | driver_version -`driver_date` | driver_date +``driver_name`` | driver name +``driver_kernel_version`` | amdgpu kernel source version, such as ``6.19.14`` +``amdgpu_driver_version`` | amdgpu module version, such as ``31400000`` +``driver_version`` | driver version, such as ``6.19.14.31400000`` +``driver_build_version`` | active DKMS build version, such as ``2370381`` +``driver_full_version`` | composed version, such as ``6.19.14.31400000-2370381`` +``driver_date`` | driver_date Exceptions that can be thrown by `amdsmi_get_gpu_driver_info` function: @@ -3060,6 +3064,8 @@ Field | Description | Units `status` | NPM status (AMDSMI_NPM_STATUS_ENABLED or AMDSMI_NPM_STATUS_DISABLED) | - `limit` | Node-level power limit | W `ubb_power_threshold` | UBB node power threshold | W +`max_node_power_limit` | The platform max bound consumed by `amdsmi_set_npm_limit()`: callers should ensure any limit passed to that function does not exceed this value | W +`current_node_power` | The current (instantaneous) node power (board/node_power), MI450+. Queried once per node rather than once per GPU | W Exceptions that can be thrown by `amdsmi_get_npm_info` function: @@ -3086,6 +3092,80 @@ try: print(npm_info['status']) print(npm_info['limit']) print(npm_info['ubb_power_threshold']) + print(npm_info['max_node_power_limit']) + print(npm_info['current_node_power']) +except amdsmi.AmdSmiException as e: + print(e) +finally: + amdsmi.amdsmi_shut_down() +``` + +### amdsmi_set_npm_limit + +Description: Set the NPM (Node Power Management) power limit for the node +associated with `node_handle` by writing to the board's +`cur_node_power_limit` sysfs interface. + +This function rejects the request with `AmdSmiLibraryException` +(`AMDSMI_STATUS_INVAL`) if NPM is disabled on the node +(`amdsmi_get_npm_info()`'s `status` == `AMDSMI_NPM_STATUS_DISABLED`), since +writing `board/cur_node_power_limit` while NPM is disabled has no defined +effect. It also validates `limit` against the platform max bound +(`amdsmi_get_npm_info()`'s `max_node_power_limit`, sourced from +`board/max_node_power_limit`) internally before ever issuing the write, +raising `AmdSmiLibraryException` with `AMDSMI_STATUS_INVAL` if `limit` is `0` +or greater than that bound. If the platform max bound itself cannot be read +(e.g. the sysfs interface is missing or returns unexpected data), this +function fails closed and raises that underlying error rather than silently +allowing an unbounded `limit` through. The amd-smi CLI's +`set --node-power-limit` additionally performs the same checks itself ahead +of calling this function, purely to fail fast and present a friendlier, +earlier user-facing error message; it is not the only validation and is not +required for correctness. + +Input parameters: + +* `node_handle` node handle obtained from `amdsmi_get_node_handle` +* `limit` new NPM power limit value to request (units match the + `board/cur_node_power_limit` sysfs interface). Must satisfy + `0 < limit <= UINT64_MAX`; out-of-range values raise + `AmdSmiParameterException` rather than silently wrapping modulo 2**64 the + way a raw `ctypes.c_uint64()` conversion would. Values that pass this local + bound check but are `0` or exceed the platform max are rejected by the + underlying library call instead (see Exceptions below) + +Output: None. This function raises an exception if the call did not succeed +(e.g. `AmdSmiLibraryException` with `AMDSMI_STATUS_INVAL` if `limit` is `0` or +exceeds the platform max bound, `AMDSMI_STATUS_NOT_SUPPORTED` if the sysfs +interface is unavailable, or `AMDSMI_STATUS_NO_PERM` if the write was +rejected) + +Exceptions that can be thrown by `amdsmi_set_npm_limit` function: + +* `AmdSmiLibraryException` +* `AmdSmiParameterException` + +#### Possible Library Exceptions + +- `AMDSMI_STATUS_INVAL` - Invalid parameters, including when `limit` is `0` + or exceeds the platform max bound (or when that bound itself cannot be + read) +- `AMDSMI_STATUS_NOT_SUPPORTED` - Feature not supported +- `AMDSMI_STATUS_NO_PERM` - Permission Denied (e.g. the write was rejected by + the driver for this guest context) + +Example: + +```python +import amdsmi +try: + amdsmi.amdsmi_init() + devices = amdsmi.amdsmi_get_processor_handles() + if len(devices) == 0: + print("No GPUs on machine") + else: + node_handle = amdsmi.amdsmi_get_node_handle(devices[0]) + amdsmi.amdsmi_set_npm_limit(node_handle, 6000) except amdsmi.AmdSmiException as e: print(e) finally: diff --git a/projects/amdsmi/docs/sphinx/requirements.in b/projects/amdsmi/docs/sphinx/requirements.in index c325d197594b..16c0d6550474 100644 --- a/projects/amdsmi/docs/sphinx/requirements.in +++ b/projects/amdsmi/docs/sphinx/requirements.in @@ -1,2 +1,2 @@ -rocm-docs-core[api_reference,llms]==1.40.0 +rocm-docs-core[api_reference,llms]==1.41.0 sphinxcontrib-mermaid diff --git a/projects/amdsmi/docs/sphinx/requirements.txt b/projects/amdsmi/docs/sphinx/requirements.txt index 59dfbe360975..84c02f55e262 100644 --- a/projects/amdsmi/docs/sphinx/requirements.txt +++ b/projects/amdsmi/docs/sphinx/requirements.txt @@ -207,8 +207,9 @@ referencing==0.37.0 requests==2.34.2 # via # pygithub + # rocm-docs-core # sphinx -rocm-docs-core[api-reference,llms]==1.40.0 +rocm-docs-core[api-reference,llms]==1.41.0 # via -r requirements.in roman-numerals==4.1.0 # via sphinx @@ -261,7 +262,7 @@ sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-mermaid==2.1.0 +sphinxcontrib-mermaid==2.1.1 # via -r requirements.in sphinxcontrib-qthelp==2.0.0 # via sphinx diff --git a/projects/amdsmi/example/amd_smi_drm_example.cc b/projects/amdsmi/example/amd_smi_drm_example.cc index 3b55c139b26e..91e2fed48678 100644 --- a/projects/amdsmi/example/amd_smi_drm_example.cc +++ b/projects/amdsmi/example/amd_smi_drm_example.cc @@ -604,7 +604,7 @@ int main() { } // Iterate through all available accelerator partition profiles - amdsmi_accelerator_partition_profile_config_t profile_config; + amdsmi_accelerator_partition_profile_config_t profile_config = {}; ret = amdsmi_get_gpu_accelerator_partition_profile_config(processor_handles[device_index], &profile_config); for (uint32_t profile_idx = 0; profile_idx < profile_config.num_profiles; profile_idx++) { @@ -620,7 +620,7 @@ int main() { << acceleratorPartitionString(updatePartition) << "): " << err_str << "\n\n"; // Get the current accelerator partition - amdsmi_accelerator_partition_profile_t profile; + amdsmi_accelerator_partition_profile_t profile = {}; uint32_t partition_id[AMDSMI_MAX_ACCELERATOR_PROFILE]; ret = amdsmi_get_gpu_accelerator_partition_profile(processor_handles[device_index], &profile, partition_id); @@ -1225,7 +1225,11 @@ int main() { CHK_AMDSMI_RET(ret) printf(" Output of amdsmi_get_gpu_driver_info:\n"); printf("\tDriver name: %s\n", driver_info.driver_name); + printf("\tamdgpu kernel source version: %s\n", driver_info.driver_kernel_version); + printf("\tamdgpu driver version: %s\n", driver_info.amdgpu_driver_version); printf("\tDriver version: %s\n", driver_info.driver_version); + printf("\tBuild version: %s\n", driver_info.driver_build_version); + printf("\tFull version: %s\n", driver_info.driver_full_version); printf("\tDriver date: %s\n\n", driver_info.driver_date); } else { printf("\tamdsmi_get_gpu_driver_info(): not available on this device.\n"); @@ -1554,8 +1558,7 @@ int main() { CHK_AMDSMI_RET(ret) } printf(" Output of amdsmi_get_temp_metric:\n"); - std::cout << "\t\tTemperature: " << std::dec << val_i64 << "C" - << "\n\n"; + std::cout << "\t\tTemperature: " << std::dec << val_i64 << "C" << "\n\n"; // Get frame buffer amdsmi_vram_usage_t vram_usage = {}; @@ -2029,8 +2032,7 @@ int main() { std::cout << "\n"; std::cout << " ** Note: Values MAX'ed out " - << "(UINTX MAX are unsupported for the version in question) ** " - << "\n\n"; + << "(UINTX MAX are unsupported for the version in question) ** " << "\n\n"; } // END GPU METRICS OUTPUTS // Get nearest GPUs diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h index 40aedf56711a..36c686b2828a 100644 --- a/projects/amdsmi/include/amd_smi/amdsmi.h +++ b/projects/amdsmi/include/amd_smi/amdsmi.h @@ -769,6 +769,7 @@ typedef enum { AMDSMI_VRAM_TYPE_HBM2E = 3, //!< High Bandwidth Memory, Generation 2 Enhanced AMDSMI_VRAM_TYPE_HBM3 = 4, //!< High Bandwidth Memory, Generation 3 AMDSMI_VRAM_TYPE_HBM3E = 5, //!< High Bandwidth Memory, Generation 3 Enhanced + AMDSMI_VRAM_TYPE_HBM4 = 6, //!< High Bandwidth Memory, Generation 4 // DDR AMDSMI_VRAM_TYPE_DDR2 = 10, //!< Double Data Rate, Generation 2 AMDSMI_VRAM_TYPE_DDR3 = 11, //!< Double Data Rate, Generation 3 @@ -1322,9 +1323,17 @@ typedef struct { * @cond @tag{gpu_bm_linux} @tag{guest_windows} @tag{host} @endcond */ typedef struct { - char driver_version[AMDSMI_MAX_STRING_LENGTH]; - char driver_date[AMDSMI_MAX_STRING_LENGTH]; - char driver_name[AMDSMI_MAX_STRING_LENGTH]; + char driver_version[AMDSMI_MAX_STRING_LENGTH]; //!< Driver version, such as + //!< 6.19.14.31400000 + char driver_date[AMDSMI_MAX_STRING_LENGTH]; //!< Driver release date + char driver_name[AMDSMI_MAX_STRING_LENGTH]; //!< Driver name + char driver_kernel_version[AMDSMI_MAX_STRING_LENGTH]; //!< amdgpu kernel source version, + //!< such as 6.19.14 + char amdgpu_driver_version[AMDSMI_MAX_STRING_LENGTH]; //!< amdgpu module version, such + //!< as 31400000 + char driver_build_version[AMDSMI_MAX_STRING_LENGTH]; //!< DKMS build, such as 2370381 + char driver_full_version[AMDSMI_MAX_STRING_LENGTH]; //!< Full version, such + //!< as 6.19.14.31400000-2370381 } amdsmi_driver_info_t; /** @@ -2709,10 +2718,16 @@ typedef enum { * @cond @tag{gpu_bm_linux} @tag{host} @endcond */ typedef struct { - amdsmi_npm_status_t status; //!< NPM status (enabled/disabled). - uint64_t limit; //!< Node-level power limit in Watts. - uint32_t ubb_power_threshold; //!< The UBB node power threshold in Watts. - uint64_t reserved[5]; + amdsmi_npm_status_t status; //!< NPM status (enabled/disabled). + uint64_t limit; //!< Node-level power limit in Watts. + uint32_t ubb_power_threshold; //!< The UBB node power threshold in Watts. + uint64_t max_node_power_limit; //!< Platform max node-level power limit in Watts + //!< (board/max_node_power_limit), used to bound + //!< amdsmi_set_npm_limit() requests. + uint32_t current_node_power; //!< The current (instantaneous) node power in W + //!< {@linux_bm}, MI450+. + uint64_t reserved[3]; //!< Reduced from reserved[5] to accommodate + //!< max_node_power_limit and current_node_power. } amdsmi_npm_info_t; /** @@ -7522,6 +7537,14 @@ amdsmi_status_t amdsmi_stop_gpu_event_notification(amdsmi_processor_handle proce * @platform{gpu_bm_linux} @platform{host} @platform{guest_1vf} @platform{guest_mvf} * @platform{guest_windows} * + * @details On Linux, driver_version is /sys/module/amdgpu/version, such as + * 6.19.14.31400000, or N/A. driver_kernel_version (6.19.14) and + * amdgpu_driver_version (31400000) are its parts, + * driver_build_version is the active DKMS build, and + * driver_full_version is driver_version-build, or driver_version + * when there is no build. The other fields are empty when + * unavailable. + * * @param[in] processor_handle Device which to query * * @param[out] info Reference to driver information structure. Must be @@ -7691,7 +7714,14 @@ amdsmi_status_t amdsmi_get_gpu_xcd_counter(amdsmi_processor_handle processor_han * * @details This function queries the NPM controller for the given node and returns whether NPM is * enabled, along with the current node-level power limit in Watts. The NPM status and limit are set - * out-of-band and reported via this API. + * out-of-band and reported via this API. The returned amdsmi_npm_info_t::max_node_power_limit is + * the platform max bound (sourced from board/max_node_power_limit) that ::amdsmi_set_npm_limit + * itself validates requests against internally before issuing the write (mirroring + * ::amdsmi_set_power_cap and ::amdsmi_get_power_cap_info); callers may still query it here ahead + * of time to fail fast / present a clean error without invoking the setter. The returned + * amdsmi_npm_info_t::current_node_power is the current (instantaneous) node-level power reading in + * Watts (sourced from board/node_power); it is node-handle scoped (queried once per node), unlike + * amdsmi_power_info_t, which is scoped per GPU handle. * * @param[in] node_handle Handle to the Node to query. * @param[out] info Pointer to amdsmi_npm_info_t structure to receive NPM status and limit. @@ -7724,6 +7754,40 @@ amdsmi_status_t amdsmi_get_npm_info(amdsmi_node_handle node_handle, amdsmi_npm_i */ amdsmi_status_t amdsmi_get_tray_info(amdsmi_node_handle node_handle, amdsmi_tray_info_t* info); +/** + * @brief Sets the node power management (NPM) power limit for the specified node. + * + * @ingroup tagNodeInfo + * + * @platform{gpu_bm_linux} @platform{host} + * + * @details This function returns ::AMDSMI_STATUS_INVAL if NPM is disabled on the node + * (amdsmi_npm_info_t::status == ::AMDSMI_NPM_STATUS_DISABLED), since writing + * board/cur_node_power_limit while NPM is disabled has no defined effect. It also validates + * `limit` against the platform max bound (amdsmi_npm_info_t::max_node_power_limit, sourced from + * board/max_node_power_limit) before ever issuing a write, returning ::AMDSMI_STATUS_INVAL if + * `limit` is `0` or greater than that bound (this mirrors ::amdsmi_set_power_cap, whose analogous + * range check against ::amdsmi_get_power_cap_info's bounds is likewise enforced internally rather + * than left purely to the caller). If the platform max bound itself cannot be read (e.g. the + * sysfs interface is missing or returns unexpected data), this function fails closed and returns + * that underlying error rather than silently allowing an unbounded `limit` through. Once + * validated, this function issues a Set NPM Limit request to GPU PMFW via the amdgpu driver for + * the given node, requesting the value of `limit` verbatim. The write path is restricted to host + * / 1 permitted VM only; the enforcement of that restriction is done by the kernel driver, not by + * this function. When both SMI and BMC set limits, GPU PMFW arbitrates by taking the minimum of + * the two, bounded by the platform max. + * + * @param[in] node_handle Handle to the Node to set the limit on. + * @param[in] limit Node-level power limit in Watts to request. + * + * @return ::AMDSMI_STATUS_SUCCESS on success. ::AMDSMI_STATUS_INVAL if NPM is disabled on this + * node, or if `limit` is `0` or exceeds the platform max bound. ::AMDSMI_STATUS_NOT_SUPPORTED if + * the sysfs interface (board/npm_status, board/cur_node_power_limit, or + * board/max_node_power_limit) is unavailable on this platform. Non-zero on other failure (e.g. + * ::AMDSMI_STATUS_NO_PERM if the driver rejects the write for this guest context). + */ +amdsmi_status_t amdsmi_set_npm_limit(amdsmi_node_handle node_handle, uint64_t limit); + /** @} End tagNodeInfo */ /*****************************************************************************/ diff --git a/projects/amdsmi/include/amd_smi/impl/amd_smi_test_internal.h b/projects/amdsmi/include/amd_smi/impl/amd_smi_test_internal.h index 8d5b08154b18..0892df8942a7 100644 --- a/projects/amdsmi/include/amd_smi/impl/amd_smi_test_internal.h +++ b/projects/amdsmi/include/amd_smi/impl/amd_smi_test_internal.h @@ -12,4 +12,22 @@ // extern-declare the rsmi_status_t function directly. amdsmi_status_t amdsmi_test_sleep(amdsmi_processor_handle processor_handle, uint32_t seconds); +// Internal test-only hook: registers |node_handle| as a valid handle in the +// same node-handle registry that amdsmi_get_npm_info()/amdsmi_set_npm_limit() +// consult (via is_registered_node_handle()) before dereferencing their +// node_handle argument. In production, that registry is only ever populated +// by amdsmi_get_node_handle(); unit tests that fabricate a raw node_handle +// directly (bypassing amdsmi_get_node_handle(), whose oam_id == 0 gate is not +// satisfiable in the test environment) must call this first so the +// registered-handle check does not reject their forged-but-otherwise-valid +// handle. Not declared in amdsmi.h -- internal use via +// amd_smi_test_internal.h only. +// Unlike the other hooks in this header, this one lets a caller mark an +// arbitrary pointer as a "valid" node handle, so it is compiled only for +// BUILD_TESTS=ON builds (see src/CMakeLists.txt) rather than shipped in +// production builds. +#ifdef BUILD_TESTS +amdsmi_status_t amdsmi_test_register_node_handle(amdsmi_node_handle node_handle); +#endif // BUILD_TESTS + #endif // AMD_SMI_INCLUDE_AMD_SMI_TEST_INTERNAL_H_ diff --git a/projects/amdsmi/include/amd_smi/impl/amd_smi_utils.h b/projects/amdsmi/include/amd_smi/impl/amd_smi_utils.h index 9232f9424446..7a88fff3a038 100644 --- a/projects/amdsmi/include/amd_smi/impl/amd_smi_utils.h +++ b/projects/amdsmi/include/amd_smi/impl/amd_smi_utils.h @@ -309,4 +309,28 @@ const char* smi_amdgpu_pp_dpm_filename_for_clk_type(amdsmi_clk_type_t clk_type); */ void init_asic_info_defaults(amdsmi_asic_info_t* info); +inline constexpr auto kAmdgpuDkmsRoot = std::string_view{"/var/lib/dkms/amdgpu"}; + +auto smi_amdgpu_parse_driver_versions(std::string_view module_version, + std::string_view package_version, amdsmi_driver_info_t* info) + -> amdsmi_status_t; + +/** + * @brief Select the DKMS package built for a running kernel. + * + * @details Reads ``kernel--`` under @p dkms_root, then + * validates only that PACKAGE_VERSION directory through the dkms.conf behind + * its source symlink. Does not scan sibling packages. + * + * @param[in] dkms_root Directory that contains kernel-* symlinks (for example + * /var/lib/dkms/amdgpu). + * @param[in] release Kernel release from uname, such as 6.8.0-124-generic. + * @param[in] machine Hardware name from uname, such as x86_64. + * @param[out] active_version Cleared on entry. Set to PACKAGE_VERSION when + * the kernel symlink points at a validated package. + */ +auto smi_amdgpu_get_active_dkms_version(std::string_view dkms_root, std::string_view release, + std::string_view machine, std::string* active_version) + -> amdsmi_status_t; + #endif // AMD_SMI_INCLUDE_AMD_SMI_UTILS_H_ diff --git a/projects/amdsmi/py-interface/CMakeLists.txt b/projects/amdsmi/py-interface/CMakeLists.txt index 79b1efba2bd1..4f8132da8f21 100644 --- a/projects/amdsmi/py-interface/CMakeLists.txt +++ b/projects/amdsmi/py-interface/CMakeLists.txt @@ -119,8 +119,15 @@ add_custom_command( ${PY_PACKAGE_DIR}/amdsmi_interface_utils.py ${PY_PACKAGE_DIR}/README.md ${PY_PACKAGE_DIR}/LICENSE + ${PY_PACKAGE_DIR}/THIRD_PARTY_LICENSES/LGPL-2.1.txt + ${PY_PACKAGE_DIR}/THIRD_PARTY_LICENSES/NOTICE.md ${PY_BUILD_DIR}/setup.py - DEPENDS python_wrapper + DEPENDS + python_wrapper + # The other copied inputs are not listed here, so editing them does not + # restage the package; the notices at least should not go stale. + ${CMAKE_CURRENT_SOURCE_DIR}/THIRD_PARTY_LICENSES/LGPL-2.1.txt + ${CMAKE_CURRENT_SOURCE_DIR}/THIRD_PARTY_LICENSES/NOTICE.md COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py ${PY_PACKAGE_DIR}/ COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/setup.py ${PY_BUILD_DIR}/ COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_exception.py ${PY_PACKAGE_DIR}/ @@ -129,6 +136,13 @@ add_custom_command( ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_interface_utils.py ${PY_PACKAGE_DIR}/ COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/README.md ${PY_PACKAGE_DIR}/ COMMAND ${CMAKE_COMMAND} -E copy_if_different ${PROJECT_SOURCE_DIR}/LICENSE ${PY_PACKAGE_DIR}/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${PY_PACKAGE_DIR}/THIRD_PARTY_LICENSES + COMMAND + ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/THIRD_PARTY_LICENSES/LGPL-2.1.txt + ${PY_PACKAGE_DIR}/THIRD_PARTY_LICENSES/ + COMMAND + ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/THIRD_PARTY_LICENSES/NOTICE.md + ${PY_PACKAGE_DIR}/THIRD_PARTY_LICENSES/ ) # copy libamd_smi.so to allow for a self-contained python package @@ -150,6 +164,8 @@ add_custom_target( ${PY_PACKAGE_DIR}/amdsmi_interface_utils.py ${PY_PACKAGE_DIR}/README.md ${PY_PACKAGE_DIR}/LICENSE + ${PY_PACKAGE_DIR}/THIRD_PARTY_LICENSES/LGPL-2.1.txt + ${PY_PACKAGE_DIR}/THIRD_PARTY_LICENSES/NOTICE.md ${PY_BUILD_DIR}/setup.py ${PY_PACKAGE_DIR}/${AMDSMI_PYTHON_LIB_NAME} ) diff --git a/projects/amdsmi/py-interface/THIRD_PARTY_LICENSES/LGPL-2.1.txt b/projects/amdsmi/py-interface/THIRD_PARTY_LICENSES/LGPL-2.1.txt new file mode 100644 index 000000000000..f6683e74e0f0 --- /dev/null +++ b/projects/amdsmi/py-interface/THIRD_PARTY_LICENSES/LGPL-2.1.txt @@ -0,0 +1,501 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, see . + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Moe Ghoul, President of Vice + +That's all there is to it! diff --git a/projects/amdsmi/py-interface/THIRD_PARTY_LICENSES/NOTICE.md b/projects/amdsmi/py-interface/THIRD_PARTY_LICENSES/NOTICE.md new file mode 100644 index 000000000000..94ba61a6e136 --- /dev/null +++ b/projects/amdsmi/py-interface/THIRD_PARTY_LICENSES/NOTICE.md @@ -0,0 +1,49 @@ +Third-Party Notices for the amdsmi Python Wheel +=============================================== + +`libamd_smi_python.so` is MIT-licensed (see `amdsmi/LICENSE`) but links at +runtime against the Netlink libraries. When the wheel is repaired for the +manylinux platform tag (`tools/build_wheel.py --repair`), `auditwheel repair` +copies the shared libraries that are not on the manylinux baseline allowlist +into `amdsmi.libs/` so the wheel is self-contained. That currently copies three +third-party shared objects: + +| Bundled file | Upstream project | Upstream source | SPDX license | +|-----------------------|-----------------------------------|----------------------------------|---------------------| +| `libnl-3.so.200` | libnl | https://github.com/thom311/libnl | `LGPL-2.1-only` | +| `libnl-genl-3.so.200` | libnl (generic-netlink component) | https://github.com/thom311/libnl | `LGPL-2.1-only` | +| `libmnl.so.0` | libmnl (Netfilter) | https://git.netfilter.org/libmnl | `LGPL-2.1-or-later` | + +Which exact build is bundled depends on the host that ran `auditwheel repair`; +the wheel's own `amdsmi.libs/` directory records what shipped in that release. + +None of the three are modified from upstream — `auditwheel` copies them +verbatim from the build host's packages. `amdsmi` links to them dynamically +(`DT_NEEDED`, not static linking or source inclusion), which keeps this inside +LGPL v2.1 Section 6 ("Combined Works"). Section 6 still requires: + +1. reproducing the license text and copyright notices — see `LGPL-2.1.txt` in + this directory, the verbatim text published at + https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt; and +2. accompanying the work with the library's corresponding source, or a written + offer valid for at least three years to provide it. + +Source availability / written offer +----------------------------------- + +libnl and libmnl are distributed as source by their upstream projects at the +URLs above and by every Linux distribution that packages them. As a written +offer under LGPL v2.1 Section 6(a): for three years from the publication date +of a given `amdsmi` wheel, AMD will, on written request to +amd-smi.support@amd.com identifying the wheel version and the SONAME of the +bundled library, either point the requester to the exact upstream source +release matching that build or provide a copy of it. + +Copyright notices +----------------- + +- libnl: Copyright (c) 2003-2012 Thomas Graf and contributors. +- libmnl: Copyright (c) 2008-2010 Pablo Neira Ayuso and + contributors. + +See each upstream repository for the full list of copyright holders. diff --git a/projects/amdsmi/py-interface/__init__.py b/projects/amdsmi/py-interface/__init__.py index 67fbba811068..15001bb3adb5 100644 --- a/projects/amdsmi/py-interface/__init__.py +++ b/projects/amdsmi/py-interface/__init__.py @@ -19,6 +19,7 @@ from .amdsmi_interface import amdsmi_get_node_handle from .amdsmi_interface import amdsmi_get_npm_info from .amdsmi_interface import amdsmi_get_tray_info +from .amdsmi_interface import amdsmi_set_npm_limit # ESMI Dependent Functions try: diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py index c206170c6d8d..52488db5b6c1 100644 --- a/projects/amdsmi/py-interface/amdsmi_interface.py +++ b/projects/amdsmi/py-interface/amdsmi_interface.py @@ -739,6 +739,8 @@ class AmdSmiVramType(IntEnum): HBM2 = amdsmi_wrapper.AMDSMI_VRAM_TYPE_HBM2 HBM2E = amdsmi_wrapper.AMDSMI_VRAM_TYPE_HBM2E HBM3 = amdsmi_wrapper.AMDSMI_VRAM_TYPE_HBM3 + HBM3E = amdsmi_wrapper.AMDSMI_VRAM_TYPE_HBM3E + HBM4 = amdsmi_wrapper.AMDSMI_VRAM_TYPE_HBM4 DDR2 = amdsmi_wrapper.AMDSMI_VRAM_TYPE_DDR2 DDR3 = amdsmi_wrapper.AMDSMI_VRAM_TYPE_DDR3 DDR4 = amdsmi_wrapper.AMDSMI_VRAM_TYPE_DDR4 @@ -3903,6 +3905,10 @@ def amdsmi_get_gpu_driver_info(processor_handle: processor_handle_t) -> Dict[str "driver_name": info.driver_name.decode("utf-8"), "driver_version": info.driver_version.decode("utf-8"), "driver_date": info.driver_date.decode("utf-8"), + "driver_kernel_version": info.driver_kernel_version.decode("utf-8"), + "amdgpu_driver_version": info.amdgpu_driver_version.decode("utf-8"), + "driver_build_version": info.driver_build_version.decode("utf-8"), + "driver_full_version": info.driver_full_version.decode("utf-8"), } for key, value in driver_info.items(): @@ -5370,6 +5376,32 @@ def amdsmi_get_node_handle(processor_handle): def amdsmi_get_npm_info(node_handle: processor_handle_t) -> Dict[str, Any]: + """ + Get NPM (Node Power Management) status and power limit info for the node + associated with node_handle. + + Parameters: + node_handle(node_handle_t): Node handle obtained via + amdsmi_get_node_handle() + + Returns: + `Dict[str, Any]`: Dictionary with keys: + - ``limit`` (`int`): Current NPM power limit in Watts. + - ``status`` (`int`): NPM status (enabled/disabled), see + ``amdsmi_npm_status_t``. + - ``ubb_power_threshold`` (`int`): UBB node power threshold in + Watts. + - ``max_node_power_limit`` (`int`): The platform max bound + consumed by amdsmi_set_npm_limit(): callers should ensure any + limit passed to that function does not exceed this value. + - ``current_node_power`` (`int`): The current (instantaneous) node + power in Watts (board/node_power), MI450+. Queried once per + node rather than once per GPU. + + Raises: + AmdSmiParameterException: If the node handle is invalid. + AmdSmiLibraryException: If the underlying library call fails. + """ if not isinstance(node_handle, amdsmi_wrapper.amdsmi_node_handle): raise AmdSmiParameterException(node_handle, amdsmi_wrapper.amdsmi_node_handle) @@ -5382,6 +5414,12 @@ def amdsmi_get_npm_info(node_handle: processor_handle_t) -> Dict[str, Any]: "ubb_power_threshold": _validate_if_max_uint( npm_info.ubb_power_threshold, MaxUIntegerTypes.UINT32_T ), + "max_node_power_limit": _validate_if_max_uint( + npm_info.max_node_power_limit, MaxUIntegerTypes.UINT64_T + ), + "current_node_power": _validate_if_max_uint( + npm_info.current_node_power, MaxUIntegerTypes.UINT32_T + ), } return dict_ret @@ -5414,6 +5452,71 @@ def amdsmi_get_tray_info( } +def amdsmi_set_npm_limit(node_handle: processor_handle_t, limit: int) -> None: + """ + Set the NPM (Node Power Management) power limit for the node associated + with node_handle by writing to the board's cur_node_power_limit sysfs + interface. + + This function rejects the request with AmdSmiLibraryException + (AMDSMI_STATUS_INVAL) if NPM is disabled on the node + (amdsmi_get_npm_info()'s "status" == AMDSMI_NPM_STATUS_DISABLED), since + writing board/cur_node_power_limit while NPM is disabled has no defined + effect. It also validates `limit` against the platform max bound + (amdsmi_get_npm_info()'s "max_node_power_limit", sourced from + board/max_node_power_limit) internally before ever issuing the write, + raising AmdSmiLibraryException with AMDSMI_STATUS_INVAL if `limit` is `0` + or greater than that bound. If the platform max bound itself cannot be + read (e.g. the sysfs interface is missing or returns unexpected data), + this function fails closed and raises that underlying error rather than + silently allowing an unbounded `limit` through. The amd-smi CLI's + `set --node-power-limit` additionally performs the same checks itself + ahead of calling this function, purely to fail fast and present a + friendlier, earlier user-facing error message; it is not the only + validation and is not required for correctness. + + Parameters: + node_handle(node_handle_t): Node handle obtained via + amdsmi_get_node_handle() + limit(int): New NPM power limit value to request (units match the + board/cur_node_power_limit sysfs interface). Must satisfy + 0 < limit <= UINT64_MAX; out-of-range values raise + AmdSmiParameterException rather than silently wrapping modulo + 2**64 the way a raw ctypes.c_uint64() conversion would. Values + that pass this local bound check but are `0` or exceed the + platform max are rejected by the underlying library call instead + (see Raises below). + + Returns: + None: This function raises an exception if the call did not succeed + (e.g. AmdSmiLibraryException with AMDSMI_STATUS_INVAL if `limit` is + `0` or exceeds the platform max bound, AMDSMI_STATUS_NOT_SUPPORTED if + the sysfs interface is unavailable, or AMDSMI_STATUS_NO_PERM if the + write was rejected) + + Raises: + AmdSmiParameterException: If `node_handle` or `limit` is the wrong + type, or `limit` is outside `0 < limit <= UINT64_MAX`. + AmdSmiLibraryException: If the underlying library call fails, + including AMDSMI_STATUS_INVAL when `limit` is `0` or exceeds the + platform max bound (or when that bound itself cannot be read). + """ + if not isinstance(node_handle, amdsmi_wrapper.amdsmi_node_handle): + raise AmdSmiParameterException(node_handle, amdsmi_wrapper.amdsmi_node_handle) + if not isinstance(limit, int): + raise AmdSmiParameterException(limit, int) + if not 0 < limit <= 0xFFFFFFFFFFFFFFFF: + # ctypes.c_uint64() silently wraps out-of-range values modulo 2**64 + # instead of raising (e.g. ctypes.c_uint64(2**64 + 12345).value == + # 12345); reject explicitly here so a bad value is never wrapped into + # an unrelated one and written to hardware. + raise AmdSmiParameterException( + limit, int, msg=f"limit must satisfy 0 < limit <= {0xFFFFFFFFFFFFFFFF} (got {limit})" + ) + + _check_res(amdsmi_wrapper.amdsmi_set_npm_limit(node_handle, ctypes.c_uint64(limit))) + + def amdsmi_get_temp_metric( processor_handle: processor_handle_t, sensor_type: AmdSmiTemperatureType, diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py index 6282ae64503f..85e3e61eb22b 100644 --- a/projects/amdsmi/py-interface/amdsmi_wrapper.py +++ b/projects/amdsmi/py-interface/amdsmi_wrapper.py @@ -909,6 +909,7 @@ class struct_amdsmi_hsmp_driver_version_t(Structure): 3: 'AMDSMI_VRAM_TYPE_HBM2E', 4: 'AMDSMI_VRAM_TYPE_HBM3', 5: 'AMDSMI_VRAM_TYPE_HBM3E', + 6: 'AMDSMI_VRAM_TYPE_HBM4', 10: 'AMDSMI_VRAM_TYPE_DDR2', 11: 'AMDSMI_VRAM_TYPE_DDR3', 12: 'AMDSMI_VRAM_TYPE_DDR4', @@ -930,6 +931,7 @@ class struct_amdsmi_hsmp_driver_version_t(Structure): AMDSMI_VRAM_TYPE_HBM2E = 3 AMDSMI_VRAM_TYPE_HBM3 = 4 AMDSMI_VRAM_TYPE_HBM3E = 5 +AMDSMI_VRAM_TYPE_HBM4 = 6 AMDSMI_VRAM_TYPE_DDR2 = 10 AMDSMI_VRAM_TYPE_DDR3 = 11 AMDSMI_VRAM_TYPE_DDR4 = 12 @@ -1504,6 +1506,10 @@ class struct_amdsmi_driver_info_t(Structure): ('driver_version', ctypes.c_char * 256), ('driver_date', ctypes.c_char * 256), ('driver_name', ctypes.c_char * 256), + ('driver_kernel_version', ctypes.c_char * 256), + ('amdgpu_driver_version', ctypes.c_char * 256), + ('driver_build_version', ctypes.c_char * 256), + ('driver_full_version', ctypes.c_char * 256), ] amdsmi_driver_info_t = struct_amdsmi_driver_info_t @@ -2726,7 +2732,10 @@ class struct_amdsmi_npm_info_t(Structure): ('limit', ctypes.c_uint64), ('ubb_power_threshold', ctypes.c_uint32), ('PADDING_1', ctypes.c_ubyte * 4), - ('reserved', ctypes.c_uint64 * 5), + ('max_node_power_limit', ctypes.c_uint64), + ('current_node_power', ctypes.c_uint32), + ('PADDING_2', ctypes.c_ubyte * 4), + ('reserved', ctypes.c_uint64 * 3), ] amdsmi_npm_info_t = struct_amdsmi_npm_info_t @@ -4345,6 +4354,12 @@ class struct_amdsmi_cper_hdr_t(Structure): amdsmi_get_tray_info.argtypes = [amdsmi_node_handle, ctypes.POINTER(struct_amdsmi_tray_info_t)] except AttributeError: pass +try: + amdsmi_set_npm_limit = _libraries['libamd_smi.so'].amdsmi_set_npm_limit + amdsmi_set_npm_limit.restype = amdsmi_status_t + amdsmi_set_npm_limit.argtypes = [amdsmi_node_handle, uint64_t] +except AttributeError: + pass try: amdsmi_get_fw_info = _libraries['libamd_smi.so'].amdsmi_get_fw_info amdsmi_get_fw_info.restype = amdsmi_status_t @@ -5313,12 +5328,12 @@ class struct_amdsmi_ttm_info_t(Structure): 'AMDSMI_VRAM_TYPE_GDDR6', 'AMDSMI_VRAM_TYPE_GDDR7', 'AMDSMI_VRAM_TYPE_HBM', 'AMDSMI_VRAM_TYPE_HBM2', 'AMDSMI_VRAM_TYPE_HBM2E', 'AMDSMI_VRAM_TYPE_HBM3', - 'AMDSMI_VRAM_TYPE_HBM3E', 'AMDSMI_VRAM_TYPE_LPDDR4', - 'AMDSMI_VRAM_TYPE_LPDDR5', 'AMDSMI_VRAM_TYPE_UNKNOWN', - 'AMDSMI_VRAM_TYPE__MAX', 'AMDSMI_WR_BW0', - 'AMDSMI_XGMI_LINK_DISABLE', 'AMDSMI_XGMI_LINK_DOWN', - 'AMDSMI_XGMI_LINK_UP', 'AMDSMI_XGMI_STATUS_ERROR', - 'AMDSMI_XGMI_STATUS_MULTIPLE_ERRORS', + 'AMDSMI_VRAM_TYPE_HBM3E', 'AMDSMI_VRAM_TYPE_HBM4', + 'AMDSMI_VRAM_TYPE_LPDDR4', 'AMDSMI_VRAM_TYPE_LPDDR5', + 'AMDSMI_VRAM_TYPE_UNKNOWN', 'AMDSMI_VRAM_TYPE__MAX', + 'AMDSMI_WR_BW0', 'AMDSMI_XGMI_LINK_DISABLE', + 'AMDSMI_XGMI_LINK_DOWN', 'AMDSMI_XGMI_LINK_UP', + 'AMDSMI_XGMI_STATUS_ERROR', 'AMDSMI_XGMI_STATUS_MULTIPLE_ERRORS', 'AMDSMI_XGMI_STATUS_NO_ERRORS', 'CLK_LIMIT_MAX', 'CLK_LIMIT_MIN', 'RD_BW0', 'WR_BW0', 'amd_metrics_table_header_t', 'amdsmi_accelerator_partition_mem_alloc_mode_t', @@ -5546,12 +5561,12 @@ class struct_amdsmi_ttm_info_t(Structure): 'amdsmi_set_gpu_perf_level', 'amdsmi_set_gpu_power_profile', 'amdsmi_set_gpu_process_isolation', 'amdsmi_set_gpu_ptl_formats', 'amdsmi_set_gpu_ptl_state', 'amdsmi_set_gpu_uma_carveout', - 'amdsmi_set_power_cap', 'amdsmi_set_soc_pstate', - 'amdsmi_set_ttm_pages_limit', 'amdsmi_set_xgmi_plpd', - 'amdsmi_shut_down', 'amdsmi_smu_fw_version_t', - 'amdsmi_sock_info_t', 'amdsmi_socket_handle', - 'amdsmi_status_code_to_string', 'amdsmi_status_t', - 'amdsmi_stop_gpu_event_notification', + 'amdsmi_set_npm_limit', 'amdsmi_set_power_cap', + 'amdsmi_set_soc_pstate', 'amdsmi_set_ttm_pages_limit', + 'amdsmi_set_xgmi_plpd', 'amdsmi_shut_down', + 'amdsmi_smu_fw_version_t', 'amdsmi_sock_info_t', + 'amdsmi_socket_handle', 'amdsmi_status_code_to_string', + 'amdsmi_status_t', 'amdsmi_stop_gpu_event_notification', 'amdsmi_temp_range_refresh_rate_t', 'amdsmi_temperature_metric_t', 'amdsmi_temperature_type_t', 'amdsmi_topo_get_link_type', 'amdsmi_topo_get_link_weight', 'amdsmi_topo_get_numa_node_number', diff --git a/projects/amdsmi/py-interface/pyproject.toml.in b/projects/amdsmi/py-interface/pyproject.toml.in index 156e3aa8bbb6..ec08da70c972 100644 --- a/projects/amdsmi/py-interface/pyproject.toml.in +++ b/projects/amdsmi/py-interface/pyproject.toml.in @@ -28,7 +28,9 @@ zip-safe = false include-package-data = true [tool.setuptools.package-data] -amdsmi = ["*.so"] +# Notices ship as package data, not PEP 639 `license-files`, which would need +# setuptools >= 77. Same glob the bundled .so already relies on. +amdsmi = ["*.so", "THIRD_PARTY_LICENSES/*"] [tool.pytest.ini_options] addopts = [ diff --git a/projects/amdsmi/pyproject.toml b/projects/amdsmi/pyproject.toml index fce157b0ce98..8ab409090455 100644 --- a/projects/amdsmi/pyproject.toml +++ b/projects/amdsmi/pyproject.toml @@ -1,5 +1,6 @@ [tool.ruff] line-length = 100 +target-version = "py37" extend-exclude = [ "build", diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h index 50b10d6b2aa0..e3b94cfbf7d6 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h @@ -621,10 +621,15 @@ typedef enum { RSMI_NPM_STATUS_DISABLED, RSMI_NPM_STATUS_ENABLED } rsmi_npm_stat * */ typedef struct { - rsmi_npm_status_t status; //!< NPM status (enabled/disabled). - uint64_t limit; //!< Node-level power limit in Watts. - uint32_t ubb_power_threshold; //!< The UBB node power threshold in Watts. - uint64_t reserved[5]; + rsmi_npm_status_t status; //!< NPM status (enabled/disabled). + uint64_t limit; //!< Node-level power limit in Watts. + uint32_t ubb_power_threshold; //!< The UBB node power threshold in Watts. + uint64_t max_node_power_limit; //!< Platform max node-level power limit in Watts + //!< (board/max_node_power_limit). + uint32_t current_node_power; //!< The current (instantaneous) node power in Watts + //!< (board/node_power). + uint64_t reserved[3]; //!< Reduced from reserved[4] to accommodate + //!< current_node_power. } rsmi_npm_info_t; /** @@ -3168,6 +3173,8 @@ rsmi_status_t rsmi_dev_fan_speed_max_get(uint32_t dv_ind, uint32_t sensor_ind, u rsmi_status_t rsmi_dev_npm_info_get(uint32_t dv_ind, uintptr_t node_handle, rsmi_npm_info_t* npm_info); +rsmi_status_t rsmi_dev_npm_limit_set(uint32_t dv_ind, uintptr_t node_handle, uint64_t limit); + rsmi_status_t rsmi_dev_baseboard_power_get(uint32_t dv_ind, uint64_t* power); /** diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_npm.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_npm.h index bdf0e3fff5f3..b44057b943d0 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_npm.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_npm.h @@ -14,8 +14,18 @@ namespace amd::smi { rsmi_status_t get_npm_board_status(const std::string& board_path, bool* enabled); rsmi_status_t get_npm_board_limit(const std::string& board_path, uint64_t* limit); +// Platform max node-level power limit query (board/max_node_power_limit), used to +// bound requests made via set_npm_board_limit()/rsmi_dev_npm_limit_set(). +rsmi_status_t get_npm_board_max_limit(const std::string& board_path, uint64_t* limit); + +// NPM board limit set (Set NPM Limit request to GPU PMFW via amdgpu driver) +rsmi_status_t set_npm_board_limit(const std::string& board_path, uint64_t limit); + // UBB (baseboard) power limit query rsmi_status_t get_ubb_power_limit(const std::string& board_path, uint64_t* limit); +// Current node power query (board/node_power) +rsmi_status_t get_npm_node_power(const std::string& board_path, uint64_t* power); + } // namespace amd::smi #endif // ROCM_SMI_INCLUDE_ROCM_SMI_ROCM_SMI_NPM_H_ diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi.cc b/projects/amdsmi/rocm_smi/src/rocm_smi.cc index df832b367f07..14810367a392 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi.cc @@ -3252,6 +3252,29 @@ rsmi_status_t rsmi_dev_npm_info_get(uint32_t dv_ind, uintptr_t node_handle, rsmi_status_t ubb_status = amd::smi::get_ubb_power_limit(*board_path_str, &ubb_power_threshold_raw); + // Get platform max node power limit (optional - don't fail if not available). + // This is descriptive metadata (used by callers, e.g. amd-smi CLI, to bound + // requests to rsmi_dev_npm_limit_set()), not on the critical read path. + uint64_t npm_max_limit = UINT64_MAX; + rsmi_status_t max_limit_status = + amd::smi::get_npm_board_max_limit(*board_path_str, &npm_max_limit); + if (max_limit_status != RSMI_STATUS_SUCCESS) { + ss << __PRETTY_FUNCTION__ << " | get_npm_board_max_limit returned " + << getRSMIStatusString(max_limit_status) << " ; using sentinel max limit"; + LOG_DEBUG(ss); + npm_max_limit = UINT64_MAX; + } + + // Get current node power (optional - don't fail if not available). This is + // descriptive telemetry, not on the critical read path. + uint64_t node_power_raw = UINT64_MAX; + rsmi_status_t node_power_status = amd::smi::get_npm_node_power(*board_path_str, &node_power_raw); + if (node_power_status != RSMI_STATUS_SUCCESS) { + ss << __PRETTY_FUNCTION__ << " | get_npm_node_power returned " + << getRSMIStatusString(node_power_status) << " ; using sentinel node power"; + LOG_DEBUG(ss); + } + // fill output std::memset(npm_info, 0, sizeof(*npm_info)); npm_info->status = npm_status ? RSMI_NPM_STATUS_ENABLED : RSMI_NPM_STATUS_DISABLED; @@ -3261,6 +3284,11 @@ rsmi_status_t rsmi_dev_npm_info_get(uint32_t dv_ind, uintptr_t node_handle, (ubb_status == RSMI_STATUS_SUCCESS && ubb_power_threshold_raw <= kU32Max) ? static_cast(ubb_power_threshold_raw) : std::numeric_limits::max(); + npm_info->max_node_power_limit = npm_max_limit; + npm_info->current_node_power = + (node_power_status == RSMI_STATUS_SUCCESS && node_power_raw <= kU32Max) + ? static_cast(node_power_raw) + : std::numeric_limits::max(); ss << __PRETTY_FUNCTION__ << " | ======= end ======= | returning " << getRSMIStatusString(RSMI_STATUS_SUCCESS); @@ -3269,6 +3297,85 @@ rsmi_status_t rsmi_dev_npm_info_get(uint32_t dv_ind, uintptr_t node_handle, CATCH } +rsmi_status_t rsmi_dev_npm_limit_set(uint32_t dv_ind, uintptr_t node_handle, uint64_t limit) { + TRY std::ostringstream ss; + ss << __PRETTY_FUNCTION__ << "| ======= start =======, dv_ind=" << dv_ind << ", limit=" << limit; + LOG_TRACE(ss); + + REQUIRE_ROOT_ACCESS + + CHECK_DV_IND_RANGE + + DEVICE_MUTEX + + if (node_handle == 0) { + ss << __PRETTY_FUNCTION__ << " | node_handle == 0 -> returning " + << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS); + LOG_ERROR(ss); + return RSMI_STATUS_INVALID_ARGS; + } + + std::string* board_path_str = reinterpret_cast(node_handle); + if (board_path_str == nullptr || board_path_str->empty()) { + ss << __PRETTY_FUNCTION__ << " | invalid/empty board path in node_handle"; + LOG_ERROR(ss); + return RSMI_STATUS_INVALID_ARGS; + } + + // Reject the write outright when NPM is disabled on this node: writing + // board/cur_node_power_limit has no defined effect in that state. This is + // the authoritative check -- the CLI's own pre-check (amdsmi_helpers.py) + // is a fail-fast convenience for that one caller, not a substitute for + // enforcing this here for every caller of rsmi_dev_npm_limit_set(). + bool npm_enabled = false; + rsmi_status_t status_ret = amd::smi::get_npm_board_status(*board_path_str, &npm_enabled); + if (status_ret != RSMI_STATUS_SUCCESS) { + ss << __PRETTY_FUNCTION__ + << " | get_npm_board_status failed: " << getRSMIStatusString(status_ret, false) + << " -> rejecting write (fail closed)"; + LOG_ERROR(ss); + return status_ret; + } + if (!npm_enabled) { + ss << __PRETTY_FUNCTION__ << " | NPM disabled on this node -> returning " + << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS); + LOG_ERROR(ss); + return RSMI_STATUS_INVALID_ARGS; + } + + // Mirror rsmi_dev_power_cap_set(): query the platform bound before ever + // touching sysfs, and reject out-of-range requests with + // RSMI_STATUS_INVALID_ARGS. Fail closed if the bound itself can't be + // read (e.g. sysfs missing/unexpected contents) -- propagate that error + // and do not fall through to the write, rather than silently allowing an + // unbounded value through, consistent with the CLI layer's own + // fail-closed handling of an unreadable platform max. + uint64_t max_limit = 0; + rsmi_status_t ret = amd::smi::get_npm_board_max_limit(*board_path_str, &max_limit); + if (ret != RSMI_STATUS_SUCCESS) { + ss << __PRETTY_FUNCTION__ + << " | get_npm_board_max_limit failed: " << getRSMIStatusString(ret, false) + << " -> rejecting write (fail closed)"; + LOG_ERROR(ss); + return ret; + } + + if (limit == 0 || limit > max_limit) { + ss << __PRETTY_FUNCTION__ << " | limit=" << limit << " out of range (valid range is " + << "1.." << max_limit << ") -> returning " << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS); + LOG_ERROR(ss); + return RSMI_STATUS_INVALID_ARGS; + } + + ret = amd::smi::set_npm_board_limit(*board_path_str, limit); + + ss << __PRETTY_FUNCTION__ << " | ======= end ======= | returning " + << getRSMIStatusString(ret, false); + LOG_TRACE(ss); + return ret; + CATCH +} + rsmi_status_t rsmi_dev_baseboard_power_get(uint32_t dv_ind, uint64_t* power) { TRY std::ostringstream ss; ss << __PRETTY_FUNCTION__ << "| ======= start ======="; diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi_gpu_metrics.cc b/projects/amdsmi/rocm_smi/src/rocm_smi_gpu_metrics.cc index 7a1922fef505..170dc35c4bca 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi_gpu_metrics.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi_gpu_metrics.cc @@ -2446,7 +2446,24 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBaseDynamic_t::copy_internal_to_externa using Dst = std::remove_reference_t; using T = std::remove_cv_t>; auto v = std::get_if>(&r.m_value); - if (!v) return; // Not a vector type, skip + if (!v) { + // A single-instance row parses as a scalar (MI450's one xGMI data + // counter); it is element 0. The driver may declare another width, so + // keep its all-ones "not reported", and values that do not fit, unset. + std::visit( + [&](const auto& x) { + using S = std::decay_t; + if constexpr (std::is_unsigned_v) { + bool unset = x == std::numeric_limits::max(); + if constexpr (sizeof(S) > sizeof(T)) { + unset = unset || x > std::numeric_limits::max(); + } + dst[0] = unset ? init_max_uint_types() : static_cast(x); + } + }, + r.m_value); + return; + } const std::size_t n = std::min(v->size(), cap); std::copy_n(v->data(), n, dst); }; diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi_npm.cc b/projects/amdsmi/rocm_smi/src/rocm_smi_npm.cc index 179501aa92bc..45ce608b907e 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi_npm.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi_npm.cc @@ -3,6 +3,7 @@ #include "rocm_smi/rocm_smi_npm.h" +#include #include #include #include @@ -70,6 +71,19 @@ static rsmi_status_t read_board_uint64(const std::string& board_path, const char rsmi_status_t r = read_npm_file(p, s); if (r != RSMI_STATUS_SUCCESS) return RSMI_STATUS_NOT_SUPPORTED; + // std::stoull() accepts a leading '-' and silently negates modulo 2^64 + // (e.g. "-1" successfully parses as UINT64_MAX with idx == s.size()), + // which is well-defined C++ behavior but would let corrupted/negative + // sysfs content masquerade as a huge-but-"valid" unsigned value instead of + // failing to parse. Reject anything that doesn't start with a digit + // before ever calling std::stoull() so negative/garbage content is always + // treated as a parse error. + // Behavior change: negative/malformed sysfs content previously parsed + // successfully as UINT64_MAX; it now returns RSMI_STATUS_UNEXPECTED_DATA. + if (s.empty() || !std::isdigit(static_cast(s[0]))) { + return RSMI_STATUS_UNEXPECTED_DATA; + } + try { size_t idx = 0; unsigned long long v = std::stoull(s, &idx, 10); @@ -87,8 +101,37 @@ rsmi_status_t get_npm_board_limit(const std::string& board_path, uint64_t* limit return read_board_uint64(board_path, "cur_node_power_limit", limit); } +rsmi_status_t get_npm_board_max_limit(const std::string& board_path, uint64_t* limit) { + return read_board_uint64(board_path, "max_node_power_limit", limit); +} + +rsmi_status_t set_npm_board_limit(const std::string& board_path, uint64_t limit) { + if (board_path.empty()) return RSMI_STATUS_INVALID_ARGS; + + fs::path bd(board_path); + if (!fs::exists(bd) || !fs::is_directory(bd)) return RSMI_STATUS_NOT_SUPPORTED; + + fs::path p = bd / "cur_node_power_limit"; + if (!fs::exists(p) || !fs::is_regular_file(p)) return RSMI_STATUS_NOT_SUPPORTED; + + // Write the numeric limit value as text to the sysfs file. This mirrors the + // WriteSysfsStr()-based pattern used elsewhere in rocm_smi for other + // write-capable attributes (e.g. power cap set path), which in turn + // triggers a Set NPM Limit request to GPU PMFW via the amdgpu driver. + int ret = WriteSysfsStr(p.string(), std::to_string(limit)); + // If the sysfs file doesn't exist, treat as not supported. + if (ret == ENOENT) { + return RSMI_STATUS_NOT_SUPPORTED; + } + return ErrnoToRsmiStatus(ret); +} + rsmi_status_t get_ubb_power_limit(const std::string& board_path, uint64_t* limit) { return read_board_uint64(board_path, "baseboard_power_limit", limit); } +rsmi_status_t get_npm_node_power(const std::string& board_path, uint64_t* power) { + return read_board_uint64(board_path, "node_power", power); +} + } // namespace amd::smi diff --git a/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs b/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs index de87274527ab..ef4f5db237a7 100644 --- a/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs +++ b/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs @@ -518,6 +518,7 @@ pub enum AmdsmiVramTypeT { AmdsmiVramTypeHbm2e = 3, AmdsmiVramTypeHbm3 = 4, AmdsmiVramTypeHbm3e = 5, + AmdsmiVramTypeHbm4 = 6, AmdsmiVramTypeDdr2 = 10, AmdsmiVramTypeDdr3 = 11, AmdsmiVramTypeDdr4 = 12, @@ -1680,10 +1681,14 @@ pub struct AmdsmiDriverInfoT { pub driver_version: [::std::os::raw::c_char; 256usize], pub driver_date: [::std::os::raw::c_char; 256usize], pub driver_name: [::std::os::raw::c_char; 256usize], + pub driver_kernel_version: [::std::os::raw::c_char; 256usize], + pub amdgpu_driver_version: [::std::os::raw::c_char; 256usize], + pub driver_build_version: [::std::os::raw::c_char; 256usize], + pub driver_full_version: [::std::os::raw::c_char; 256usize], } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of AmdsmiDriverInfoT"][::std::mem::size_of::() - 768usize]; + ["Size of AmdsmiDriverInfoT"][::std::mem::size_of::() - 1792usize]; ["Alignment of AmdsmiDriverInfoT"][::std::mem::align_of::() - 1usize]; ["Offset of field: AmdsmiDriverInfoT::driver_version"] [::std::mem::offset_of!(AmdsmiDriverInfoT, driver_version) - 0usize]; @@ -1691,6 +1696,14 @@ const _: () = { [::std::mem::offset_of!(AmdsmiDriverInfoT, driver_date) - 256usize]; ["Offset of field: AmdsmiDriverInfoT::driver_name"] [::std::mem::offset_of!(AmdsmiDriverInfoT, driver_name) - 512usize]; + ["Offset of field: AmdsmiDriverInfoT::driver_kernel_version"] + [::std::mem::offset_of!(AmdsmiDriverInfoT, driver_kernel_version) - 768usize]; + ["Offset of field: AmdsmiDriverInfoT::amdgpu_driver_version"] + [::std::mem::offset_of!(AmdsmiDriverInfoT, amdgpu_driver_version) - 1024usize]; + ["Offset of field: AmdsmiDriverInfoT::driver_build_version"] + [::std::mem::offset_of!(AmdsmiDriverInfoT, driver_build_version) - 1280usize]; + ["Offset of field: AmdsmiDriverInfoT::driver_full_version"] + [::std::mem::offset_of!(AmdsmiDriverInfoT, driver_full_version) - 1536usize]; }; #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -3409,7 +3422,9 @@ pub struct AmdsmiNpmInfoT { pub status: AmdsmiNpmStatusT, pub limit: u64, pub ubb_power_threshold: u32, - pub reserved: [u64; 5usize], + pub max_node_power_limit: u64, + pub current_node_power: u32, + pub reserved: [u64; 3usize], } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { @@ -3421,8 +3436,12 @@ const _: () = { [::std::mem::offset_of!(AmdsmiNpmInfoT, limit) - 8usize]; ["Offset of field: AmdsmiNpmInfoT::ubb_power_threshold"] [::std::mem::offset_of!(AmdsmiNpmInfoT, ubb_power_threshold) - 16usize]; + ["Offset of field: AmdsmiNpmInfoT::max_node_power_limit"] + [::std::mem::offset_of!(AmdsmiNpmInfoT, max_node_power_limit) - 24usize]; + ["Offset of field: AmdsmiNpmInfoT::current_node_power"] + [::std::mem::offset_of!(AmdsmiNpmInfoT, current_node_power) - 32usize]; ["Offset of field: AmdsmiNpmInfoT::reserved"] - [::std::mem::offset_of!(AmdsmiNpmInfoT, reserved) - 24usize]; + [::std::mem::offset_of!(AmdsmiNpmInfoT, reserved) - 40usize]; }; #[repr(u32)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -5145,6 +5164,9 @@ extern "C" { info: *mut AmdsmiTrayInfoT, ) -> AmdsmiStatusT; } +extern "C" { + pub fn amdsmi_set_npm_limit(node_handle: AmdsmiNodeHandle, limit: u64) -> AmdsmiStatusT; +} extern "C" { pub fn amdsmi_get_fw_info( processor_handle: AmdsmiProcessorHandle, diff --git a/projects/amdsmi/rust-interface/src/utils.rs b/projects/amdsmi/rust-interface/src/utils.rs index 3bde4a563b28..4fb42e25af26 100644 --- a/projects/amdsmi/rust-interface/src/utils.rs +++ b/projects/amdsmi/rust-interface/src/utils.rs @@ -156,7 +156,16 @@ impl_cstr_getters!( ); // Implement the getters for the C string fields in AmdsmiDriverInfoT -impl_cstr_getters!(AmdsmiDriverInfoT, driver_version, driver_date, driver_name); +impl_cstr_getters!( + AmdsmiDriverInfoT, + driver_version, + driver_date, + driver_name, + driver_kernel_version, + amdgpu_driver_version, + driver_build_version, + driver_full_version +); // Implement the getters for the C string fields in AmdsmiNameValueT impl_cstr_getters!(AmdsmiNameValueT, name); diff --git a/projects/amdsmi/src/CMakeLists.txt b/projects/amdsmi/src/CMakeLists.txt index bd01bef0818e..7107e74d47ba 100644 --- a/projects/amdsmi/src/CMakeLists.txt +++ b/projects/amdsmi/src/CMakeLists.txt @@ -216,6 +216,12 @@ function(amdsmi_configure_common_target _TARGET) target_include_directories(${_TARGET} PRIVATE ${AMDCUID_INCLUDE_DIR}) endif() + # Compiles in test-only escape hatches (e.g. amdsmi_test_register_node_handle()) + # that the GTest suite needs but that must not ship in production builds. + if(BUILD_TESTS) + target_compile_definitions(${_TARGET} PRIVATE BUILD_TESTS) + endif() + target_link_directories(${_TARGET} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/nic/ai-nic/amdsmi_unified/build/) endfunction() diff --git a/projects/amdsmi/src/amd_smi/amd_smi.cc b/projects/amdsmi/src/amd_smi/amd_smi.cc index 91887e422539..9f1905a8620d 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi.cc @@ -647,6 +647,39 @@ amdsmi_status_t amdsmi_get_switch_processor_handles(amdsmi_socket_handle socket_ return AMDSMI_STATUS_SUCCESS; } +// Registry of amdsmi_node_handle values that were actually handed out by +// amdsmi_get_node_handle() (or, in unit tests only, injected via +// amdsmi_test_register_node_handle() -- see amd_smi_test_internal.h). An +// amdsmi_node_handle is internally a std::string* to a board sysfs path, but +// callers only ever see it as an opaque void*; a caller can pass any +// garbage-but-nonzero value through the public API. is_registered_node_handle() +// lets consumers of a node_handle (amdsmi_get_npm_info(), +// amdsmi_set_npm_limit()) confirm the pointer is one this library actually +// vended *before* reinterpret_cast'ing and dereferencing it, so an +// unregistered/garbage handle is rejected as AMDSMI_STATUS_INVAL instead of +// being dereferenced as an arbitrary pointer. +// +// g_node_registered_pointers is checked by pointer identity only (no +// dereference of the candidate handle), which is what makes the check safe +// to run on a completely untrusted node_handle value. It is kept separate +// from (but updated alongside) g_node_registry: the latter owns the +// std::string storage for handles amdsmi_get_node_handle() itself allocated, +// while the pointer set also accommodates test-injected handles that this +// library does not own. +namespace { +std::mutex g_node_mu; +std::map> g_node_registry; +std::set g_node_registered_pointers; + +bool is_registered_node_handle(amdsmi_node_handle node_handle) { + if (node_handle == nullptr) { + return false; + } + std::lock_guard lk(g_node_mu); + return g_node_registered_pointers.find(node_handle) != g_node_registered_pointers.end(); +} +} // namespace + amdsmi_status_t amdsmi_get_node_handle(amdsmi_processor_handle processor_handle, amdsmi_node_handle* node_handle) { AMDSMI_CHECK_INIT(); @@ -698,9 +731,10 @@ amdsmi_status_t amdsmi_get_node_handle(amdsmi_processor_handle processor_handle, } // Store board path so node handle remains valid for library lifetime. - static std::mutex g_node_mu; - static std::map> g_node_registry; - + // g_node_mu/g_node_registry/g_node_registered_pointers are file-scope (see + // the anonymous namespace above amdsmi_get_node_handle()) so that + // is_registered_node_handle() can validate a node_handle from any caller, + // not just from within this function. std::string board_path = found_board.string(); { std::lock_guard lk(g_node_mu); @@ -709,6 +743,7 @@ amdsmi_status_t amdsmi_get_node_handle(amdsmi_processor_handle processor_handle, auto ptr = std::make_unique(board_path); amdsmi_node_handle h = reinterpret_cast(ptr.get()); g_node_registry.emplace(board_path, std::move(ptr)); + g_node_registered_pointers.insert(h); *node_handle = h; } else { *node_handle = reinterpret_cast(it->second.get()); @@ -718,6 +753,28 @@ amdsmi_status_t amdsmi_get_node_handle(amdsmi_processor_handle processor_handle, return AMDSMI_STATUS_SUCCESS; } +// Test-only wrapper: registers a raw, test-fabricated amdsmi_node_handle as +// valid in the same registry amdsmi_get_node_handle() populates, so that +// is_registered_node_handle() (consulted by amdsmi_get_npm_info() and +// amdsmi_set_npm_limit()) accepts it. Not declared in amdsmi.h -- internal +// use via amd_smi_test_internal.h only. See that header for why this exists: +// unit tests fabricate node_handle values directly (bypassing +// amdsmi_get_node_handle()'s oam_id == 0 gate, which the test environment +// cannot satisfy) and need a way to make that fabricated pointer pass the +// production registered-handle check without weakening the check itself. +// Guarded by BUILD_TESTS so this handle-forging escape hatch never ships in +// a production build (see amd_smi_test_internal.h and src/CMakeLists.txt). +#ifdef BUILD_TESTS +amdsmi_status_t amdsmi_test_register_node_handle(amdsmi_node_handle node_handle) { + if (node_handle == nullptr) { + return AMDSMI_STATUS_INVAL; + } + std::lock_guard lk(g_node_mu); + g_node_registered_pointers.insert(node_handle); + return AMDSMI_STATUS_SUCCESS; +} +#endif // BUILD_TESTS + amdsmi_status_t amdsmi_get_processor_count_from_handles(amdsmi_processor_handle* processor_handles, uint32_t* processor_count, uint32_t* nr_cpusockets, @@ -1532,10 +1589,9 @@ amdsmi_status_t amdsmi_get_gpu_board_info(amdsmi_processor_handle processor_hand std::ostringstream ss; ss << __PRETTY_FUNCTION__ << "[Before rocm smi correction] " - << "Returning status = AMDSMI_STATUS_SUCCESS" - << "\n; info->model_number: |" << board_info->model_number << "|" - << "\n; info->product_serial: |" << board_info->product_serial << "|" - << "\n; info->fru_id: |" << board_info->fru_id << "|" + << "Returning status = AMDSMI_STATUS_SUCCESS" << "\n; info->model_number: |" + << board_info->model_number << "|" << "\n; info->product_serial: |" + << board_info->product_serial << "|" << "\n; info->fru_id: |" << board_info->fru_id << "|" << "\n; info->manufacturer_name: |" << board_info->manufacturer_name << "|" << "\n; info->product_name: |" << board_info->product_name << "|"; LOG_INFO(ss); @@ -1586,10 +1642,9 @@ amdsmi_status_t amdsmi_get_gpu_board_info(amdsmi_processor_handle processor_hand } ss << __PRETTY_FUNCTION__ << " | [After rocm smi correction] " - << "Returning status = AMDSMI_STATUS_SUCCESS" - << "\n; info->model_number: |" << board_info->model_number << "|" - << "\n; info->product_serial: |" << board_info->product_serial << "|" - << "\n; info->fru_id: |" << board_info->fru_id << "|" + << "Returning status = AMDSMI_STATUS_SUCCESS" << "\n; info->model_number: |" + << board_info->model_number << "|" << "\n; info->product_serial: |" + << board_info->product_serial << "|" << "\n; info->fru_id: |" << board_info->fru_id << "|" << "\n; info->manufacturer_name: |" << board_info->manufacturer_name << "|" << "\n; info->product_name: |" << board_info->product_name << "|"; LOG_INFO(ss); @@ -1675,6 +1730,25 @@ amdsmi_status_t amdsmi_get_temp_metric(amdsmi_processor_handle processor_handle, return amdsmi_status; } +// amdsmi_get_npm_info() memcpy's rsmi_npm_info_t directly into +// amdsmi_npm_info_t; keep their shared fields byte-compatible so that copy +// stays correct if either struct changes. +static_assert(sizeof(amdsmi_npm_info_t) == sizeof(rsmi_npm_info_t), + "amdsmi_npm_info_t and rsmi_npm_info_t must stay the same size"); +static_assert(offsetof(amdsmi_npm_info_t, status) == offsetof(rsmi_npm_info_t, status), + "status offset mismatch between amdsmi_npm_info_t and rsmi_npm_info_t"); +static_assert(offsetof(amdsmi_npm_info_t, limit) == offsetof(rsmi_npm_info_t, limit), + "limit offset mismatch between amdsmi_npm_info_t and rsmi_npm_info_t"); +static_assert(offsetof(amdsmi_npm_info_t, ubb_power_threshold) == + offsetof(rsmi_npm_info_t, ubb_power_threshold), + "ubb_power_threshold offset mismatch between amdsmi_npm_info_t and rsmi_npm_info_t"); +static_assert(offsetof(amdsmi_npm_info_t, max_node_power_limit) == + offsetof(rsmi_npm_info_t, max_node_power_limit), + "max_node_power_limit offset mismatch between amdsmi_npm_info_t and rsmi_npm_info_t"); +static_assert(offsetof(amdsmi_npm_info_t, current_node_power) == + offsetof(rsmi_npm_info_t, current_node_power), + "current_node_power offset mismatch between amdsmi_npm_info_t and rsmi_npm_info_t"); + amdsmi_status_t amdsmi_get_npm_info(amdsmi_node_handle node_handle, amdsmi_npm_info_t* npm_info) { AMDSMI_CHECK_INIT(); @@ -1682,6 +1756,19 @@ amdsmi_status_t amdsmi_get_npm_info(amdsmi_node_handle node_handle, amdsmi_npm_i return AMDSMI_STATUS_INVAL; } + // Reject any node_handle this library did not itself hand out (via + // amdsmi_get_node_handle(), or a test-registered stand-in -- see + // is_registered_node_handle()) *before* the cast/dereference below: an + // unprivileged caller could otherwise pass an arbitrary non-null value and + // have it dereferenced as a std::string* here, ahead of any privilege + // check further down the call chain. + // Behavior change: previously any non-null node_handle was dereferenced + // here; a caller-supplied garbage handle now fails with + // AMDSMI_STATUS_INVAL instead of crashing. + if (!is_registered_node_handle(node_handle)) { + return AMDSMI_STATUS_INVAL; + } + // Verify board path from node_handle auto board_path_str = reinterpret_cast(node_handle); if (board_path_str == nullptr || board_path_str->empty()) { @@ -1710,6 +1797,41 @@ amdsmi_status_t amdsmi_get_npm_info(amdsmi_node_handle node_handle, amdsmi_npm_i return AMDSMI_STATUS_SUCCESS; } +amdsmi_status_t amdsmi_set_npm_limit(amdsmi_node_handle node_handle, uint64_t limit) { + AMDSMI_CHECK_INIT(); + + if (node_handle == nullptr) { + return AMDSMI_STATUS_INVAL; + } + + // Reject any node_handle this library did not itself hand out (via + // amdsmi_get_node_handle(), or a test-registered stand-in -- see + // is_registered_node_handle()) *before* the cast/dereference below: an + // unprivileged caller could otherwise pass an arbitrary non-null value and + // have it dereferenced as a std::string* here, ahead of the + // REQUIRE_ROOT_ACCESS privilege check that only exists one layer down in + // rsmi_dev_npm_limit_set(). + if (!is_registered_node_handle(node_handle)) { + return AMDSMI_STATUS_INVAL; + } + + // Verify board path from node_handle + auto board_path_str = reinterpret_cast(node_handle); + if (board_path_str == nullptr || board_path_str->empty()) { + return AMDSMI_STATUS_INVAL; + } + +#ifdef ENABLE_WSL_BACKEND + if (amd::smi::WSLGPUBackend::IsActive()) { + return AMDSMI_STATUS_NOT_SUPPORTED; + } +#endif + + rsmi_status_t rstatus = + rsmi_dev_npm_limit_set(0, reinterpret_cast(node_handle), limit); + return amd::smi::rsmi_to_amdsmi_status(rstatus); +} + amdsmi_status_t amdsmi_get_gpu_vram_usage(amdsmi_processor_handle processor_handle, amdsmi_vram_usage_t* vram_info) { AMDSMI_CHECK_INIT(); @@ -1836,14 +1958,14 @@ static void system_wait(int milli_seconds) { // 1 ms = 1000 us int waitTime = milli_seconds * 1000; - ss << __PRETTY_FUNCTION__ << " | " - << "** Waiting for " << std::dec << waitTime << " us (" << waitTime / 1000 << " seconds) **"; + ss << __PRETTY_FUNCTION__ << " | " << "** Waiting for " << std::dec << waitTime << " us (" + << waitTime / 1000 << " seconds) **"; LOG_DEBUG(ss); usleep(static_cast(waitTime)); auto stop = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(stop - start); - ss << __PRETTY_FUNCTION__ << " | " - << "** Waiting took " << duration.count() / 1000 << " milli-seconds **"; + ss << __PRETTY_FUNCTION__ << " | " << "** Waiting took " << duration.count() / 1000 + << " milli-seconds **"; LOG_DEBUG(ss); } @@ -2080,9 +2202,9 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha } else { violation_status->active_ppt_pwr = 0; } - ss << __PRETTY_FUNCTION__ << " | " - << "ENTERED ppt_residency_acc | per_ppt_pwr: " << std::dec << violation_status->per_ppt_pwr - << "%; active_ppt_pwr = " << std::dec << violation_status->active_ppt_pwr << "\n"; + ss << __PRETTY_FUNCTION__ << " | " << "ENTERED ppt_residency_acc | per_ppt_pwr: " << std::dec + << violation_status->per_ppt_pwr << "%; active_ppt_pwr = " << std::dec + << violation_status->active_ppt_pwr << "\n"; LOG_DEBUG(ss); } if ((metric_info_b.socket_thm_residency_acc != std::numeric_limits::max() || @@ -2117,8 +2239,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha } else { violation_status->active_vr_thrm = 0; } - ss << __PRETTY_FUNCTION__ << " | " - << "ENTERED vr_thm_residency_acc | per_vr_thrm: " << std::dec + ss << __PRETTY_FUNCTION__ << " | " << "ENTERED vr_thm_residency_acc | per_vr_thrm: " << std::dec << violation_status->per_vr_thrm << "%; active_ppt_pwr = " << std::dec << violation_status->active_vr_thrm << "\n"; LOG_DEBUG(ss); @@ -2247,8 +2368,7 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha std::ref(violation_status->active_low_utilization), std::ref(violation_status->active_gfx_clk_below_host_limit_total)}); - ss << __PRETTY_FUNCTION__ << " | " - << "RETURNING AMDSMI_STATUS_SUCCESS | " + ss << __PRETTY_FUNCTION__ << " | " << "RETURNING AMDSMI_STATUS_SUCCESS | " << "violation_status->reference_timestamp (time since epoch): " << std::dec << violation_status->reference_timestamp << "; violation_status->violation_timestamp (ms): " << std::dec @@ -3357,8 +3477,7 @@ amdsmi_status_t amdsmi_get_gpu_memory_partition_config(amdsmi_processor_handle p 0, memory_caps, kLenCapsSize); ss << __PRETTY_FUNCTION__ << " | rsmi_dev_memory_partition_capabilities_get Returning: " << smi_amdgpu_get_status_string(status_mem_caps, false) - << " | Type: memory_partition_capabilities" - << " | Data: " << memory_caps; + << " | Type: memory_partition_capabilities" << " | Data: " << memory_caps; LOG_DEBUG(ss); std::string memory_caps_str = "N/A"; if (status_mem_caps == AMDSMI_STATUS_SUCCESS) { // older kernels may not support this @@ -4091,8 +4210,7 @@ amdsmi_status_t amdsmi_set_gpu_accelerator_partition_profile( partition_type_str = it->second; } - ss << __PRETTY_FUNCTION__ << " | " - << "config.profiles[" << i + ss << __PRETTY_FUNCTION__ << " | " << "config.profiles[" << i << "].profile_type: " << static_cast(config.profiles[i].profile_type) << "\n" << "| config.profiles[" << i << "].profile_type (str): " << partition_type_str << "\n" << "| config.profiles[" << i @@ -5670,10 +5788,22 @@ amdsmi_status_t amdsmi_get_gpu_driver_info(amdsmi_processor_handle processor_han return AMDSMI_STATUS_INVAL; } + std::memset(info, 0, sizeof(*info)); int length = AMDSMI_MAX_STRING_LENGTH; + char module_version[AMDSMI_MAX_STRING_LENGTH] = {}; // Get the driver version - status = smi_amdgpu_get_driver_version(gpu_device, &length, info->driver_version); + status = smi_amdgpu_get_driver_version(gpu_device, &length, module_version); + if (status != AMDSMI_STATUS_SUCCESS) { + return status; + } + + auto package_version = std::string{}; + auto uts = utsname{}; + if (uname(&uts) == 0) { + smi_amdgpu_get_active_dkms_version(kAmdgpuDkmsRoot, uts.release, uts.machine, &package_version); + } + status = smi_amdgpu_parse_driver_versions(module_version, package_version, info); if (status != AMDSMI_STATUS_SUCCESS) { return status; } @@ -5751,7 +5881,11 @@ amdsmi_status_t amdsmi_get_gpu_driver_info(amdsmi_processor_handle processor_han snprintf(info->driver_name, AMDSMI_MAX_STRING_LENGTH, "%s", driver_name.c_str()); drm_free_version(version); libdrm.unload(); - ss << __PRETTY_FUNCTION__ << " | Driver version: " << info->driver_version << "\n" + ss << __PRETTY_FUNCTION__ << " | Driver kernel version: " << info->driver_kernel_version << "\n" + << " | amdgpu driver version: " << info->amdgpu_driver_version << "\n" + << " | Driver version: " << info->driver_version << "\n" + << " | Driver build version: " << info->driver_build_version << "\n" + << " | Driver full version: " << info->driver_full_version << "\n" << " | Driver date: " << info->driver_date << "\n" << " | Driver name: " << info->driver_name << "\n" << " | Returning: " << smi_amdgpu_get_status_string(status, false); diff --git a/projects/amdsmi/src/amd_smi/amd_smi_cper.cc b/projects/amdsmi/src/amd_smi/amd_smi_cper.cc index 45c08a44f46a..998958c7fff6 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi_cper.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi_cper.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -48,8 +49,13 @@ class CperReader { template const Tp* at(size_t offset) const { - return is_buffer_fit(offset, sizeof(Tp)) ? reinterpret_cast(base_ + offset) - : nullptr; + return at(offset, sizeof(Tp)); + } + + // For a Tp whose trailing union the record only fills in part. + template + const Tp* at(size_t offset, size_t extent) const { + return is_buffer_fit(offset, extent) ? reinterpret_cast(base_ + offset) : nullptr; } private: @@ -371,6 +377,15 @@ static int cper_dump_nonstd_err(const struct cper_sec_nonstd_err* nonstd_err, section->revision_major, body->err_ctx.reg_ctx_type); } +// Crashdump sections end at whichever dump member amdgpu wrote: runtime fatal +// records stop after fatal_err, 32 bytes short of sizeof(cper_sec_crashdump). +constexpr size_t kCrashdumpDumpOffset = + (offsetof(struct cper_sec_crashdump, data) + offsetof(struct cper_sec_crashdump_data, dump)); +constexpr size_t kFatalCrashdumpExtent = + (kCrashdumpDumpOffset + sizeof(cper_sec_crashdump_data{}.dump.fatal_err)); +constexpr size_t kBootCrashdumpExtent = + (kCrashdumpDumpOffset + sizeof(cper_sec_crashdump_data{}.dump.boot_err)); + static int cper_dump_cr_fatal(const struct cper_sec_crashdump* crashdump, const cper_sec_desc* section) { std::ostringstream ss; @@ -637,13 +652,14 @@ std::vector cper_decode(const amdsmi_cper_hdr_t* cper, size_t buf_size) { int afid = -1; if (cper_is_cr(sec_guid)) { - const struct cper_sec_crashdump* crashdump = - rec.at(section->sec_offset); + const bool is_boot = cper_is_bt(cper_guid); + const struct cper_sec_crashdump* crashdump = rec.at( + section->sec_offset, is_boot ? kBootCrashdumpExtent : kFatalCrashdumpExtent); if (!crashdump) { ss << __PRETTY_FUNCTION__ << "\n:" << __LINE__ << "[AFIDS] crash dump section: " << i << " out of bounds; skipping\n"; LOG_ERROR(ss); - } else if (cper_is_bt(cper_guid)) { + } else if (is_boot) { ss << __PRETTY_FUNCTION__ << "\n:" << __LINE__ << "[AFIDS] decoding boot crash dump\n"; LOG_DEBUG(ss); afid = cper_dump_cr_boot(crashdump, section); diff --git a/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc b/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc index f04c7db5d11c..f10d5627a86f 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc @@ -174,7 +174,10 @@ struct ComputeProcessCache { }; std::unordered_map process_info_cache_map; -std::unordered_map compute_process_cache_map; +// Never destroyed: a thread can still be inside get_compute_process_list_impl() +// while the library's static destructors run at exit. +auto& compute_process_cache_map = + *new std::unordered_map>(); std::mutex compute_process_list_mutex; static const std::chrono::milliseconds kComputeProcessCacheDuration = std::chrono::milliseconds(read_env_ms("AMDSMI_PROCESS_INFO_CACHE_MS", 1)); @@ -187,9 +190,9 @@ int32_t AMDSmiGPUDevice::get_compute_process_list_impl( { std::lock_guard lock(compute_process_list_mutex); if (compute_process_cache_map.find(gpu_id_) == compute_process_cache_map.end()) { - compute_process_cache_map[gpu_id_] = new ComputeProcessCache(); + compute_process_cache_map[gpu_id_] = std::make_unique(); } - cache_ptr = compute_process_cache_map[gpu_id_]; + cache_ptr = compute_process_cache_map[gpu_id_].get(); } /** diff --git a/projects/amdsmi/src/amd_smi/amd_smi_utils.cc b/projects/amdsmi/src/amd_smi/amd_smi_utils.cc index 0bfc35e21e25..ed30488d4dcd 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi_utils.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi_utils.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -226,8 +227,7 @@ amdsmi_status_t smi_amdgpu_get_board_info(amd::smi::AMDSmiGPUDevice* device, AMDSMI_MAX_STRING_LENGTH, false); std::ostringstream ss; - ss << __PRETTY_FUNCTION__ << "[Before correction] " - << "Returning status = AMDSMI_STATUS_SUCCESS" + ss << __PRETTY_FUNCTION__ << "[Before correction] " << "Returning status = AMDSMI_STATUS_SUCCESS" << " | model_number_path = |" << model_number_path << "|\n" << "; info->model_number: |" << info->model_number << "|\n" << "; ret_mod = " << ret_mod << "|\n" @@ -241,8 +241,8 @@ amdsmi_status_t smi_amdgpu_get_board_info(amd::smi::AMDSmiGPUDevice* device, << "; info->manufacturer_name: |" << info->manufacturer_name << "|\n" << "; ret_man = " << ret_man << "|\n" << "\n product_name_path = |" << product_name_path << "|\n" - << "; info->product_name: |" << info->product_name << "|" - << "; ret_prod = " << ret_prod << "|\n"; + << "; info->product_name: |" << info->product_name << "|" << "; ret_prod = " << ret_prod + << "|\n"; LOG_INFO(ss); return AMDSMI_STATUS_SUCCESS; @@ -1478,3 +1478,279 @@ void init_asic_info_defaults(amdsmi_asic_info_t* info) { info->chip_rev_id = std::numeric_limits::max(); info->external_rev_id = std::numeric_limits::max(); } + +namespace { + +namespace fs = std::filesystem; + +/** DKMS registers amdgpu under /var/lib/dkms/amdgpu//; package + * metadata lives in source/dkms.conf behind the source symlink + */ +constexpr std::string_view kAmdgpuDkmsPackageName = "amdgpu"; +constexpr std::string_view kAmdgpuDkmsSourceSymlinkName = "source"; +constexpr std::string_view kDkmsAmdgpuKernelPrefix = "kernel-"; +constexpr std::string_view kDkmsAmdgpuDkmsConfName = "dkms.conf"; +constexpr std::string_view kDkmsAmdgpuDkmsConfPackageName = "PACKAGE_NAME"; +constexpr std::string_view kDkmsAmdgpuDkmsConfPackageVersion = "PACKAGE_VERSION"; + +// Ubuntu appends "." (e.g. "-2370381.24.04"); other distros' +// PACKAGE_VERSION may end at the build number, so only that part is required. +const auto kAmdgpuDkmsVersionDirRegex = std::regex{R"(^\d+\.\d+\.\d+-\d+(\..*)?$)"}; +// Custom and older amdgpu builds report only major.minor.patch. +const auto kAmdgpuModuleVersionRegex = std::regex{R"(^(\d+\.\d+\.\d+)(?:\.(\d+))?$)"}; + +using smi_amdgpu_dkms_conf_t = std::pair; + +auto log_dkms_debug(std::string_view caller, std::string_view message) -> void { + auto outstream = std::ostringstream{}; + outstream << caller << " | " << message; + LOG_DEBUG(outstream); +} + +auto is_amdgpu_dkms_version_dir_name(std::string_view dir_name) -> bool { + return std::regex_match(dir_name.begin(), dir_name.end(), kAmdgpuDkmsVersionDirRegex); +} + +auto has_prefix(std::string_view str, std::string_view prefix) -> bool { + return ((str.size() >= prefix.size()) && (str.compare(0, prefix.size(), prefix) == 0)); +} + +auto try_parse_dkms_conf_assignment(std::string_view line, std::string_view key) + -> std::optional { + const auto trimmed_line = trim(line); + if ((trimmed_line.size() <= (key.size() + 1)) || (!has_prefix(trimmed_line, key))) { + return std::nullopt; + } + + if (trimmed_line[key.size()] != '=') { + return std::nullopt; + } + + const auto value_part = trim(trimmed_line.substr((key.size() + 1))); + if ((value_part.size() < 2) || (value_part.front() != '"') || (value_part.back() != '"')) { + return std::nullopt; + } + + return value_part.substr(1, (value_part.size() - 2)); +} + +auto resolve_symlink_target_path(const fs::path& source_link, const fs::path& target, + std::error_code& err_code) -> fs::path { + if (target.is_absolute()) { + return fs::weakly_canonical(target, err_code); + } + return fs::weakly_canonical(source_link.parent_path() / target, err_code); +} + +auto read_dkms_conf(std::string_view caller, const fs::path& conf_path) + -> std::optional { + auto instream = std::ifstream{conf_path}; + if (instream.is_open()) { + auto package_name = std::string{}; + auto package_version = std::string{}; + auto line_str = std::string{}; + + while (std::getline(instream, line_str)) { + if ((!line_str.empty()) && (line_str.back() == '\r')) { + line_str.pop_back(); + } + + if (const auto parsed_name = + try_parse_dkms_conf_assignment(line_str, kDkmsAmdgpuDkmsConfPackageName)) { + package_name = std::string{*parsed_name}; + continue; + } + + if (const auto parsed_version = + try_parse_dkms_conf_assignment(line_str, kDkmsAmdgpuDkmsConfPackageVersion)) { + package_version = std::string{*parsed_version}; + } + } + + if ((!package_name.empty()) && (!package_version.empty())) { + return std::make_pair(package_name, package_version); + } + + auto outstream = std::ostringstream{}; + outstream << kDkmsAmdgpuDkmsConfName << " at: " << conf_path << " has no " + << kDkmsAmdgpuDkmsConfPackageName << " or " << kDkmsAmdgpuDkmsConfPackageVersion; + log_dkms_debug(caller, outstream.str()); + return std::nullopt; + } + + auto outstream = std::ostringstream{}; + outstream << "Cannot open " << kDkmsAmdgpuDkmsConfName << " at: " << conf_path; + log_dkms_debug(caller, outstream.str()); + return std::nullopt; +} + +auto make_kernel_symlink_name(std::string_view release, std::string_view machine) -> std::string { + auto name = std::string{kDkmsAmdgpuKernelPrefix}; + name.append(release); + name.push_back('-'); + name.append(machine); + return name; +} + +auto package_version_from_kernel_symlink(std::string_view caller, const fs::path& dkms_root, + const fs::path& kernel_link) + -> std::optional { + auto err_code = std::error_code{}; + if ((!fs::is_symlink(kernel_link, err_code)) || (err_code)) { + auto outstream = std::ostringstream{}; + outstream << "No kernel symlink at: " << kernel_link; + log_dkms_debug(caller, outstream.str()); + return std::nullopt; + } + + const auto target = fs::read_symlink(kernel_link, err_code); + if ((err_code) || (target.empty())) { + auto outstream = std::ostringstream{}; + outstream << "Cannot read kernel symlink at: " << kernel_link << ": " << err_code.message(); + log_dkms_debug(caller, outstream.str()); + return std::nullopt; + } + + auto version = std::string{}; + if (target.is_relative()) { + version = target.begin()->string(); + } else { + const auto resolved = resolve_symlink_target_path(kernel_link, target, err_code); + if (err_code) { + auto outstream = std::ostringstream{}; + outstream << "Cannot resolve kernel symlink at: " << kernel_link << ": " + << err_code.message(); + log_dkms_debug(caller, outstream.str()); + return std::nullopt; + } + + const auto relative = fs::relative(resolved, dkms_root, err_code); + if ((err_code) || (relative.empty()) || (relative.has_root_path())) { + auto outstream = std::ostringstream{}; + outstream << "Kernel symlink at: " << kernel_link + << " is outside the DKMS root: " << dkms_root; + log_dkms_debug(caller, outstream.str()); + return std::nullopt; + } + version = relative.begin()->string(); + } + + if (!is_amdgpu_dkms_version_dir_name(version)) { + auto outstream = std::ostringstream{}; + outstream << "Kernel symlink at: " << kernel_link + << " target is not version-shaped: " << target; + log_dkms_debug(caller, outstream.str()); + return std::nullopt; + } + return version; +} + +auto is_valid_dkms_package(std::string_view caller, const fs::path& version_dir, + std::string_view dir_name) -> bool { + // DKMS source_tree is configurable, so follow the source symlink instead of assuming /usr/src. + const auto dkms_conf_path = + (version_dir / kAmdgpuDkmsSourceSymlinkName / kDkmsAmdgpuDkmsConfName); + const auto package_info = read_dkms_conf(caller, dkms_conf_path); + if (!package_info.has_value()) { + return false; + } + + const auto& package_name = package_info->first; + const auto& package_version = package_info->second; + if (std::string_view{package_name} != kAmdgpuDkmsPackageName) { + auto outstream = std::ostringstream{}; + outstream << kDkmsAmdgpuDkmsConfName << " at: " << dkms_conf_path + << " sets PACKAGE_NAME to: " << package_name + << "; expected: " << kAmdgpuDkmsPackageName; + log_dkms_debug(caller, outstream.str()); + return false; + } + + if (package_version != dir_name) { + auto outstream = std::ostringstream{}; + outstream << kDkmsAmdgpuDkmsConfName << " at: " << dkms_conf_path << " sets PACKAGE_VERSION to " + << package_version << "; directory name is " << dir_name; + log_dkms_debug(caller, outstream.str()); + return false; + } + + return true; +} + +} // namespace + +auto smi_amdgpu_parse_driver_versions(std::string_view module_version, + std::string_view package_version, amdsmi_driver_info_t* info) + -> amdsmi_status_t { + if (info == nullptr) { + return AMDSMI_STATUS_INVAL; + } + + const auto module_version_string = std::string{module_version}; + smi_clear_char_and_reinitialize(info->driver_version, AMDSMI_MAX_STRING_LENGTH, + module_version_string); + info->driver_kernel_version[0] = '\0'; + info->amdgpu_driver_version[0] = '\0'; + info->driver_build_version[0] = '\0'; + info->driver_full_version[0] = '\0'; + + if (module_version.empty() || (module_version == "N/A")) { + return AMDSMI_STATUS_SUCCESS; + } + + auto full_version = module_version_string; + auto match = std::smatch{}; + if (std::regex_match(module_version_string, match, kAmdgpuModuleVersionRegex)) { + const auto kernel_version = match[1].str(); + smi_clear_char_and_reinitialize(info->driver_kernel_version, AMDSMI_MAX_STRING_LENGTH, + kernel_version); + smi_clear_char_and_reinitialize(info->amdgpu_driver_version, AMDSMI_MAX_STRING_LENGTH, + match[2].str()); + // A module built outside this DKMS package (custom kernel) must not report its build. + if (is_amdgpu_dkms_version_dir_name(package_version) && + (package_version.substr(0, package_version.find('-')) == kernel_version)) { + const auto build_start = (package_version.find('-') + 1); + const auto build_end = package_version.find('.', build_start); + const auto build_version = + std::string{package_version.substr(build_start, (build_end - build_start))}; + smi_clear_char_and_reinitialize(info->driver_build_version, AMDSMI_MAX_STRING_LENGTH, + build_version); + full_version.push_back('-'); + full_version.append(build_version); + } + } + smi_clear_char_and_reinitialize(info->driver_full_version, AMDSMI_MAX_STRING_LENGTH, + full_version); + + return AMDSMI_STATUS_SUCCESS; +} + +auto smi_amdgpu_get_active_dkms_version(std::string_view dkms_root, std::string_view release, + std::string_view machine, std::string* active_version) + -> amdsmi_status_t { + if (active_version == nullptr) { + return AMDSMI_STATUS_INVAL; + } + active_version->clear(); + + if ((release.empty()) || (machine.empty())) { + return AMDSMI_STATUS_NOT_SUPPORTED; + } + + const auto root = fs::path{std::string{dkms_root}}; + const auto kernel_link = root / make_kernel_symlink_name(release, machine); + const auto selected = package_version_from_kernel_symlink(__func__, root, kernel_link); + if (!selected.has_value()) { + return AMDSMI_STATUS_NOT_SUPPORTED; + } + + if (!is_valid_dkms_package(__func__, root / *selected, *selected)) { + auto outstream = std::ostringstream{}; + outstream << "Kernel symlink package " << *selected << " is not a validated DKMS package"; + log_dkms_debug(__func__, outstream.str()); + return AMDSMI_STATUS_NOT_SUPPORTED; + } + + *active_version = *selected; + return AMDSMI_STATUS_SUCCESS; +} diff --git a/projects/amdsmi/src/amd_smi/amd_smi_wsl_device.cc b/projects/amdsmi/src/amd_smi/amd_smi_wsl_device.cc index 2e81a280f2f5..e88342e1b3d9 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi_wsl_device.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi_wsl_device.cc @@ -509,6 +509,9 @@ amdsmi_status_t WSLGPUBackend::GetDriverInfo(amdsmi_driver_info_t* info) { copy_rocdxg_string(info->driver_version, device_info_.driver.driver_version); copy_rocdxg_string(info->driver_date, device_info_.driver.driver_date); copy_rocdxg_string(info->driver_name, device_info_.driver.driver_name); + // WSL has no amdgpu sysfs/DKMS fields. CLI AMDGPU Version reads + // driver_full_version, so reuse the WDDM driver_version string. + copy_rocdxg_string(info->driver_full_version, device_info_.driver.driver_version); return AMDSMI_STATUS_SUCCESS; } diff --git a/projects/amdsmi/tests/amd_smi_test/CMakeLists.txt b/projects/amdsmi/tests/amd_smi_test/CMakeLists.txt index 5ab65f95dd0a..c5d42b665400 100644 --- a/projects/amdsmi/tests/amd_smi_test/CMakeLists.txt +++ b/projects/amdsmi/tests/amd_smi_test/CMakeLists.txt @@ -104,6 +104,11 @@ target_include_directories( # Absolute path to the committed mock CPER fixtures (unit/gpu/mock_cper). target_compile_definitions(${TEST} PRIVATE AMDSMI_TEST_MOCK_DIR="${CMAKE_CURRENT_SOURCE_DIR}/unit/gpu/mock_cper") +# This directory is only added when BUILD_TESTS is ON (see top-level +# CMakeLists.txt), so the test-only amdsmi_test_register_node_handle() +# declaration (amd_smi_test_internal.h) is always needed here. +target_compile_definitions(${TEST} PRIVATE BUILD_TESTS) + if(ENABLE_WSL_BACKEND) target_compile_definitions(${TEST} PRIVATE ENABLE_WSL_BACKEND) target_include_directories(${TEST} PRIVATE ${PROJECT_SOURCE_DIR}/../rocr-runtime/libhsakmt/include) diff --git a/projects/amdsmi/tests/amd_smi_test/lsan.supp b/projects/amdsmi/tests/amd_smi_test/lsan.supp new file mode 100644 index 000000000000..209dd1692c7a --- /dev/null +++ b/projects/amdsmi/tests/amd_smi_test/lsan.supp @@ -0,0 +1,3 @@ +# LeakSanitizer suppressions for amdsmitst (.github/workflows/amdsmi-asan.yml). +# Known, unfixed leaks only: delete an entry with its fix. Entries that still +# match are listed under "Suppressions used:" at the end of the test output. diff --git a/projects/amdsmi/tests/amd_smi_test/unit/gpu/amdsmi_npm_limit_test.cc b/projects/amdsmi/tests/amd_smi_test/unit/gpu/amdsmi_npm_limit_test.cc new file mode 100644 index 000000000000..b7f18ab48759 --- /dev/null +++ b/projects/amdsmi/tests/amd_smi_test/unit/gpu/amdsmi_npm_limit_test.cc @@ -0,0 +1,574 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +// Unit tests for the public amdsmi_set_npm_limit() API (amd_smi.cc) and the +// current_node_power round-trip in amdsmi_get_npm_info(). +// +// amdsmi_set_npm_limit() takes an amdsmi_node_handle, which is internally a +// `std::string*` board-path pointer (see amdsmi_get_node_handle() in +// amd_smi.cc). The public acquisition path (amdsmi_get_node_handle()) gates +// on `asic_info.oam_id != 0`, which is not satisfiable on this test +// environment's hardware/mock setup (pre-existing, out of scope for this +// feature). To exercise amdsmi_set_npm_limit() itself we construct the +// node_handle directly by reinterpret_cast'ing the address of a local +// std::string, exactly as amdsmi_get_node_handle() would have handed back, +// bypassing only the acquisition helper -- not the function under test. This +// mirrors the approach used in the feature's own smoke test (see the +// implementer's handoff notes). +// +// amdsmi_get_npm_info()/amdsmi_set_npm_limit() only accept a node_handle that +// this library actually vended (tracked in a registry populated by +// amdsmi_get_node_handle(); see is_registered_node_handle() in amd_smi.cc, +// fixing a pre-cast/dereference validation gap). Since tests fabricate their +// node_handle directly instead of calling amdsmi_get_node_handle(), every +// fabricated handle below is registered via the test-only +// amdsmi_test_register_node_handle() hook (amd_smi_test_internal.h) +// immediately after construction, so it is treated as valid by that check. +// The NpmInfoRejectsUnregisteredHandle/SetNpmLimitRejectsUnregisteredHandle +// tests below are the ones that specifically exercise the *absence* of that +// registration call, and must not call amdsmi_test_register_node_handle(). +// +// rsmi_dev_npm_limit_set() (rocm_smi.cc) applies REQUIRE_ROOT_ACCESS before +// touching any file, so: +// - non-root callers always observe AMDSMI_STATUS_NO_PERM, regardless of +// whether the board_path/file exists -- this is verifiable without root +// and is asserted unconditionally below. +// - success / not-supported branches (reached only past the root gate) +// require root and are gated with GTEST_SKIP_ following the existing +// is_sudo_user() convention used throughout tests/amd_smi_test/main.cc. +// Since set_npm_board_limit()/rsmi_dev_npm_limit_set() operate on an +// arbitrary board_path string (not hardcoded to /sys), the root-gated cases +// use a plain std::filesystem temp directory instead of real /sys mocking. + +#include + +#include +#include +#include +#include +#include +#include + +#include "amd_smi/amdsmi.h" +#include "amd_smi/impl/amd_smi_test_internal.h" +#include "config/amd_smi_config.h" +#include "rocm_smi/rocm_smi_utils.h" + +#if defined(ENABLE_WSL_BACKEND) +#include "amd_smi/impl/amd_smi_wsl_device.h" +#endif // ENABLE_WSL_BACKEND + +namespace fs = std::filesystem; + +namespace { + +// RAII helper identical in spirit to the one in rocm_smi_npm_test.cc; kept +// local to this file to avoid coupling across unit-test translation units. +class TempBoardDir { + public: + TempBoardDir() { + path_ = fs::temp_directory_path() / + fs::path("amdsmi_npm_limit_test_" + std::to_string(reinterpret_cast(this))); + fs::create_directories(path_); + } + ~TempBoardDir() { + std::error_code ec; + fs::remove_all(path_, ec); + } + const fs::path& path() const { return path_; } + void WriteFile(const std::string& filename, const std::string& contents) const { + std::ofstream ofs(path_ / filename); + ofs << contents; + } + std::string ReadFile(const std::string& filename) const { + std::ifstream ifs(path_ / filename); + std::string line; + std::getline(ifs, line); + return line; + } + + private: + fs::path path_; +}; + +// Small RAII wrapper around amdsmi_init()/amdsmi_shut_down() so each test is +// self-contained regardless of test execution order (ref-counted, so nesting +// with other tests' init/shutdown pairs elsewhere in the binary is safe). +class ScopedAmdSmiInit { + public: + ScopedAmdSmiInit() { status_ = amdsmi_init(AMDSMI_INIT_AMD_GPUS); } + ~ScopedAmdSmiInit() { + if (status_ == AMDSMI_STATUS_SUCCESS) { + amdsmi_shut_down(); + } + } + amdsmi_status_t status() const { return status_; } + + private: + amdsmi_status_t status_; +}; + +} // namespace + +// --------------------------------------------------------------------- +// amdsmi_set_npm_limit() +// --------------------------------------------------------------------- + +TEST(GpuUnit, SetNpmLimitNullHandleIsInval) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(nullptr, 100), AMDSMI_STATUS_INVAL); +} + +TEST(GpuUnit, SetNpmLimitNonRootIsNoPerm) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (amd::smi::is_sudo_user()) { + GTEST_SKIP_( + "Running as root: REQUIRE_ROOT_ACCESS gate cannot be exercised here; " + "see SetNpmLimitRoot* tests instead"); + } + + // Any non-empty board path is sufficient: REQUIRE_ROOT_ACCESS in + // rsmi_dev_npm_limit_set() rejects before the path is ever touched. + std::string board_path = "/tmp/amdsmi_npm_limit_test_nonroot_probe"; + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 100), AMDSMI_STATUS_NO_PERM); +} + +TEST(GpuUnit, SetNpmLimitRootSuccessWritesValue) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + TempBoardDir board; + // NPM must be enabled on this node now that rsmi_dev_npm_limit_set() + // rejects the write outright when it is not. + board.WriteFile("npm_status", "enabled"); + // max_node_power_limit must be present and >= the requested limit now that + // rsmi_dev_npm_limit_set() range-checks against it before writing (mirrors + // rsmi_dev_power_cap_set()'s range_get()-before-write pattern). + board.WriteFile("max_node_power_limit", "6400"); + board.WriteFile("cur_node_power_limit", "0"); + std::string board_path = board.path().string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 250), AMDSMI_STATUS_SUCCESS); + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "250"); +} + +// --------------------------------------------------------------------- +// amdsmi_set_npm_limit() -- NPM-disabled enforcement +// +// rsmi_dev_npm_limit_set() rejects the write outright when NPM is disabled +// on the node (before even reading the platform max): writing +// cur_node_power_limit while NPM is disabled has no defined effect. +// --------------------------------------------------------------------- + +TEST(GpuUnit, SetNpmLimitRootRejectsWhenNpmDisabled) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + TempBoardDir board; + board.WriteFile("npm_status", "disabled"); + board.WriteFile("max_node_power_limit", "6400"); + board.WriteFile("cur_node_power_limit", "0"); + std::string board_path = board.path().string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 250), AMDSMI_STATUS_INVAL); + // The write must not have happened. + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "0"); +} + +// --------------------------------------------------------------------- +// amdsmi_set_npm_limit() -- platform-max range enforcement (F-5 fix) +// +// rsmi_dev_npm_limit_set() now mirrors rsmi_dev_power_cap_set(): it queries +// the platform max via get_npm_board_max_limit() and rejects out-of-range +// requests with AMDSMI_STATUS_INVAL before ever touching +// cur_node_power_limit, and fails closed (propagates the underlying error) +// if the max itself can't be read. +// --------------------------------------------------------------------- + +TEST(GpuUnit, SetNpmLimitRootRejectsZero) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + TempBoardDir board; + board.WriteFile("npm_status", "enabled"); + board.WriteFile("max_node_power_limit", "6400"); + board.WriteFile("cur_node_power_limit", "6000"); + std::string board_path = board.path().string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 0), AMDSMI_STATUS_INVAL); + // The write must not have happened. + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "6000"); +} + +TEST(GpuUnit, SetNpmLimitRootRejectsOverMax) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + TempBoardDir board; + board.WriteFile("npm_status", "enabled"); + board.WriteFile("max_node_power_limit", "6400"); + board.WriteFile("cur_node_power_limit", "6000"); + std::string board_path = board.path().string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 6401), AMDSMI_STATUS_INVAL); + // The write must not have happened. + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "6000"); +} + +TEST(GpuUnit, SetNpmLimitRootRejectsWhenMaxUnreadable) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + TempBoardDir board; + board.WriteFile("npm_status", "enabled"); + // Deliberately do not create max_node_power_limit: the platform max is + // unreadable, so the request must be rejected (fail closed) rather than + // falling through to an unbounded write. + board.WriteFile("cur_node_power_limit", "6000"); + std::string board_path = board.path().string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 250), AMDSMI_STATUS_NOT_SUPPORTED); + // The write must not have happened. + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "6000"); +} + +TEST(GpuUnit, SetNpmLimitRootRejectsWhenMaxCorrupted) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + TempBoardDir board; + // "-1" is a regression case, not generic garbage: std::stoull("-1", ...) + // is well-defined C++ behavior that successfully parses to UINT64_MAX + // (leading '-' negates modulo 2^64) rather than throwing, so a naive + // parser would treat this corrupted content as a "successfully read" + // huge platform max -- which would let any limit through the + // `limit > max_limit` check. The platform max must fail to parse and be + // treated as unreadable (fail closed), exactly like the missing-file case + // in SetNpmLimitRootRejectsWhenMaxUnreadable above, not silently accepted + // as UINT64_MAX. + board.WriteFile("npm_status", "enabled"); + board.WriteFile("max_node_power_limit", "-1"); + board.WriteFile("cur_node_power_limit", "6000"); + std::string board_path = board.path().string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + // Unlike the missing-file case above (RSMI_STATUS_NOT_SUPPORTED), a + // present-but-corrupted file is a parse failure: + // get_npm_board_max_limit()/read_board_uint64() returns + // RSMI_STATUS_UNEXPECTED_DATA, which rsmi_dev_npm_limit_set() propagates + // as-is (fail closed) and which maps to AMDSMI_STATUS_UNEXPECTED_DATA. + EXPECT_EQ(amdsmi_set_npm_limit(handle, 250), AMDSMI_STATUS_UNEXPECTED_DATA); + // The write must not have happened. + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "6000"); +} + +TEST(GpuUnit, SetNpmLimitRootAcceptsInRange) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + TempBoardDir board; + board.WriteFile("npm_status", "enabled"); + board.WriteFile("max_node_power_limit", "6400"); + board.WriteFile("cur_node_power_limit", "0"); + std::string board_path = board.path().string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 6400), AMDSMI_STATUS_SUCCESS); + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "6400"); +} + +TEST(GpuUnit, SetNpmLimitRootMissingFileIsNotSupported) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + TempBoardDir board; + board.WriteFile("npm_status", "enabled"); + // max_node_power_limit present (so the new range check itself succeeds), + // but cur_node_power_limit deliberately absent -- exercises the write-time + // NOT_SUPPORTED path in set_npm_board_limit(), distinct from + // SetNpmLimitRootRejectsWhenMaxUnreadable above. + board.WriteFile("max_node_power_limit", "6400"); + std::string board_path = board.path().string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 250), AMDSMI_STATUS_NOT_SUPPORTED); +} + +TEST(GpuUnit, SetNpmLimitRootMissingBoardDirIsNotSupported) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::is_sudo_user()) { + GTEST_SKIP_("Invalid permission - Must run as super user"); + } + + fs::path missing = fs::temp_directory_path() / "amdsmi_npm_limit_test_missing_board_dir"; + std::error_code ec; + fs::remove_all(missing, ec); + std::string board_path = missing.string(); + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 250), AMDSMI_STATUS_NOT_SUPPORTED); +} + +// --------------------------------------------------------------------- +// Registered-node_handle validation (F-2 fix) +// +// amdsmi_get_npm_info()/amdsmi_set_npm_limit() previously +// reinterpret_cast()'d and immediately dereferenced +// (->empty()) their caller-supplied node_handle with no check that the +// pointer was ever actually vended by this library. An unprivileged caller +// could pass any non-null garbage value and hit that dereference *before* +// any privilege check (REQUIRE_ROOT_ACCESS lives one layer down, only inside +// rsmi_dev_npm_limit_set(), and not at all in the read path). Both functions +// now call is_registered_node_handle() first and reject anything not found +// in the node_handle registry with AMDSMI_STATUS_INVAL, never dereferencing +// an unregistered pointer. +// +// These tests deliberately do NOT call amdsmi_test_register_node_handle(): +// the handle below points at real, validly-constructed stack/heap memory +// (so a would-be crash is not guaranteed/portable to reproduce in CI), but +// it was never registered, which is exactly the condition the fix targets. +// Asserting AMDSMI_STATUS_INVAL (rather than merely "did not crash") +// demonstrates the registry check itself is what stopped execution, not +// incidental luck about what garbage bytes happened to be at that address. +// --------------------------------------------------------------------- + +TEST(GpuUnit, GetNpmInfoRejectsUnregisteredHandle) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + // Heap-allocated (not stack-allocated) so this address cannot alias a + // stack slot some other test in this binary already passed to + // amdsmi_test_register_node_handle(): registered pointers live for the + // rest of the process, and other tests here register the address of a + // same-shaped stack-local std::string, which the allocator/compiler can + // legitimately hand back again for a later test's stack frame. A fresh + // heap allocation that nothing in this file ever registers cannot collide + // with that set. + auto unregistered_board_path = + std::make_unique("/tmp/amdsmi_npm_limit_test_unregistered_probe"); + amdsmi_node_handle handle = reinterpret_cast(unregistered_board_path.get()); + + amdsmi_npm_info_t npm_info{}; + EXPECT_EQ(amdsmi_get_npm_info(handle, &npm_info), AMDSMI_STATUS_INVAL); +} + +TEST(GpuUnit, SetNpmLimitRejectsUnregisteredHandle) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + // See GetNpmInfoRejectsUnregisteredHandle above for why this must be + // heap-allocated rather than a stack-local std::string. + auto unregistered_board_path = + std::make_unique("/tmp/amdsmi_npm_limit_test_unregistered_probe2"); + amdsmi_node_handle handle = reinterpret_cast(unregistered_board_path.get()); + + // AMDSMI_STATUS_INVAL must be returned regardless of caller privilege: the + // registry check runs before REQUIRE_ROOT_ACCESS is ever reached, whether + // or not this test process happens to be root. + EXPECT_EQ(amdsmi_set_npm_limit(handle, 100), AMDSMI_STATUS_INVAL); +} + +TEST(GpuUnit, SetNpmLimitAcceptsHandleAfterTestRegistration) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (amd::smi::is_sudo_user()) { + GTEST_SKIP_( + "Running as root: this test isolates the registry check from " + "REQUIRE_ROOT_ACCESS by asserting NO_PERM (not INVAL); see " + "SetNpmLimitNonRootIsNoPerm for the rationale"); + } + + // Same handle shape as SetNpmLimitRejectsUnregisteredHandle above, but + // registered first: the registry check must pass, and the request must + // proceed to (and be stopped by) REQUIRE_ROOT_ACCESS instead of being + // rejected as INVAL -- demonstrating the fix does not reject legitimately + // vended handles. + std::string board_path = "/tmp/amdsmi_npm_limit_test_registered_probe"; + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 100), AMDSMI_STATUS_NO_PERM); +} + +#if defined(ENABLE_WSL_BACKEND) +// When the WSL backend is active, amdsmi_set_npm_limit() must short-circuit +// to AMDSMI_STATUS_NOT_SUPPORTED before ever touching board_path/sysfs (see +// the `#ifdef ENABLE_WSL_BACKEND` / WSLGPUBackend::IsActive() guard at the +// top of amdsmi_set_npm_limit() in amd_smi.cc). This is only observable on a +// real WSL2 machine where amdsmi_init() -> populate_amd_gpu_devices() -> +// WSLGPUBackend::TryPopulate() actually succeeded (requires /dev/dxg and a +// loadable librocdxg); otherwise IsActive() stays false even when compiled +// with ENABLE_WSL_BACKEND=ON, so this test skips itself in that case rather +// than asserting anything about the (untaken) native-Linux path. +TEST(GpuUnit, SetNpmLimitNotSupportedWhenWslBackendActive) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + if (!amd::smi::WSLGPUBackend::IsActive()) { + GTEST_SKIP_( + "WSLGPUBackend is not active (not a real WSL2 machine, or " + "librocdxg/rocdxg node enumeration unavailable) -- the " + "NOT_SUPPORTED short-circuit cannot be exercised here"); + } + + // Any board path is sufficient: the WSL-backend guard runs before the + // path is ever dereferenced. + std::string board_path = "/tmp/amdsmi_npm_limit_test_wsl_probe"; + amdsmi_node_handle handle = reinterpret_cast(&board_path); + ASSERT_EQ(amdsmi_test_register_node_handle(handle), AMDSMI_STATUS_SUCCESS); + + EXPECT_EQ(amdsmi_set_npm_limit(handle, 100), AMDSMI_STATUS_NOT_SUPPORTED); +} +#endif // ENABLE_WSL_BACKEND + +// --------------------------------------------------------------------- +// amdsmi_get_npm_info() -- current_node_power round trip +// --------------------------------------------------------------------- + +// current_node_power moved from amdsmi_power_info_t (per-GPU, amd-smi metric +// --power) to amdsmi_npm_info_t (per-node, amd-smi node -p / +// amdsmi_get_npm_info()) -- see the "Set npm power limit" design doc. It is +// still populated by resolving the board directory from the real device's +// enumeration info (drm_render) via amdsmi_get_node_handle(), not from a +// caller-supplied path, so it cannot be pointed at a temp directory the way +// amdsmi_set_npm_limit() can. This test therefore validates against whatever +// the current hardware reports: either a concrete value (if +// board/node_power exists) or the UINT32_MAX sentinel/"N/A" case (if it +// doesn't, e.g. no board/ directory on this platform) -- both are valid, +// contract-conformant outcomes, and the test asserts the round trip is +// internally consistent either way. +// +// amdsmi_get_node_handle() additionally gates on asic_info.oam_id == 0 (see +// the file header comment above), so on hardware/mocks where that gate is +// not satisfied this test will skip rather than fail -- it is exercising the +// real acquisition path end-to-end, not a test-fabricated handle. +TEST(GpuUnit, GetNpmInfoCurrentNodePowerRoundTrip) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + amdsmi_socket_handle sockets[16]; + uint32_t socket_count = 16; + ASSERT_EQ(amdsmi_get_socket_handles(&socket_count, sockets), AMDSMI_STATUS_SUCCESS); + if (socket_count == 0) { + GTEST_SKIP_("No sockets discovered on this system"); + } + + bool exercised_at_least_one_node = false; + constexpr uint32_t kSentinel = std::numeric_limits::max(); + + for (uint32_t s = 0; s < socket_count; ++s) { + amdsmi_processor_handle procs[16]; + uint32_t proc_count = 16; + if (amdsmi_get_processor_handles(sockets[s], &proc_count, procs) != AMDSMI_STATUS_SUCCESS) { + continue; + } + for (uint32_t p = 0; p < proc_count; ++p) { + amdsmi_node_handle node_handle = nullptr; + if (amdsmi_get_node_handle(procs[p], &node_handle) != AMDSMI_STATUS_SUCCESS) { + continue; + } + + amdsmi_npm_info_t npm_info; + std::memset(&npm_info, 0, sizeof(npm_info)); + amdsmi_status_t r = amdsmi_get_npm_info(node_handle, &npm_info); + if (r != AMDSMI_STATUS_SUCCESS) { + continue; + } + exercised_at_least_one_node = true; + + // Contract: current_node_power is either the sentinel (falls back + // silently, e.g. no board/node_power file) or a concrete uint32 value + // read from board/node_power -- never left uninitialized (e.g. + // garbage/0 as a side effect of a partially-implemented read). + if (npm_info.current_node_power != kSentinel) { + // A "real" reading was obtained: sanity-check it is a plausible + // wattage rather than accidentally aliasing the sentinel's bit + // pattern or being negative-when-reinterpreted (uint32_t can't be + // negative, but guard against absurd overflow wrap regardless). + EXPECT_LT(npm_info.current_node_power, kSentinel); + } + } + } + + if (!exercised_at_least_one_node) { + GTEST_SKIP_( + "amdsmi_get_node_handle()/amdsmi_get_npm_info() did not succeed for any processor on " + "this system"); + } +} + +TEST(GpuUnit, GetPowerInfoNullInfoIsInval) { + ScopedAmdSmiInit init; + ASSERT_EQ(init.status(), AMDSMI_STATUS_SUCCESS); + + amdsmi_socket_handle sockets[16]; + uint32_t socket_count = 16; + ASSERT_EQ(amdsmi_get_socket_handles(&socket_count, sockets), AMDSMI_STATUS_SUCCESS); + if (socket_count == 0) { + GTEST_SKIP_("No sockets discovered on this system"); + } + amdsmi_processor_handle procs[16]; + uint32_t proc_count = 16; + ASSERT_EQ(amdsmi_get_processor_handles(sockets[0], &proc_count, procs), AMDSMI_STATUS_SUCCESS); + if (proc_count == 0) { + GTEST_SKIP_("No processors discovered on this system"); + } + + EXPECT_EQ(amdsmi_get_power_info(procs[0], nullptr), AMDSMI_STATUS_INVAL); +} diff --git a/projects/amdsmi/tests/amd_smi_test/unit/gpu/cper_read_test.cc b/projects/amdsmi/tests/amd_smi_test/unit/gpu/cper_read_test.cc index ee994ebc5d3f..d9075fe6f246 100644 --- a/projects/amdsmi/tests/amd_smi_test/unit/gpu/cper_read_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/unit/gpu/cper_read_test.cc @@ -431,6 +431,13 @@ constexpr amdsmi_cper_guid_t kNonStandardGuid = // AMD_GPU_NONSTANDARD_ERROR MakeGuid(0x32AC0C78, 0x2623, 0x48F6, 0x81, 0xA2, 0xAC, 0x69, 0x17, 0x80, 0x55, 0x1D); constexpr amdsmi_cper_guid_t kProcErrGuid = // PROC_ERR_SECTION_TYPE MakeGuid(0xDC3EA0B0, 0xA144, 0x4797, 0xB9, 0x5B, 0x53, 0xFA, 0x24, 0x2B, 0x6E, 0x1D); +constexpr amdsmi_cper_guid_t kBootNotifyGuid = // BOOT_TYPE + MakeGuid(0x3D61A466, 0xAB40, 0x409A, 0xA6, 0x98, 0xF3, 0x62, 0xD4, 0x64, 0xB3, 0x8F); + +// Crashdump section lengths as amdgpu writes them: the dump union carries fatal_err +// (32 bytes) in runtime records and boot_err (64 bytes) in boot records. +constexpr uint32_t kAmdgpuFatalCrashdumpSecLen = 0xB0; +constexpr uint32_t kAmdgpuBootCrashdumpSecLen = 0xD0; // Tracks the production static_assert that ties the reg_dump capacity to the // register count decode_afid accepts, so both stay on the same constant. @@ -503,6 +510,27 @@ std::vector MakeCrashdumpRecord(uint32_t record_length, size_t sec_pad = 0 return buf; } +// [header][desc0][crashdump] with record_length ending sec_len bytes into the +// section, the shape amdgpu emits. The buffer always holds the full struct so an +// overread past record_length stays inside it; only the reported length varies. +std::vector MakeAmdgpuCrashdumpRecord(uint32_t sec_len, bool boot) { + const size_t sec_offset = (kDescTableOffset + sizeof(struct cper_sec_desc)); + std::vector buf(sec_offset + sizeof(struct cper_sec_crashdump), 0); + InitHeader(&buf, /*sec_cnt=*/1, static_cast(sec_offset + sec_len)); + if (boot) { + reinterpret_cast(buf.data())->notify_type = kBootNotifyGuid; + } + struct cper_sec_desc* desc = DescAt(&buf, 0); + desc->sec_type = kCrashdumpGuid; + desc->sec_offset = static_cast(sec_offset); + desc->sec_length = sec_len; + auto* crashdump = reinterpret_cast(buf.data() + sec_offset); + crashdump->data.reg_ctx_type = kAcaRegisterContext; + std::memcpy(&crashdump->data.dump.fatal_err, kFatalRegisterPattern, + sizeof(kFatalRegisterPattern)); + return buf; +} + // [header][desc0][sec_pad][non-standard section]. reg_arr_size is // caller-controlled so a test can claim more registers than reg_dump holds, and // reg_ctx_type so a test can pick the branch that reads them. sec_pad slides the @@ -604,7 +632,7 @@ TEST(GpuUnit, CperDecodeSkipsSectionWithOutOfBoundsOffset) { TEST(GpuUnit, CperDecodeSkipsSectionThatStartsInsideButOverrunsTheRecord) { std::vector buf = MakeCrashdumpRecord(static_cast(kCrashdumpRecordSize)); auto* hdr = reinterpret_cast(buf.data()); - // One byte of room at sec_offset, against a section that needs the full struct. + // One byte of room at sec_offset, against a section that needs at least fatal_err. hdr->record_length = static_cast(kCrashdumpSecOffset + 1); // Forked for the same reason as the wild-offset case: unguarded, the section @@ -617,6 +645,41 @@ TEST(GpuUnit, CperDecodeSkipsSectionThatStartsInsideButOverrunsTheRecord) { testing::ExitedWithCode(0), ""); } +// A runtime fatal crashdump ends after fatal_err, 32 bytes short of the dump +// union's boot_err-sized footprint. Bounding it by sizeof(cper_sec_crashdump) +// rejects every fatal record amdgpu writes. +TEST(GpuUnit, CperDecodeDecodesFatalCrashdumpSizedAsAmdgpuWritesIt) { + std::vector buf = MakeAmdgpuCrashdumpRecord(kAmdgpuFatalCrashdumpSecLen, /*boot=*/false); + const auto* hdr = reinterpret_cast(buf.data()); + + EXPECT_EQ(cper_decode(hdr, buf.size()).size(), 1u); +} + +// One byte short of fatal_err is still an overrun. +TEST(GpuUnit, CperDecodeSkipsFatalCrashdumpOneByteShortOfFatalErr) { + std::vector buf = + MakeAmdgpuCrashdumpRecord(kAmdgpuFatalCrashdumpSecLen - 1, /*boot=*/false); + const auto* hdr = reinterpret_cast(buf.data()); + + EXPECT_TRUE(cper_decode(hdr, buf.size()).empty()); +} + +// Boot records are read through boot_err, so the fatal extent is not enough. +TEST(GpuUnit, CperDecodeSkipsBootCrashdumpSizedOnlyForFatalErr) { + std::vector buf = MakeAmdgpuCrashdumpRecord(kAmdgpuFatalCrashdumpSecLen, /*boot=*/true); + const auto* hdr = reinterpret_cast(buf.data()); + + EXPECT_TRUE(cper_decode(hdr, buf.size()).empty()); +} + +// Positive control for the boot reject above. +TEST(GpuUnit, CperDecodeDecodesBootCrashdumpSizedAsAmdgpuWritesIt) { + std::vector buf = MakeAmdgpuCrashdumpRecord(kAmdgpuBootCrashdumpSecLen, /*boot=*/true); + const auto* hdr = reinterpret_cast(buf.data()); + + EXPECT_EQ(cper_decode(hdr, buf.size()).size(), 1u); +} + // The same overrun on the non-standard path, which reaches the bound through an // explicit extent rather than through at<> because the body is a flexible array // member. diff --git a/projects/amdsmi/tests/amd_smi_test/unit/gpu/dynamic_metrics_test.cc b/projects/amdsmi/tests/amd_smi_test/unit/gpu/dynamic_metrics_test.cc index 42846ef34b0a..7401ec662ddb 100644 --- a/projects/amdsmi/tests/amd_smi_test/unit/gpu/dynamic_metrics_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/unit/gpu/dynamic_metrics_test.cc @@ -4,9 +4,13 @@ #include #include +#include #include +#include #include #include +#include +#include #include #include "rocm_smi/rocm_smi_gpu_metrics.h" @@ -96,6 +100,47 @@ auto WriteBlobToTempFile(const std::vector& blob, return file_path; } +using AttrId = amd::smi::details::AMDGpuMetricAttributeId_t; + +using AttrType = amd::smi::details::AMDGpuMetricAttributeType_t; + +// Serializes a dynamic (v1.9) gpu_metrics table, laid out as the driver emits +// it: header, attribute count, then encoded id + values, all of one type. +auto BuildDynamicBlob(const std::vector>>& attrs, + AttrType type = AttrType::TYPE_UINT64) -> std::vector { + std::vector blob; + auto put = [&blob](const auto& v) { + const auto* p = reinterpret_cast(&v); + blob.insert(blob.end(), p, p + sizeof(v)); + }; + put(amd::smi::details::AMDGpuDynamicMetricsHeader_v1_t{0, 1, 9}); + put(static_cast(attrs.size())); + for (const auto& [id, values] : attrs) { + put(amd::smi::details::amdgpu_metrics_encode_attr(0, static_cast(type), + static_cast(id), values.size())); + for (const auto v : values) { + if (type == AttrType::TYPE_UINT32) { + put(static_cast(v)); + } else { + put(v); + } + } + } + const auto size = static_cast(blob.size()); + std::memcpy(blob.data(), &size, sizeof(size)); + return blob; +} + +auto ToPublicMetrics(const std::vector& blob) -> amd::smi::AMGpuMetricsPublicLatest_t { + amd::smi::AMDGpuDynamicMetrics_t dyn; + EXPECT_EQ(dyn.parse_from_buffer(blob.data(), blob.size()), RSMI_STATUS_SUCCESS); + amd::smi::GpuMetricsBaseDynamic_t metrics; + metrics.set_parsed_dynamic(std::move(dyn)); + const auto [status, out] = metrics.copy_internal_to_external_metrics(); + EXPECT_EQ(status, RSMI_STATUS_SUCCESS); + return out; +} + } // namespace TEST(GpuUnit, GPUMetricDynamicVersionSupported) { @@ -189,3 +234,53 @@ TEST(GpuUnit, XCPMetricDynamicVersionSupported) { std::filesystem::remove(fake_path); } } + +// MI450 publishes one xGMI read/write data counter (its GPU-CPU link). It must +// land in link 0 rather than be dropped, leaving every link "not reported". +TEST(GpuUnit, DynamicMetricsXgmiSingleInstanceFillsFirstLink) { + const auto out = ToPublicMetrics(BuildDynamicBlob({ + {AttrId::XGMI_READ_DATA_ACC, {449353}}, + {AttrId::XGMI_WRITE_DATA_ACC, {0}}, + })); + EXPECT_EQ(out.xgmi_read_data_acc[0], 449353u); + EXPECT_EQ(out.xgmi_write_data_acc[0], 0u); + EXPECT_EQ(out.xgmi_read_data_acc[1], UINT64_MAX); + EXPECT_EQ(out.xgmi_write_data_acc[1], UINT64_MAX); +} + +// Per-link arrays keep mapping element by element. +TEST(GpuUnit, DynamicMetricsXgmiPerLinkArrayCopiesEachLink) { + const auto out = ToPublicMetrics(BuildDynamicBlob({ + {AttrId::XGMI_READ_DATA_ACC, {10, 20, 30}}, + })); + EXPECT_EQ(out.xgmi_read_data_acc[0], 10u); + EXPECT_EQ(out.xgmi_read_data_acc[1], 20u); + EXPECT_EQ(out.xgmi_read_data_acc[2], 30u); + EXPECT_EQ(out.xgmi_read_data_acc[3], UINT64_MAX); +} + +// A driver may declare the counter narrower than the public field; the value +// still lands in link 0 rather than being dropped. +TEST(GpuUnit, DynamicMetricsXgmiSingleInstanceNarrowerTypeFillsFirstLink) { + const auto out = ToPublicMetrics( + BuildDynamicBlob({{AttrId::XGMI_READ_DATA_ACC, {1234}}}, AttrType::TYPE_UINT32)); + EXPECT_EQ(out.xgmi_read_data_acc[0], 1234u); + EXPECT_EQ(out.xgmi_read_data_acc[1], UINT64_MAX); +} + +// A narrower all-ones "not reported" value must stay unavailable instead of +// widening into a real-looking 4294967295. +TEST(GpuUnit, DynamicMetricsXgmiSingleInstanceNarrowerUnsetStaysUnset) { + const auto out = ToPublicMetrics( + BuildDynamicBlob({{AttrId::XGMI_READ_DATA_ACC, {UINT32_MAX}}}, AttrType::TYPE_UINT32)); + EXPECT_EQ(out.xgmi_read_data_acc[0], UINT64_MAX); +} + +// A value wider than the public field must not be truncated into a +// plausible-looking reading; a value that fits maps through. +TEST(GpuUnit, DynamicMetricsSingleInstanceOutOfRangeStaysUnset) { + const auto wide = ToPublicMetrics(BuildDynamicBlob({{AttrId::XGMI_LINK_STATUS, {70000}}})); + EXPECT_EQ(wide.xgmi_link_status[0], UINT16_MAX); + const auto fits = ToPublicMetrics(BuildDynamicBlob({{AttrId::XGMI_LINK_STATUS, {1}}})); + EXPECT_EQ(fits.xgmi_link_status[0], 1u); +} diff --git a/projects/amdsmi/tests/amd_smi_test/unit/gpu/mock_cper/README.md b/projects/amdsmi/tests/amd_smi_test/unit/gpu/mock_cper/README.md index 5a4584c0080c..e918a7bc67f7 100644 --- a/projects/amdsmi/tests/amd_smi_test/unit/gpu/mock_cper/README.md +++ b/projects/amdsmi/tests/amd_smi_test/unit/gpu/mock_cper/README.md @@ -29,7 +29,10 @@ Scrubbed: a second record when the fixtures are concatenated. The read path under test only counts records and filters by severity, so the -zeroed body does not affect the tests. The `serial_number` was already `N/A` and +zeroed body does not affect the tests. The AFID decode test uses +`cper_fatal.cper` for its real 0xB0-byte fatal crashdump section and writes a +known register set back into the zeroed body before decoding. The +`serial_number` was already `N/A` and the FRU id already a generic `OAM0` in the captures. ## Files diff --git a/projects/amdsmi/tests/amd_smi_test/unit/gpu/mock_cper_test.cc b/projects/amdsmi/tests/amd_smi_test/unit/gpu/mock_cper_test.cc index f88d8a18970a..563953e24e0a 100644 --- a/projects/amdsmi/tests/amd_smi_test/unit/gpu/mock_cper_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/unit/gpu/mock_cper_test.cc @@ -10,14 +10,18 @@ // the build-tree fallback for in-tree runs. // Single-record parsing is also covered synthetically in cper_read_test.cc; these // fixtures add real-capture coverage: header byte alignment, multi-record -// rings, and severity_mask filtering. +// rings, severity_mask filtering, and amdgpu's crashdump section length. #include #include #include #include +#include #include +#include +#include +#include #include #include @@ -56,6 +60,11 @@ std::string MockDir() { std::string MockPath(const char* name) { return MockDir() + "/" + name; } +std::vector ReadFixture(const char* name) { + std::ifstream in(MockPath(name), std::ios::binary); + return {std::istreambuf_iterator(in), std::istreambuf_iterator()}; +} + // Reads a mock fixture and returns the parsed severities of the accepted // records (those passing severity_mask). entry_count and buf_size are reported // via out-params. @@ -193,3 +202,30 @@ TEST(GpuUnit, CperMockSeverityMaskRejectAll) { EXPECT_EQ(buf_size, 0u); EXPECT_TRUE(sevs.empty()); } + +// The fixture's crashdump section is amdgpu's 0xB0-byte fatal shape, shorter than +// sizeof(cper_sec_crashdump). Its payload is scrubbed, so the registers of a real +// MI308X fatal record (SMU GfxMmhubError, AFID 30) are written back in. +TEST(GpuUnit, CperMockFatalRecordDecodesAfid) { + std::vector rec = ReadFixture("cper_fatal.cper"); + ASSERT_GE(rec.size(), sizeof(amdsmi_cper_hdr_t) + sizeof(struct cper_sec_desc)); + const auto* hdr = reinterpret_cast(rec.data()); + ASSERT_EQ(hdr->record_length, rec.size()); + ASSERT_EQ(hdr->sec_cnt, 1); + const auto* desc = + reinterpret_cast(rec.data() + sizeof(amdsmi_cper_hdr_t)); + ASSERT_EQ(desc->sec_length, 0xB0u); + ASSERT_EQ(desc->sec_offset + desc->sec_length, hdr->record_length); + + EXPECT_TRUE(cper_decode(hdr, rec.size()).empty()) << "scrubbed payload decoded an AFID"; + + constexpr uint16_t kAcaRegisterContext = 1; + constexpr uint64_t kRegs[] = {0xBAA00000003B0000ULL, 0, 0x0001100103B30401ULL, 0x905ULL}; + const size_t data_off = desc->sec_offset + offsetof(struct cper_sec_crashdump, data); + const size_t dump_off = data_off + offsetof(struct cper_sec_crashdump_data, dump); + ASSERT_LE(dump_off + sizeof(kRegs), rec.size()); + std::memcpy(rec.data() + data_off, &kAcaRegisterContext, sizeof(kAcaRegisterContext)); + std::memcpy(rec.data() + dump_off, kRegs, sizeof(kRegs)); + + EXPECT_EQ(cper_decode(hdr, rec.size()), std::vector{30}); +} diff --git a/projects/amdsmi/tests/amd_smi_test/unit/gpu/rocm_smi_npm_test.cc b/projects/amdsmi/tests/amd_smi_test/unit/gpu/rocm_smi_npm_test.cc new file mode 100644 index 000000000000..0ea313bf0e90 --- /dev/null +++ b/projects/amdsmi/tests/amd_smi_test/unit/gpu/rocm_smi_npm_test.cc @@ -0,0 +1,211 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +// Hardware-free unit tests for the NPM (Node Power Management) helpers in +// rocm_smi_npm.cc: set_npm_board_limit() and get_npm_node_power(). +// +// These functions take an arbitrary board_path string and operate on plain +// files underneath it (they do not require /sys or any specific sysfs +// layout), so we exercise them against a real temporary directory created +// via std::filesystem instead of mocking /sys. This mirrors the existing +// (unexercised prior to this change) get_npm_board_limit()/ +// get_npm_board_status() logic in the same translation unit, all of which +// share the same "board_path directory + single-value file" convention. + +#include "rocm_smi/rocm_smi_npm.h" + +#include + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +// RAII helper: creates a unique temp directory for the duration of a test +// and removes it (recursively) on destruction. +class TempBoardDir { + public: + TempBoardDir() { + path_ = fs::temp_directory_path() / + fs::path("amdsmi_npm_test_" + + std::to_string(::testing::UnitTest::GetInstance()->random_seed()) + "_" + + std::to_string(reinterpret_cast(this))); + fs::create_directories(path_); + } + + ~TempBoardDir() { + std::error_code ec; + fs::remove_all(path_, ec); + } + + const fs::path& path() const { return path_; } + + void WriteFile(const std::string& filename, const std::string& contents) const { + std::ofstream ofs(path_ / filename); + ofs << contents; + } + + std::string ReadFile(const std::string& filename) const { + std::ifstream ifs(path_ / filename); + std::string line; + std::getline(ifs, line); + return line; + } + + private: + fs::path path_; +}; + +} // namespace + +// --------------------------------------------------------------------- +// set_npm_board_limit() +// --------------------------------------------------------------------- + +TEST(GpuUnit, SetNpmBoardLimitEmptyPathIsInvalidArgs) { + EXPECT_EQ(amd::smi::set_npm_board_limit("", 100), RSMI_STATUS_INVALID_ARGS); +} + +TEST(GpuUnit, SetNpmBoardLimitMissingBoardDirIsNotSupported) { + fs::path missing = fs::temp_directory_path() / "amdsmi_npm_test_definitely_missing_dir_xyz"; + std::error_code ec; + fs::remove_all(missing, ec); // ensure it really doesn't exist + + EXPECT_EQ(amd::smi::set_npm_board_limit(missing.string(), 100), RSMI_STATUS_NOT_SUPPORTED); +} + +TEST(GpuUnit, SetNpmBoardLimitMissingLimitFileIsNotSupported) { + TempBoardDir board; + // Deliberately do not create cur_node_power_limit. + EXPECT_EQ(amd::smi::set_npm_board_limit(board.path().string(), 100), RSMI_STATUS_NOT_SUPPORTED); +} + +TEST(GpuUnit, SetNpmBoardLimitSuccessWritesValue) { + TempBoardDir board; + board.WriteFile("cur_node_power_limit", "0"); + + EXPECT_EQ(amd::smi::set_npm_board_limit(board.path().string(), 250), RSMI_STATUS_SUCCESS); + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "250"); +} + +TEST(GpuUnit, SetNpmBoardLimitSuccessOverwritesPreviousValue) { + TempBoardDir board; + board.WriteFile("cur_node_power_limit", "999"); + + EXPECT_EQ(amd::smi::set_npm_board_limit(board.path().string(), 42), RSMI_STATUS_SUCCESS); + EXPECT_EQ(board.ReadFile("cur_node_power_limit"), "42"); +} + +TEST(GpuUnit, SetNpmBoardLimitDirectoryInPlaceOfFileIsNotSupported) { + TempBoardDir board; + // Create cur_node_power_limit as a directory, not a regular file, to + // exercise the is_regular_file() guard. + fs::create_directory(board.path() / "cur_node_power_limit"); + + EXPECT_EQ(amd::smi::set_npm_board_limit(board.path().string(), 100), RSMI_STATUS_NOT_SUPPORTED); +} + +// --------------------------------------------------------------------- +// get_npm_node_power() +// --------------------------------------------------------------------- + +TEST(GpuUnit, GetNpmNodePowerNullOutPtrIsInvalidArgs) { + TempBoardDir board; + board.WriteFile("node_power", "123"); + EXPECT_EQ(amd::smi::get_npm_node_power(board.path().string(), nullptr), RSMI_STATUS_INVALID_ARGS); +} + +TEST(GpuUnit, GetNpmNodePowerEmptyPathIsInvalidArgs) { + uint64_t power = 0; + EXPECT_EQ(amd::smi::get_npm_node_power("", &power), RSMI_STATUS_INVALID_ARGS); +} + +TEST(GpuUnit, GetNpmNodePowerMissingBoardDirIsNotSupported) { + fs::path missing = fs::temp_directory_path() / "amdsmi_npm_test_definitely_missing_dir_pwr"; + std::error_code ec; + fs::remove_all(missing, ec); + + uint64_t power = 0; + EXPECT_EQ(amd::smi::get_npm_node_power(missing.string(), &power), RSMI_STATUS_NOT_SUPPORTED); +} + +TEST(GpuUnit, GetNpmNodePowerMissingFileIsNotSupported) { + TempBoardDir board; + // Deliberately do not create node_power. + uint64_t power = 0; + EXPECT_EQ(amd::smi::get_npm_node_power(board.path().string(), &power), RSMI_STATUS_NOT_SUPPORTED); +} + +TEST(GpuUnit, GetNpmNodePowerSuccessReadsValue) { + TempBoardDir board; + board.WriteFile("node_power", "777"); + + uint64_t power = 0; + EXPECT_EQ(amd::smi::get_npm_node_power(board.path().string(), &power), RSMI_STATUS_SUCCESS); + EXPECT_EQ(power, 777u); +} + +TEST(GpuUnit, GetNpmNodePowerNonNumericContentsIsUnexpectedData) { + TempBoardDir board; + board.WriteFile("node_power", "not-a-number"); + + uint64_t power = 0; + EXPECT_EQ(amd::smi::get_npm_node_power(board.path().string(), &power), + RSMI_STATUS_UNEXPECTED_DATA); +} + +// --------------------------------------------------------------------- +// get_npm_board_max_limit() +// --------------------------------------------------------------------- + +TEST(GpuUnit, GetNpmBoardMaxLimitNullOutPtrIsInvalidArgs) { + TempBoardDir board; + board.WriteFile("max_node_power_limit", "6400"); + EXPECT_EQ(amd::smi::get_npm_board_max_limit(board.path().string(), nullptr), + RSMI_STATUS_INVALID_ARGS); +} + +TEST(GpuUnit, GetNpmBoardMaxLimitEmptyPathIsInvalidArgs) { + uint64_t limit = 0; + EXPECT_EQ(amd::smi::get_npm_board_max_limit("", &limit), RSMI_STATUS_INVALID_ARGS); +} + +TEST(GpuUnit, GetNpmBoardMaxLimitMissingBoardDirIsNotSupported) { + fs::path missing = fs::temp_directory_path() / "amdsmi_npm_test_definitely_missing_dir_max"; + std::error_code ec; + fs::remove_all(missing, ec); + + uint64_t limit = 0; + EXPECT_EQ(amd::smi::get_npm_board_max_limit(missing.string(), &limit), RSMI_STATUS_NOT_SUPPORTED); +} + +TEST(GpuUnit, GetNpmBoardMaxLimitMissingFileIsNotSupported) { + TempBoardDir board; + // Deliberately do not create max_node_power_limit. + uint64_t limit = 0; + EXPECT_EQ(amd::smi::get_npm_board_max_limit(board.path().string(), &limit), + RSMI_STATUS_NOT_SUPPORTED); +} + +TEST(GpuUnit, GetNpmBoardMaxLimitSuccessReadsValue) { + TempBoardDir board; + board.WriteFile("max_node_power_limit", "6400"); + + uint64_t limit = 0; + EXPECT_EQ(amd::smi::get_npm_board_max_limit(board.path().string(), &limit), RSMI_STATUS_SUCCESS); + EXPECT_EQ(limit, 6400u); +} + +TEST(GpuUnit, GetNpmBoardMaxLimitNonNumericContentsIsUnexpectedData) { + TempBoardDir board; + board.WriteFile("max_node_power_limit", "not-a-number"); + + uint64_t limit = 0; + EXPECT_EQ(amd::smi::get_npm_board_max_limit(board.path().string(), &limit), + RSMI_STATUS_UNEXPECTED_DATA); +} diff --git a/projects/amdsmi/tests/amd_smi_test/unit/system/dkms_versions_test.cc b/projects/amdsmi/tests/amd_smi_test/unit/system/dkms_versions_test.cc new file mode 100644 index 000000000000..450b95fd4ec9 --- /dev/null +++ b/projects/amdsmi/tests/amd_smi_test/unit/system/dkms_versions_test.cc @@ -0,0 +1,422 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +// Validates DKMS package discovery against a temporary tree. No GPU required. + +#include + +#include +#include +#include +#include +#include + +#include "amd_smi/impl/amd_smi_utils.h" + +namespace { + +namespace fs = std::filesystem; + +constexpr auto kPackageName = "amdgpu"; +constexpr auto kOtherPackageName = "not-amdgpu"; +constexpr auto kVersionA = "6.19.13-2370380.24.04"; +constexpr auto kVersionB = "6.19.20-2450390.24.04"; +constexpr auto kKernelRelease = "6.8.0-124-generic"; +constexpr auto kKernelMachine = "x86_64"; +constexpr auto kKernelSymlinkPrefix = "kernel-"; + +class DkmsTreeFixture { + public: + DkmsTreeFixture() { + auto base_template = (fs::temp_directory_path() / "amdsmi_dkms_versions_test_XXXXXX").string(); + if (mkdtemp(base_template.data()) == nullptr) { + throw std::runtime_error{"mkdtemp failed for the DKMS test tree"}; + } + base_ = base_template; + dkms_root_ = base_ / "dkms"; + source_tree_prefix_ = (base_ / "usr_src/amdgpu-").string(); + fs::create_directories(dkms_root_); + fs::create_directories(base_ / "usr_src"); + } + + DkmsTreeFixture(const DkmsTreeFixture&) = delete; + auto operator=(const DkmsTreeFixture&) -> DkmsTreeFixture& = delete; + DkmsTreeFixture(DkmsTreeFixture&&) = delete; + auto operator=(DkmsTreeFixture&&) -> DkmsTreeFixture& = delete; + + ~DkmsTreeFixture() { + auto err_code = std::error_code{}; + fs::remove_all(base_, err_code); + } + + auto dkms_root() const -> const fs::path& { return dkms_root_; } + + auto source_tree_prefix() const -> std::string_view { return source_tree_prefix_; } + + auto add_package(const std::string& version) -> void { + const auto paths = write_package_files(version); + fs::create_directory_symlink(paths.source_tree, paths.version_dir / "source"); + } + + auto add_package_with_relative_source_symlink(const std::string& version) -> void { + const auto paths = write_package_files(version); + const auto relative_source = fs::relative(paths.source_tree, paths.version_dir); + fs::create_directory_symlink(relative_source, paths.version_dir / "source"); + } + + auto add_kernel_symlink(const std::string& version, std::string_view release = kKernelRelease, + std::string_view machine = kKernelMachine) -> void { + const auto target = ((dkms_root_ / version) / std::string{release}) / std::string{machine}; + fs::create_directories(target); + fs::create_directory_symlink(fs::relative(target, dkms_root_), + dkms_root_ / make_kernel_link_name(release, machine)); + } + + auto add_absolute_kernel_symlink(const std::string& version, + std::string_view release = kKernelRelease, + std::string_view machine = kKernelMachine) -> void { + const auto target = ((dkms_root_ / version) / std::string{release}) / std::string{machine}; + fs::create_directories(target); + fs::create_directory_symlink(target, dkms_root_ / make_kernel_link_name(release, machine)); + } + + auto add_malformed_kernel_symlink(std::string_view target, + std::string_view release = kKernelRelease, + std::string_view machine = kKernelMachine) -> void { + fs::create_directory_symlink(fs::path{std::string{target}}, + dkms_root_ / make_kernel_link_name(release, machine)); + } + + private: + struct PackagePaths { + fs::path version_dir; + fs::path source_tree; + }; + + auto write_package_files(const std::string& version) -> PackagePaths { + auto paths = PackagePaths{}; + paths.version_dir = dkms_root_ / version; + paths.source_tree = fs::path{source_tree_prefix_ + version}; + const auto dkms_conf_dir = paths.source_tree / "amd/dkms"; + fs::create_directories(paths.version_dir); + fs::create_directories(dkms_conf_dir); + + auto conf_stream = std::ofstream{dkms_conf_dir / "dkms.conf"}; + conf_stream << "PACKAGE_NAME=\"" << kPackageName << "\"\n"; + conf_stream << "PACKAGE_VERSION=\"" << version << "\"\n"; + fs::create_symlink("amd/dkms/dkms.conf", paths.source_tree / "dkms.conf"); + return paths; + } + + auto make_kernel_link_name(std::string_view release, std::string_view machine) const + -> std::string { + auto link_name = std::string{kKernelSymlinkPrefix}; + link_name.append(release); + link_name.push_back('-'); + link_name.append(machine); + return link_name; + } + + fs::path base_{}; + fs::path dkms_root_{}; + std::string source_tree_prefix_{}; +}; + +} // namespace + +TEST(SystemUnit, DriverVersionsSplitModuleAndPackageVersions) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.19.14.31400000", "6.19.14-2370381.24.04", &info), + AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, "6.19.14"); + EXPECT_STREQ(info.amdgpu_driver_version, "31400000"); + EXPECT_STREQ(info.driver_version, "6.19.14.31400000"); + EXPECT_STREQ(info.driver_build_version, "2370381"); + EXPECT_STREQ(info.driver_full_version, "6.19.14.31400000-2370381"); +} + +TEST(SystemUnit, DriverVersionsKeepBuildEmptyWithoutActiveDkmsPackage) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.19.14.31400000", "", &info), AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, "6.19.14"); + EXPECT_STREQ(info.amdgpu_driver_version, "31400000"); + EXPECT_STREQ(info.driver_version, "6.19.14.31400000"); + EXPECT_STREQ(info.driver_build_version, ""); + EXPECT_STREQ(info.driver_full_version, "6.19.14.31400000"); +} + +TEST(SystemUnit, DriverVersionsExtractBuildWithoutDistroSuffix) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.19.14.31400000", "6.19.14-2370381", &info), + AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, "6.19.14"); + EXPECT_STREQ(info.amdgpu_driver_version, "31400000"); + EXPECT_STREQ(info.driver_version, "6.19.14.31400000"); + EXPECT_STREQ(info.driver_build_version, "2370381"); + EXPECT_STREQ(info.driver_full_version, "6.19.14.31400000-2370381"); +} + +TEST(SystemUnit, DriverVersionsExtractBuildWithNonUbuntuDistroSuffix) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.19.14.31400000", "6.19.14-2370381.el8", &info), + AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, "6.19.14"); + EXPECT_STREQ(info.amdgpu_driver_version, "31400000"); + EXPECT_STREQ(info.driver_version, "6.19.14.31400000"); + EXPECT_STREQ(info.driver_build_version, "2370381"); + EXPECT_STREQ(info.driver_full_version, "6.19.14.31400000-2370381"); +} + +TEST(SystemUnit, DriverVersionsPassThroughUnexpectedFormats) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.8.0-60-generic", "6.19.14-2370381.24.04", &info), + AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, ""); + EXPECT_STREQ(info.amdgpu_driver_version, ""); + EXPECT_STREQ(info.driver_version, "6.8.0-60-generic"); + EXPECT_STREQ(info.driver_build_version, ""); + EXPECT_STREQ(info.driver_full_version, "6.8.0-60-generic"); +} + +TEST(SystemUnit, DriverVersionsThreePartModuleVersionWithoutBuild) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.19.4", "", &info), AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, "6.19.4"); + EXPECT_STREQ(info.amdgpu_driver_version, ""); + EXPECT_STREQ(info.driver_version, "6.19.4"); + EXPECT_STREQ(info.driver_build_version, ""); + EXPECT_STREQ(info.driver_full_version, "6.19.4"); +} + +TEST(SystemUnit, DriverVersionsThreePartModuleVersionWithBuild) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.19.4", "6.19.4-2193512.24.04", &info), + AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, "6.19.4"); + EXPECT_STREQ(info.amdgpu_driver_version, ""); + EXPECT_STREQ(info.driver_version, "6.19.4"); + EXPECT_STREQ(info.driver_build_version, "2193512"); + EXPECT_STREQ(info.driver_full_version, "6.19.4-2193512"); +} + +TEST(SystemUnit, DriverVersionsIgnoreBuildFromOtherDkmsPackage) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.19.4", "6.19.14-2370381.24.04", &info), + AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, "6.19.4"); + EXPECT_STREQ(info.amdgpu_driver_version, ""); + EXPECT_STREQ(info.driver_version, "6.19.4"); + EXPECT_STREQ(info.driver_build_version, ""); + EXPECT_STREQ(info.driver_full_version, "6.19.4"); +} + +TEST(SystemUnit, DriverVersionsModuleVersionUnavailable) { + auto info = amdsmi_driver_info_t{}; + + EXPECT_EQ(smi_amdgpu_parse_driver_versions("N/A", "6.19.14-2370381.24.04", &info), + AMDSMI_STATUS_SUCCESS); + EXPECT_STREQ(info.driver_kernel_version, ""); + EXPECT_STREQ(info.amdgpu_driver_version, ""); + EXPECT_STREQ(info.driver_version, "N/A"); + EXPECT_STREQ(info.driver_build_version, ""); + EXPECT_STREQ(info.driver_full_version, ""); +} + +TEST(SystemUnit, DriverVersionsRejectNullOutput) { + EXPECT_EQ(smi_amdgpu_parse_driver_versions("6.19.14.31400000", "6.19.14-2370381.24.04", nullptr), + AMDSMI_STATUS_INVAL); +} + +TEST(SystemUnit, ActiveDkmsVersionSelectsKernelSymlinkPackage) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package(kVersionA); + fixture.add_package(kVersionB); + fixture.add_kernel_symlink(kVersionA); + + auto active_version = std::string{}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_SUCCESS); + EXPECT_EQ(active_version, kVersionA); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsMissingKernelSymlink) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package(kVersionA); + + auto active_version = std::string{"unchanged"}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_NOT_SUPPORTED); + EXPECT_TRUE(active_version.empty()); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsUnvalidatedKernelSymlinkTarget) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package(kVersionA); + fixture.add_kernel_symlink(kVersionB); + + auto active_version = std::string{"unchanged"}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_NOT_SUPPORTED); + EXPECT_TRUE(active_version.empty()); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsNullOutput) { + auto fixture = DkmsTreeFixture{}; + + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, nullptr), + AMDSMI_STATUS_INVAL); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsEmptyRelease) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package(kVersionA); + fixture.add_kernel_symlink(kVersionA); + + auto active_version = std::string{"unchanged"}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), "", kKernelMachine, + &active_version), + AMDSMI_STATUS_NOT_SUPPORTED); + EXPECT_TRUE(active_version.empty()); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsEmptyMachine) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package(kVersionA); + fixture.add_kernel_symlink(kVersionA); + + auto active_version = std::string{"unchanged"}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, "", + &active_version), + AMDSMI_STATUS_NOT_SUPPORTED); + EXPECT_TRUE(active_version.empty()); +} + +TEST(SystemUnit, ActiveDkmsVersionSelectsAbsoluteKernelSymlink) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package(kVersionA); + fixture.add_absolute_kernel_symlink(kVersionA); + + auto active_version = std::string{}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_SUCCESS); + EXPECT_EQ(active_version, kVersionA); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsMalformedKernelSymlinkTarget) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package(kVersionA); + fixture.add_malformed_kernel_symlink( + (std::string{"not-a-version/"} + kKernelRelease + "/" + kKernelMachine)); + + auto active_version = std::string{"unchanged"}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_NOT_SUPPORTED); + EXPECT_TRUE(active_version.empty()); +} + +TEST(SystemUnit, ActiveDkmsVersionAcceptsRelativeSourceSymlink) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package_with_relative_source_symlink(kVersionA); + fixture.add_kernel_symlink(kVersionA); + + auto active_version = std::string{}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_SUCCESS); + EXPECT_EQ(active_version, kVersionA); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsMissingDkmsConf) { + auto fixture = DkmsTreeFixture{}; + const auto version_dir = fixture.dkms_root() / kVersionA; + const auto source_tree = fs::path{std::string{fixture.source_tree_prefix()} + kVersionA}; + fs::create_directories(version_dir); + fs::create_directories(source_tree); + fs::create_directory_symlink(source_tree, version_dir / "source"); + fixture.add_kernel_symlink(kVersionA); + + auto active_version = std::string{"unchanged"}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_NOT_SUPPORTED); + EXPECT_TRUE(active_version.empty()); +} + +TEST(SystemUnit, ActiveDkmsVersionAcceptsSourceTreeOutsideDefaultPrefix) { + auto fixture = DkmsTreeFixture{}; + fixture.add_package(kVersionA); + const auto version_dir = fixture.dkms_root() / kVersionA; + const auto default_tree = fs::path{std::string{fixture.source_tree_prefix()} + kVersionA}; + const auto custom_tree = fixture.dkms_root().parent_path() / "custom_src" / kVersionA; + fs::create_directories(custom_tree.parent_path()); + fs::rename(default_tree, custom_tree); + fs::remove(version_dir / "source"); + fs::create_directory_symlink(custom_tree, version_dir / "source"); + fixture.add_kernel_symlink(kVersionA); + + auto active_version = std::string{}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_SUCCESS); + EXPECT_EQ(active_version, kVersionA); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsPackageNameMismatch) { + auto fixture = DkmsTreeFixture{}; + const auto version_dir = fixture.dkms_root() / kVersionA; + const auto source_tree = fs::path{std::string{fixture.source_tree_prefix()} + kVersionA}; + const auto dkms_conf_dir = source_tree / "amd/dkms"; + fs::create_directories(version_dir); + fs::create_directories(dkms_conf_dir); + + auto conf_stream = std::ofstream{dkms_conf_dir / "dkms.conf"}; + conf_stream << "PACKAGE_NAME=\"" << kOtherPackageName << "\"\n"; + conf_stream << "PACKAGE_VERSION=\"" << kVersionA << "\"\n"; + conf_stream.close(); + fs::create_symlink("amd/dkms/dkms.conf", source_tree / "dkms.conf"); + fs::create_directory_symlink(source_tree, version_dir / "source"); + fixture.add_kernel_symlink(kVersionA); + + auto active_version = std::string{"unchanged"}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_NOT_SUPPORTED); + EXPECT_TRUE(active_version.empty()); +} + +TEST(SystemUnit, ActiveDkmsVersionRejectsPackageVersionMismatch) { + auto fixture = DkmsTreeFixture{}; + const auto version_dir = fixture.dkms_root() / kVersionA; + const auto source_tree = fs::path{std::string{fixture.source_tree_prefix()} + kVersionA}; + const auto dkms_conf_dir = source_tree / "amd/dkms"; + fs::create_directories(version_dir); + fs::create_directories(dkms_conf_dir); + + auto conf_stream = std::ofstream{dkms_conf_dir / "dkms.conf"}; + conf_stream << "PACKAGE_NAME=\"" << kPackageName << "\"\n"; + conf_stream << "PACKAGE_VERSION=\"" << kVersionB << "\"\n"; + conf_stream.close(); + fs::create_symlink("amd/dkms/dkms.conf", source_tree / "dkms.conf"); + fs::create_directory_symlink(source_tree, version_dir / "source"); + fixture.add_kernel_symlink(kVersionA); + + auto active_version = std::string{"unchanged"}; + EXPECT_EQ(smi_amdgpu_get_active_dkms_version(fixture.dkms_root().string(), kKernelRelease, + kKernelMachine, &active_version), + AMDSMI_STATUS_NOT_SUPPORTED); + EXPECT_TRUE(active_version.empty()); +} diff --git a/projects/amdsmi/tests/python/README.md b/projects/amdsmi/tests/python/README.md index 8b5c22b3d130..fc578f2e3387 100644 --- a/projects/amdsmi/tests/python/README.md +++ b/projects/amdsmi/tests/python/README.md @@ -213,7 +213,7 @@ Legend: . = pass, s = skipped, F = fail, E = error [----------] 40 tests ran. (512 ms total) [ PASSED ] 38 tests. -[ FAILED ] 1 test, listed below: +[ FAILED ] 1 failure, listed below: [ FAILED ] TestPower.test_power_cap [ SKIPPED ] 1 test, listed below: [ SKIPPED ] TestOverdrive.test_overdrive_write @@ -237,7 +237,10 @@ prints one character per test: - `[----------] N tests ran. (X ms total)` — total count and wall-clock time. - `[ PASSED ] N tests.` — count that passed. -- `[ FAILED ] ...` — one line per failing test, listed by `Class.method`. +- `[ FAILED ] ...` — one line per failure, listed by `Class.method`. Counted in + failures rather than tests: one test can report several (a cleanup failing + after the body, or one per `subTest`), and class-level `setUpClass` / + `tearDownClass` errors appear here without being tests at all. Errors and unexpected `@expectedFailure` successes are counted here too. - `[ SKIPPED ] ...` — one line per skipped test. diff --git a/projects/amdsmi/tests/python/cli/base.py b/projects/amdsmi/tests/python/cli/base.py index f1fc055df9cb..bd09e82f3969 100644 --- a/projects/amdsmi/tests/python/cli/base.py +++ b/projects/amdsmi/tests/python/cli/base.py @@ -11,13 +11,99 @@ files each subclass ``TestCliBase`` and add their own test_* methods. """ +import collections import json +import math import os import stat +import sys import unittest import common.common as common import common.runcmd as runcmd +from common.common import amdsmi + + +def _load_cli_exceptions(): + """The CLI's own exception module, from whichever amd-smi is under test.""" + # These tests spawn the amd-smi binary from the install tree (see runcmd), so + # the exit codes have to come from that same install, not from the checkout. + cli_dir = common.find_cli_dir( + *common.cli_search_order(os.path.dirname(os.path.abspath(__file__)), prefer_install=True) + ) + if cli_dir and cli_dir not in sys.path: + sys.path.append(cli_dir) + # Tests name AmdSmiExitCode, so a miss has to fail here rather than as an + # attribute error on None once a test is already running. + try: + import amdsmi_cli_exceptions + except ImportError as exc: + raise ImportError( + f"amd-smi CLI not found under {cli_dir!r}; the CLI tests need it for exit codes" + ) from exc + + return amdsmi_cli_exceptions + + +AmdSmiExitCode = _load_cli_exceptions().AmdSmiExitCode + + +def _build_exit_code_names(): + """Exit code -> name, e.g. ``NOT_SUPPORTED``.""" + names = {} + # Folding a status to its low byte can collide, so the first member wins. + for member in amdsmi.AmdSmiStatus: + names.setdefault(abs(int(member.value)) & 0xFF, member.name) + # CLI codes are exact, so they override any collision above. + for member in AmdSmiExitCode: + if member.value: + names[member.value] = member.name + return names + + +EXIT_CODE_NAMES = _build_exit_code_names() + + +def _describe_codes(codes): + """Accepted exit codes, named, for a failure message.""" + named = ", ".join(f"{EXIT_CODE_NAMES.get(code, 'UNKNOWN')}({code})" for code in codes) + return f"[{named}]" + + +# One command's verdict: the line RunCmds prints, and whether it counts as a +# test failure -- which is not the same as the command failing, since a command +# declared to fail counts as a pass when it does. +Verdict = collections.namedtuple("Verdict", "message counts_as_failure") + +# --clk-level placeholder -> the clock name the CLI expects. PCIE is sized from +# a different source than the rest; see _clk_level_count(). +CLK_LEVEL_TYPES = { + "{clk_level_sclk}": "SCLK", + "{clk_level_mclk}": "MCLK", + "{clk_level_fclk}": "FCLK", + "{clk_level_socclk}": "SOCCLK", + "{clk_level_pcie}": "PCIE", +} + +# --clk-limit placeholder -> (clock name the CLI expects, which bound to set). +# The metric key and the value key both follow from these, so they cannot drift +# apart the way separately written entries did. +CLK_LIMIT_TARGETS = { + "{clk_limit_sclk_max}": ("SCLK", "MAX"), + "{clk_limit_sclk_min}": ("SCLK", "MIN"), + "{clk_limit_mclk_max}": ("MCLK", "MAX"), + "{clk_limit_mclk_min}": ("MCLK", "MIN"), + "{clk_limit_fclk_max}": ("FCLK", "MAX"), + "{clk_limit_fclk_min}": ("FCLK", "MIN"), +} + +# Clock name -> the ``metric --json`` key reporting its min_clk/max_clk. SCLK is +# the graphics clock (the CLI maps it to AmdSmiClkType.GFX), not the SOC clock. +CLK_LIMIT_METRIC_KEYS = {"SCLK": "gfx_0", "MCLK": "mem_0", "FCLK": "fclk_0"} + +# --clk-limit probe values are rounded up to a multiple of this. Cosmetic only: +# the driver accepts any MHz within range. +CLK_LIMIT_MHZ_STEP = 10 class TestCliBase(unittest.TestCase): @@ -32,6 +118,39 @@ class TestCliBase(unittest.TestCase): TMP_FILENAME = "_tmp.log" TMP_FOLDER = "_tmp" + # TODO(amdsmi_team): drop these once the CLI tests supports automated + # input + SWEEP_EXCLUDED_ARGS = { + "set": {"--compute-partition": "waits for confirmation"}, + "reset": { + # --gtt reaches reset.py's prompt_reboot(), which calls `systemctl reboot` on "y" + "--gtt": "prompts to reboot the host", + "--gpureset": "can hang the host", + }, + } + + # TODO(amdsmi_team): User input is not a problem, we just need to test above ^ + PROMPT_ANSWERS = {"set": {"--fan": "y\n", "--memory-partition": "y\n"}} + + # Commands whose sweep also accepts NOT_SUPPORTED. A write may legitimately + # refuse a feature the ASIC lacks, but a read answering NOT_SUPPORTED is a + # regression the sweep should catch. + NOT_SUPPORTED_TOLERANT_COMMANDS = {"set", "reset"} + + # Exit codes a single argument may also answer with, beyond NOT_SUPPORTED + SWEEP_TOLERATED_CODES = { + # Not all ASICs support setting all modes, which is an invalid set option + "set": {"--memory-partition": amdsmi.AmdSmiStatus.INVAL}, + # This reset is a read -- the kernel clears xgmi_error when it is read -- + # so the read's errno becomes the reset's status, and some drivers reject + # it with EINVAL. + "reset": {"--xgmierr": amdsmi.AmdSmiStatus.INVAL}, + } + + # Order for the amd-smi reset commands, putting --clocks last since + # it resets clocks and overdrive to default + SWEEP_ARG_ORDER = {"reset": {"first": "--fans", "last": "--clocks"}} + # Scaffolding shared across every CLI test class. setUpClass populates # these once, directly on ``TestCliBase``; subclasses then resolve them # natively through normal attribute inheritance -- no dynamic ``setattr`` @@ -43,6 +162,7 @@ class TestCliBase(unittest.TestCase): static_data: dict metric_data: dict partition_data: dict + clk_freq: list gpus: list sub_args: dict @@ -68,17 +188,11 @@ def setUpClass(cls): TestCliBase.static_data = baseline["static_data"] TestCliBase.list_data = baseline["list_data"] TestCliBase.partition_data = baseline["partition_data"] + TestCliBase.clk_freq = baseline["clk_freq"] TestCliBase.gpus = baseline["gpus"] TestCliBase.sub_args = baseline["sub_args"] TestCliBase._initialized = True - # TODO: Remove this condition when CLI supports User automated input - # Commands that need User permission to run - TestCliBase.cmds_need_permission = { - "set": ["--fan", "--memory-partition", "--compute-partition"], - "reset": ["--gtt"], - } - @classmethod def _build_baseline(cls): baseline = {} @@ -121,12 +235,16 @@ def _build_baseline(cls): gpus.append(entry["uuid"]) baseline["gpus"] = gpus - # When parsing, expand each arg with array element + # When parsing, expand each arg with its value(s). + # CPU/Core are given default values because their sub-arguments require a value. + # Otherwise, tests will populate the args incorrectly on CPU/APU-capable systems. baseline["sub_args"] = { "CLOCK": ["SYS", "DF", "DCEF", "SOC", "MEM", "VCLK0", "VCLK1", "DCLK0", "DCLK1", "ALL"], "PID": [123], "NAME": ["AMD"], "GPU": gpus, + "CPU": ["all", "0"], + "CORE": ["all", "0"], "FILE": [ cls.TMP_FILENAME, f"{cls.TMP_FILENAME} --overwrite", @@ -137,8 +255,60 @@ def _build_baseline(cls): "FILE_LIMIT": [10], #'LEVEL': ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], } + baseline["clk_freq"] = cls._read_clk_freq() return baseline + @classmethod + def _read_clk_freq(cls): + """Per-GPU DPM clock tables, read from the library rather than the CLI. + + Neither CLI command can say how many levels are settable, and combining them + does not close the gap either: + + - ``amd-smi static --clock`` lists every row of the table as ``Level N``, + including the non-settable deep-sleep row, and does not mark which one it + is. A three-entry list is therefore either three settable levels, or two + plus an ``S`` row, and the two cases are indistinguishable. + - ``amd-smi metric --clock`` does report a per-clock ``deep_sleep``, but it + discards the library value and recomputes it as (current clk < min clk). + That is a runtime state which flips as the GPU idles, so it cannot identify + the row static left ambiguous -- the same device answers differently + depending on how busy it was when the command ran. + + ``amdsmi_get_clock_info`` returns ``clk_deep_sleep`` before the CLI recomputes + it, and it is populated only when the table carries the ``S`` row, which is + what decides the settable level indices. + + Returns: + list: one dict per GPU, keyed by the CLI's clock names; a value is None + when the device does not expose that clock. + """ + clk_types = { + "SCLK": amdsmi.AmdSmiClkType.SYS, + "MCLK": amdsmi.AmdSmiClkType.MEM, + "FCLK": amdsmi.AmdSmiClkType.DF, + "SOCCLK": amdsmi.AmdSmiClkType.SOC, + } + per_gpu = [] + # Common.__init__ shuts the library down once it has read its own data. + cls.common.amdsmi_smart_init() + try: + for processor in amdsmi.amdsmi_get_processor_handles(): + entry = {} + for name, clk_type in clk_types.items(): + try: + freq = amdsmi.amdsmi_get_clk_freq(processor, clk_type) + info = amdsmi.amdsmi_get_clock_info(processor, clk_type) + except amdsmi.AmdSmiLibraryException: + entry[name] = None + continue + freq["has_deep_sleep"] = info["clk_deep_sleep"] != "N/A" + entry[name] = freq + per_gpu.append(entry) + finally: + amdsmi.amdsmi_shut_down() + return per_gpu + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.Debug = False @@ -150,7 +320,7 @@ def __init__(self, *args, **kwargs): self.AddWatchArgs = True self.AddLogLevel = "--loglevel DEBUG" - self.PASS = 0 + self.PASS = amdsmi.AmdSmiStatus.SUCCESS self.FAIL = 1 self.tab = " " self.tmp_filename = self.TMP_FILENAME @@ -185,8 +355,8 @@ def __init__(self, *args, **kwargs): self.memory_partition_modes = ["NPS1", "NPS2", "NPS4", "NPS8"] self.power_types = ["ppt0", "ppt1"] self.ptl_formats = ["I8", "F16", "BF16", "F32", "F64", "F8", "VECTOR"] - self.clk_limits = ["SCLK", "MCLK"] - self.limit_types = ["MIN", "MAX"] + self.clk_limits = ["SCLK", "MCLK", "FCLK"] + self.limit_types = ["MAX", "MIN"] self.clk_levels = ["SCLK", "MCLK", "FCLK", "SOCCLK", "PCIE"] # When parsing, ignore these entries as they are abnormal @@ -351,18 +521,26 @@ def FindArgs(self, cmd, match_str): elif ( sub_arg == "CLK_TYPE" and "limit" in items[item_index] ): # arg --clk-limit - options.append(f"{items[item_index]} {{clk_limit_sclk_min}}") + # Raise the ceiling before the floor: a MIN above the + # current MAX is refused, so MIN-first cannot widen a + # range upwards. options.append(f"{items[item_index]} {{clk_limit_sclk_max}}") - options.append(f"{items[item_index]} {{clk_limit_mclk_min}}") + options.append(f"{items[item_index]} {{clk_limit_sclk_min}}") options.append(f"{items[item_index]} {{clk_limit_mclk_max}}") + options.append(f"{items[item_index]} {{clk_limit_mclk_min}}") + options.append(f"{items[item_index]} {{clk_limit_fclk_max}}") + options.append(f"{items[item_index]} {{clk_limit_fclk_min}}") elif ( sub_arg == "STATUS" and "process" in items[item_index] ): # arg --process-isolation options.append(f"{items[item_index]} 0") options.append(f"{items[item_index]} 1") else: - print( - f"TODO: set {items[item_index]} sub_arg={sub_arg} match_str={match_str}" + # Reached when no branch above recognises the + # placeholder, so no values can be generated. + self.common.print( + f"{self.tab}Untested (no sweep values for {sub_arg}): " + f"amd-smi {cmd.split()[1]} {items[item_index]}" ) if not sub_found: # Put in sub_arg if it was not found @@ -372,13 +550,14 @@ def FindArgs(self, cmd, match_str): options.append(items[item_index]) if match_str in line: found = True + if not options: return ["pass"] return options def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): cmd = f"amd-smi {cmd_name} --help" - list1_args = self.FindArgs(cmd, list1_name) + list1_args = self._order_sweep_args(cmd_name, self.FindArgs(cmd, list1_name)) list2_args = self.FindArgs(cmd, list2_name) list3_args = self.FindArgs(cmd, list3_name) list4_args = self.FindArgs(cmd, list4_name) @@ -394,33 +573,33 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): cmds = [] cmd = f"amd-smi {cmd_name}" + ok = [self.PASS] + if cmd_name in self.NOT_SUPPORTED_TOLERANT_COMMANDS: + ok.append(amdsmi.AmdSmiStatus.NOT_SUPPORTED) for list1_arg in list1_args: if list1_arg != "pass": - cmds.append((f"{cmd} {list1_arg} {self.AddLogLevel}", self.PASS)) + cmds.append((f"{cmd} {list1_arg} {self.AddLogLevel}", ok)) if not list1_arg: - cmds.append((f"{cmd} --file {self.tmp_filename} {self.AddLogLevel}", self.PASS)) - cmds.append((f"{cmd} {{json}} {self.AddLogLevel}", self.PASS)) - cmds.append((f"{cmd} {{json_file}} {self.AddLogLevel}", self.PASS)) - cmds.append((f"{cmd} {{json_file_append}} {self.AddLogLevel}", self.PASS)) - cmds.append((f"{cmd} {{json_file_overwrite}} {self.AddLogLevel}", self.PASS)) - cmds.append((f"{cmd} {{csv}} {self.AddLogLevel}", self.PASS)) - cmds.append((f"{cmd} {{csv_file}} {self.AddLogLevel}", self.PASS)) - cmds.append((f"{cmd} {{csv_file_append}} {self.AddLogLevel}", self.PASS)) - cmds.append((f"{cmd} {{csv_file_overwrite}} {self.AddLogLevel}", self.PASS)) + cmds.append((f"{cmd} --file {self.tmp_filename} {self.AddLogLevel}", ok)) + cmds.append((f"{cmd} {{json}} {self.AddLogLevel}", ok)) + cmds.append((f"{cmd} {{json_file}} {self.AddLogLevel}", ok)) + cmds.append((f"{cmd} {{json_file_append}} {self.AddLogLevel}", ok)) + cmds.append((f"{cmd} {{json_file_overwrite}} {self.AddLogLevel}", ok)) + cmds.append((f"{cmd} {{csv}} {self.AddLogLevel}", ok)) + cmds.append((f"{cmd} {{csv_file}} {self.AddLogLevel}", ok)) + cmds.append((f"{cmd} {{csv_file_append}} {self.AddLogLevel}", ok)) + cmds.append((f"{cmd} {{csv_file_overwrite}} {self.AddLogLevel}", ok)) else: list1_arg = "" for list2_arg in list2_args: if list2_arg != "pass": - cmds.append((f"{cmd} {list1_arg} {list2_arg} {self.AddLogLevel}", self.PASS)) + cmds.append((f"{cmd} {list1_arg} {list2_arg} {self.AddLogLevel}", ok)) else: list2_arg = "" for list3_arg in list3_args: if list3_arg != "pass": cmds.append( - ( - f"{cmd} {list1_arg} {list2_arg} {list3_arg} {self.AddLogLevel}", - self.PASS, - ) + (f"{cmd} {list1_arg} {list2_arg} {list3_arg} {self.AddLogLevel}", ok) ) else: list3_arg = "" @@ -429,7 +608,7 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): cmds.append( ( f"{cmd} {list1_arg} {list2_arg} {list3_arg} {list4_arg} {self.AddLogLevel}", - self.PASS, + ok, ) ) @@ -437,14 +616,19 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): # Removes cmds that are invalid for index, cmd_cond in enumerate(cmds): cmd, cond = cmd_cond + tolerated = self._lookup(self.SWEEP_TOLERATED_CODES, cmd) + if tolerated is not None: + cond = [*ok, tolerated] while self.openCurlyBrace in cmd: items = cmd.split() - # Find gpu index + # Find gpu index. No --gpu, "all" or a bdf all target every device. + explicit_gpu = False try: i = items.index("--gpu") gpu = items[i + 1] if gpu.isdigit(): gpu_index = int(gpu) + explicit_gpu = True else: gpu_index = 0 except ValueError: @@ -499,14 +683,15 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): nameStr == "{min_power}" or nameStr == "{avg_power}" or nameStr == "{max_power}" ): # For setting --power-cap - # Find power_type + # Stop at the match; a later iteration would overwrite it with "N/A". for power_type in self.power_types: if power_type in cmd: power_type = self.static_data["gpu_data"][gpu_index]["limit"][ power_type ] - else: - power_type = "N/A" + break + else: + power_type = "N/A" if ( power_type == "N/A" or not isinstance(power_type, dict) @@ -515,7 +700,14 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): ): cmd = "" else: - min_power = power_type["min_power_limit"]["value"] + # A current cap of 0 means the CLI refuses every write to this + # sensor, whatever value is asked for. + current_cap = power_type["socket_power_limit"] + if isinstance(current_cap, dict) and current_cap["value"] == 0: + cond = [*ok, AmdSmiExitCode.INVALID_PARAMETER_VALUE] + # The CLI clamps its lower bound to 1, so a reported min of 0 + # is refused rather than being the smallest settable cap. + min_power = max(power_type["min_power_limit"]["value"], 1) max_power = power_type["max_power_limit"]["value"] avg_power = int((min_power + max_power) / 2) if nameStr == "{min_power}": @@ -536,69 +728,29 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): else: cmd = "" elif "clk_limit" in nameStr: - clock = self.metric_data["gpu_data"][gpu_index]["clock"] - clk_type = clk_type_name = limit_type = clk_limit_name = "" - if nameStr == "{clk_limit_sclk_min}": - clk_type = "SCLK" - clk_type_name = "socclk_0" - limit_type = "MIN" - clk_limit_name = "min_clk" - elif nameStr == "{clk_limit_sclk_max}": - clk_type = "SCLK" - clk_type_name = "socclk_0" - limit_type = "MAX" - clk_limit_name = "max_clk" - elif nameStr == "{clk_limit_mclk_min}": - clk_type = "MCLK" - clk_type_name = "mem_0" - limit_type = "MAX" - clk_limit_name = "min_clk" - elif nameStr == "{clk_limit_mclk_max}": - clk_type = "MCLK" - clk_type_name = "mem_0" - limit_type = "MIN" - clk_limit_name = "max_clk" - clk_type_limit_name = clock[clk_type_name][clk_limit_name] - if type(clk_type_limit_name) is dict: - value = clk_type_limit_name["value"] - cmd = cmd.replace(nameStr, f"{clk_type} {limit_type} {value}", 1) - else: + clk_type, bound = CLK_LIMIT_TARGETS[nameStr] + value = self._clk_limit_probe_value( + self.metric_data["gpu_data"][gpu_index]["clock"], clk_type, bound + ) + if value is None: cmd = "" + else: + # The sweep writes MAX before MIN, and that write can narrow the + # range, so a value probed from the snapshot may no longer fit. + cond = [*ok, AmdSmiExitCode.INVALID_PARAMETER_VALUE] + cmd = cmd.replace(nameStr, f"{clk_type} {bound} {value}", 1) elif "clk_level" in nameStr: - clock = self.static_data["gpu_data"][gpu_index]["clock"] - value = -1 - clk_type = clk_type_name = "" - if nameStr == "{clk_level_sclk}": - clk_type = "SCLK" - clk_type_name = "sys" - elif nameStr == "{clk_level_mclk}": - clk_type = "MCLK" - clk_type_name = "mem" - elif nameStr == "{clk_level_fclk}": - clk_type = "FCLK" - clk_type_name = "df" - elif nameStr == "{clk_level_socclk}": - clk_type = "SOCCLK" - clk_type_name = "soc" - elif nameStr == "{clk_level_pcie}": - bus = self.static_data["gpu_data"][gpu_index]["bus"] - clk_type = "PCIE" - pcie_levels = bus["pcie_levels"] - if type(pcie_levels) is dict: - value = len(pcie_levels) - if value > 0: - value = 0 - if clk_type != "PCIE" and value < 0: - clk_type_name = clock[clk_type_name] - if type(clk_type_name) is dict: - current_level = clk_type_name["current_level"] - freq_levels = clk_type_name["frequency_levels"] - if current_level == 0: - value = len(freq_levels) - 1 - else: - value = 0 - if value >= 0: - cmd = cmd.replace(nameStr, f"{clk_type} {value}", 1) + clk_type = CLK_LEVEL_TYPES[nameStr] + count = self._clk_level_count(clk_type, gpu_index, explicit_gpu) + if clk_type != "PCIE": + # A DPM table's length is a runtime value, not a fixed property: + # the same device reports fewer levels when the domain is idle. + # The CLI re-reads it as the command runs, so a mask sized here + # can name a level it no longer accepts. + cond = [*ok, AmdSmiExitCode.INVALID_PARAMETER_VALUE] + if count: + levels = " ".join(str(level) for level in range(count)) + cmd = cmd.replace(nameStr, f"{clk_type} {levels}", 1) else: cmd = "" elif nameStr == "{soc_pstate}": @@ -654,7 +806,7 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): # No explicit gpu infers a gpu=0 gpu_index = "0" - if "--gpu" in cmd: + if "--gpu" in items: try: i = items.index("--gpu") gpu_index = items[i + 1] @@ -667,7 +819,8 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): if cmd and found_sub_arg: sub_arg = items[2] if sub_arg != found_sub_arg: - if "--gpu" in cmd: + # Whole word matches only: i.e. remove "--gpu", not "--gpureset" + if "--gpu" in items: cmd = "" # Remove all file and watch modifiers except for gpu 0 @@ -699,25 +852,22 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): cmds[index] = (cmd, cond) - # Remove commands requiring input - _skipped_cmds = [] + # Remove commands the sweep must not run + announced = set() for index, cmd_cond in enumerate(cmds): cmd, cond = cmd_cond if not cmd: continue - items = cmd.split() - if len(items) < 3: + reason = self._lookup(self.SWEEP_EXCLUDED_ARGS, cmd) + if reason is None: continue - if items[1] in self.cmds_need_permission: - if items[2] in self.cmds_need_permission[items[1]]: - cmd = "" - key = f"{items[1]}_{items[2]}" - if key not in _skipped_cmds: - print(f"TODO: Skip needs permission: {items[1]} {items[2]}") - _skipped_cmds.append(key) - # Update cmds when cmd has changed - if not cmd: - cmds[index] = (cmd, cond) + # Name what went untested; a sweep that drops commands silently + # reads as coverage it never had. + key = " ".join(cmd.split()[1:3]) + if key not in announced: + announced.add(key) + self.common.print(f"{self.tab}Excluded ({reason}): amd-smi {key}") + cmds[index] = ("", cond) # Remove empty (cmd,cond) arguments cmds = [cmd_cond for cmd_cond in cmds if cmd_cond[0] != ""] @@ -733,6 +883,190 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): print(json.dumps(cmds, sort_keys=False, indent=4), flush=True) return cmds + def _order_sweep_args(self, cmd_name, args): + """Move one of *args* to the front and one to the back, per SWEEP_ARG_ORDER. + + Args: + cmd_name: the amd-smi command being swept, e.g. ``"reset"``. + args: that command's own arguments as FindArgs returned them, each + either a bare flag (``"--gpureset"``) or a flag and its value + (``"--fan 50%"``). + + Returns: + list: the same arguments, reordered. Anything the table does not + name keeps the position the help text gave it. + """ + order = self.SWEEP_ARG_ORDER.get(cmd_name) + if not order: + return args + + goes_first = order.get("first") + goes_last = order.get("last") + head, middle, tail = [], [], [] + for arg in args: + flag = arg.split(" ")[0] + if flag == goes_first: + head.append(arg) + elif flag == goes_last: + tail.append(arg) + else: + middle.append(arg) + return head + middle + tail + + @staticmethod + def _lookup(table, cmd): + """Value for *cmd* in a ``{command: {sub-arg: value}}`` table, else None. + + Matches on the words the command contains rather than where they sit, so + an option that moves (``set --gpu 0 --fan 50``) is still found. + """ + words = cmd.split() + for command, sub_args in table.items(): + if command not in words: + continue + for sub_arg, value in sub_args.items(): + if sub_arg in words: + return value + return None + + def _prompt_answer_for(self, cmd): + """Reply to pipe to a command whose parser prompts, else None.""" + return self._lookup(self.PROMPT_ANSWERS, cmd) + + def _clk_level_count(self, clk_type, gpu_index, explicit_gpu): + """How many levels a ``--clk-level`` mask for *clk_type* may name. + + Args: + clk_type: clock name as the CLI spells it, e.g. ``SOCCLK``. + gpu_index: device the command reads its values from. + explicit_gpu: whether the command named that device, as opposed to + defaulting to it while actually running on every device. + + Returns: + int: the highest settable level plus one, or 0 when the device does + not expose the clock -- in which case the caller drops the command. + """ + if clk_type == "PCIE": + pcie_levels = self.static_data["gpu_data"][gpu_index]["bus"]["pcie_levels"] + return len(pcie_levels) if isinstance(pcie_levels, dict) else 0 + + # Without an explicit --gpu the command runs on every device, so the mask + # has to be one the smallest table also accepts. + tables = [self.clk_freq[gpu_index]] if explicit_gpu else self.clk_freq + per_gpu = [] + for table in tables: + freq = table.get(clk_type) + if freq and freq["num_supported"]: + # The deep-sleep row occupies a table slot but is not settable. + per_gpu.append(freq["num_supported"] - int(freq["has_deep_sleep"])) + return min(per_gpu) if per_gpu else 0 + + @staticmethod + def _clk_limit_value(clocks, clk_type, bound): + """The clock's currently reported *bound*, or None when it is not readable. + + Args: + clocks: the ``clock`` object from ``metric --json`` for one device. + clk_type: clock name as the CLI spells it, e.g. ``SCLK``. + bound: ``"MIN"`` or ``"MAX"``. + + Returns: + int | None: the reported value, or None when the device does not + report it. + """ + entry = clocks.get(CLK_LIMIT_METRIC_KEYS[clk_type]) + if not isinstance(entry, dict): + return None + value = entry["min_clk" if bound == "MIN" else "max_clk"] + return value["value"] if isinstance(value, dict) else None + + @classmethod + def _clk_limit_probe_value(cls, clocks, clk_type, bound): + """A value inside the clock's reported range, or None when there is none. + + Replaying the reported bound is a no-op -- the CLI answers "already set" + -- so a sweep built from it never shows that a limit can move. + + Args: + clocks: the ``clock`` object from ``metric --json`` for one device. + clk_type: clock name as the CLI spells it, e.g. ``SCLK``. + bound: ``"MIN"`` or ``"MAX"``. + + Returns: + int | None: the value to send, or None when the clock is unreadable + or its range is too narrow to step into. + """ + low = cls._clk_limit_value(clocks, clk_type, "MIN") + high = cls._clk_limit_value(clocks, clk_type, "MAX") + if low is None or high is None: + return None + + # A quarter in from each end leaves MIN below MAX whichever is written first. + quarter = (high - low) // 4 + if quarter <= 0: + return None + target = low + quarter if bound == "MIN" else high - quarter + + # Rounding up can carry a narrow range past its own ceiling. It cannot fall + # below the floor: quarter is at least 1, so target already exceeds it. + target = math.ceil(target / CLK_LIMIT_MHZ_STEP) * CLK_LIMIT_MHZ_STEP + return min(target, high) + + def _accepted_codes(self, cond): + """Exit codes *cond* names, as a tuple; a bare code becomes a 1-tuple. + + Args: + cond: one exit code, or a list/tuple of the codes that are acceptable. + + Returns: + tuple: the accepted codes, in the order they were declared. + """ + if isinstance(cond, (list, tuple)): + return tuple(cond) + return (cond,) + + def _grade(self, cond, rc, file_error, detail): + """Judge one command result against the outcome it declared. + + Args: + cond: the declared outcome -- an exit code, a list of acceptable + codes, or ``self.FAIL`` for "any non-zero code will do". + rc: the exit code the command actually returned. + file_error: complaint about the ``--file`` output, or None. + detail: the ``rc=N NAME`` fragment every verdict line ends with. + + Returns: + Verdict: the message to print and whether it counts as a failure. + """ + # The one condition naming no specific code, so it cannot go through + # the accepted-code comparison below. + if cond == self.FAIL: + if not rc: + return Verdict(" Failure: Received PASS (0), expected FAIL (!0)", True) + return Verdict(f" Success: Received and Expected FAIL ({detail})", False) + + accept = self._accepted_codes(cond) + if rc not in accept: + received = "PASS" if not rc else "FAIL" + return Verdict( + f" Failure: Received {received} ({detail}), expected {_describe_codes(accept)}", + True, + ) + + tolerates_pass = self.PASS in accept + if file_error is not None and tolerates_pass: + return Verdict(f" Failure: {file_error}", True) + + if not rc: + if accept == (self.PASS,): + return Verdict(f" Success: Received and Expected PASS ({detail})", False) + return Verdict(f" Success: Received PASS ({detail})", False) + # A non-zero code accepted alongside PASS is leniency, not a declared outcome. + if tolerates_pass: + prefix = "Unsupported" if rc == amdsmi.AmdSmiStatus.NOT_SUPPORTED else "Success" + return Verdict(f" {prefix}: Received and Allowed FAIL ({detail})", False) + return Verdict(f" Success: Received and Expected FAIL ({detail})", False) + def RunCmds(self, cmds): errors = [] msg_len = 0 @@ -746,50 +1080,37 @@ def RunCmds(self, cmds): print(f"cmd={cmd}") if self.PrintCmdsOnly: continue - (rc, std_out, std_err) = self.util.RunCmdSync(cmd) + # Remove any stale output file so amd-smi does not block on its + # interactive "file exists, overwrite?" prompt (e.g. a leftover + # from a previously interrupted run). + if "--file" in cmd and os.path.exists(self.tmp_filename): + os.chmod(self.tmp_filename, stat.S_IWRITE) + os.remove(self.tmp_filename) + (rc, std_out, std_err) = self.util.RunCmdSync(cmd, msg_in=self._prompt_answer_for(cmd)) + # Name the code from the same tables the CLI maps it with. Scraping + # stderr yielded its last word instead ("...without setting value"), + # and most failures write nothing to stderr at all. error_code = rc - if rc and std_err: - items = std_err.split() - if "amdsmi_exception" in std_err: - # error code from amdsmi library exception - for index, item in enumerate(items): - if item == "Error": - error_code_str = items[index + 4] - error_code = error_code_str - # break - else: - # error code from amd-smi CLI - error_code = items[-1] - # Check for parse error 'choice' - if "CRITICAL" in error_code: - error_code = "Bad loglevel" + reason = EXIT_CODE_NAMES.get(rc, "") if rc else "" + detail = f"rc={error_code}" + (f" {reason}" if reason else "") msg = f"{cmd:{msg_len}s}:" + file_error = None if "--file" in cmd: if not os.path.exists(self.tmp_filename): - _msg = f"{msg} Failure: File {self.tmp_filename} does not exist" - errors.append(_msg) + file_error = f"File {self.tmp_filename} does not exist" else: with open(self.tmp_filename, "r") as fin: std_out = fin.read() if not len(std_out): - _msg = f"{msg} Failure: File {self.tmp_filename} was empty" - errors.append(_msg) + file_error = f"File {self.tmp_filename} was empty" os.chmod(self.tmp_filename, stat.S_IWRITE) os.remove(self.tmp_filename) - if rc and cond == self.PASS: - msg += f" Failure: Received FAIL ({error_code}), expected PASS (0)" - errors.append(msg) - elif not rc and cond != self.PASS: - msg += " Failure: Received PASS (0), expected FAIL (!0)" + verdict = self._grade(cond, rc, file_error, detail) + msg += verdict.message + if verdict.counts_as_failure: errors.append(msg) - else: - if not rc: - expected = "PASS" - else: - expected = "FAIL" - msg += f" Success: Received and Expected {expected} ({error_code})" self.common.print(f"{self.tab}{msg}") if self.Debug: diff --git a/projects/amdsmi/tests/python/cli/test_general.py b/projects/amdsmi/tests/python/cli/test_general.py index 95fb083ef2f9..8b6084f2e28a 100644 --- a/projects/amdsmi/tests/python/cli/test_general.py +++ b/projects/amdsmi/tests/python/cli/test_general.py @@ -71,7 +71,7 @@ def test_invalid(self): ("amd-smi list --invalid", self.FAIL), ("amd-smi static --invalid", self.FAIL), ("amd-smi firmware --invalid", self.FAIL), - ("amd-smi bad_pages --invalid", self.FAIL), + ("amd-smi bad-pages --invalid", self.FAIL), ("amd-smi metric --invalid", self.FAIL), ("amd-smi process --invalid", self.FAIL), ("amd-smi event --invalid", self.FAIL), diff --git a/projects/amdsmi/tests/python/cli/test_ras.py b/projects/amdsmi/tests/python/cli/test_ras.py index bac419c71c6b..edb6a72203c9 100644 --- a/projects/amdsmi/tests/python/cli/test_ras.py +++ b/projects/amdsmi/tests/python/cli/test_ras.py @@ -45,8 +45,8 @@ def test_afid_folder(self): tmp_dir = tempfile.mkdtemp(prefix="amdsmi_ras_afid_") try: - # A real but undecodable .cper (non-empty garbage bytes): decodes to - # "decode failed" in the table, but the command still exits 0. + # A real but undecodable .cper (non-empty garbage bytes): reported as + # a decode error in the table, and the command exits with that status. garbage = os.path.join(tmp_dir, "garbage.cper") with open(garbage, "wb") as fout: fout.write(b"\x00" * 64) @@ -65,9 +65,9 @@ def test_afid_folder(self): os.symlink(target_dir, symlink_dir) cmds = [ - # Folder with a (garbage) .cper: undecodable rows are reported but - # the command exits 0. - (f"amd-smi ras --afid --folder {tmp_dir}", self.PASS), + # Folder with a (garbage) .cper: the undecodable file is reported + # and the command exits non-zero with the decode status. + (f"amd-smi ras --afid --folder {tmp_dir}", self.FAIL), # --cper-file and --folder are mutually exclusive under --afid. (f"amd-smi ras --afid --cper-file {garbage} --folder {tmp_dir}", self.FAIL), # Nonexistent folder. @@ -84,19 +84,35 @@ def test_afid_folder(self): # assigned to self.output a second time. cmd = f"amd-smi ras --afid --folder {tmp_dir} --json" (rc, data, std_err) = self.util.RunCmdSync(cmd) - self.assertEqual(rc, self.PASS, f"Command '{cmd}' failed with rc={rc}") + # The garbage .cper fails to decode, so the command exits non-zero; + # the JSON is still emitted for the caller to parse. + self.assertNotEqual(rc, self.PASS, f"Command '{cmd}' unexpectedly passed (rc={rc})") self.assertIsNotNone(data, f"Command '{cmd}' produced no output") json_data = json.loads(data) self.assertIsInstance(json_data, list, f"'{cmd}' did not emit a JSON list") + # The folder holds a .cper, so an empty list would skip every check below. + self.assertTrue(json_data, f"'{cmd}' emitted an empty JSON list") for entry in json_data: self.assertIsInstance( entry, dict, f"'{cmd}' emitted a non-object element (double-wrapped?): {entry!r}", ) + # Per-file schema: the pre-existing cper_file/afids/decode_failed + # plus the status fields. afids stays a list so a consumer doing + # len(entry["afids"]) still counts AFIDs, not characters. self.assertIn("cper_file", entry) self.assertIn("afids", entry) + self.assertIsInstance(entry["afids"], list) self.assertIn("decode_failed", entry) + self.assertIn("status", entry) + self.assertIn("message", entry) + self.assertIn("code", entry) + # Every fixture this folder holds is 64 zero bytes, so each entry + # must report the decode failure rather than only those that did. + self.assertNotEqual(entry["status"], "AMDSMI_STATUS_SUCCESS") + self.assertNotEqual(entry["code"], 0) + self.assertTrue(entry["decode_failed"]) finally: shutil.rmtree(tmp_dir, ignore_errors=True) return diff --git a/projects/amdsmi/tests/python/cli/test_set.py b/projects/amdsmi/tests/python/cli/test_set.py index 63ff8207ef73..b4dc5f23e435 100644 --- a/projects/amdsmi/tests/python/cli/test_set.py +++ b/projects/amdsmi/tests/python/cli/test_set.py @@ -36,9 +36,15 @@ def test_command(self): cmds = self.CreateCmds( "set", "Set Arguments:", "Device Arguments:", "Command Modifiers:", "" ) + # Registered before the sweep: RunCmds raises on the first failure, and a + # sweep that aborts partway is exactly when the GPU is left mid-change. + self.addCleanup(self._restore_starting_values, power_profile) self.RunCmds(cmds) - # Restore starting values + return + + def _restore_starting_values(self, power_profile): + """Put the values the sweep changed back to what setUpClass recorded.""" cmds = [] for index, gpu in enumerate(self.common.processors): # Validate max fan speed is sensible; gpu_od GPUs must report <= 100 @@ -60,12 +66,6 @@ def test_command(self): if fan_speed != "N/A": cmds.append((f"amd-smi reset --fans --gpu {index}", self.PASS)) - # set --perf-level defaults - perf_level = self.metric_data["gpu_data"][index]["perf_level"] - if perf_level != "N/A": - perf_level = _strip_prefix(perf_level, "AMDSMI_DEV_PERF_LEVEL_") - cmds.append((f"amd-smi set --perf-level {perf_level} --gpu {index}", self.PASS)) - # set --profile defaults if power_profile[index]: profile = _strip_prefix(power_profile[index]["current"], "AMDSMI_PWR_PROF_PRST_") @@ -77,24 +77,31 @@ def test_command(self): num = len(clock_sys["frequency_levels"]) level = f"Level {num - 1}" clock_freq = int(clock_sys["frequency_levels"][level]["value"]) + # Readable clock levels do not imply the determinism feature exists. cmds.append( - (f"amd-smi set --perf-determinism {clock_freq} --gpu {index}", self.PASS) - ) - - # set --compute-partition defaults - accelerator_type = self.partition_data["current_partition"][index]["accelerator_type"] - if accelerator_type != "N/A": - cmds.append( - (f"amd-smi set --compute-partition {accelerator_type} --gpu {index}", self.PASS) + ( + f"amd-smi set --perf-determinism {clock_freq} --gpu {index}", + [self.PASS, amdsmi.AmdSmiStatus.NOT_SUPPORTED], + ) ) # set --memory-partition defaults + # Safe to write back: the mode only takes effect after a driver reload. memory_partition = self.partition_data["current_partition"][index]["memory"] if memory_partition != "N/A": cmds.append( (f"amd-smi set --memory-partition {memory_partition} --gpu {index}", self.PASS) ) + # set --compute-partition defaults + # Must come after memory partition sets, since the memory partition set can override + # this setting (their defaults may differ). + accelerator_type = self.partition_data["current_partition"][index]["accelerator_type"] + if accelerator_type != "N/A": + cmds.append( + (f"amd-smi set --compute-partition {accelerator_type} --gpu {index}", self.PASS) + ) + # set --compute-partition-mem-alloc-mode defaults try: mem_alloc_mode = self.static_data["gpu_data"][index]["partition"][ @@ -110,35 +117,11 @@ def test_command(self): ) ) - # set --power-cap defaults - for power_type in self.power_types: - _power_type = self.static_data["gpu_data"][index]["limit"][power_type] - socket_power_limit = _power_type["socket_power_limit"] - if socket_power_limit != "N/A": - socket_power = socket_power_limit["value"] - cmds.append( - ( - f"amd-smi set --power-cap {socket_power} {power_type} --gpu {index}", - self.PASS, - ) - ) - # Both bounds are inclusive: the exact min and max must - # succeed. A reported min of 0 means the technical minimum - # is 1, since setting 0 reads back the current cap. - min_power = max(_power_type["min_power_limit"]["value"], 1) - max_power = _power_type["max_power_limit"]["value"] - cmds.append( - ( - f"amd-smi set --power-cap {min_power} {power_type} --gpu {index}", - self.PASS, - ) - ) - cmds.append( - ( - f"amd-smi set --power-cap {max_power} {power_type} --gpu {index}", - self.PASS, - ) - ) + # reset --power-cap + # Writes each supported sensor's default_power_cap. Replaying the recorded + # value did not restore it: min and max were written after it, so the cap + # was left at max. + cmds.append((f"amd-smi reset --power-cap --gpu {index}", self.PASS)) # set --soc-pstate defaults soc_pstate = self.static_data["gpu_data"][index]["soc_pstate"] @@ -172,68 +155,50 @@ def test_command(self): # set --clk-limit defaults clock = self.metric_data["gpu_data"][index]["clock"] for clk_type in self.clk_limits: - if clk_type == "SCLK": - clk_type_name = "socclk_0" - else: - clk_type_name = "mem_0" for limit_type in self.limit_types: - if limit_type == "MIN": - clk_limit_name = "min_clk" - else: - clk_limit_name = "max_clk" - clk_type_limit_name = clock[clk_type_name][clk_limit_name] - if type(clk_type_limit_name) is dict: - value = clk_type_limit_name["value"] + value = self._clk_limit_value(clock, clk_type, limit_type) + if value is not None: + # A readable min/max clock does not imply the device + # supports setting a limit on it. cmds.append( ( f"amd-smi set --clk-limit {clk_type} {limit_type} {value} --gpu {index}", - self.PASS, + [self.PASS, amdsmi.AmdSmiStatus.NOT_SUPPORTED], ) ) - # set --clk-level defaults - clock = self.static_data["gpu_data"][index]["clock"] - for clk_type in self.clk_levels: - value = -1 - clk_type_name = "" - if clk_type == "SCLK": - clk_type_name = "sys" - elif clk_type == "MCLK": - clk_type_name = "mem" - elif clk_type == "FCLK": - clk_type_name = "df" - elif clk_type == "SOCCLK": - clk_type_name = "soc" - else: - bus = self.static_data["gpu_data"][index]["bus"] - pcie_levels = bus["pcie_levels"] - if type(pcie_levels) is dict: - value = len(pcie_levels) - if value > 0: - value -= 1 - if clk_type != "PCIE" and value < 0: - clk_type_name = clock[clk_type_name] - if type(clk_type_name) is dict: - current_level = clk_type_name["current_level"] - value = current_level - if value >= 0: - cmds.append( - (f"amd-smi set --clk-level {clk_type} {value} --gpu {index}", self.PASS) - ) - # set --process-isolation defaults - process_isolation = self.static_data["gpu_data"][index]["process_isolation"] - if process_isolation == "Disabled": - process_isolation_value = 0 - else: - process_isolation_value = 1 + # reset --clocks + # A clk-level write is a bitmask, so replaying the recorded level would pin DPM + # to that one level; only AUTO hands every level back. Overdrive is reset too, + # and reports NOT_SUPPORTED on parts that lack it even though the rest worked. cmds.append( ( - f"amd-smi set --process-isolation {process_isolation_value} --gpu {index}", - self.PASS, + f"amd-smi reset --clocks --gpu {index}", + [self.PASS, amdsmi.AmdSmiStatus.NOT_SUPPORTED], ) ) - print("Restore Starting Values") + # set --perf-level defaults + # Last of the clock writes: the reset above forces AUTO, and --clk-limit and + # --perf-determinism each leave the device on a level of their own. + perf_level = self.metric_data["gpu_data"][index]["perf_level"] + if perf_level not in ("N/A", "AMDSMI_DEV_PERF_LEVEL_UNKNOWN"): + perf_level = _strip_prefix(perf_level, "AMDSMI_DEV_PERF_LEVEL_") + cmds.append((f"amd-smi set --perf-level {perf_level} --gpu {index}", self.PASS)) + # set --process-isolation defaults + process_isolation = self.static_data["gpu_data"][index]["process_isolation"] + # Put back whatever value the sweep left; "N/A" means unreadable, so + # there is nothing to restore. + if process_isolation != "N/A": + original = 0 if process_isolation == "Disabled" else 1 + cmds.append( + ( + f"amd-smi set --process-isolation {original} --gpu {index}", + [self.PASS, amdsmi.AmdSmiStatus.NOT_SUPPORTED], + ) + ) + + self.common.print("Restore Starting Values") self.RunCmds(cmds) return diff --git a/projects/amdsmi/tests/python/common/common.py b/projects/amdsmi/tests/python/common/common.py index 8dd2c5077777..f908577907a8 100644 --- a/projects/amdsmi/tests/python/common/common.py +++ b/projects/amdsmi/tests/python/common/common.py @@ -21,6 +21,7 @@ """ import contextlib +import importlib.util import inspect import io import json @@ -98,9 +99,175 @@ def print_shadow_error(script, loaded_from, expected_path, file=sys.stderr): print(f"\nRefer to `{script} -h` for more details.", file=file) -amdsmi_path = os.environ.get("AMDSMI_PATH") or os.path.join( - os.environ.get("ROCM_HOME") or os.environ.get("ROCM_PATH") or "/opt/rocm", "share/amd_smi" -) +def find_cli_dir(*start_dirs): + """Return the amd-smi CLI dir (the one holding amdsmi_cli_exceptions.py), or None. + + Walks up from each start dir, checking the installed (``libexec/amdsmi_cli``) + and source (``amdsmi_cli``) layouts, and returns the first hit -- so argument + order *is* priority. Callers should not hand-order the start dirs; spread + ``cli_search_order()`` instead, which encodes the install-vs-source rule in + one place. Callers skip the suite when the result is None. + """ + sentinel = "amdsmi_cli_exceptions.py" + for start in start_dirs: + if not start: + continue + # abspath rather than Path.resolve(): a symlinked ROCm root + # (/opt/rocm -> /opt/rocm-x.y.z) must stay as the caller spelled it. + start_path = pathlib.Path(os.path.abspath(start)) + for directory in (start_path, *start_path.parents): + for rel in (("libexec", "amdsmi_cli"), ("amdsmi_cli",)): + candidate = directory.joinpath(*rel) + if (candidate / sentinel).is_file(): + return str(candidate) + return None + + +def cli_search_order(source_dir, prefer_install=False): + """Return ``find_cli_dir`` start dirs, install-first only when one was asked for. + + Spread it: ``find_cli_dir(*cli_search_order(_THIS_DIR))``. The priority flips + on whether the caller pinned an install: + + * AMDSMI_PATH / ROCM_HOME / ROCM_PATH set -- that install wins, so a run + pinned to an install exercises *its* CLI rather than whatever source tree + the test file happens to live in. Pointing AMDSMI_PATH at a build tree + still selects the source CLI, because find_cli_dir() walks up out of the + build dir and finds ``amdsmi_cli`` in the checkout. + * nothing set -- the source tree wins. Otherwise a checkout silently tests + /opt/rocm: a stale install imports fine, so an ImportError guard never + trips, and the suite either errors on missing attributes or passes while + exercising code that is not the one under review. + + ``prefer_install`` forces install-first even with nothing set. It is for + black-box callers that import CLI modules only to describe a binary they + spawn out of the install tree; white-box callers importing the code under + test want the default. + """ + if prefer_install or amdsmi_path_is_explicit: + return (amdsmi_path, source_dir) + return (source_dir, amdsmi_path) + + +def generated_version_stub(version="0.0.0+source"): + """Stub entry for the CLI's ``_version``, or empty when the real one resolves. + + ``_version.py`` is generated by CMake from ``_version.py.in``, so a source + checkout carries only the template and any CLI module doing + ``from _version import __version__`` fails to import. Merge the result into a + stub mapping, or into ``sys.modules`` directly -- it is empty whenever the + real module is importable, so it never shadows a build. No test reads it. + """ + if importlib.util.find_spec("_version") is not None: + return {} + return {"_version": fake_module("_version", __version__=version)} + + +def load_cli_module(name, path, sys_path_dir=None): + """Import the CLI file at *path* as module *name*, or skip the test. + + Skipping is the point: the module-level ``*_PATH`` constants are None when + find_cli_dir() came up empty, and ``os.path.isfile(None)`` raises TypeError + rather than returning False, so an unguarded loader reports a hard error on + a machine that simply has no CLI installed. + + Pass *sys_path_dir* when the loaded file imports its CLI siblings by bare + name, so those resolve to the real modules. + """ + if not path or not os.path.isfile(path): + raise unittest.SkipTest(f"amd-smi CLI file not available: {name} ({path})") + if sys_path_dir and sys_path_dir not in sys.path: + sys.path.insert(0, sys_path_dir) + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise unittest.SkipTest(f"amd-smi CLI file is not loadable: {name} ({path})") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def add_class_cleanup(test_cls, func, *args): + """Run ``func`` after ``test_cls`` finishes. + + Not ``addClassCleanup`` (3.8+, and Debian 10 ships 3.7). This runs off + ``tearDownClass``, which unittest skips when ``setUpClass`` raises, so a + class that dies mid-setup leaves its cleanups unrun. + """ + # Own the list rather than inherit a base class's, so subclasses do not share. + cleanups = test_cls.__dict__.get("_pending_class_cleanups") + if cleanups is None: + cleanups = test_cls._pending_class_cleanups = [] + previous = test_cls.tearDownClass.__func__ + + def _run_class_cleanups(cls): + try: + previous(cls) + finally: + while cleanups: # last registered runs first, as addClassCleanup does + cleanup, cleanup_args = cleanups.pop() + cleanup(*cleanup_args) + + test_cls.tearDownClass = classmethod(_run_class_cleanups) + + cleanups.append((func, args)) + + +def fake_module(name, **attrs): + """Module object carrying *attrs*, for stubbing an import a test cannot satisfy. + + setattr rather than direct assignment: a stub invents attributes the real + module type does not declare, which a type checker flags on every line. + Pass the result to ``stub_modules`` so it is restored afterwards. + """ + module = types.ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + return module + + +def stub_modules_at_import(modules): + """Replace ``sys.modules`` entries now; returns the callable that restores them. + + ``modules`` maps a module name to its replacement; a ``None`` value removes the + name instead, so the next import re-resolves it. The caller owns the restore -- + drive it from ``tearDownModule`` when the stub has to exist before the module's + own imports run (the CLI does ``from amdsmi_init import *`` at load, which is + too early for ``setUpClass``). Prefer ``stub_modules`` when a class can own it. + """ + saved = {name: sys.modules.get(name) for name in modules} + + def _restore(): + for name, module in saved.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + for name, module in modules.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + return _restore + + +def stub_modules(test_cls, modules): + """Replace ``sys.modules`` entries for the lifetime of ``test_cls``. + + Same contract as ``stub_modules_at_import``, with the restore tied to the class + teardown instead of the caller. A stub that outlives its class corrupts every + later test module in the same interpreter, which is what the runner's isolation + guard reports as a leak. + """ + add_class_cleanup(test_cls, stub_modules_at_import(modules)) + + +# Provenance, not just the value: an explicit AMDSMI_PATH can name the same path +# as the default, and cli_search_order() has to tell those two apart. +_env_share = os.environ.get("AMDSMI_PATH") +_env_root = os.environ.get("ROCM_HOME") or os.environ.get("ROCM_PATH") +amdsmi_path_is_explicit = bool(_env_share or _env_root) +amdsmi_path = _env_share or os.path.join(_env_root or "/opt/rocm", "share/amd_smi") if not os.path.exists(amdsmi_path): raise FileNotFoundError( f'amdsmi path "{amdsmi_path}" does not exist. ' @@ -304,10 +471,16 @@ def build_type_lists(): cond = PASS if member.name in ["UNKNOWN"]: cond = FAIL - elif member.name in ["NPS4", "NPS8"]: - # NPS4/NPS8 are hardware-dependent; accept success or invalid depending on support - # BTW - no asic supports NPS8... - cond = [PASS, FAIL] + elif member.name in ["NPS1", "NPS2", "NPS4", "NPS8"]: + # Hardware-dependent for every mode: + # - NOT_SUPPORTED: when the partition sysfs is absent (or write permissions denied) + # - INVAL: when the driver rejects the mode + # No asic supports NPS8. + cond = [ + amdsmi.AmdSmiStatus.SUCCESS, + amdsmi.AmdSmiStatus.INVAL, + amdsmi.AmdSmiStatus.NOT_SUPPORTED, + ] memory_partition_types.append( (member.name, amdsmi.AmdSmiMemoryPartitionType(member.value), cond) ) @@ -631,6 +804,178 @@ def run_test_dir(subdir, title, top_level_dir): sys.exit(0 if result.wasSuccessful() else 1) +# Sentinel distinguishing "module was absent from the baseline" from "present but +# mapped to None" when snapshotting sys.modules. +_ISO_MISSING = object() + + +class _IsolationLeakMarker: + """Stand-in "test" for attributing a sys.modules leak when the offending + class cannot be identified (rare fallback). Implements the minimal + ``TestResult`` surface (``id``/``shortDescription``/``str``).""" + + def __init__(self, label): + self._label = label + + def id(self): + return f"{self._label} (sys.modules cleanup)" + + def shortDescription(self): + return None + + def __str__(self): + return self.id() + + +class _ModuleIsolationResult(unittest.TextTestResult): + """Fail a test class that leaves ``sys.modules`` dirty (cross-test pollution guard). + + A unit test that stubs a module into ``sys.modules`` -- e.g. to import a CLI + module without hardware or the compiled ``amdsmi`` package -- MUST restore it: + snapshot the originals in ``setUpClass`` and put them back in + ``tearDownClass``. If it forgets, the stub leaks into every later test module + run by the same interpreter (a "dirty test invocation"). That is exactly how + an empty ``AMDSMIHelpers`` stub once hid ``handle_gpus`` and broke every + ``test_cli_exit_codes`` set/reset test -- but only in a full run, never when + the class ran alone, which makes it painful to diagnose. + + Leaks are judged only at the *end* of the run, after every ``tearDownClass`` + has had its chance. Whatever module is still replaced or stubbed then is a + leak, and it is blamed on the test that *first introduced the object* now + occupying that name (tracked by object identity as each test runs). This is + robust against save/restore cascades: if one class leaks a stub and a later, + well-behaved class snapshots and restores that same leaked object, the blame + stays with the original leaker -- never the innocent bystander that merely + ran afterward. A class that stubs a name and properly restores it is never + blamed, because its object is no longer present at the end. + """ + + def startTestRun(self): + super().startTestRun() + # Baseline of already-imported modules; genuine imports carry a __spec__. + self._iso_baseline = dict(sys.modules) + # id(object) -> the first test seen with that stub/replacement in sys.modules. + # A leak is blamed on whoever FIRST introduced the object still sitting in + # sys.modules at the end of the run. This survives save/restore cascades: + # when a leak persists and a later, innocent class re-exposes the same leaked + # object (e.g. by snapshotting then restoring it), the blame stays with the + # class that originally created that object -- not the bystander. + self._iso_first_seen = {} + # Hold a reference to every object we have attributed so its id() cannot be + # reused by a later, unrelated object once the original is scrubbed/replaced. + self._iso_seen_objs = [] + # (class label, [leaked module names]) per offending class, so the runner + # can print a distinct root-cause banner after the normal summary. + self._iso_leaks = [] + + @staticmethod + def _iso_looks_like_stub(mod): + # A hand-built module (``types.ModuleType("x")``) has no import spec and + # no backing file; a genuinely imported module always has a __spec__. + return ( + isinstance(mod, types.ModuleType) + and getattr(mod, "__spec__", None) is None + and not getattr(mod, "__file__", None) + ) + + def _iso_dirty(self): + """name -> module for every entry that differs from the clean baseline: a + baseline module replaced by a different object, or a new hand-built stub.""" + baseline = self._iso_baseline + dirty = {} + for name, mod in list(sys.modules.items()): + # Skip Python-internal dunder modules (e.g. __main__, and __mp_main__ + # which multiprocessing installs): they look stub-like but are never + # test-installed stubs. + if name.startswith("__"): + continue + original = baseline.get(name, _ISO_MISSING) + if original is _ISO_MISSING: + if self._iso_looks_like_stub(mod): + dirty[name] = mod + elif mod is not original: + dirty[name] = mod + return dirty + + def _iso_note(self, test): + """Record the first test to expose each currently-dirty module object. + + Sampled at the start and end of every test. ``setUpClass`` has already run + by the first ``startTest``, so a class's stubs are attributed to it. The + first test to show a given object keeps it for the rest of the run, even + if a later class momentarily replaces and then restores that same object. + """ + for _name, mod in self._iso_dirty().items(): + key = id(mod) + if key not in self._iso_first_seen: + self._iso_first_seen[key] = test + self._iso_seen_objs.append(mod) + + def startTest(self, test): + # setUpClass ran just before this, so its stubs are attributed to `test`. + self._iso_note(test) + super().startTest(test) + + def stopTest(self, test): + # Also sample after the body, in case the test itself installed a stub. + super().stopTest(test) + self._iso_note(test) + + def stopTestRun(self): + # Every tearDownClass has run by now; whatever is still dirty is a real + # leak. Blame each leaked name on the first test that introduced the object + # now occupying it, scrub it so a re-run starts clean, and fail that class. + dirty = self._iso_dirty() + if dirty: + baseline = self._iso_baseline + by_test = {} + for name, mod in dirty.items(): + by_test.setdefault(self._iso_first_seen.get(id(mod)), []).append(name) + for name in dirty: + original = baseline.get(name, _ISO_MISSING) + if original is _ISO_MISSING: + sys.modules.pop(name, None) + else: + sys.modules[name] = original + for test, names in by_test.items(): + label = ( + f"{type(test).__module__}.{type(test).__qualname__}" + if test is not None + else None + ) + self._iso_record_leak(label, test, names) + super().stopTestRun() + + def _iso_record_leak(self, label, test, names): + who = label or "a test class" + sorted_names = sorted(names) + # Remember the leak so the runner can surface it in a root-cause banner; + # a leak usually triggers a cascade of downstream failures, and this + # banner keeps the actual culprit from being lost in that noise. + self._iso_leaks.append((who, sorted_names)) + msg = ( + f"{who} left sys.modules dirty: {', '.join(sorted_names)}.\n" + "It stubbed or replaced these modules and did not restore them, so the\n" + "stub leaks into later test modules run by the same interpreter and\n" + "corrupts them (a 'dirty test invocation' -- e.g. an empty AMDSMIHelpers\n" + "stub hides handle_gpus from test_cli_exit_codes). It surfaces only when\n" + "this class runs before the affected test in the same run; a filtered\n" + "run that excludes this class (the runner's -k/-x) hides the problem.\n" + "Fix: in setUpClass snapshot the originals\n" + " cls._saved = {n: sys.modules.get(n) for n in NAMES}\n" + "and in tearDownClass restore them (put the originals back, or pop the\n" + "names you added). See tests/python/unit/gpu/test_cli_metric_partition.py.\n" + "(The runner scrubs the leak after the run so a re-run starts clean.)" + ) + try: + raise AssertionError(msg) + except AssertionError: + # Attribute to a representative test of the offending class when known; + # otherwise fall back to a synthetic id so the run still fails loudly. + target = test if test is not None else _IsolationLeakMarker(who) + self.addFailure(target, sys.exc_info()) + + class GTestSummaryRunner(unittest.TextTestRunner): """TextTestRunner that appends a GTest-style pass/skip/fail summary. @@ -660,10 +1005,16 @@ class GTestSummaryRunner(unittest.TextTestRunner): runner = common.GTestSummaryRunner(stream=sys.stderr, verbosity=common.make_runner_verbosity(verbose)) """ + # Enforce per-class sys.modules hygiene: a test that stubs a module and + # forgets to restore it fails (see _ModuleIsolationResult) instead of + # silently corrupting later test modules. + resultclass = _ModuleIsolationResult + _CYAN = "\033[36m" _GREEN = "\033[32m" _YELLOW = "\033[33m" _RED = "\033[31m" + _MAGENTA = "\033[35m" _RESET = "\033[0m" @staticmethod @@ -685,6 +1036,17 @@ def _test_label(test): return test_id[len("__main__.") :] return test_id + @staticmethod + def _owning_test_id(test): + """The id of the test that owns *test*, folding subTests onto their parent. + + unittest reports a failing subTest as a separate object whose id is the parent + id plus its parameters -- ``Class.test_modes (mode='NPS1')`` -- while a plain + test id never contains a space. Cutting at the first space therefore yields the + owning test in both cases. + """ + return test.id().split(" ", 1)[0] + def _color(self, code, text): """Wrap *text* in *code* only when colors are appropriate. @@ -713,8 +1075,17 @@ def _print_gtest_summary(self, result, elapsed_ms): # They cause wasSuccessful() to return False, so count them as failures so # the summary accurately reflects the overall run status. unexpectedSuccesses = len(getattr(result, "unexpectedSuccesses", [])) - failures = len(result.failures) + len(result.errors) + unexpectedSuccesses - passed = result.testsRun - skipped - failures + # Failures come in two units: a *record* is one reported failure, and a single + # test can add several (a cleanup failing after the body, or one per subTest). + # Records are what gets listed below; distinct tests are what testsRun counts, + # so that is the unit passed has to net out. Class-level errors and leak + # markers arrive as stand-ins that testsRun never counted, so skip those. + failure_records = result.failures + result.errors + failed_tests = { + self._owning_test_id(t) for t, _ in failure_records if isinstance(t, unittest.TestCase) + } + failures = len(failure_records) + unexpectedSuccesses + passed = result.testsRun - skipped - len(failed_tests) - unexpectedSuccesses stream.writeln() stream.writeln( @@ -731,12 +1102,10 @@ def _print_gtest_summary(self, result, elapsed_ms): stream.writeln( self._color( self._RED, - f"[ FAILED ] {failures} test{self._plural(failures)}, listed below:", + f"[ FAILED ] {failures} failure{self._plural(failures)}, listed below:", ) ) - for t, _ in result.failures: - stream.writeln(self._color(self._RED, f"[ FAILED ] {self._test_label(t)}")) - for t, _ in result.errors: + for t, _ in failure_records: stream.writeln(self._color(self._RED, f"[ FAILED ] {self._test_label(t)}")) # unexpectedSuccesses items are bare TestCase instances (not (test, traceback) tuples). for t in getattr(result, "unexpectedSuccesses", []): @@ -755,6 +1124,40 @@ def _print_gtest_summary(self, result, elapsed_ms): for t, _ in result.skipped: stream.writeln(self._color(self._YELLOW, f"[ SKIPPED ] {self._test_label(t)}")) stream.writeln() + self._print_leak_banner(result) + + def _print_leak_banner(self, result): + """Print a distinct root-cause banner for any sys.modules leaks. + + A leaked stub typically causes a cascade of unrelated downstream + failures; this banner prints last so the real culprit (the class that + forgot to restore sys.modules) is the final, unmissable thing on screen. + """ + leaks = getattr(result, "_iso_leaks", None) + if not leaks: + return + bar = "=" * 72 + stream = self.stream + stream.writeln(self._color(self._MAGENTA, bar)) + stream.writeln( + self._color( + self._MAGENTA, + "[ LEAK ] sys.modules pollution detected -- likely the ROOT CAUSE of", + ) + ) + stream.writeln( + self._color( + self._MAGENTA, + "[ LEAK ] any failures above. Fix the class(es) below first, then re-run:", + ) + ) + for who, names in leaks: + stream.writeln(self._color(self._MAGENTA, f"[ LEAK ] {who}")) + stream.writeln( + self._color(self._MAGENTA, f"[ LEAK ] leaked: {', '.join(names)}") + ) + stream.writeln(self._color(self._MAGENTA, bar)) + stream.writeln() def has_gpu_od_interface(bdf): @@ -778,12 +1181,10 @@ def has_gpu_od_interface(bdf): # to remove the dependency on amdsmi_helpers from this common module. # Exposing non-public SYSFS API interfaces in the CLI (and in general) # is a bad design pattern and needs to be addressed in the future. - amdsmi_cli_path = os.path.normpath( - os.path.join(amdsmi_path, "..", "..", "libexec", "amdsmi_cli") - ) - if not os.path.exists(amdsmi_cli_path): + amdsmi_cli_path = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) + if not amdsmi_cli_path: raise FileNotFoundError( - f'amdsmi_cli path "{amdsmi_cli_path}" does not exist. ' + f'amdsmi_cli directory not found under "{amdsmi_path}". ' f"Ensure the AMD-SMI CLI is installed, or set AMDSMI_PATH correctly." ) if amdsmi_cli_path not in sys.path: @@ -950,6 +1351,7 @@ def _normalize_expected(expected): def check_ret(self, msg, exc, expected_code_name=None, printIt=True): # Returns True if the test FAILED (i.e. the result did not match expected). # Callers use the pattern: `if self.check_ret(...): raise_exception = e` + expected_code_name = self._normalize_expected(expected_code_name) if isinstance(exc, str) and len(exc) == 0: error_code_name = expected_code_name error_code = "-1" @@ -972,8 +1374,13 @@ def check_ret(self, msg, exc, expected_code_name=None, printIt=True): if self.verbose > VERBOSITY_QUIET and printIt: if msg: print(f"{msg}\n", end="") - ec_code = self.get_error_code_from_name(ec_name) - print(f"\tTEST SUCCESS, AMDSMI API Returned {ec_code:>2s}, {ec_name}") + status_msg = f"\tTEST SUCCESS, AMDSMI API Returned {error_code:>2s}, {error_code_name}" + for name in expected_code_name: + code = str(self.get_error_code_from_name(name)) + status_msg += ( + f"\n\t AMDSMI API Expected {code:>2s}, {name}" + ) + print(status_msg) return False # No expected result matched - print failure (respects same guards as single-condition path) @@ -984,10 +1391,8 @@ def check_ret(self, msg, exc, expected_code_name=None, printIt=True): f"\tTEST FAILURE, AMDSMI API Returned {error_code:>2s}, {error_code_name}\n" ) for ec_name in expected_code_name: - ec_code = self.get_error_code_from_name(ec_name) - status_msg += ( - f"\t AMDSMI API Expected {ec_code:>2s}, {expected_code_name}" - ) + ec_code = str(self.get_error_code_from_name(ec_name)) + status_msg += f"\t AMDSMI API Expected {ec_code:>2s}, {ec_name}" return True # Check for single passing condition @@ -1156,7 +1561,7 @@ def _print_status_outcome(self, outcome): lines.append(f"\t AMDSMI API Accepted {code:>2s}, {name}") print("\n".join(lines), flush=True) - def _check_amdgpu_driver(self): + def check_amdgpu_driver(self): """Returns true if amdgpu is found in the list of initialized modules""" amd_gpu_status_file = pathlib.Path("/sys/module/amdgpu/initstate") if amd_gpu_status_file.exists(): @@ -1252,20 +1657,22 @@ def amdsmi_smart_init(self): Raises: AmdSmiLibraryException: If initialization fails for reasons other than driver not loaded AmdSmiParameterException: If invalid parameters are passed to amdsmi_init - unittest.SkipTest: If no compatible AMD drivers are detected on the system + RuntimeError: If no compatible AMD drivers are detected on the system """ # Build init_flag additively from detected drivers (mirrors CLI amdsmi_init.py pattern). # Using INIT_AMD_APUS when both GPU and CPU drivers are present is wrong on discrete-GPU # servers — use OR-combined flags instead, exactly as the CLI does. init_flag = 0 - if self._check_amdgpu_driver(): + if self.check_amdgpu_driver(): init_flag |= amdsmi.AmdSmiInitFlags.INIT_AMD_GPUS if self._check_amd_hsmp_driver(): init_flag |= amdsmi.AmdSmiInitFlags.INIT_AMD_CPUS if init_flag == 0: + # Errors rather than skips: a driverless box cannot exercise anything, + # and skipping reports OK for a suite that never ran. msg = "Drivers not loaded (amdgpu, amd_hsmp or hsmp_acpi drivers not found in modules)" self.print(msg) - raise unittest.SkipTest(msg) + raise RuntimeError(msg) msg = "AMDSMI init failed for detected drivers" ret = self._init_with_flag(init_flag, msg) diff --git a/projects/amdsmi/tests/python/common/runcmd.py b/projects/amdsmi/tests/python/common/runcmd.py index ee0e1d1916b2..9892c98b5e22 100644 --- a/projects/amdsmi/tests/python/common/runcmd.py +++ b/projects/amdsmi/tests/python/common/runcmd.py @@ -10,7 +10,6 @@ tests; it has no amdsmi or unittest dependencies. """ -import argparse import datetime import locale import os @@ -30,10 +29,17 @@ def __init__(self, debug_level="ERROR"): # Build the subprocess environment once and cache it. Prepends the ROCm # bin directory so amd-smi is found even when invoked via sudo, which - # strips non-standard PATH entries. Honors ROCM_HOME/ROCM_PATH when not - # running under sudo; defaults to /opt/rocm otherwise (sudo strips those - # vars so the fallback is what actually takes effect in that case). - rocm_root = os.getenv("ROCM_HOME") or os.getenv("ROCM_PATH") or "/opt/rocm" + # strips non-standard PATH entries. AMDSMI_PATH (which names + # /share/amd_smi) wins, then ROCM_HOME/ROCM_PATH, matching + # common.cli_search_order so the binary spawned here and the exit codes + # cli/base.py imports come from one install. Defaults to /opt/rocm + # otherwise (sudo strips those vars so the fallback is what actually + # takes effect in that case). + amdsmi_path = os.getenv("AMDSMI_PATH") + if amdsmi_path: + rocm_root = os.path.abspath(os.path.join(amdsmi_path, os.pardir, os.pardir)) + else: + rocm_root = os.getenv("ROCM_HOME") or os.getenv("ROCM_PATH") or "/opt/rocm" rocm_bin = os.path.join(rocm_root, "bin") self._subprocess_env = os.environ.copy() existing_path = self._subprocess_env.get("PATH", "") @@ -223,35 +229,3 @@ def RunCmdAsync(self, cmd, use_shell=False, msg_in=None): rc, std_out, std_err, proc = self._RunCmd(cmd, use_shell, msg_in, time_out=None, wait=False) return (rc, std_out, std_err, proc) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Utility") - parser.add_argument( - "--version", action="version", version=version_number, help="Show version and exit" - ) - parser.add_argument("--build", action="version", version=build_date, help="Show build and exit") - parser.add_argument( - "--verbose", - choices=verbose_choices, - type=str, - default="WARNING", - help="Level of information to output, default=%(default)s", - ) - parser.add_argument("--cmd", type=str, default=None, help="Run cmd, default=%(default)s") - args = parser.parse_args() - - util = Util(args.verbose) - - if args.cmd: - cmd = args.cmd - else: - cmd = "amd-smi" - - (rc, std_out, std_err) = util.RunCmdSync(cmd) - print(f"output:{cmd}") - print(f"\trc={rc}") - print(f"\tstd_out={std_out}") - print(f"\tstd_err={std_err}") - - sys.exit(rc) diff --git a/projects/amdsmi/tests/python/functional/gpu/test_asic_revision_ids.py b/projects/amdsmi/tests/python/functional/gpu/test_asic_revision_ids.py index e77c33e4538a..e31b54c25907 100644 --- a/projects/amdsmi/tests/python/functional/gpu/test_asic_revision_ids.py +++ b/projects/amdsmi/tests/python/functional/gpu/test_asic_revision_ids.py @@ -123,7 +123,3 @@ def test_revision_ids_match_drm_dev_info(self): if checked == 0: self.skipTest(f"no render node was readable: {unreachable}") - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/functional/gpu/test_benchmark.py b/projects/amdsmi/tests/python/functional/gpu/test_benchmark.py index 2ddb0d856fcf..76690b0b226e 100644 --- a/projects/amdsmi/tests/python/functional/gpu/test_benchmark.py +++ b/projects/amdsmi/tests/python/functional/gpu/test_benchmark.py @@ -1714,8 +1714,9 @@ def test_performance_get_gpu_ras_feature_info(self): def test_performance_get_gpu_reg_table_info(self): self.common.print_func_name("") - if self.TODO_SKIP_FAIL: - self.skipTest("Skipping test_performance_get_gpu_reg_table_info as it fails on MI300.") + self.skipTest( + "Skipping test_performance_get_gpu_reg_table_info as it causes resets on some MI3x systems." + ) for i, processor in enumerate(self.processors): for reg_type_name, reg_type, reg_type_cond in common.REG_TYPES: diff --git a/projects/amdsmi/tests/python/functional/gpu/test_memory_partition_cycle.py b/projects/amdsmi/tests/python/functional/gpu/test_memory_partition_cycle.py new file mode 100644 index 000000000000..77d8f1c9ef30 --- /dev/null +++ b/projects/amdsmi/tests/python/functional/gpu/test_memory_partition_cycle.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Cycle a GPU through every NPS mode: set -> driver reload -> verify -> restore. + +Laid out in phases, mirroring example/amd_smi_partition_example.py: + + Phase 1 Report the current mode and the modes the device advertises. + Phase 2 For each mode: stage it, reload amdgpu, re-enumerate, verify. + Phase 3 Put the device back on the mode the run started with. + +Every call is issued against every handle amdsmi enumerates, and a mode counts as +applied only once all handles that can report one agree on it. In a partitioned +layout the sub-partition handles return NOT_SUPPORTED for the memory partition +APIs; those are skipped rather than counted as a mismatch. + +The driver is only reloaded when a reload can actually change something -- a +mode the device rejects, or a mode that is already active, is reported and +skipped. Every external wait (modprobe, device re-enumeration) is bounded by a +timeout so a stuck reload fails the test instead of hanging the suite. + +NOTE: Partition changes alter device topology -- AMD SMI must re-initialize. +AMD SMI builds its device table at amdsmi_init() and does not update it live, so +every processor handle is valid only inside the session that produced it. Each +phase below therefore opens its own AmdsmiSession rather than caching a handle. + +Requirements +------------ +- Root privileges (modprobe requires root) +- kmod installed (modprobe must be on PATH, only if AMDSMI_ALLOW_DESTRUCTIVE_TESTS=1) +- No GPU workloads running during the run +- AMDSMI_ALLOW_DESTRUCTIVE_TESTS=1 (test_cycle_memory_partition_modes only -- + without it, every mode is still staged and its status checked, but the + driver is never reloaded, so nothing is actually applied or verified) + +Usage +----- + 1. Install amd-smi-lib & amd-smi-lib-test packages + 2. sudo /opt/rocm/share/amd_smi/tests/python_unittest/integration_test.py \ + -k test_cycle_memory_partition_modes -v + Add AMDSMI_ALLOW_DESTRUCTIVE_TESTS=1 to also reload the driver and verify + each mode was actually applied. +""" + +import os +import shutil +import subprocess +import time +import unittest + +import common.common as common +from common.common import amdsmi + +# modprobe is slow on large hives and the devices then come back asynchronously, +# so the unload/load call and the re-enumeration poll get separate bounds. +_MODPROBE_TIMEOUT_SEC = 120 +_DEVICE_READY_TIMEOUT_SEC = 60 +_DEVICE_POLL_SEC = 2 +# A modprobe stuck in the driver stays in uninterruptible sleep and ignores +# SIGKILL, so the reap after a timeout needs its own bound. +_REAP_TIMEOUT_SEC = 10 + +# Debug: reload even when the mode was rejected or already active. Only takes +# effect alongside AMDSMI_ALLOW_DESTRUCTIVE_TESTS=1 below -- that gate is +# checked first everywhere this is read, so this alone never touches modprobe. +_ALWAYS_RELOAD = os.environ.get("AMDSMI_TEST_ALWAYS_RELOAD") == "1" + +# Opt-in: test_cycle_memory_partition_modes unloads/reloads amdgpu on live +# hardware. Without this, it still stages every mode and checks the status +# each write returns, but never actually reloads the driver. +_ALLOW_DESTRUCTIVE_TESTS = os.environ.get("AMDSMI_ALLOW_DESTRUCTIVE_TESTS") == "1" + +_BANNER_WIDTH = 70 + +# name -> enum. UNKNOWN is a reporting sentinel, not settable. The shared table's +# status column is not used: _stage_mode states what it accepts itself. +_NPS_MODES = { + name: partition_type + for name, partition_type, _ in common.MEMORY_PARTITION_TYPES + if name != "UNKNOWN" +} + + +class AmdsmiSession: + """Scoped amdsmi_init / amdsmi_shut_down pair yielding the handle list.""" + + def __enter__(self): + amdsmi.amdsmi_init() + try: + return amdsmi.amdsmi_get_processor_handles() + except amdsmi.AmdSmiException: + # __exit__ never runs when __enter__ raises, so unwind the init here. + amdsmi.amdsmi_shut_down() + raise + + def __exit__(self, *_): + try: + amdsmi.amdsmi_shut_down() + except amdsmi.AmdSmiLibraryException: + pass + + +class TestGpuMemoryPartitionCycle(unittest.TestCase): + @classmethod + def setUpClass(cls): + if os.geteuid() != 0: + raise unittest.SkipTest("Memory partition tests need root/admin") + cls.common = common.Common(common.verbose) + + def tearDown(self): + try: + amdsmi.amdsmi_shut_down() + except amdsmi.AmdSmiLibraryException: + pass + + # -- output ------------------------------------------------------------- + + def _separator(self, title): + header = f"--- {title} " + self.common.print(f"\n{header}{'-' * (_BANNER_WIDTH - len(header))}") + + # -- queries ------------------------------------------------------------ + + def _physical_gpu_count(self): + """Physical GPUs in the system. Caller must hold an open session. + + A socket is keyed on the BD of the BDF, so every logical partition of a + GPU folds into one socket. The processor handle list does not: a reload + resets the accelerator partition to the default for the new NPS mode, so + the handle count legitimately changes (SPX -> CPX multiplies it). + """ + return len(amdsmi.amdsmi_get_socket_handles()) + + def _for_each_gpu(self, call_name, api, accept, params=""): + """Run *api* on every handle in one session; return ``{index: result}``. + + Only handles that answered appear in the result. In a partitioned layout + amdsmi enumerates one handle per logical partition, and the sub-partition + handles report NOT_SUPPORTED for the memory partition APIs -- they drop + out here rather than counting as a mismatch when modes are compared. + """ + answered = {} + with AmdsmiSession() as gpus: + # Printed before the ### marker so the api_summary parser still sees + # the status line as the one following it. + self.common.print( + f" >> Device Count: {self._physical_gpu_count()} physical GPUs, " + f"{len(gpus)} (processor) partition handles" + ) + self.assertTrue(gpus, "No GPU handles returned after amdsmi_init()") + # One sweep per call so a device that reports an unaccepted status + # cannot hide the devices after it. + with self.common.status_sweep(): + for i, gpu in enumerate(gpus): + msg = f"\t### {call_name}(gpu={i}{params}):" + with self.common.expect_status(msg, accept): + answered[i] = api(gpu) + # Setters return nothing; only a getter has a result to show. + if answered.get(i) is not None: + self.common.print(f"\t\t{answered[i]}") + return answered + + def _read_modes(self): + """``{index: NPS mode}`` for every handle that reports one.""" + return self._for_each_gpu( + "amdsmi_get_gpu_memory_partition", + amdsmi.amdsmi_get_gpu_memory_partition, + accept=[amdsmi.AmdSmiStatus.SUCCESS, amdsmi.AmdSmiStatus.NOT_SUPPORTED], + ) + + @staticmethod + def _agreed_mode(modes): + """The mode every readable handle agrees on, or None when they differ.""" + distinct = set(modes.values()) + return distinct.pop() if len(distinct) == 1 else None + + def _read_caps(self): + """Modes to attempt: the advertised caps, else every known NPS mode. + + Memory partition is hive-wide, so the first handle that answers speaks + for the system. The caps read is advisory only -- a device that reports + nothing still gets each mode attempted, since the set call is the real + authority. + """ + configs = self._for_each_gpu( + "amdsmi_get_gpu_memory_partition_config", + amdsmi.amdsmi_get_gpu_memory_partition_config, + accept=[amdsmi.AmdSmiStatus.SUCCESS, amdsmi.AmdSmiStatus.NOT_SUPPORTED], + ) + for config in configs.values(): + caps = [c for c in config.get("partition_caps", []) if c in _NPS_MODES] + if caps: + return caps + return list(_NPS_MODES) + + @staticmethod + def _profile_of(profile): + """``(index, name)`` for a handle that owns an accelerator profile, else None. + + This getter reports NOT_SUPPORTED in-band as "N/A" instead of raising, so + sub-partition handles have to be filtered on the value rather than dropping + out of _for_each_gpu the way the memory partition getters do. + """ + partition = profile["partition_profile"] + index = partition["profile_index"] + return (index, partition["profile_type"]) if isinstance(index, int) else None + + @staticmethod + def _describe_profiles(profiles): + """One-line summary, e.g. ``SPX (index 0) on 8 GPUs``.""" + if not profiles: + return "none reported" + distinct = sorted({f"{name} (index {index})" for index, name in profiles}) + return f"{', '.join(distinct)} on {len(profiles)} GPUs" + + def _read_accelerator_profiles(self): + """``[(index, name), ...]``, one entry per physical GPU, in handle order.""" + profiles = self._for_each_gpu( + "amdsmi_get_gpu_accelerator_partition_profile", + amdsmi.amdsmi_get_gpu_accelerator_partition_profile, + accept=[amdsmi.AmdSmiStatus.SUCCESS, amdsmi.AmdSmiStatus.NOT_SUPPORTED], + ) + return [p for p in map(self._profile_of, profiles.values()) if p is not None] + + # -- policy ------------------------------------------------------------- + + @staticmethod + def _reload_skip_reason(mode, staged, current): + """Why a reload would change nothing, or None when one is needed.""" + if not _ALLOW_DESTRUCTIVE_TESTS: + return "AMDSMI_ALLOW_DESTRUCTIVE_TESTS not set; staged only, skipping driver reload" + if _ALWAYS_RELOAD: + return None + if not staged: + return f"{mode} rejected; skipping driver reload" + if mode == current: + return f"{mode} already active; skipping driver reload" + return None + + # -- mutations ---------------------------------------------------------- + + def _stage_mode(self, mode): + """Stage *mode* on every device. False when no device accepted it. + + A hardware-dependent rejection is an expected status rather than a test + failure. + """ + # INVAL is how a device declines a mode it cannot do, and a handle without + # the partition APIs answers NOT_SUPPORTED; neither is a test failure. + accept = [ + amdsmi.AmdSmiStatus.SUCCESS, + amdsmi.AmdSmiStatus.INVAL, + amdsmi.AmdSmiStatus.NOT_SUPPORTED, + ] + successfully_set = self._for_each_gpu( + "amdsmi_set_gpu_memory_partition_mode", + lambda gpu: amdsmi.amdsmi_set_gpu_memory_partition_mode(gpu, _NPS_MODES[mode]), + accept=accept, + params=f", memory_partition_type={mode}", + ) + return bool(successfully_set) + + def _reload_driver(self): + """Unload then reload amdgpu. shell=False, so there is no injection surface. + + Mandatory for a staged memory partition to take effect. Callers must hold no + open session, and the baseline count below is read in a scoped one -- amdsmi + may hold /dev/dri descriptors that would make ``modprobe -r`` fail with + "module in use". + """ + self._separator("Reload driver") + with AmdsmiSession(): + expected = self._physical_gpu_count() + self.common.print(" Reloading driver, this may take some time...") + started = time.monotonic() + for argv in (["modprobe", "-r", "amdgpu"], ["modprobe", "amdgpu"]): + command = " ".join(argv) # log text only; never handed to a shell + self.common.print(f" {command} (timeout {_MODPROBE_TIMEOUT_SEC}s)") + leg_started = time.monotonic() + proc = subprocess.Popen(argv) + try: + returncode = proc.wait(timeout=_MODPROBE_TIMEOUT_SEC) + except subprocess.TimeoutExpired: + proc.kill() + try: + proc.wait(timeout=_REAP_TIMEOUT_SEC) + except subprocess.TimeoutExpired: + self.fail( + f"{command} timed out after {_MODPROBE_TIMEOUT_SEC}s and ignored " + f"SIGKILL: the module load is stuck inside the driver and cannot be " + f"cancelled, so the node needs a reboot. See dmesg and " + f"/sys/module/amdgpu/initstate." + ) + self.fail(f"{command} timed out after {_MODPROBE_TIMEOUT_SEC}s") + if returncode != 0: + self.fail(f"{command} exited {returncode}; check dmesg") + self.common.print(f" took {time.monotonic() - leg_started:.1f}s") + probe_started = time.monotonic() + self._wait_for_devices(expected) + self.common.print( + f" {expected} physical GPUs re-enumerated in {time.monotonic() - probe_started:.1f}s" + ) + self.common.print(f" Driver reloaded in {time.monotonic() - started:.1f}s total") + + def _wait_for_devices(self, expected): + """Block until *expected* physical GPUs (sockets) re-enumerate after a reload. + + modprobe returns once the module is inserted; the devices then probe + asynchronously and one at a time, so poll rather than assume a fixed + settle time. We used socket count as a check, since the partitioned + devices can fluctuate based on the NPS mode. + """ + deadline = time.monotonic() + _DEVICE_READY_TIMEOUT_SEC + found = 0 + while True: + time.sleep(_DEVICE_POLL_SEC) + try: + with AmdsmiSession(): + found = self._physical_gpu_count() + if found >= expected: + return + except amdsmi.AmdSmiException: + pass + if time.monotonic() >= deadline: + self.fail( + f"Only {found} of {expected} physical GPUs enumerated within " + f"{_DEVICE_READY_TIMEOUT_SEC}s of reloading amdgpu" + ) + + def _restore_memory_partition(self, mode): + """Put the device back on the memory partition mode the run started with.""" + self._separator(f"Restore memory partition ({mode})") + if not self.common.check_amdgpu_driver(): + self.common.print(f" amdgpu is not loaded; cannot restore {mode} -- node left as-is") + return + if not _ALLOW_DESTRUCTIVE_TESTS: + # No reload ever ran, so the active mode says nothing about what is + # staged. Always re-stage `mode` so a later reload (by anyone) comes + # up on it, even if the run aborted before reaching its own restage. + if not self._stage_mode(mode): + self.fail(f"could not stage restore back to {mode}") + self.common.print(f" staged {mode}; no reload performed (destructive tests disabled)") + return + if self._agreed_mode(self._read_modes()) == mode and not _ALWAYS_RELOAD: + self.common.print(f" already on {mode}; nothing to restore") + else: + if not self._stage_mode(mode): + self.fail(f"could not stage restore back to {mode}") + self._reload_driver() + self._assert_all_on(self._read_modes(), mode, "restore failed") + + def _restore_accelerator_profiles(self, profiles): + """Re-apply the accelerator profiles captured before the cycle. + + A memory partition reload resets the accelerator profile to the new mode's + default, and profile_index only means anything against that mode's config + table -- so this runs after the memory partition is back on the original. + """ + if not profiles: + return + self._separator("Restore accelerator partition") + if not self.common.check_amdgpu_driver(): + self.common.print(" amdgpu is not loaded; cannot restore the accelerator profile") + return + self.common.print(f" Restoring : {self._describe_profiles(profiles)}") + wanted = iter(profiles) + # Handles without a profile are skipped below, so only a handle that + # lacks the API entirely may decline the write. + accept = [amdsmi.AmdSmiStatus.SUCCESS, amdsmi.AmdSmiStatus.NOT_SUPPORTED] + with AmdsmiSession() as gpus, self.common.status_sweep(): + for i, gpu in enumerate(gpus): + current = amdsmi.amdsmi_get_gpu_accelerator_partition_profile(gpu) + if self._profile_of(current) is None: + continue + target = next(wanted, None) + if target is None: + break + index, name = target + msg = ( + "\t### amdsmi_set_gpu_accelerator_partition_profile(" + f"gpu={i}, profile={name}, index={index}):" + ) + with self.common.expect_status(msg, accept): + amdsmi.amdsmi_set_gpu_accelerator_partition_profile(gpu, index) + + def _assert_all_on(self, modes, mode, context): + """Fail unless every handle that reported a mode reports *mode*.""" + self.assertTrue(modes, f"{context}: no device reported a memory partition mode") + mismatched = {i: m for i, m in modes.items() if m != mode} + self.assertFalse( + mismatched, + f"{context}: expected all {len(modes)} readable devices on {mode}, " + f"but {len(mismatched)} differ: {mismatched}", + ) + + # -- tests --------------------------------------------------------------- + + def test_applying_memory_partition_needs_reload(self): + """Staging a mode must not change the active mode until the driver reloads. + + Documented in docs/conceptual/partition.md: the set call alone does not apply + the change, and the system keeps the old configuration until the next driver + load. Nothing here reloads, so nothing should change. + """ + self.common.print_func_name("") + self._separator("Set without reload") + + before = self._read_modes() + original = self._agreed_mode(before) + # A staged mode stays pending until the next driver load, re-staging the original + # clears it. Registered before any set so a failure below cannot leave it armed. + if original in _NPS_MODES: + self.addCleanup(self._stage_mode, original) + + for mode in _NPS_MODES: + with self.subTest(mode=mode): + staged = self._stage_mode(mode) + self.assertEqual( + self._read_modes(), + before, + f"{mode} was staged without a driver reload; the reported mode must not change", + ) + if staged: + self.common.print(f" [PASS] {mode} staged; reported mode unchanged") + else: + self.common.print(f" [INFO] {mode} was rejected; reported mode unchanged") + + def test_cycle_memory_partition_modes(self): + self.common.print_func_name("") + if not _ALLOW_DESTRUCTIVE_TESTS: + self.common.print( + " AMDSMI_ALLOW_DESTRUCTIVE_TESTS not set; staging + status checks run, but the " + "driver is never reloaded, so nothing is actually applied or verified.\n" + " Set AMDSMI_ALLOW_DESTRUCTIVE_TESTS=1 to also reload the driver and verify " + "each mode." + ) + elif not shutil.which("modprobe"): + self.skipTest("Cycling memory partition modes needs modprobe (install kmod)") + + # -- Phase 1: what the device reports today ------------------------- + self._separator("Current memory partition") + modes = self._read_modes() + original = self._agreed_mode(modes) + caps = self._read_caps() + accelerator = self._read_accelerator_profiles() + + # Every advertised mode in order, then the starting mode once more so the + # run ends where it began. The leading pass over the starting mode is + # already active, so it stages without a reload. + candidates = list(caps) + if original in caps: + candidates.append(original) + + self.common.print(f" Current mode : {original} (on {len(modes)} sockets)") + self.common.print(f" Accelerator : {self._describe_profiles(accelerator)}") + self.common.print(f" Attempting : {', '.join(candidates)}") + if original is None and modes: + self.common.print(f" Devices disagree on the current mode: {modes}") + + if original in _NPS_MODES: + # Cleanups run LIFO, so registering the accelerator profile first runs + # it last -- the memory reload resets it to the new mode's default. + self.addCleanup(self._restore_accelerator_profiles, accelerator) + # A no-op when nothing changed, so a device that rejects every mode + # is never reloaded on the way out. + self.addCleanup(self._restore_memory_partition, original) + elif original is not None: + self.common.print(f" {original!r} is not a settable mode; it cannot be restored") + + # -- Phase 2: stage / reload / verify, once per mode ---------------- + current = original + applied = [] + for mode in candidates: + self._separator(f"Set memory partition -> {mode}") + staged = self._stage_mode(mode) + + skipped = self._reload_skip_reason(mode, staged, current) + if skipped: + self.common.print(f" {skipped}") + continue + + # Outside the subTest below: subTest swallows failures, and continuing + # the cycle after a broken reload only buries the real cause. + self._reload_driver() + modes = self._read_modes() + current = self._agreed_mode(modes) + + with self.subTest(mode=mode): + if not staged: + self.common.print(f" [INFO] {mode} was rejected; active mode is {current}") + else: + self._assert_all_on(modes, mode, f"after reload to {mode}") + applied.append(mode) + self.common.print( + f" [PASS] {mode} applied and verified on {len(modes)} devices" + ) + + # -- Phase 3: leave the device as the run found it ------------------ + self._separator("Summary") + if applied: + self.common.print(f" Modes applied : {', '.join(applied)}") + elif not _ALLOW_DESTRUCTIVE_TESTS: + self.common.print( + " AMDSMI_ALLOW_DESTRUCTIVE_TESTS not set; staged every mode but skipped " + "reload/verify" + ) + else: + self.common.print(" No mode was settable; validated the reported status instead") diff --git a/projects/amdsmi/tests/python/functional/gpu/test_ptl.py b/projects/amdsmi/tests/python/functional/gpu/test_ptl.py index ff2f1bd0836e..1a5b648c14b4 100644 --- a/projects/amdsmi/tests/python/functional/gpu/test_ptl.py +++ b/projects/amdsmi/tests/python/functional/gpu/test_ptl.py @@ -96,7 +96,3 @@ def test_set_gpu_ptl_state(self): self.fail("PTL set did not take effect:\n " + "\n ".join(mismatches)) if tested == 0: self.skipTest("No PTL-capable GPUs found") - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/functional/ifoe/test_discovery.py b/projects/amdsmi/tests/python/functional/ifoe/test_discovery.py index 2b682dc72c96..6ed61b5b1d3f 100644 --- a/projects/amdsmi/tests/python/functional/ifoe/test_discovery.py +++ b/projects/amdsmi/tests/python/functional/ifoe/test_discovery.py @@ -9,7 +9,3 @@ class TestIfoeDiscovery(unittest.TestCase): pass # Add IFoE discovery tests here - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/functional/ifoe/test_fabric_lazy_init.py b/projects/amdsmi/tests/python/functional/ifoe/test_fabric_lazy_init.py index 6c22fea76c61..063b38dc55ed 100644 --- a/projects/amdsmi/tests/python/functional/ifoe/test_fabric_lazy_init.py +++ b/projects/amdsmi/tests/python/functional/ifoe/test_fabric_lazy_init.py @@ -81,7 +81,3 @@ def test_fabric_query_resolves_without_hanging(self): pass elapsed = time.monotonic() - start self.assertLess(elapsed, MAX_SECONDS, f"Fabric queries took {elapsed:.2f}s") - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/projects/amdsmi/tests/python/functional/ifoe/test_identity.py b/projects/amdsmi/tests/python/functional/ifoe/test_identity.py index 44ac367d9ade..940b7c0a2ee9 100644 --- a/projects/amdsmi/tests/python/functional/ifoe/test_identity.py +++ b/projects/amdsmi/tests/python/functional/ifoe/test_identity.py @@ -9,7 +9,3 @@ class TestIfoeIdentity(unittest.TestCase): pass # Add NIC/IFoE identity tests here - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/functional/nic/test_identity.py b/projects/amdsmi/tests/python/functional/nic/test_identity.py index 6c965a4d3449..23443d0652e3 100644 --- a/projects/amdsmi/tests/python/functional/nic/test_identity.py +++ b/projects/amdsmi/tests/python/functional/nic/test_identity.py @@ -9,7 +9,3 @@ class TestNicIdentity(unittest.TestCase): pass # Add NIC/IFoE identity tests here - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/integration/test_memory_partition_lifecycle.py b/projects/amdsmi/tests/python/integration/test_memory_partition_lifecycle.py deleted file mode 100644 index 6ec4e0142d98..000000000000 --- a/projects/amdsmi/tests/python/integration/test_memory_partition_lifecycle.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python3 -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Memory partition driver lifecycle integration test. - -Exercises the full set -> driver reload -> verify round-trip that cannot be -covered by the standard functional test suite. - -Requirements ------------- -- Root privileges (modprobe requires root) -- kmod installed (modprobe must be on PATH) -- No GPU workloads running during the test -- Run in isolation -- NOT via the standard unittest/pytest runner - -Usage ------ - sudo python3 tests/python/integration/test_memory_partition_lifecycle.py [--target NPS2] - -Exit codes ----------- - 0 all phases passed - 1 a phase failed (reason printed to stdout) - 2 precondition not met (no root, no kmod, bad --target) - 3 unexpected exception -""" - -import argparse -import os -import shutil -import subprocess -import sys -import time - -import amdsmi - -_MODPROBE_TIMEOUT_SEC = 120 -_DRIVER_SETTLE_SEC = 5 - -_NPS_NAME_TO_TYPE = { - "NPS1": amdsmi.AmdSmiMemoryPartitionType.NPS1, - "NPS2": amdsmi.AmdSmiMemoryPartitionType.NPS2, - "NPS4": amdsmi.AmdSmiMemoryPartitionType.NPS4, - "NPS8": amdsmi.AmdSmiMemoryPartitionType.NPS8, -} - - -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - - -def _print(msg: str) -> None: - print(msg, flush=True) - - -def _fail(msg: str) -> int: - _print(f" [FAIL] {msg}") - return 1 - - -def _check_preconditions(target_mode: str) -> int: - """Return 0 if all preconditions are satisfied, 2 otherwise.""" - if target_mode not in _NPS_NAME_TO_TYPE: - _print( - f"[ERROR] Unknown target mode '{target_mode}'. Choose from: {list(_NPS_NAME_TO_TYPE)}" - ) - return 2 - - if os.geteuid() != 0: - _print("[ERROR] This test requires root privileges (run with sudo).") - return 2 - - if not shutil.which("modprobe"): - _print("[ERROR] modprobe not found. Install kmod and retry.") - return 2 - - return 0 - - -def _reload_driver() -> bool: - """ - Unload then reload the amdgpu kernel module. - - Uses two separate subprocess.run() calls with shell=False so the arguments - are passed directly to execvp -- no shell is involved and there is no - injection surface. Returns True on success. - """ - _print(" Running: modprobe -r amdgpu") - try: - subprocess.run(["modprobe", "-r", "amdgpu"], check=True, timeout=_MODPROBE_TIMEOUT_SEC) - except subprocess.CalledProcessError as e: - _print(f" [FAIL] modprobe -r amdgpu exited with code {e.returncode}; check dmesg") - return False - except subprocess.TimeoutExpired: - _print(f" [FAIL] modprobe -r amdgpu timed out after {_MODPROBE_TIMEOUT_SEC}s") - return False - - _print(" Running: modprobe amdgpu") - try: - subprocess.run(["modprobe", "amdgpu"], check=True, timeout=_MODPROBE_TIMEOUT_SEC) - except subprocess.CalledProcessError as e: - _print(f" [FAIL] modprobe amdgpu exited with code {e.returncode}; check dmesg") - return False - except subprocess.TimeoutExpired: - _print(f" [FAIL] modprobe amdgpu timed out after {_MODPROBE_TIMEOUT_SEC}s") - return False - - _print(f" Driver reloaded. Waiting {_DRIVER_SETTLE_SEC}s for devices to settle...") - time.sleep(_DRIVER_SETTLE_SEC) - return True - - -def _init_and_get_primary() -> tuple: - """ - Init amdsmi and return (gpu_handle, partition_string) for the first GPU. - Caller is responsible for calling amdsmi.amdsmi_shut_down() when done. - Raises RuntimeError if no GPUs are found. - """ - amdsmi.amdsmi_init() - gpus = amdsmi.amdsmi_get_processor_handles() - if not gpus: - amdsmi.amdsmi_shut_down() - raise RuntimeError("No GPU handles returned after amdsmi_init()") - gpu = gpus[0] - partition = amdsmi.amdsmi_get_gpu_memory_partition(gpu) - return gpu, partition - - -# --------------------------------------------------------------------------- -# Test phases -# --------------------------------------------------------------------------- - - -def _phase1_read_original() -> tuple: - """Return (gpu, original_partition). Shuts down amdsmi before returning.""" - _print("\n=== Phase 1: Read original partition ===") - gpu, original_partition = _init_and_get_primary() - _print(f" Original partition: {original_partition}") - amdsmi.amdsmi_shut_down() - return gpu, original_partition - - -def _phase2_check_capabilities(target_mode: str) -> bool: - """ - Return True if target_mode is advertised by hardware capabilities. - Returns True unconditionally if the config API is unavailable (older kernel). - Shuts down amdsmi before returning. - """ - _print("\n=== Phase 2: Check hardware capabilities ===") - gpu, _ = _init_and_get_primary() - supported = True - try: - config = amdsmi.amdsmi_get_gpu_memory_partition_config(gpu) - caps = config.get("partition_caps", []) - _print(f" Hardware capabilities: {caps}") - if target_mode not in caps: - _print(f" [SKIP] {target_mode} not advertised in partition_caps") - supported = False - except amdsmi.AmdSmiLibraryException as e: - _print(f" [WARN] amdsmi_get_gpu_memory_partition_config unavailable: {e}") - _print(" Proceeding without capability pre-check.") - amdsmi.amdsmi_shut_down() - return supported - - -def _phase3_stage_change(target_mode: str) -> int: - """Stage the partition change. Returns 0 on success, 1 on failure.""" - _print(f"\n=== Phase 3: Stage partition change to {target_mode} ===") - gpu, _ = _init_and_get_primary() - try: - amdsmi.amdsmi_set_gpu_memory_partition(gpu, _NPS_NAME_TO_TYPE[target_mode]) - _print(f" amdsmi_set_gpu_memory_partition({target_mode}): SUCCESS") - except amdsmi.AmdSmiLibraryException as e: - amdsmi.amdsmi_shut_down() - return _fail(f"amdsmi_set_gpu_memory_partition failed: {e}") - amdsmi.amdsmi_shut_down() - return 0 - - -def _phase4_reload() -> int: - """Reload the driver. Returns 0 on success, 1 on failure.""" - _print("\n=== Phase 4: Reload driver to apply change ===") - return 0 if _reload_driver() else _fail("Driver reload failed") - - -def _phase5_verify(expected_mode: str) -> int: - """Re-init and verify the active partition matches expected_mode. Returns 0 or 1.""" - _print(f"\n=== Phase 5: Re-init and verify partition is {expected_mode} ===") - _, post_partition = _init_and_get_primary() - _print(f" Post-reload partition: {post_partition}") - amdsmi.amdsmi_shut_down() - if post_partition != expected_mode: - return _fail(f"Expected {expected_mode}, got {post_partition}") - _print(f" [PASS] Partition is {post_partition}") - return 0 - - -def _phase6_restore(original_partition: str) -> int: - """Stage and apply restore to original_partition. Returns 0 or 1.""" - _print(f"\n=== Phase 6: Restore to {original_partition} ===") - gpu, _ = _init_and_get_primary() - try: - amdsmi.amdsmi_set_gpu_memory_partition(gpu, _NPS_NAME_TO_TYPE[original_partition]) - _print(f" amdsmi_set_gpu_memory_partition({original_partition}): SUCCESS") - except amdsmi.AmdSmiLibraryException as e: - amdsmi.amdsmi_shut_down() - return _fail(f"Restore set failed: {e}") - amdsmi.amdsmi_shut_down() - - if not _reload_driver(): - return _fail("Driver reload for restore failed") - - _, restored = _init_and_get_primary() - amdsmi.amdsmi_shut_down() - _print(f" Post-restore partition: {restored}") - if restored != original_partition: - return _fail(f"Restore failed: expected {original_partition}, got {restored}") - _print(f" [PASS] Partition restored to {restored}") - return 0 - - -# --------------------------------------------------------------------------- -# Entry point -# --------------------------------------------------------------------------- - - -def run(target_mode: str) -> int: - rc = _check_preconditions(target_mode) - if rc != 0: - return rc - - try: - _, original_partition = _phase1_read_original() - - if original_partition == target_mode: - _print( - f"\n[SKIP] Device is already in {target_mode}. " - "Choose a different --target to exercise a partition change." - ) - return 0 - - if original_partition not in _NPS_NAME_TO_TYPE: - _print(f"\n[ERROR] Original partition '{original_partition}' is not a known NPS mode.") - return 1 - - if not _phase2_check_capabilities(target_mode): - return 0 # hardware does not support this mode -- skip, not fail - - rc = _phase3_stage_change(target_mode) - if rc != 0: - return rc - - rc = _phase4_reload() - if rc != 0: - # Driver did not reload -- attempt restore so the system is not left - # with a staged-but-unapplied partition change. - _print(" Attempting restore despite reload failure...") - _phase6_restore(original_partition) - return rc - - rc = _phase5_verify(target_mode) - # Always attempt restore, even if verify failed. - restore_rc = _phase6_restore(original_partition) - - if rc != 0: - return rc - if restore_rc != 0: - return restore_rc - - except Exception as e: - _print(f"\n[ERROR] Unexpected exception: {e}") - try: - amdsmi.amdsmi_shut_down() - except Exception: - pass - return 3 - - _print( - f"\n[PASS] Full lifecycle test passed: " - f"{original_partition} -> {target_mode} -> {original_partition}" - ) - return 0 - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Memory partition driver lifecycle integration test." - ) - parser.add_argument( - "--target", - default="NPS2", - choices=list(_NPS_NAME_TO_TYPE), - help="Partition mode to change to (default: NPS2)", - ) - args = parser.parse_args() - sys.exit(run(args.target)) diff --git a/projects/amdsmi/tests/python/unit/gpu/test_apu_metrics.py b/projects/amdsmi/tests/python/unit/gpu/test_apu_metrics.py index b044daf43465..89cb6fd15abd 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_apu_metrics.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_apu_metrics.py @@ -70,7 +70,3 @@ def test_na_dict_symmetry(self): } self.assertEqual(na_dict_keys, na_full_keys) self.assertEqual(na_dict_keys, live_keys) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_asic_info_layout.py b/projects/amdsmi/tests/python/unit/gpu/test_asic_info_layout.py index 658538c7b423..24b737a2f8c2 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_asic_info_layout.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_asic_info_layout.py @@ -95,7 +95,3 @@ def test_library_without_the_fields_renders_na(self): def test_rev_id_not_supported_renders_na(self): # rev_id shares the not-supported value, so it must not leak it raw. self.assertEqual(_asic_info_with(0x47, 0x47)["rev_id"], "N/A") - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_cache_labels.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_cache_labels.py index 2d2ff2d6a9ed..858e4d09d1f8 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_cli_cache_labels.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_cache_labels.py @@ -13,21 +13,28 @@ * each entry carries a ``total_cache_size`` equal to the per-instance ``cache_size`` times ``num_cache_instance``. -``static.py`` is loaded from the source tree so the test exercises the code -under development rather than a possibly-stale installed copy. +``static.py`` is loaded through cli_search_order, so a plain checkout +exercises the code under development; an explicit AMDSMI_PATH pins that install. """ import argparse import copy -import importlib.util import os -import sys -import types import unittest -_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -_REPO_ROOT = os.path.abspath(os.path.join(_THIS_DIR, "..", "..", "..", "..")) -STATIC_PATH = os.path.join(_REPO_ROOT, "amdsmi_cli", "subcommands", "static.py") +from common.common import ( + amdsmi_path, + cli_search_order, + fake_module, + find_cli_dir, + load_cli_module, + stub_modules, +) + +# Locate the CLI dir; cli_search_order() decides whether the install or this +# checkout wins. None -> setUpClass skips. +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +STATIC_PATH = os.path.join(_CLI_DIR, "subcommands", "static.py") if _CLI_DIR else None # gfx950 (MI350) cache layout captured live: four L1 variants plus L2 and L3. # Mirrors the per-entry dict shape returned by ``amdsmi_get_gpu_cache_info``. @@ -89,47 +96,38 @@ def get_error_info(self): return str(self) -def _install_fake_modules(): - """Register a stub ``amdsmi`` package plus the sibling CLI modules. +def _build_fake_modules(): + """Build a stub ``amdsmi`` package plus the sibling CLI modules. - Returns the fake ``amdsmi_interface`` so tests can swap the cache payload. + Returns the name -> module mapping for ``common.stub_modules``. """ - amdsmi_pkg = types.ModuleType("amdsmi") - interface = types.ModuleType("amdsmi.amdsmi_interface") - exception = types.ModuleType("amdsmi.amdsmi_exception") def _get_cache_info(_handle): return {"cache": copy.deepcopy(_GFX950_CACHE)} - interface.amdsmi_get_gpu_cache_info = _get_cache_info - exception.AmdSmiLibraryException = _FakeLibraryException - - amdsmi_pkg.amdsmi_interface = interface - amdsmi_pkg.amdsmi_exception = exception - sys.modules["amdsmi"] = amdsmi_pkg - sys.modules["amdsmi.amdsmi_interface"] = interface - sys.modules["amdsmi.amdsmi_exception"] = exception + interface = fake_module("amdsmi.amdsmi_interface", amdsmi_get_gpu_cache_info=_get_cache_info) + exception = fake_module("amdsmi.amdsmi_exception", AmdSmiLibraryException=_FakeLibraryException) + amdsmi_pkg = fake_module("amdsmi", amdsmi_interface=interface, amdsmi_exception=exception) # ``static.py`` imports these sibling names at load time; the cache path # never instantiates them (the test injects a fake helpers object). - helpers_mod = types.ModuleType("amdsmi_helpers") - helpers_mod.AMDSMIHelpers = object - sys.modules["amdsmi_helpers"] = helpers_mod - - exceptions_mod = types.ModuleType("amdsmi_cli_exceptions") - exceptions_mod.AmdSmiInvalidParameterException = type( - "AmdSmiInvalidParameterException", (Exception,), {} + helpers_mod = fake_module("amdsmi_helpers", AMDSMIHelpers=object) + exceptions_mod = fake_module( + "amdsmi_cli_exceptions", + AmdSmiInvalidParameterException=type("AmdSmiInvalidParameterException", (Exception,), {}), ) - sys.modules["amdsmi_cli_exceptions"] = exceptions_mod - return interface + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + "amdsmi_helpers": helpers_mod, + "amdsmi_cli_exceptions": exceptions_mod, + } def _load_static_module(): - spec = importlib.util.spec_from_file_location("static_under_test", STATIC_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("static_under_test", STATIC_PATH) class _FakeLogger: @@ -210,9 +208,13 @@ def _build_args(): class TestCliCacheLabels(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(STATIC_PATH): - raise unittest.SkipTest(f"amd-smi CLI static.py not found at {STATIC_PATH}") - cls.interface = _install_fake_modules() + if not STATIC_PATH or not os.path.isfile(STATIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI static.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _build_fake_modules() + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] cls.static_module = _load_static_module() def _run_cache(self, fmt): @@ -270,7 +272,3 @@ def test_json_labels_and_totals(self): # Existing per-instance size wrapping is unchanged. self.assertEqual(cache_info[0]["cache_size"], {"value": 32, "unit": "KB"}) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_current_node_power_management.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_current_node_power_management.py new file mode 100644 index 000000000000..6c038aec54b6 --- /dev/null +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_current_node_power_management.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Mock-based unit tests for the ``CURRENT_NODE_POWER`` field in ``amd-smi node -p``. + +current_node_power moved from amdsmi_power_info_t (per-GPU, amd-smi metric +--power) to amdsmi_npm_info_t (per-node, amd-smi node -p / +amdsmi_get_npm_info()) -- see the "Set npm power limit" design doc. This file +replaces the previous test_metric_power_node_power.py, which exercised the +now-removed amd-smi metric --power NODE_POWER surface. + +``NodeCommands.node()`` in ``amdsmi_cli/subcommands/node.py`` now seeds +``npm_dict["current_node_power"] = "N/A"`` and, on a successful +``amdsmi_get_npm_info()`` call, overwrites it with +``npm_info.get("current_node_power", "N/A")`` -- directly mirroring the +pre-existing ``threshold``/``limit`` fields it sits next to. These tests +drive the real ``NodeCommands.node()`` dispatch (stubbing only the +underlying ``amdsmi`` package, per the +``test_vcn_busy_navi.py``/``test_cli_set_clk_limit.py`` stub-and-importlib +pattern) and assert ``current_node_power`` is present with the correct +format/value across all output modes: + +* human-readable: unit-suffixed string printed as ``CURRENT_NODE_POWER: 5800 W``. +* JSON/file output: ``{"value": 5800, "unit": "W"}`` (unit_format's json + branch), nested under ``power_management``. +* CSV: the raw value (unit_format is not applied on the CSV path). + +Also covers the "N/A" sentinel pass-through (key absent from the library's +dict, or explicitly UINT32_MAX-turned-"N/A" upstream in +``amdsmi_get_npm_info()``'s Python wrapper) and the +``AmdSmiLibraryException`` fallback path (whole npm_dict stays at its "N/A" +defaults, current_node_power included). +""" + +import argparse +import importlib.util +import os +import unittest + +from common.common import cli_search_order, fake_module, find_cli_dir, stub_modules + +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +NODE_PATH = os.path.join(_CLI_DIR, "subcommands", "node.py") if _CLI_DIR else "" + + +class _FakeLibraryException(Exception): + def __init__(self, message="mock error"): + super().__init__(message) + self._message = message + + def get_error_info(self): + return self._message + + +_DEFAULT_NPM_INFO = { + "limit": 6000, + "status": 0, # AMDSMI_NPM_STATUS_ENABLED + "ubb_power_threshold": 6200, + "max_node_power_limit": 6400, + "current_node_power": 5800, +} + + +def _build_fake_modules(): + wrapper = fake_module("amdsmi.amdsmi_wrapper", AMDSMI_NPM_STATUS_DISABLED=1) + interface = fake_module( + "amdsmi.amdsmi_interface", + amdsmi_wrapper=wrapper, + amdsmi_get_npm_info=lambda _h: dict(_DEFAULT_NPM_INFO), + ) + exception = fake_module("amdsmi.amdsmi_exception", AmdSmiLibraryException=_FakeLibraryException) + amdsmi_pkg = fake_module( + "amdsmi", amdsmi_interface=interface, amdsmi_exception=exception, amdsmi_wrapper=wrapper + ) + + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + "amdsmi.amdsmi_wrapper": wrapper, + } + + +def _load_node_module(): + spec = importlib.util.spec_from_file_location("node_under_test_power", NODE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _FakeLogger: + def __init__(self, output_format="human", destination="stdout"): + self._format = output_format + self.destination = destination + self.output = None + + def is_json_format(self): + return self._format == "json" + + def is_csv_format(self): + return self._format == "csv" + + def is_human_readable_format(self): + return self._format == "human" + + def print_output(self, *args, **kwargs): + pass + + def store_multiple_device_output(self): + pass + + +class _FakeHelpers: + def check_required_groups(self): + pass + + def unit_format(self, logger, value, unit): + # Mirrors the real AMDSMIHelpers.unit_format's output-mode branches + # so the CLI-under-test's formatting choices are exercised + # faithfully, not just stubbed to a passthrough. + if value == "N/A": + return "N/A" + if logger.is_json_format(): + return {"value": value, "unit": unit} if unit else value + if logger.is_csv_format(): + return value + if logger.is_human_readable_format(): + return f"{value} {unit}".rstrip() if unit else f"{value}".rstrip() + return f"{value}" + + +def _build_node_args(**overrides): + defaults = dict( + nodes=None, power_management=True, base_board_temps=False, gtt=False, tray=False + ) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + +def _run_node(node_module, output_format="human", destination="stdout"): + helpers = _FakeHelpers() + logger = _FakeLogger(output_format=output_format, destination=destination) + commands = object.__new__(node_module.NodeCommands) + commands.logger = logger + commands.helpers = helpers + commands.group_check_printed = True + commands.node_handle = object() + + commands.node(_build_node_args()) + return logger, commands + + +class TestNodePowerManagementCurrentNodePower(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not os.path.isfile(NODE_PATH): + raise unittest.SkipTest(f"amd-smi CLI node.py not found at {NODE_PATH}") + modules = _build_fake_modules() + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] + + def setUp(self): + # Reload fresh each test so per-test monkeypatches on the shared + # interface module (e.g. amdsmi_get_npm_info raising) don't leak. + self.interface.amdsmi_get_npm_info = lambda _h: dict(_DEFAULT_NPM_INFO) + self.node_module = _load_node_module() + + def test_current_node_power_human_readable(self): + import io + from contextlib import redirect_stdout + + buf = io.StringIO() + with redirect_stdout(buf): + _run_node(self.node_module, output_format="human", destination="stdout") + + self.assertIn("CURRENT_NODE_POWER: 5800 W", buf.getvalue()) + + def test_current_node_power_json_format(self): + _, commands = _run_node(self.node_module, output_format="json", destination="file") + self.assertEqual( + commands.logger.output["node"]["power_management"]["current_node_power"], + {"value": 5800, "unit": "W"}, + ) + + def test_current_node_power_csv_format(self): + _, commands = _run_node(self.node_module, output_format="csv", destination="file") + # CSV path stores raw values, no unit_format applied. + self.assertEqual(commands.logger.output["current_node_power"], 5800) + # CSV rows always lead with a device identifier, mirroring gpu/cpu/nic. + self.assertEqual(commands.logger.output["node"], 0) + + def test_current_node_power_na_key_absent_from_library_dict(self): + # If current_node_power is missing from the dict entirely (e.g. an + # older amdsmi build predating this field), the CLI's .get(..., "N/A") + # default must apply rather than raising a KeyError. + npm_info_without_current_node_power = dict(_DEFAULT_NPM_INFO) + del npm_info_without_current_node_power["current_node_power"] + self.interface.amdsmi_get_npm_info = lambda _h: npm_info_without_current_node_power + + _, commands = _run_node(self.node_module, output_format="json", destination="file") + + self.assertEqual( + commands.logger.output["node"]["power_management"]["current_node_power"], "N/A" + ) + + def test_current_node_power_na_sentinel_value(self): + # amdsmi_get_npm_info()'s Python wrapper already converts the + # UINT32_MAX sentinel to the string "N/A" before the dict reaches the + # CLI; confirm that value passes through unchanged rather than being + # formatted as "N/A W". + npm_info_na = dict(_DEFAULT_NPM_INFO) + npm_info_na["current_node_power"] = "N/A" + self.interface.amdsmi_get_npm_info = lambda _h: npm_info_na + + _, commands = _run_node(self.node_module, output_format="json", destination="file") + + self.assertEqual( + commands.logger.output["node"]["power_management"]["current_node_power"], "N/A" + ) + + def test_current_node_power_defaults_to_na_on_library_exception(self): + # When amdsmi_get_npm_info() itself raises, the whole npm_dict + # (seeded with "N/A" defaults, current_node_power included) is left + # untouched -- no partial/crashed state. + def _raise(_h): + raise _FakeLibraryException("npm info unavailable") + + self.interface.amdsmi_get_npm_info = _raise + + _, commands = _run_node(self.node_module, output_format="json", destination="file") + + power_management = commands.logger.output["node"]["power_management"] + self.assertEqual(power_management["current_node_power"], "N/A") + self.assertEqual(power_management["limit"], "N/A") diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_apu_sections.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_apu_sections.py index 0adf8ebcb80d..a7a5522e5a55 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_apu_sections.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_apu_sections.py @@ -16,26 +16,27 @@ """ import argparse -import importlib.util import os -import sys import types import unittest -from common.common import amdsmi_path +from common.common import amdsmi_path, cli_search_order, find_cli_dir, load_cli_module, stub_modules -# The amd-smi CLI ships alongside the amdsmi package: ``common`` resolves -# ``amdsmi_path`` to ``/share/amd_smi`` and the CLI installs to the sibling -# ``/libexec/amdsmi_cli``. ``setUpClass`` skips the suite if it is absent. -_ROCM_ROOT = os.path.dirname(os.path.dirname(amdsmi_path)) -METRIC_PATH = os.path.join(_ROCM_ROOT, "libexec", "amdsmi_cli", "subcommands", "metric.py") +# Locate the CLI dir; cli_search_order() decides whether the install or this +# checkout wins. None -> setUpClass skips. +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +METRIC_PATH = os.path.join(_CLI_DIR, "subcommands", "metric.py") if _CLI_DIR else None # Sections gated on APU parts, and the value each reports when named explicitly. _SCALAR_NA_SECTIONS = ("ecc_blocks", "overdrive", "xgmi_err", "energy") _DICT_NA_SECTIONS = ("pcie", "voltage_curve", "voltage") -class _FakeLibraryException(Exception): +class _FakeAmdSmiException(Exception): + """Stands in for ``AmdSmiException``, the base metric.py catches on the header read.""" + + +class _FakeLibraryException(_FakeAmdSmiException): def __init__(self, message="AMDSMI_STATUS_NOT_SUPPORTED"): super().__init__(message) self._message = message @@ -67,8 +68,8 @@ class _FakeEnum(metaclass=_EnumMeta): pass -def _install_fake_amdsmi(): - """Register a stub ``amdsmi`` package so ``metric.py`` imports cleanly. +def _build_fake_amdsmi(): + """Build a stub ``amdsmi`` package so ``metric.py`` imports cleanly. Any ``amdsmi_*`` entry point that a test does not stub explicitly raises ``_FakeLibraryException``, standing in for a sensor the APU does not expose. @@ -111,22 +112,21 @@ def _raise(*_args, **_kwargs): interface.__getattr__ = _unsupported + exception.AmdSmiException = _FakeAmdSmiException exception.AmdSmiLibraryException = _FakeLibraryException amdsmi_pkg.amdsmi_interface = interface amdsmi_pkg.amdsmi_exception = exception - sys.modules["amdsmi"] = amdsmi_pkg - sys.modules["amdsmi.amdsmi_interface"] = interface - sys.modules["amdsmi.amdsmi_exception"] = exception - return interface + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + } def _load_metric_module(): - spec = importlib.util.spec_from_file_location("metric_apu_under_test", METRIC_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("metric_apu_under_test", METRIC_PATH, sys_path_dir=_CLI_DIR) class _FakeLogger: @@ -249,9 +249,13 @@ def _build_args(**overrides): class TestCliMetricApuSections(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(METRIC_PATH): - raise unittest.SkipTest(f"amd-smi CLI metric.py not found at {METRIC_PATH}") - cls.interface = _install_fake_amdsmi() + if not METRIC_PATH or not os.path.isfile(METRIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI metric.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _build_fake_amdsmi() + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] cls.metric_module = _load_metric_module() def _run_metric(self, metrics=None, **arg_overrides): @@ -341,7 +345,3 @@ def test_zero_bandwidth_still_carries_unit(self): usage = self._run_metric(metrics={"apu_metrics.average_ipu_reads": 0}, usage=True)["usage"] self.assertEqual(usage["apu_average_ipu_reads"], "0 MB/s") - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_partition.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_partition.py index dd22110288c4..72a86b4e670f 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_partition.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_partition.py @@ -23,20 +23,23 @@ """ import argparse -import importlib.util import os -import sys import types import unittest -from common.common import amdsmi_path +from common.common import ( + amdsmi_path, + cli_search_order, + fake_module, + find_cli_dir, + load_cli_module, + stub_modules, +) -# The amd-smi CLI ships alongside the amdsmi package: ``common`` resolves -# ``amdsmi_path`` to ``/share/amd_smi`` and the CLI installs to the sibling -# ``/libexec/amdsmi_cli``. ``setUpClass`` skips the suite if it is absent -# (e.g. an unusual layout where only the package, not the CLI, is present). -_ROCM_ROOT = os.path.dirname(os.path.dirname(amdsmi_path)) -METRIC_PATH = os.path.join(_ROCM_ROOT, "libexec", "amdsmi_cli", "subcommands", "metric.py") +# Locate the CLI dir; cli_search_order() decides whether the install or this +# checkout wins. None -> setUpClass skips. +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +METRIC_PATH = os.path.join(_CLI_DIR, "subcommands", "metric.py") if _CLI_DIR else None class _FakeClkType: @@ -50,7 +53,11 @@ class _FakeClkType: DF = "DF" -class _FakeLibraryException(Exception): +class _FakeAmdSmiException(Exception): + """Stands in for ``AmdSmiException``, the base metric.py catches on the header read.""" + + +class _FakeLibraryException(_FakeAmdSmiException): def __init__(self, message="mock error"): super().__init__(message) self._message = message @@ -81,47 +88,54 @@ def _patch_interface(testcase, interface, **overrides): testcase.addCleanup(_restore_attr, interface, name, original) -def _install_fake_amdsmi(): +def _build_fake_amdsmi(**interface_overrides): """Register a stub ``amdsmi`` package so ``metric.py`` imports cleanly. + ``interface_overrides`` replace or add ``amdsmi_interface`` attributes, for a + class that needs a different default (e.g. a call that must raise). + Returns the fake ``amdsmi_interface`` module so individual tests can swap in per-case return values for the C-library entry points. """ - amdsmi_pkg = types.ModuleType("amdsmi") - interface = types.ModuleType("amdsmi.amdsmi_interface") - exception = types.ModuleType("amdsmi.amdsmi_exception") - - interface.AMDSMI_MAX_NUM_GFX_CLKS = 8 - interface.AMDSMI_MAX_NUM_CLKS = 4 - interface.AMDSMI_MAX_RAIL_INDEX = 7 - interface.AmdSmiClkType = _FakeClkType # Default clock-limit payload reused by every AmdSmiClkType lookup. def _get_clock_info(_handle, _clk_type): return {"min_clk": 400, "max_clk": 2100, "clk_deep_sleep": "DISABLED"} - interface.amdsmi_get_clock_info = _get_clock_info - interface.amdsmi_get_gpu_metrics_info = lambda _handle: {} - interface._NA_amdsmi_get_gpu_metrics_info = lambda: {} - # Set per-test; default keeps the partition path inert. - interface.amdsmi_get_gpu_partition_metrics_info = lambda _handle: None - - exception.AmdSmiLibraryException = _FakeLibraryException - - amdsmi_pkg.amdsmi_interface = interface - amdsmi_pkg.amdsmi_exception = exception + interface = fake_module( + "amdsmi.amdsmi_interface", + AMDSMI_MAX_NUM_GFX_CLKS=8, + AMDSMI_MAX_NUM_CLKS=4, + AMDSMI_MAX_RAIL_INDEX=7, + AmdSmiClkType=_FakeClkType, + amdsmi_get_clock_info=_get_clock_info, + amdsmi_get_gpu_metrics_info=lambda _handle: {}, + _NA_amdsmi_get_gpu_metrics_info=lambda: {}, + # An unmapped version, so unsupported-field filtering suppresses nothing and + # these tests see the partition behavior alone. + amdsmi_get_gpu_metrics_header_info=lambda _handle: {}, + # Set per-test; default keeps the partition path inert. + amdsmi_get_gpu_partition_metrics_info=lambda _handle: None, + **interface_overrides, + ) + exception = fake_module( + "amdsmi.amdsmi_exception", + AmdSmiException=_FakeAmdSmiException, + AmdSmiLibraryException=_FakeLibraryException, + ) + amdsmi_pkg = fake_module("amdsmi", amdsmi_interface=interface, amdsmi_exception=exception) - sys.modules["amdsmi"] = amdsmi_pkg - sys.modules["amdsmi.amdsmi_interface"] = interface - sys.modules["amdsmi.amdsmi_exception"] = exception - return interface + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + # The metric module is loaded against these fakes, so it goes with them. + "metric_under_test": None, + } def _load_metric_module(): - spec = importlib.util.spec_from_file_location("metric_under_test", METRIC_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("metric_under_test", METRIC_PATH, sys_path_dir=_CLI_DIR) class _FakeLogger: @@ -161,7 +175,7 @@ def is_hypervisor(self): def is_windows(self): return False - def is_baremetal(self): + def is_baremetal(self) -> bool: return True def is_linux(self): @@ -176,7 +190,7 @@ def get_gpu_id_from_device_handle(self, _handle): def os_info(self): return "mock-os" - def _get_metric_version_and_partition_info(self, *args, **kwargs): + def _get_metric_version_and_partition_info(self, *args, **kwargs) -> dict: return {"num_partition": 1} def unit_format(self, logger, value, unit): @@ -194,10 +208,10 @@ def unit_format(self, logger, value, unit): class _FakeHelpersVirtualOS(_FakeHelpers): """Virtual-OS stub: ``is_baremetal()`` is False, so ``--partition`` is unregistered.""" - def is_baremetal(self): + def is_baremetal(self) -> bool: return False - def _get_metric_version_and_partition_info(self, *args, **kwargs): + def _get_metric_version_and_partition_info(self, *args, **kwargs) -> dict: return {"num_partition": "N/A"} @@ -266,14 +280,22 @@ def _build_virtual_os_args(**overrides): class TestCliMetricPartitionClock(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(METRIC_PATH): - raise unittest.SkipTest(f"amd-smi CLI metric.py not found at {METRIC_PATH}") - cls.interface = _install_fake_amdsmi() + if not METRIC_PATH or not os.path.isfile(METRIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI metric.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _build_fake_amdsmi() + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] cls.metric_module = _load_metric_module() def _run_clock_partition(self, partition_metrics): """Drive ``metric_gpu`` for ``--clock --partition`` and return ``clocks``.""" - self.interface.amdsmi_get_gpu_partition_metrics_info = lambda _handle: partition_metrics + setattr( + self.interface, + "amdsmi_get_gpu_partition_metrics_info", + lambda _handle: partition_metrics, + ) commands = object.__new__(self.metric_module.MetricCommands) commands.logger = _FakeLogger() @@ -401,7 +423,10 @@ def _run_baremetal_section(self, args): commands.group_check_printed = True commands.device_handles = [] commands.metric_gpu(args) - return commands.logger.captured_values + captured = commands.logger.captured_values + self.assertIsNotNone(captured, "metric_gpu did not store a values payload") + assert captured is not None # narrows the Optional for the callers below + return captured def test_usage_partition_branch_uses_partition_metrics(self): # partition=True drives the fetch; gpu_partition_metrics is non-None, so @@ -456,22 +481,25 @@ class TestCliMetricPartitionVirtualOS(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(METRIC_PATH): - raise unittest.SkipTest(f"amd-smi CLI metric.py not found at {METRIC_PATH}") - cls.interface = _install_fake_amdsmi() - cls.metric_module = _load_metric_module() - # fclk is unavailable so the non-partition clock exception handler fires. - cls.interface.amdsmi_get_clk_freq = _raise_lib_exc - # Temperature enum access is not guarded, so the enums must exist; the - # sensor fetches degrade to N/A. - cls.interface.amdsmi_get_gpu_activity = _raise_lib_exc - cls.interface.amdsmi_get_temp_metric = _raise_lib_exc - cls.interface.AmdSmiTemperatureType = types.SimpleNamespace( - EDGE="EDGE", HOTSPOT="HOTSPOT", VRAM="VRAM" - ) - cls.interface.AmdSmiTemperatureMetric = types.SimpleNamespace( - CURRENT="CURRENT", CRITICAL="CRITICAL" + if not METRIC_PATH or not os.path.isfile(METRIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI metric.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _build_fake_amdsmi( + # fclk is unavailable so the non-partition clock exception handler fires. + amdsmi_get_clk_freq=_raise_lib_exc, + # Temperature enum access is not guarded, so the enums must exist; the + # sensor fetches degrade to N/A. + amdsmi_get_gpu_activity=_raise_lib_exc, + amdsmi_get_temp_metric=_raise_lib_exc, + AmdSmiTemperatureType=types.SimpleNamespace( + EDGE="EDGE", HOTSPOT="HOTSPOT", VRAM="VRAM" + ), + AmdSmiTemperatureMetric=types.SimpleNamespace(CURRENT="CURRENT", CRITICAL="CRITICAL"), ) + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] + cls.metric_module = _load_metric_module() def _run_metric(self, args): partition_calls = [] @@ -480,7 +508,9 @@ def _tracking_partition_metrics(handle): partition_calls.append(handle) return None - self.interface.amdsmi_get_gpu_partition_metrics_info = _tracking_partition_metrics + setattr( + self.interface, "amdsmi_get_gpu_partition_metrics_info", _tracking_partition_metrics + ) commands = object.__new__(self.metric_module.MetricCommands) commands.logger = _FakeLogger() @@ -531,7 +561,3 @@ def test_usage_and_temperature_sections_do_not_read_partition(self): self.assertIn("usage", captured) self.assertIn("temperature", captured) self.assertEqual(partition_calls, []) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_unsupported_filter.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_unsupported_filter.py new file mode 100644 index 000000000000..a2d8b187541a --- /dev/null +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_metric_unsupported_filter.py @@ -0,0 +1,943 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""End-to-end output-shape tests for ``amd-smi metric`` unsupported-field filtering. + +Drives ``MetricCommands.metric_gpu`` with the C library, logger and helpers +stubbed, so the whole path runs without GPU hardware: the header read, the +suppression set, the all-N/A gate that decides what reaches the logger, and the +rule that a section named on the command line is never emptied. The helper-level +tests in ``test_cli_metrics_field_support`` exercise those pieces in isolation +and cannot see a regression in how they are wired together, so every assertion +here compares default output against ``--show-unsupported`` output. + +Filtering applies to human-readable output only. ``--json`` and ``--csv`` are +the machine-readable contract, so every class below pairs its human-format +assertions with the claim that the two machine formats are byte-for-byte what +``--show-unsupported`` produces, which is also what the feature replaced. + +The stubbing pattern is the one in ``test_cli_metric_partition`` and +``test_vcn_busy_navi``. +""" + +import argparse +import contextlib +import copy +import csv +import importlib.util +import io +import logging +import os +import sys +import types +import unittest + +from common.common import amdsmi_path, cli_search_order, find_cli_dir, stub_modules + +CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +METRIC_PATH = os.path.join(CLI_DIR, "subcommands", "metric.py") if CLI_DIR else "" + +# Skip only when no CLI resolves; a resolved CLI missing metric.py fails on load. +_RUNNABLE = bool(CLI_DIR) +_SKIP_REASON = f"amd-smi CLI not found (looked in {amdsmi_path})" + +NA = "N/A" + + +class _FakeClkType: + GFX = "GFX" + MEM = "MEM" + VCLK0 = "VCLK0" + DCLK0 = "DCLK0" + SOC = "SOC" + DF = "DF" + + +class _FakeTemperatureType: + EDGE = "EDGE" + HOTSPOT = "HOTSPOT" + VRAM = "VRAM" + + +class _FakeTemperatureMetric: + CURRENT = "CURRENT" + CRITICAL = "CRITICAL" + + +class _FakeException(Exception): + """Stands in for ``AmdSmiException``, the base of the two below.""" + + +class _FakeLibraryException(_FakeException): + def __init__(self, message="mock error"): + super().__init__(message) + self._message = message + + def get_error_info(self): + return self._message + + +class _FakeParameterException(_FakeException): + """Sibling of the library exception, and like the real one has no error info.""" + + +def _raise_lib_exc(*_args, **_kwargs): + raise _FakeLibraryException("mock error") + + +def _raise_param_exc(*_args, **_kwargs): + raise _FakeParameterException("mock bad parameter") + + +class _FakeNamedValues: + """Stands in for an interface enum: any member resolves to its own name.""" + + def __getattr__(self, name): + return types.SimpleNamespace(name=name, value=name) + + def __getitem__(self, name): + return getattr(self, name) + + +class _FakeInterfaceModule(types.ModuleType): + """Any ``amdsmi_*`` call not stubbed below behaves as an unsupported call. + + Plain ``amd-smi metric`` turns on every section, most of which these tests do + not care about. Failing their reads is both the shortest stub and a faithful + one: the CLI already prints "N/A" for a library call it cannot make. + """ + + def __getattr__(self, name): + if name.startswith("amdsmi_"): + return _raise_lib_exc + raise AttributeError(name) + + +# Throttle rows whose residency accumulators report real numbers, and rows that +# report nothing. The second group is what a metrics version can make impossible. +_LIVE_THROTTLE_ROWS = ("prochot_thrm", "ppt_pwr", "socket_thrm", "vr_thrm", "hbm_thrm") +_EMPTY_THROTTLE_ROWS = ( + "gfx_clk_below_host_limit", + "gfx_clk_below_host_limit_pwr", + "gfx_clk_below_host_limit_thm", + "gfx_clk_below_host_limit_total", + "low_utilization", +) + + +def _violation_status(*_args, **_kwargs): + # build_xcp_dict() mutates what it is handed, so hand it a fresh dict. + status = {"acc_counter": 1000} + for row in _LIVE_THROTTLE_ROWS: + status[f"acc_{row}"] = 5 + status[f"active_{row}"] = False + status[f"per_{row}"] = 1 + for row in _EMPTY_THROTTLE_ROWS: + status[f"acc_{row}"] = NA + status[f"active_{row}"] = NA + status[f"per_{row}"] = NA + return status + + +def _gpu_metrics(*_args, **_kwargs): + return { + "vcn_activity": NA, + "jpeg_activity": NA, + "temperature_hbm_stacks": NA, + "temperature_mid": NA, + "temperature_aid": NA, + "xcp_stats.temperature_xcd": NA, + } + + +def _pcie_info(*_args, **_kwargs): + return { + "pcie_metric": { + "pcie_width": 16, + "pcie_speed": 32000, + "pcie_bandwidth": NA, + "pcie_replay_count": 3, + "pcie_l0_to_recovery_count": NA, + "pcie_replay_roll_over_count": NA, + "pcie_nak_received_count": NA, + "pcie_nak_sent_count": NA, + "pcie_lc_perf_other_end_recovery_count": NA, + } + } + + +def _install_fake_amdsmi(test_cls): + amdsmi_pkg = types.ModuleType("amdsmi") + interface = _FakeInterfaceModule("amdsmi.amdsmi_interface") + exception = types.ModuleType("amdsmi.amdsmi_exception") + + interface.AMDSMI_MAX_NUM_GFX_CLKS = 8 + interface.AMDSMI_MAX_NUM_CLKS = 4 + interface.AMDSMI_MAX_RAIL_INDEX = 7 + interface.AMDSMI_NUM_VOLTAGE_CURVE_POINTS = 3 + interface.AmdSmiClkType = _FakeClkType + interface.AmdSmiTemperatureType = _FakeTemperatureType + interface.AmdSmiTemperatureMetric = _FakeTemperatureMetric + interface.AmdSmiVoltageType = _FakeNamedValues() + interface.AmdSmiVoltageMetric = _FakeNamedValues() + interface.AmdSmiMemoryType = _FakeNamedValues() + interface.AmdSmiRasErrState = _FakeNamedValues() + interface.AmdSmiGpuBlock = _FakeNamedValues() + interface.AmdSmiLibraryException = _FakeLibraryException + + interface.amdsmi_get_gpu_metrics_info = _gpu_metrics + interface._NA_amdsmi_get_gpu_metrics_info = lambda: {} + interface.amdsmi_get_gpu_partition_metrics_info = lambda _h: None + interface.amdsmi_get_gpu_activity = lambda _h: { + "gfx_activity": 30, + "umc_activity": 12, + "mm_activity": NA, + } + # Navi-style sysfs fallback is unavailable, so usage.vcn_busy reads "N/A". + # It is unmapped, which makes it the control for "all-N/A but never dropped". + interface.amdsmi_get_vcn_busy_percent = _raise_lib_exc + interface.amdsmi_get_pcie_info = _pcie_info + interface.amdsmi_get_gpu_pci_throughput = _raise_lib_exc + interface.amdsmi_get_temp_metric = lambda _h, _type, _metric: 55 + interface.amdsmi_get_violation_status = _violation_status + # Replaced per test with the version under test. + interface.amdsmi_get_gpu_metrics_header_info = lambda _h: {} + + exception.AmdSmiException = _FakeException + exception.AmdSmiLibraryException = _FakeLibraryException + exception.AmdSmiParameterException = _FakeParameterException + + amdsmi_pkg.amdsmi_interface = interface + amdsmi_pkg.amdsmi_exception = exception + + stub_modules( + test_cls, + { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + }, + ) + return interface + + +def _load_metric_module(): + # metric.py imports its CLI-level siblings by bare name. + if CLI_DIR not in sys.path: + sys.path.insert(0, CLI_DIR) + spec = importlib.util.spec_from_file_location("metric_under_test_unsupported", METRIC_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_logger_module(test_cls): + """The real ``AMDSMILogger``, with only its helpers import replaced. + + The CSV column behavior asserted below belongs to the logger, so a + hand-rolled stand-in would prove nothing. ``amdsmi_helpers`` pulls in + ``amdsmi_init``, which talks to the driver, so that one import is stubbed. + """ + if "amdsmi_helpers" not in sys.modules: + helpers_module = types.ModuleType("amdsmi_helpers") + helpers_module.AMDSMIHelpers = type("AMDSMIHelpers", (), {}) + stub_modules(test_cls, {"amdsmi_helpers": helpers_module}) + spec = importlib.util.spec_from_file_location( + "amdsmi_logger_under_test_unsupported", os.path.join(CLI_DIR, "amdsmi_logger.py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _FakeLogger: + """Captures the ``values`` payload per device handle, in one output format.""" + + FORMATS = ("human", "json", "csv") + MACHINE_FORMATS = ("json", "csv") + + def __init__(self, output_format="human"): + self.output_format = output_format + self.captured = {} + self.store_gpu_json_output = [] + + def is_json_format(self): + return self.output_format == "json" + + def is_csv_format(self): + return self.output_format == "csv" + + def is_human_readable_format(self): + return self.output_format == "human" + + def store_output(self, gpu, key, value): + if key == "values": + self.captured[id(gpu)] = value + + def print_output(self, *args, **kwargs): + pass + + def store_multiple_device_output(self): + pass + + def store_watch_output(self, *args, **kwargs): + pass + + +class _FakeHelpers: + def __init__(self, gpu_ids=None): + self._gpu_ids = gpu_ids or {} + + def is_hypervisor(self): + return False + + def is_windows(self): + return False + + def is_baremetal(self): + return True + + def is_linux(self): + return True + + def check_required_groups(self): + pass + + def get_gpu_id_from_device_handle(self, handle): + return self._gpu_ids.get(id(handle), 0) + + def os_info(self): + return "mock-os" + + def _get_metric_version_and_partition_info(self, *args, **kwargs): + return {"num_partition": NA} + + # Board temperatures come from a helper rather than the metrics blobs, and + # an empty dict is how metric.py already spells "nothing to show". + def get_gpu_board_temperatures(self, *_args, **_kwargs): + return {} + + def get_base_board_temperatures(self, *_args, **_kwargs): + return {} + + def build_xcp_dict(self, key, violation_status, num_partition): + value = violation_status[key] + if isinstance(value, list): + return {f"xcp_{index}": value[index] for index in range(num_partition)} + if "active_" in key and value != NA: + return "ACTIVE" if value is True else "NOT ACTIVE" + return value + + def unit_format(self, logger, value, unit): + if isinstance(value, list): + return [self.unit_format(logger, item, unit) for item in value] + if value == NA: + return NA + if logger.is_json_format(): + return {"value": value, "unit": unit} if unit else value + if unit: + return f"{value} {unit}".rstrip() + return f"{value}".rstrip() + + +# Every section flag `metric` registers on baremetal Linux, in the order +# metric_gpu builds them. +_SECTION_ARGS = ( + "mem_usage", + "usage", + "power", + "clock", + "temperature", + "voltage", + "pcie", + "ecc", + "ecc_blocks", + "base_board", + "gpu_board", + "fan", + "voltage_curve", + "overdrive", + "perf_level", + "xgmi_err", + "energy", + "throttle", +) + +# The metrics-backed sections, named explicitly, which is the common case here. +_DEFAULT_SECTIONS = ("usage", "pcie", "temperature", "throttle") + + +def _build_args(sections=_DEFAULT_SECTIONS, **overrides): + """Namespace mirroring argparse on baremetal Linux. + + ``sections`` is what the user named; an empty tuple is plain ``amd-smi + metric``, which metric_gpu then expands to every section. + """ + defaults = dict( + gpu=None, + watch=False, + watch_time=None, + iterations=None, + loglevel="INFO", + show_unsupported=False, + partition=False, + ) + defaults.update({arg: arg in sections for arg in _SECTION_ARGS}) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + +def _header(version): + return {"structure_size": 512, "format_revision": version[0], "content_revision": version[1]} + + +def _key_paths(payload, prefix=""): + """Every dotted key path in a values payload, for key-set comparisons.""" + for key, value in payload.items(): + path = f"{prefix}.{key}" if prefix else key + yield path + if isinstance(value, dict): + yield from _key_paths(value, path) + + +@unittest.skipUnless(_RUNNABLE, _SKIP_REASON) +class _MetricGpuHarness(unittest.TestCase): + """Runs ``metric_gpu`` against the stubs and hands back the stored payload.""" + + @classmethod + def setUpClass(cls): + cls.interface = _install_fake_amdsmi(cls) + cls.metric_module = _load_metric_module() + + def setUp(self): + self.interface.amdsmi_get_violation_status = _violation_status + + def run_metric( + self, + *, + show_unsupported=False, + version=(1, 6), + output="human", + reader=None, + sections=_DEFAULT_SECTIONS, + loglevel="INFO", + partition=False, + ): + """Drive one GPU and return the ``values`` payload the logger received.""" + self.header_calls = [] + + def default_reader(handle): + self.header_calls.append(handle) + return _header(version) + + self.interface.amdsmi_get_gpu_metrics_header_info = reader or default_reader + + handle = object() + commands = object.__new__(self.metric_module.MetricCommands) + commands.logger = _FakeLogger(output) + commands.helpers = _FakeHelpers() + commands.group_check_printed = True + commands.device_handles = [] + commands.metric_gpu( + _build_args( + sections=sections, + gpu=handle, + show_unsupported=show_unsupported, + loglevel=loglevel, + partition=partition, + ) + ) + + captured = commands.logger.captured.get(id(handle)) + self.assertIsNotNone(captured, "metric_gpu did not store a values payload") + return captured + + def assertKeyPresent(self, values, section, key): + self.assertIn(section, values, f"section {section} missing") + self.assertIn(key, values[section], f"{section}.{key} was dropped") + + def assertKeyAbsent(self, values, section, key): + self.assertNotIn(key, values.get(section, {}), f"{section}.{key} was kept") + + def assertMachineFormatsUnfiltered(self, *, version, sections=_DEFAULT_SECTIONS): + """``--json`` and ``--csv`` must equal what ``--show-unsupported`` produces. + + Compared as whole payloads rather than key sets, so a machine format + cannot start filtering by blanking a value instead of dropping a key. + """ + for output in _FakeLogger.MACHINE_FORMATS: + with self.subTest(output=output): + unfiltered = self.run_metric( + show_unsupported=True, version=version, output=output, sections=sections + ) + default = self.run_metric(version=version, output=output, sections=sections) + self.assertEqual(default, unfiltered, f"--{output} was filtered") + + +class TestMachineFormatsAreNeverFiltered(_MetricGpuHarness): + """``--json`` and ``--csv`` are a machine-readable contract and never filter. + + Scripts index fixed keys and columns, so the key set has to stay complete + whatever the gpu_metrics version reports. These fail if the format gate in + ``metric_gpu`` is ever dropped. + """ + + VERSIONS = ((1, 3), (1, 6), (1, 7), (1, 9), (9, 9)) + + def test_machine_output_equals_show_unsupported_on_every_version(self): + for version in self.VERSIONS: + with self.subTest(version=version): + self.assertMachineFormatsUnfiltered(version=version) + + def test_machine_output_does_not_depend_on_the_version(self): + for output in _FakeLogger.MACHINE_FORMATS: + with self.subTest(output=output): + on_v13 = self.run_metric(version=(1, 3), output=output) + on_v16 = self.run_metric(version=(1, 6), output=output) + self.assertEqual(on_v13, on_v16) + + def test_machine_output_never_reads_the_header(self): + # Nothing can consume it there, so metric_gpu must not pay for the read. + for output in _FakeLogger.MACHINE_FORMATS: + with self.subTest(output=output): + self.run_metric(version=(1, 3), output=output) + self.assertEqual(self.header_calls, []) + + def test_machine_output_still_reads_the_header_under_debug(self): + for output in _FakeLogger.FORMATS: + with self.subTest(output=output): + with self.assertLogs(level=logging.DEBUG): + self.run_metric( + version=(1, 3), output=output, show_unsupported=True, loglevel="DEBUG" + ) + self.assertNotEqual(self.header_calls, [], "DEBUG lost the version log") + + def test_the_flag_is_a_no_op_in_machine_formats(self): + for output in _FakeLogger.MACHINE_FORMATS: + with self.subTest(output=output): + values = self.run_metric(version=(1, 3), output=output, sections=()) + with_flag = self.run_metric( + version=(1, 3), output=output, sections=(), show_unsupported=True + ) + self.assertEqual(values, with_flag) + + def test_a_section_the_version_empties_survives_in_machine_formats(self): + # v1.3 can populate no throttle row, so human output drops the section. + for output in _FakeLogger.MACHINE_FORMATS: + with self.subTest(output=output): + values = self.run_metric(version=(1, 3), output=output, sections=()) + self.assertIn("throttle", values) + + +class TestShowUnsupportedRestoresTheOldOutput(_MetricGpuHarness): + def test_show_unsupported_never_reads_the_header(self): + # The opt-out is total: with the flag, and outside DEBUG logging, + # metric_gpu does not even query the version. + for output in _FakeLogger.FORMATS: + with self.subTest(output=output): + self.run_metric(show_unsupported=True, version=(1, 6), output=output) + self.assertEqual(self.header_calls, []) + + def test_show_unsupported_output_does_not_depend_on_the_version(self): + for output in _FakeLogger.FORMATS: + with self.subTest(output=output): + on_v13 = self.run_metric(show_unsupported=True, version=(1, 3), output=output) + on_v16 = self.run_metric(show_unsupported=True, version=(1, 6), output=output) + self.assertEqual(on_v13, on_v16) + + def test_show_unsupported_keeps_every_key_the_default_drops(self): + kept = self.run_metric(show_unsupported=True, version=(1, 3), output="human") + dropped = self.run_metric(show_unsupported=False, version=(1, 3), output="human") + self.assertNotEqual(kept, dropped, "the default suppressed nothing at all") + for section, section_value in dropped.items(): + if isinstance(section_value, dict): + self.assertLessEqual(set(section_value), set(kept[section])) + self.assertLessEqual(set(dropped), set(kept)) + + +class TestDefaultFilteringOnKnownVersion(_MetricGpuHarness): + def test_v13_drops_only_the_impossible_all_na_keys(self): + values = self.run_metric(version=(1, 3), output="human") + + # v1.3 has no hbm_stacks/mid/aid array and no XCD sensor. + for key in ("hbm_stacks", "mid", "aid", "xcd"): + self.assertKeyAbsent(values, "temperature", key) + # The hwmon-backed sensors carry readings and are never mapped. + for key in ("edge", "hotspot", "mem"): + self.assertKeyPresent(values, "temperature", key) + + # v1.3 predates the instantaneous PCIe counters. + for key in ( + "bandwidth", + "l0_to_recovery_count", + "replay_roll_over_count", + "nak_sent_count", + "nak_received_count", + "lc_perf_other_end_recovery_count", + ): + self.assertKeyAbsent(values, "pcie", key) + for key in ("width", "speed", "replay_count"): + self.assertKeyPresent(values, "pcie", key) + + # v1.3 carries mm_activity and vcn_activity but not jpeg_activity. + self.assertKeyAbsent(values, "usage", "jpeg_activity") + for key in ("mm_activity", "vcn_activity"): + self.assertKeyPresent(values, "usage", key) + + self.assertMachineFormatsUnfiltered(version=(1, 3)) + + def test_v16_drops_the_legacy_keys_and_keeps_the_new_ones(self): + values = self.run_metric(version=(1, 6), output="human") + + # v1.6 dropped these three from the table outright. + for key in ("mm_activity", "vcn_activity", "jpeg_activity"): + self.assertKeyAbsent(values, "usage", key) + # ...and added the XCP busy arrays, so those stay even at "N/A". + for key in ("gfx_busy_inst", "jpeg_busy"): + self.assertKeyPresent(values, "usage", key) + + self.assertMachineFormatsUnfiltered(version=(1, 6)) + + def test_a_suppressed_path_holding_a_reading_is_still_printed(self): + # The all-N/A gate, end to end: v1.3 has none of the residency + # accumulators, yet these rows report numbers and must survive. + for output in _FakeLogger.FORMATS: + with self.subTest(output=output): + values = self.run_metric(version=(1, 3), output=output) + self.assertKeyPresent(values, "throttle", "accumulation_counter") + for row in ("prochot", "ppt", "socket_thermal", "vr_thermal", "hbm_thermal"): + for form in ("accumulated", "violation_status", "violation_activity"): + self.assertKeyPresent(values, "throttle", f"{row}_{form}") + + def test_an_unmapped_all_na_key_is_never_dropped(self): + for output in _FakeLogger.FORMATS: + with self.subTest(output=output): + for version in ((1, 3), (1, 6), (1, 9)): + values = self.run_metric(version=version, output=output) + self.assertEqual(values["usage"]["vcn_busy"], NA) + + +class TestDerivedThrottleActivityRows(_MetricGpuHarness): + """Derived throttle-activity rows: a percentage needs both its accumulator and the counter.""" + + _GFX_LIMIT_ROWS = ( + "gfx_clk_below_host_limit", + "gfx_clk_below_host_limit_power", + "gfx_clk_below_host_limit_thermal", + "total_gfx_clk_below_host_limit", + "low_utilization", + ) + # v1.6 carries accumulation_counter but none of the gfx-limit accumulators. + # v1.7 adds the deprecated bare one back, so only that row survives. + SUPPRESSED_ROWS = { + (1, 6): frozenset(_GFX_LIMIT_ROWS), + (1, 7): frozenset(_GFX_LIMIT_ROWS) - {"gfx_clk_below_host_limit"}, + } + + def test_every_form_of_an_impossible_row_is_dropped(self): + for version, rows in sorted(self.SUPPRESSED_ROWS.items()): + with self.subTest(version=version): + values = self.run_metric(version=version, output="human") + for row in self._GFX_LIMIT_ROWS: + for form in ("accumulated", "violation_status", "violation_activity"): + key = f"{row}_{form}" + if row in rows: + self.assertKeyAbsent(values, "throttle", key) + else: + self.assertKeyPresent(values, "throttle", key) + + def test_machine_formats_keep_every_form_of_an_impossible_row(self): + for version in sorted(self.SUPPRESSED_ROWS): + with self.subTest(version=version): + self.assertMachineFormatsUnfiltered(version=version) + for output in _FakeLogger.MACHINE_FORMATS: + values = self.run_metric(version=version, output=output) + for row in self._GFX_LIMIT_ROWS: + for form in ("accumulated", "violation_status", "violation_activity"): + self.assertKeyPresent(values, "throttle", f"{row}_{form}") + + +class TestSuppressionDegradesSafely(_MetricGpuHarness): + def test_unknown_version_suppresses_nothing(self): + for output in _FakeLogger.FORMATS: + with self.subTest(output=output): + baseline = self.run_metric(show_unsupported=True, output=output) + values = self.run_metric(version=(9, 9), output=output) + self.assertEqual(values, baseline) + + def test_unreadable_header_suppresses_nothing(self): + # AmdSmiParameterException is a sibling of AmdSmiLibraryException, so + # catching only the latter lets a bad handle escape to the user. + for raiser in (_raise_lib_exc, _raise_param_exc): + for output in _FakeLogger.FORMATS: + with self.subTest(raiser=raiser.__name__, output=output): + baseline = self.run_metric(show_unsupported=True, output=output) + with self.assertLogs(level=logging.DEBUG) as captured_logs: + values = self.run_metric(output=output, reader=raiser) + self.assertEqual(values, baseline) + self.assertEqual( + [record for record in captured_logs.records if record.exc_info], + [], + "the failed header read produced a traceback", + ) + + +class TestExplicitSectionIsNeverEmptied(_MetricGpuHarness): + """A section the user named by flag must never come back empty. + + On v1.3 every throttle row is version-absent, and with the violation API + unavailable every one of them reads "N/A", so filtering would leave nothing + at all. That is the wall-of-N/A case the feature exists for when the user + asked for everything, and an unacceptable answer when they asked for this. + """ + + def setUp(self): + super().setUp() + self.interface.amdsmi_get_violation_status = _raise_lib_exc + + def test_plain_metric_drops_a_section_filtering_empties(self): + values = self.run_metric(version=(1, 3), output="human", sections=()) + self.assertNotIn("throttle", values) + self.assertMachineFormatsUnfiltered(version=(1, 3), sections=()) + + def test_a_named_section_survives_whole(self): + for output in _FakeLogger.FORMATS: + with self.subTest(output=output): + asked = self.run_metric(version=(1, 3), output=output, sections=("throttle",)) + unfiltered = self.run_metric( + version=(1, 3), output=output, sections=("throttle",), show_unsupported=True + ) + self.assertTrue(asked.get("throttle"), "--throttle answered with nothing") + self.assertEqual(asked["throttle"], unfiltered["throttle"]) + + def test_a_named_section_that_is_only_partly_suppressed_is_still_filtered(self): + values = self.run_metric(version=(1, 6), output="human", sections=("usage",)) + for key in ("mm_activity", "vcn_activity", "jpeg_activity"): + self.assertKeyAbsent(values, "usage", key) + self.assertKeyPresent(values, "usage", "gfx_busy_inst") + self.assertMachineFormatsUnfiltered(version=(1, 6), sections=("usage",)) + + def test_partition_is_not_a_section_selector(self): + """``--partition`` scopes the data, it does not name a section. + + It contributes nothing to the protected set, so the invocation filters + exactly like plain ``amd-smi metric`` and drops the emptied section + rather than protecting it the way a real section flag would. + """ + plain = self.run_metric(version=(1, 3), output="human", sections=()) + with_partition = self.run_metric( + version=(1, 3), output="human", sections=(), partition=True + ) + self.assertEqual(with_partition, plain) + self.assertNotIn("throttle", with_partition) + # Contrast: a real section flag protects the section it names. + named = self.run_metric(version=(1, 3), output="human", sections=("throttle",)) + self.assertTrue(named.get("throttle")) + + +class _WatchingLogger(_FakeLogger): + """Keeps every iteration's payload, not just the last one.""" + + def __init__(self, output_format="human"): + super().__init__(output_format) + self.iterations = [] + + def store_output(self, gpu, key, value): + super().store_output(gpu, key, value) + if key == "values": + self.iterations.append(value) + + +class _WatchHelpers(_FakeHelpers): + """``AMDSMIHelpers.handle_watch`` without the sleeps. + + Mirrors the real loop in the one respect that matters here: it clears the + watch args and then calls the subcommand repeatedly with the *same* args + object. + """ + + def __init__(self, iterations, gpu_ids=None): + super().__init__(gpu_ids) + self.iterations = iterations + + def handle_watch(self, args, subcommand, logger): + args.watch = None + args.watch_time = None + args.iterations = None + for _ in range(self.iterations): + subcommand(args, watching_output=True) + return 1 + + +@unittest.skipUnless(_RUNNABLE, _SKIP_REASON) +class TestWatchIterationsFilterIdentically(unittest.TestCase): + """``amd-smi metric --watch`` must filter the same way on every sample. + + Watch reuses one args object, and metric_gpu turns every section arg on when + the user named none. Read back on the next iteration that is indistinguishable + from the user having named every section, which would protect them all and + silently stop collapsing sections after the first sample. The protected set is + therefore captured before that mutation and memoized on the args; these tests + are what drives the second iteration that the memo exists for. + """ + + @classmethod + def setUpClass(cls): + cls.interface = _install_fake_amdsmi(cls) + cls.metric_module = _load_metric_module() + + def setUp(self): + # v1.3 can populate no throttle row, and with the violation API + # unavailable every row reads "N/A", so the section is a collapse + # candidate on both iterations. + self.interface.amdsmi_get_violation_status = _raise_lib_exc + + def run_watch(self, *, version=(1, 3), sections=(), iterations=2): + handle = object() + header_calls = [] + + def reader(device_handle): + header_calls.append(device_handle) + return _header(version) + + self.interface.amdsmi_get_gpu_metrics_header_info = reader + + commands = object.__new__(self.metric_module.MetricCommands) + commands.logger = _WatchingLogger("human") + commands.helpers = _WatchHelpers(iterations) + commands.group_check_printed = True + commands.device_handles = [] + commands.metric_gpu( + _build_args(sections=sections, gpu=handle, watch=1, iterations=iterations) + ) + + payloads = commands.logger.iterations + self.assertEqual(len(payloads), iterations, "watch did not run every iteration") + return payloads, header_calls + + def test_plain_metric_collapses_the_section_on_every_iteration(self): + payloads, _ = self.run_watch(sections=()) + for index, payload in enumerate(payloads): + with self.subTest(iteration=index): + self.assertNotIn("throttle", payload, "section collapse stopped after iteration 1") + + def test_a_named_section_stays_protected_on_every_iteration(self): + payloads, _ = self.run_watch(sections=("throttle",)) + for index, payload in enumerate(payloads): + with self.subTest(iteration=index): + self.assertTrue(payload.get("throttle"), "--throttle answered with nothing") + + def test_every_iteration_produces_the_same_payload(self): + for sections in ((), ("throttle",), ("usage",)): + with self.subTest(sections=sections): + payloads, _ = self.run_watch(version=(1, 6), sections=sections, iterations=3) + for index, payload in enumerate(payloads[1:], start=1): + self.assertEqual(payload, payloads[0], f"iteration {index} drifted") + + def test_the_header_is_read_once_per_handle(self): + # The header cannot change under a handle, and the read pulls a metrics + # blob, so watching must not re-read it every sample. + _, header_calls = self.run_watch(iterations=4) + self.assertEqual(len(header_calls), 1) + + +@unittest.skipUnless(_RUNNABLE, _SKIP_REASON) +class TestPerGpuSuppression(unittest.TestCase): + """Two GPUs on different metrics versions each get their own suppression set.""" + + @classmethod + def setUpClass(cls): + cls.interface = _install_fake_amdsmi(cls) + cls.metric_module = _load_metric_module() + cls.logger_module = _load_logger_module(cls) + + def _drive_two_gpus(self, versions, make_logger, show_unsupported=False): + handles = [object(), object()] + by_handle = {id(handle): version for handle, version in zip(handles, versions)} + self.interface.amdsmi_get_gpu_metrics_header_info = lambda handle: _header( + by_handle[id(handle)] + ) + helpers = _FakeHelpers({id(handle): index for index, handle in enumerate(handles)}) + + commands = object.__new__(self.metric_module.MetricCommands) + commands.logger = make_logger(helpers) + commands.helpers = helpers + commands.group_check_printed = True + commands.device_handles = [] + commands.metric_gpu(_build_args(gpu=copy.copy(handles), show_unsupported=show_unsupported)) + return handles, commands.logger + + def _run_two_gpus(self, output, versions, show_unsupported=False): + handles, logger = self._drive_two_gpus( + versions, lambda _helpers: _FakeLogger(output), show_unsupported + ) + self.assertEqual(len(logger.captured), 2, "expected one payload per GPU") + return [logger.captured[id(handle)] for handle in handles] + + def _csv_rows(self, versions, show_unsupported=False): + """One CSV row per GPU, off the real logger rather than a stand-in.""" + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self._drive_two_gpus( + versions, + lambda helpers: self.logger_module.AMDSMILogger(format="csv", helpers=helpers), + show_unsupported, + ) + return list(csv.DictReader(io.StringIO(stdout.getvalue()))) + + def test_each_gpu_is_filtered_against_its_own_version(self): + v13, v16 = self._run_two_gpus("human", [(1, 3), (1, 6)]) + + # jpeg_activity is impossible on both; mm/vcn_activity only on v1.6. + self.assertNotIn("jpeg_activity", v13["usage"]) + self.assertNotIn("jpeg_activity", v16["usage"]) + for key in ("mm_activity", "vcn_activity"): + self.assertIn(key, v13["usage"]) + self.assertNotIn(key, v16["usage"]) + + # v1.6 gained the PCIe counters that v1.3 cannot report. + self.assertNotIn("bandwidth", v13["pcie"]) + self.assertIn("bandwidth", v16["pcie"]) + + self.assertNotEqual(set(v13["usage"]), set(v16["usage"])) + + def test_machine_formats_give_both_gpus_the_same_key_set(self): + # Mixed versions are what would make a filtered machine format go ragged. + for output in _FakeLogger.MACHINE_FORMATS: + with self.subTest(output=output): + v13, v16 = self._run_two_gpus(output, [(1, 3), (1, 6)]) + self.assertEqual(set(_key_paths(v13)), set(_key_paths(v16))) + for key in ("mm_activity", "vcn_activity", "jpeg_activity"): + self.assertIn(key, v13["usage"]) + self.assertIn(key, v16["usage"]) + self.assertIn("bandwidth", v13["pcie"]) + self.assertIn("bandwidth", v16["pcie"]) + + # Version-absent on the v1.6 GPU only, on the v1.3 GPU only, and on both. + _VERSION_ABSENT_COLUMNS = ("mm_activity", "vcn_activity", "bandwidth", "jpeg_activity") + + def test_csv_is_not_filtered_when_the_gpus_share_a_version(self): + rows = self._csv_rows([(1, 3), (1, 3)]) + self.assertEqual(len(rows), 2, "expected one CSV row per GPU") + for row in rows: + for column in self._VERSION_ABSENT_COLUMNS: + self.assertIn(column, row, f"{column} was filtered out of CSV") + self.assertEqual(row[column], NA) + + def test_csv_is_not_filtered_when_the_gpus_are_on_different_versions(self): + rows = self._csv_rows([(1, 3), (1, 6)]) + self.assertEqual(len(rows), 2, "expected one CSV row per GPU") + self.assertEqual([row["gpu"] for row in rows], ["0", "1"]) + self.assertEqual(list(rows[0]), list(rows[1]), "the two GPUs produced ragged columns") + for row in rows: + for column in self._VERSION_ABSENT_COLUMNS: + self.assertIn(column, row, f"{column} was filtered out of CSV") + + def test_csv_header_does_not_depend_on_the_flag(self): + for versions in ([(1, 3), (1, 3)], [(1, 3), (1, 6)]): + with self.subTest(versions=versions): + self.assertEqual( + list(self._csv_rows(versions)[0]), + list(self._csv_rows(versions, show_unsupported=True)[0]), + ) diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_metrics_field_support.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_metrics_field_support.py new file mode 100644 index 000000000000..cf36e4f5bd0a --- /dev/null +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_metrics_field_support.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Hardware-free unit tests for ``amdsmi_metrics_field_support``. + +The module backs the default field filtering of ``amd-smi metric``: it maps a +detected ``gpu_metrics`` header version to the public fields the library can +populate, then drops the CLI output paths that version can never fill. + +Two properties matter more than the size of the table: + +* an unmapped version suppresses nothing, so a future metrics version degrades + to today's output rather than losing data; +* a path is only ever dropped when every leaf under it already reads "N/A", + so a mis-mapped path can under-report but can never hide a real value. +""" + +import importlib.util +import os +import re +import sys +import unittest + +from common.common import amdsmi_path, cli_search_order, find_cli_dir + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CLI_DIR = find_cli_dir(*cli_search_order(_THIS_DIR)) +_MODULE_FILENAME = "amdsmi_metrics_field_support.py" + +# The C++ the version table mirrors. Present in a source checkout only. +_CPP_SOURCE_PATH = os.path.normpath( + os.path.join(_THIS_DIR, "..", "..", "..", "..", "rocm_smi", "src", "rocm_smi_gpu_metrics.cc") +) + +MODULE_PATH = os.path.join(_CLI_DIR, _MODULE_FILENAME) if _CLI_DIR else "" + +# Skip only when no CLI resolves; a resolved CLI missing the module fails on load. +_RUNNABLE = bool(_CLI_DIR) +_SKIP_REASON = f"amd-smi CLI not found (looked in {amdsmi_path})" + + +def _load_module(): + if not os.path.isfile(MODULE_PATH): + raise AssertionError(f"{_MODULE_FILENAME} not found in {_CLI_DIR!r}") + spec = importlib.util.spec_from_file_location("amdsmi_metrics_field_support", MODULE_PATH) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +NA = "N/A" + + +@unittest.skipUnless(_RUNNABLE, _SKIP_REASON) +class TestVersionFieldTable(unittest.TestCase): + """The per-version field sets must match the C++ copy_internal_to_external_metrics().""" + + @classmethod + def setUpClass(cls): + cls.support = _load_module() + + def fields(self, fmt, content): + return self.support.populatable_fields(fmt, content) + + def test_v13_carries_fields_that_v14_drops(self): + v13 = self.fields(1, 3) + v14 = self.fields(1, 4) + for field in ("temperature_edge", "current_fan_speed", "average_socket_power"): + self.assertIn(field, v13) + self.assertNotIn(field, v14) + + def test_throttle_status_dropped_after_v15(self): + self.assertIn("throttle_status", self.fields(1, 5)) + for content in (6, 7, 8): + self.assertNotIn("throttle_status", self.fields(1, content)) + self.assertNotIn("throttle_status", self.fields(1, 9)) + + def test_jpeg_activity_only_in_v15(self): + self.assertNotIn("jpeg_activity", self.fields(1, 4)) + self.assertIn("jpeg_activity", self.fields(1, 5)) + for content in (6, 7, 8, 9): + self.assertNotIn("jpeg_activity", self.fields(1, content)) + + def test_vcn_activity_dropped_after_v15(self): + for content in (3, 4, 5): + self.assertIn("vcn_activity", self.fields(1, content)) + for content in (6, 7, 8, 9): + self.assertNotIn("vcn_activity", self.fields(1, content)) + + def test_vram_max_bandwidth_is_the_public_name_on_v17_and_v18(self): + # v1.8 renamed the private member to mem_max_bandwidth but still copies + # it into the public vram_max_bandwidth slot. + self.assertNotIn("vram_max_bandwidth", self.fields(1, 6)) + for content in (7, 8, 9): + self.assertIn("vram_max_bandwidth", self.fields(1, content)) + self.assertNotIn("mem_max_bandwidth", self.fields(1, content)) + + def test_dynamic_version_field_set(self): + for content in (9, 10, 42): + dynamic = self.fields(1, content) + self.assertIsNotNone(dynamic) + for present in ( + "temperature_hbm_stacks", + "current_socclks_mid", + "current_uclk_aid", + "xgmi_link_status", + ): + self.assertIn(present, dynamic) + for absent in ("temperature_edge", "jpeg_activity", "num_partition"): + self.assertNotIn(absent, dynamic) + + def test_xcp_stats_sub_fields_are_version_scoped(self): + v16 = self.fields(1, 6) + for present in ("xcp_stats.gfx_busy_inst", "xcp_stats.jpeg_busy", "xcp_stats.vcn_busy"): + self.assertIn(present, v16) + self.assertNotIn("xcp_stats.temperature_xcd", v16) + self.assertNotIn("xcp_stats.gfx_busy_inst", self.fields(1, 5)) + self.assertIn("xcp_stats.temperature_xcd", self.fields(1, 8)) + self.assertIn("xcp_stats.temperature_xcd", self.fields(1, 9)) + + def test_apu_versions_differ(self): + v24 = self.fields(2, 4) + v30 = self.fields(3, 0) + self.assertNotEqual(v24, v30) + for only_v24 in ("throttle_status", "current_gfxclk", "current_uclk", "current_dclk0"): + self.assertIn(only_v24, v24) + self.assertNotIn(only_v24, v30) + self.assertIn("average_gfx_activity", v30) + + def test_unknown_version_returns_none(self): + for version in ((9, 9), (0, 0), (4, 0), (2, 5), (1, "N/A")): + self.assertIsNone(self.fields(*version)) + + def test_future_content_revision_reuses_the_dynamic_bound(self): + # amdgpu_metrics_factory() routes every (1, >=9) header to the dynamic + # reader, so a newer content revision is mapped rather than unknown. + self.assertEqual(self.fields(1, 99999), self.fields(1, 9)) + + +@unittest.skipUnless(_RUNNABLE, _SKIP_REASON) +class TestSuppressionSet(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.support = _load_module() + + def suppressed(self, fmt, content): + return self.support.build_suppression_set( + {"structure_size": 512, "format_revision": fmt, "content_revision": content} + ) + + def test_unknown_version_suppresses_nothing(self): + self.assertEqual(self.suppressed(9, 9), frozenset()) + + def test_missing_header_suppresses_nothing(self): + for header in (None, {}, "N/A", {"format_revision": "N/A", "content_revision": "N/A"}): + self.assertEqual(self.support.build_suppression_set(header), frozenset()) + + def test_v13_suppresses_v16_only_paths(self): + v13 = self.suppressed(1, 3) + self.assertIn(("usage", "jpeg_activity"), v13) + self.assertIn(("throttle", "prochot_accumulated"), v13) + self.assertNotIn(("usage", "vcn_activity"), v13) + self.assertNotIn(("temperature", "edge"), v13) + + def test_v19_suppresses_removed_legacy_paths(self): + v19 = self.suppressed(1, 9) + for path in ( + ("usage", "jpeg_activity"), + ("usage", "vcn_activity"), + ("usage", "mm_activity"), + ("power", "throttle_status"), + ): + self.assertIn(path, v19) + + def test_paths_with_a_non_metrics_source_are_never_mapped(self): + """Only paths sourced solely from the metrics blobs may be suppressed; anything + metric.py can also fill from hwmon, sysfs or another API must stay.""" + for path in ( + ("temperature", "edge"), + ("temperature", "hotspot"), + ("temperature", "mem"), + ("fan", "speed"), + ("fan", "max"), + ("fan", "rpm"), + ("fan", "usage"), + ("power", "gfx_voltage"), + ("power", "soc_voltage"), + ("power", "mem_voltage"), + ("pcie", "replay_count"), + ("clock", "mem_0"), + ("clock", "socclk_0"), + ("usage", "vcn_busy"), + ): + self.assertNotIn(path, self.support._CLI_PATH_REQUIRED_SOURCES) + for fmt, content in ((1, 0), (1, 3), (1, 6), (1, 9), (2, 4), (3, 0)): + self.assertNotIn(path, self.suppressed(fmt, content)) + + def test_unmapped_paths_are_never_suppressed(self): + for fmt, content in ((1, 0), (1, 3), (1, 9), (3, 0)): + suppressed = self.suppressed(fmt, content) + for path in ( + ("mem_usage", "total_vram"), + ("ecc", "total_correctable_count"), + ("clock", "fclk_0"), + ("energy", "total_energy_consumption"), + ): + self.assertNotIn(path, suppressed) + + +@unittest.skipUnless(_RUNNABLE, _SKIP_REASON) +class TestEverySourceIsRequired(unittest.TestCase): + """Sources listed for a CLI path are required together, not alternatives. + + The multi-source entries are the derived throttle rows: the activity is a + percentage of a residency accumulator over ``accumulation_counter`` and the + status is that percentage tested against zero, so a version carrying only + one of the two can never fill either form. Reading the sources as + alternatives ("suppress only when none is present") left those rows printing + "N/A" under the flag. + """ + + DERIVED_FORMS = ("_violation_activity", "_violation_status") + + # Rows a version cannot fill because its accumulator is absent while + # accumulation_counter is present. Pinned so the predicate cannot silently + # flip back to the alternatives reading. + _GFX_LIMIT_ROWS = ( + "gfx_clk_below_host_limit", + "gfx_clk_below_host_limit_power", + "gfx_clk_below_host_limit_thermal", + "total_gfx_clk_below_host_limit", + "low_utilization", + ) + SUPPRESSED_DERIVED_ROWS = { + (1, 6): frozenset(_GFX_LIMIT_ROWS), + (1, 7): frozenset(_GFX_LIMIT_ROWS) - {"gfx_clk_below_host_limit"}, + (1, 8): frozenset({"gfx_clk_below_host_limit"}), + (1, 9): frozenset({"gfx_clk_below_host_limit"}), + } + + @classmethod + def setUpClass(cls): + cls.support = _load_module() + + def suppressed(self, fmt, content): + return self.support.build_suppression_set( + {"format_revision": fmt, "content_revision": content} + ) + + def test_multi_source_entries_are_all_accumulator_over_counter(self): + # The pinned counts below only mean what they say while this holds. + for path, sources in self.support._CLI_PATH_REQUIRED_SOURCES.items(): + if len(sources) > 1: + self.assertEqual(sources[-1], "accumulation_counter", path) + self.assertEqual(len(sources), 2, path) + + def test_only_the_derived_throttle_forms_need_the_counter(self): + for row, source in self.support._THROTTLE_SOURCES.items(): + self.assertEqual( + self.support._CLI_PATH_REQUIRED_SOURCES[("throttle", f"{row}_accumulated")], + (source,), + ) + for form in self.DERIVED_FORMS: + self.assertEqual( + self.support._CLI_PATH_REQUIRED_SOURCES[("throttle", f"{row}{form}")], + (source, "accumulation_counter"), + ) + + def test_derived_rows_missing_an_accumulator_are_suppressed(self): + for version, expected in sorted(self.SUPPRESSED_DERIVED_ROWS.items()): + for form in self.DERIVED_FORMS: + with self.subTest(version=version, form=form): + # These versions carry the counter, so only the required- + # source reading can suppress these rows. + self.assertIn("accumulation_counter", self.support.populatable_fields(*version)) + actual = frozenset( + key[: -len(form)] + for section, key in self.suppressed(*version) + if section == "throttle" and key.endswith(form) + ) + self.assertEqual(actual, expected) + + def test_a_derived_row_never_outlives_its_accumulator(self): + suffix = "_accumulated" + for version in sorted(self.support.known_versions()) + [(1, 9), (1, 12)]: + with self.subTest(version=version): + suppressed = self.suppressed(*version) + for section, key in suppressed: + if section == "throttle" and key.endswith(suffix): + row = key[: -len(suffix)] + for form in self.DERIVED_FORMS: + self.assertIn(("throttle", f"{row}{form}"), suppressed) + + +@unittest.skipUnless(_RUNNABLE, _SKIP_REASON) +class TestFilterUnsupported(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.support = _load_module() + + def test_drops_suppressed_na_leaves(self): + values = { + "gpu": 0, + "usage": {"gfx_activity": 42, "jpeg_activity": [NA, NA], "vcn_activity": [NA]}, + } + filtered = self.support.filter_unsupported( + values, frozenset({("usage", "jpeg_activity"), ("usage", "vcn_activity")}) + ) + self.assertEqual(filtered, {"gpu": 0, "usage": {"gfx_activity": 42}}) + + def test_indexed_clock_slots_are_matched_literally(self): + # Suppression is by exact (section, key); a family name matches nothing. + na_clock = {"clk": NA, "min_clk": NA, "max_clk": NA, "clk_locked": NA, "deep_sleep": NA} + values = {"clock": {"gfx_0": dict(na_clock), "gfx_1": dict(na_clock)}} + self.assertEqual( + self.support.filter_unsupported(values, frozenset({("clock", "gfx")})), values + ) + + def test_collapses_section_left_empty(self): + values = {"gpu": 1, "usage": {"mm_activity": NA, "jpeg_activity": NA}} + suppressed = frozenset({("usage", "mm_activity"), ("usage", "jpeg_activity")}) + self.assertEqual(self.support.filter_unsupported(values, suppressed), {"gpu": 1}) + + def test_protected_section_survives_whole_instead_of_collapsing(self): + values = {"gpu": 1, "usage": {"mm_activity": NA, "jpeg_activity": NA}} + suppressed = frozenset({("usage", "mm_activity"), ("usage", "jpeg_activity")}) + self.assertEqual( + self.support.filter_unsupported(values, suppressed, frozenset({"usage"})), values + ) + + def test_protected_section_is_still_filtered_when_something_survives(self): + values = {"usage": {"gfx_activity": 42, "jpeg_activity": NA}} + suppressed = frozenset({("usage", "jpeg_activity")}) + self.assertEqual( + self.support.filter_unsupported(values, suppressed, frozenset({"usage"})), + {"usage": {"gfx_activity": 42}}, + ) + + def test_keeps_section_that_was_already_empty(self): + values = {"ecc_blocks": {}} + self.assertEqual( + self.support.filter_unsupported(values, frozenset({("usage", "mm_activity")})), values + ) + + def test_recognizes_human_readable_na_list_string(self): + values = { + "usage": {"jpeg_activity": "[N/A, N/A, N/A]", "vcn_activity": "[N/A, 5 %]"}, + "temperature": {"hbm_stacks": "[]"}, + } + suppressed = frozenset( + {("usage", "jpeg_activity"), ("usage", "vcn_activity"), ("temperature", "hbm_stacks")} + ) + filtered = self.support.filter_unsupported(values, suppressed) + self.assertNotIn("jpeg_activity", filtered["usage"]) + self.assertEqual(filtered["usage"]["vcn_activity"], "[N/A, 5 %]") + self.assertEqual(filtered["temperature"]["hbm_stacks"], "[]") + + def test_does_not_mutate_input(self): + values = {"usage": {"gfx_activity": 42, "jpeg_activity": [NA, NA]}} + before = {"usage": {"gfx_activity": 42, "jpeg_activity": [NA, NA]}} + self.support.filter_unsupported(values, frozenset({("usage", "jpeg_activity")})) + self.assertEqual(values, before) + + def test_empty_suppression_set_is_identity(self): + values = {"gpu": 0, "usage": {"gfx_activity": NA}, "fan": {"speed": NA}} + self.assertEqual(self.support.filter_unsupported(values, frozenset()), values) + + def test_never_drops_a_real_value(self): + """Safety invariant: every mapped path suppressed for every known version, + with a real value in place, must survive the filter.""" + real_values = ( + 7, + 0, + "ENABLED", + [1, 2, 3], + [NA, 4], + "[N/A, 4 %]", + {"value": 3, "unit": "%"}, + {"xcp_0": [NA, 9]}, + ) + versions = list(self.support.known_versions()) + [(1, 9), (1, 12)] + for fmt, content in versions: + suppressed = self.support.build_suppression_set( + {"format_revision": fmt, "content_revision": content} + ) + for index, real in enumerate(real_values): + values = {} + for section, key in suppressed: + values.setdefault(section, {})[key] = real + filtered = self.support.filter_unsupported(values, suppressed) + self.assertEqual( + filtered, + values, + f"version {fmt}.{content} dropped real value #{index}: {real!r}", + ) + + +@unittest.skipUnless(_RUNNABLE, _SKIP_REASON) +class TestVersionTableMatchesCppSource(unittest.TestCase): + """Drift guard: the hand-written table must track the C++ it mirrors. + + Derives each version's public field set from the per-version + ``copy_internal_to_external_metrics()`` in ``rocm_smi_gpu_metrics.cc`` and + diffs it against ``_VERSION_FIELDS``. Skipped where the C++ source is not + shipped, such as an installed package. + """ + + # Class name -> the (format_revision, content_revision) it reads. + CLASS_VERSIONS = {f"GpuMetricsBase_v1{content}_t": (1, content) for content in range(9)} + DYNAMIC_CLASS = "GpuMetricsBaseDynamic_t" + # APU versions (2, 4) and (3, 0) share one class and are told apart by a + # branch inside it, so CLASS_VERSIONS cannot map them; + # test_apu_versions_match_their_branches covers them instead. + APU_CLASS = "ApuMetricsBase_v30_t" + + # common_header is the version stamp itself; apu_metrics is the internal + # source struct, not a public member. xcp_stats sub-fields are assigned in a + # copy loop the extractor cannot see, so the bare name is dropped here and + # the sub-fields are excluded from every comparison by _comparable(). + IGNORED_ROOTS = frozenset({"common_header", "apu_metrics", "xcp_stats"}) + + _FUNCTION_RE = re.compile( + r"^AMGpuMetricsPublicLatestTupl_t\s+(\w+)::copy_internal_to_external_metrics\(\)\s*\{", + re.MULTILINE, + ) + # A body ends at its own closing brace in column 0. Without this the last + # class in the file would absorb every later assignment in the file. + _FUNCTION_END_RE = re.compile(r"^\}", re.MULTILINE) + # v1.x classes copy into a local named metrics_public_init; the dynamic + # reader copies into one named out. + _ASSIGNMENT_RE = re.compile(r"\b(?:metrics_public_init|out)\.([A-Za-z_][A-Za-z0-9_.]*)") + _LINE_COMMENT_RE = re.compile(r"//[^\n]*") + # ApuMetricsBase_v30_t serves both APU versions from one function: a v2.4 + # branch and an else branch for v3.0. Comparing the merged body would let a + # deleted v3.0 field pass, so each branch is extracted and compared alone. + _APU_V24_BRANCH_RE = re.compile( + r"if\s*\(\s*get_gpu_metrics_version_used\(\)\s*==\s*[\w:]*kApuMetricV24\s*\)\s*\{" + ) + # Branch braces sit one level in; the bodies themselves are deeper. + _APU_ELSE_RE = re.compile(r"^ \} else \{", re.MULTILINE) + _APU_BRANCH_END_RE = re.compile(r"^ \}", re.MULTILINE) + + @classmethod + def setUpClass(cls): + if not os.path.isfile(_CPP_SOURCE_PATH): + raise unittest.SkipTest(f"C++ source not available at {_CPP_SOURCE_PATH}") + cls.support = _load_module() + with open(_CPP_SOURCE_PATH, encoding="utf-8", errors="replace") as source: + cls.bodies = cls._derive_bodies(source.read()) + cls.derived = {name: cls._fields_in(body) for name, body in cls.bodies.items()} + + @classmethod + def _derive_bodies(cls, text): + functions = list(cls._FUNCTION_RE.finditer(text)) + bodies = {} + for index, match in enumerate(functions): + limit = functions[index + 1].start() if index + 1 < len(functions) else len(text) + closing = cls._FUNCTION_END_RE.search(text, match.end(), limit) + end = closing.end() if closing else limit + bodies[match.group(1)] = cls._LINE_COMMENT_RE.sub("", text[match.start() : end]) + return bodies + + @classmethod + def _fields_in(cls, body): + return { + name + for name in cls._ASSIGNMENT_RE.findall(body) + if name.split(".")[0] not in cls.IGNORED_ROOTS + } + + @classmethod + def _apu_branch_fields(cls): + """(v2.4 fields, v3.0 fields), or None if the branches cannot be located.""" + body = cls.bodies.get(cls.APU_CLASS, "") + opening = cls._APU_V24_BRANCH_RE.search(body) + if opening is None: + return None + otherwise = cls._APU_ELSE_RE.search(body, opening.end()) + if otherwise is None: + return None + closing = cls._APU_BRANCH_END_RE.search(body, otherwise.end()) + if closing is None: + return None + return ( + cls._fields_in(body[opening.end() : otherwise.start()]), + cls._fields_in(body[otherwise.end() : closing.start()]), + ) + + @staticmethod + def _comparable(fields): + # xcp_stats.* is deliberately outside the drift guard: the C++ assigns + # those sub-fields inside a per-partition copy loop, so a line-based + # extractor cannot attribute them to a version. Guarding them needs a + # branch-aware extractor. Until then 18 of the 49 mapped CLI paths are + # unguarded and rest on the explicit xcp_stats tests above. + return {name for name in fields if not name.startswith("xcp_stats.")} + + def _assert_matches(self, class_name, table_fields): + self.assertIn( + class_name, self.derived, f"{class_name}::copy_internal_to_external_metrics() not found" + ) + expected = self._comparable(self.derived[class_name]) + actual = self._comparable(table_fields) + self.assertEqual( + actual, + expected, + f"{class_name} drifted from the table | " + f"only in table: {sorted(actual - expected)} | " + f"only in C++: {sorted(expected - actual)}", + ) + + def test_every_mapped_class_is_present_in_the_source(self): + missing = sorted( + name + for name in list(self.CLASS_VERSIONS) + [self.DYNAMIC_CLASS, self.APU_CLASS] + if name not in self.derived + ) + self.assertEqual(missing, [], f"classes renamed or removed in C++: {missing}") + + def test_explicit_versions_match_the_cpp_source(self): + for class_name, version in sorted(self.CLASS_VERSIONS.items(), key=lambda item: item[1]): + with self.subTest(version=version): + table_fields = self.support.populatable_fields(*version) + self.assertIsNotNone(table_fields, f"{version} is missing from _VERSION_FIELDS") + self._assert_matches(class_name, table_fields) + + def test_dynamic_bound_matches_the_cpp_source(self): + self._assert_matches( + self.DYNAMIC_CLASS, + self.support.populatable_fields(1, self.support._DYNAMIC_MIN_CONTENT_REVISION), + ) + + def test_apu_versions_match_their_branches(self): + branches = self._apu_branch_fields() + self.assertIsNotNone( + branches, + f"could not split {self.APU_CLASS}::copy_internal_to_external_metrics() into its " + "kApuMetricV24 and else branches; the branch shape changed", + ) + for version, branch in zip(((2, 4), (3, 0)), branches): + with self.subTest(version=version): + expected = self._comparable(branch) + actual = self._comparable(self.support.populatable_fields(*version)) + self.assertEqual( + actual, + expected, + f"{version} drifted from its {self.APU_CLASS} branch | " + f"only in table: {sorted(actual - expected)} | " + f"only in C++: {sorted(expected - actual)}", + ) diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_ras_cper_json.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_ras_cper_json.py index b5e12aa36fb8..66c7d5ba7ac5 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_cli_ras_cper_json.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_ras_cper_json.py @@ -20,7 +20,6 @@ import argparse import contextlib -import importlib.util import io import json import os @@ -29,16 +28,21 @@ import types import unittest +from common.common import ( + add_class_cleanup, + amdsmi_path, + cli_search_order, + find_cli_dir, + load_cli_module, + stub_modules, +) -_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -# tests/python/unit/gpu -> repo root is four levels up. -_REPO_ROOT = os.path.abspath(os.path.join(_THIS_DIR, "..", "..", "..", "..")) -_CLI_DIR = os.path.join(_REPO_ROOT, "amdsmi_cli") -_RAS_SRC = os.path.join(_CLI_DIR, "subcommands", "ras.py") +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +_RAS_SRC = os.path.join(_CLI_DIR, "subcommands", "ras.py") if _CLI_DIR else None # Modules imported (directly or transitively) when the source CLI loads against -# the faked ``amdsmi`` package. They are snapshotted and cleared around the suite -# so a real/installed copy loaded by a sibling test is never shadowed. +# the faked ``amdsmi`` package. They are cleared around the suite so a +# real/installed copy loaded by a sibling test is never shadowed. _CLI_MODULES = ( "amdsmi", "amdsmi.amdsmi_interface", @@ -112,7 +116,7 @@ class _FakeWrapper: amdsmi_processor_handle = _FakeHandle -def _install_fake_amdsmi(): +def _build_fake_amdsmi(): """Register a stub ``amdsmi`` package so the source CLI imports without the compiled library, and return the fake ``amdsmi_interface`` for per-test wiring.""" amdsmi_pkg = types.ModuleType("amdsmi") @@ -149,17 +153,15 @@ def _bdf_unavailable(_handle): amdsmi_pkg.amdsmi_interface = interface amdsmi_pkg.amdsmi_exception = exception - sys.modules["amdsmi"] = amdsmi_pkg - sys.modules["amdsmi.amdsmi_interface"] = interface - sys.modules["amdsmi.amdsmi_exception"] = exception - return interface + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + } def _load_ras_module(): - spec = importlib.util.spec_from_file_location("ras_cper_json_under_test", _RAS_SRC) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("ras_cper_json_under_test", _RAS_SRC) def _build_ras_args(handles, **overrides): @@ -180,19 +182,21 @@ def _build_ras_args(handles, **overrides): class TestCliRasCperJson(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(_RAS_SRC): - raise unittest.SkipTest(f"amd-smi CLI ras.py not found at {_RAS_SRC}") - - # Clear the CLI modules so the fake amdsmi wins the import race, keeping a - # snapshot to restore afterwards. - cls._saved_modules = {name: sys.modules.get(name) for name in _CLI_MODULES} - for name in _CLI_MODULES: - sys.modules.pop(name, None) - cls._path_added = _CLI_DIR not in sys.path - if cls._path_added: + if not _RAS_SRC or not os.path.isfile(_RAS_SRC): + raise unittest.SkipTest( + f"amd-smi CLI ras.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + + # Clear the CLI modules so the fake amdsmi wins the import race; the fakes + # replace the entries they own. stub_modules restores every name afterwards. + modules = dict.fromkeys(_CLI_MODULES) + modules.update(_build_fake_amdsmi()) + stub_modules(cls, modules) + if _CLI_DIR not in sys.path: sys.path.insert(0, _CLI_DIR) + add_class_cleanup(cls, sys.path.remove, _CLI_DIR) - cls.interface = _install_fake_amdsmi() + cls.interface = modules["amdsmi.amdsmi_interface"] import amdsmi_helpers import amdsmi_logger @@ -200,16 +204,6 @@ def setUpClass(cls): cls.logger_cls = amdsmi_logger.AMDSMILogger cls.ras_module = _load_ras_module() - @classmethod - def tearDownClass(cls): - if getattr(cls, "_path_added", False) and _CLI_DIR in sys.path: - sys.path.remove(_CLI_DIR) - for name, mod in getattr(cls, "_saved_modules", {}).items(): - if mod is None: - sys.modules.pop(name, None) - else: - sys.modules[name] = mod - def setUp(self): self._handles = [_FakeHandle(10), _FakeHandle(20)] self.interface.amdsmi_get_processor_handles = lambda: list(self._handles) @@ -317,7 +311,3 @@ def test_afid_folder_all_files_skipped_emits_empty_json(self): self.assertEqual(captured.strip(), "[]") self.assertEqual(json.loads(captured), []) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_set_clk_limit.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_set_clk_limit.py index ce6593b52724..96c4f0e0b645 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_cli_set_clk_limit.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_set_clk_limit.py @@ -11,47 +11,33 @@ """ import collections -import importlib.util import os -import sys import types import unittest # ``common.common`` bootstraps the real amdsmi package at import time (sys.path # insert + ``import amdsmi`` + module-level ``build_type_lists()``), which fails -# on a stale or mismatched install. This suite fully stubs ``amdsmi`` itself and -# only needs ``amdsmi_path`` to locate the *installed* CLI fallback, so degrade -# gracefully: if the shared harness cannot load, drop to ``None`` and rely on -# the in-tree source checkout (resolved first below). Keeps this file runnable -# from a plain checkout even when no matching amdsmi is installed. +# on a stale or mismatched install. This suite fully stubs ``amdsmi`` itself, so +# degrade gracefully rather than erroring at import; without the harness there is +# no resolver, and the suite skips with the reason below. try: - from common.common import amdsmi_path + from common.common import ( + amdsmi_path, + cli_search_order, + find_cli_dir, + load_cli_module, + stub_modules, + ) except (ImportError, FileNotFoundError): # pragma: no cover - harness/install unavailable amdsmi_path = None + cli_search_order = None + find_cli_dir = None -# set_value.py lives in the amd-smi CLI, which exists in two layouts: -# * source checkout: /projects/amdsmi/amdsmi_cli (sibling of tests/) -# * installed: /libexec/amdsmi_cli (amdsmi_path is the sibling -# /share/amd_smi) -# Prefer the in-tree source when running from a checkout so the test exercises -# the code under review; fall back to the installed CLI otherwise. -_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -_SOURCE_CLI_DIR = os.path.normpath(os.path.join(_THIS_DIR, "..", "..", "..", "..", "amdsmi_cli")) -_INSTALLED_CLI_DIR = ( - os.path.join(os.path.dirname(os.path.dirname(amdsmi_path)), "libexec", "amdsmi_cli") - if amdsmi_path - else "" +_CLI_DIR = ( + find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) + if find_cli_dir and cli_search_order + else None ) - - -def _resolve_cli_dir(): - for cli_dir in (_SOURCE_CLI_DIR, _INSTALLED_CLI_DIR): - if cli_dir and os.path.isfile(os.path.join(cli_dir, "subcommands", "set_value.py")): - return cli_dir - return None - - -_CLI_DIR = _resolve_cli_dir() SET_VALUE_PATH = os.path.join(_CLI_DIR, "subcommands", "set_value.py") if _CLI_DIR else "" # AMDSMI_STATUS_NOT_SUPPORTED sentinel used by the stubbed library-error path. @@ -77,11 +63,11 @@ def get_error_info(self): return self._message -def _install_fake_amdsmi(): - """Register a stub ``amdsmi`` package so ``set_value.py`` imports cleanly. +def _build_fake_amdsmi(): + """Build a stub ``amdsmi`` package so ``set_value.py`` imports cleanly. - Returns the fake ``amdsmi_interface`` module so individual tests can swap in - per-case ``amdsmi_get_clk_freq`` return values or side effects. + Returns the name -> module mapping for ``common.stub_modules``; tests reach the + fake ``amdsmi_interface`` through it to swap per-case return values. """ amdsmi_pkg = types.ModuleType("amdsmi") interface = types.ModuleType("amdsmi.amdsmi_interface") @@ -101,22 +87,18 @@ def _install_fake_amdsmi(): amdsmi_pkg.amdsmi_interface = interface amdsmi_pkg.amdsmi_exception = exception - sys.modules["amdsmi"] = amdsmi_pkg - sys.modules["amdsmi.amdsmi_interface"] = interface - sys.modules["amdsmi.amdsmi_exception"] = exception - sys.modules["amdsmi.amdsmi_wrapper"] = wrapper - return interface + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + "amdsmi.amdsmi_wrapper": wrapper, + } def _load_set_value_module(): # set_value.py imports the sibling ``amdsmi_cli_exceptions`` module by bare # name, so the CLI dir must be importable before the module is executed. - if _CLI_DIR and _CLI_DIR not in sys.path: - sys.path.insert(0, _CLI_DIR) - spec = importlib.util.spec_from_file_location("set_value_under_test", SET_VALUE_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("set_value_under_test", SET_VALUE_PATH, sys_path_dir=_CLI_DIR) class TestSnapClkLimitToDpm(unittest.TestCase): @@ -126,31 +108,17 @@ class TestSnapClkLimitToDpm(unittest.TestCase): # entry to prove the helper sorts the levels and filters out f <= 0. FCLK_DPM_HZ = [1600_000_000, 0, 1200_000_000, 2000_000_000, 1900_000_000] - _SAVED_MODULE_NAMES = ( - "amdsmi", - "amdsmi.amdsmi_interface", - "amdsmi.amdsmi_exception", - "amdsmi.amdsmi_wrapper", - ) - @classmethod def setUpClass(cls): - if not SET_VALUE_PATH: - raise unittest.SkipTest("amd-smi CLI set_value.py not found (source or installed)") - # Snapshot any real amdsmi already loaded so the stub does not leak into - # sibling suites sharing the interpreter; restored in tearDownClass. - cls._saved_modules = {name: sys.modules.get(name) for name in cls._SAVED_MODULE_NAMES} - cls.interface = _install_fake_amdsmi() + if not SET_VALUE_PATH or not os.path.isfile(SET_VALUE_PATH): + raise unittest.SkipTest( + f"amd-smi CLI set_value.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _build_fake_amdsmi() + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] cls.module = _load_set_value_module() - @classmethod - def tearDownClass(cls): - for name, saved in cls._saved_modules.items(): - if saved is None: - sys.modules.pop(name, None) - else: - sys.modules[name] = saved - def _snap(self, requested_mhz, frequency, min_mhz=None): # Shared harness: point the stubbed C entry point at a fixed DPM table # (Hz) so every case drives the real helper with controlled input. @@ -269,6 +237,9 @@ def last_output(self, key): class _StubHelpers: """Minimal ``self.helpers`` stub for the GPU ``set`` path.""" + def __init__(self): + self.recorded_errors = [] + def check_required_groups(self): pass @@ -284,6 +255,12 @@ def is_baremetal(self): def get_gpu_id_from_device_handle(self, handle): return 0 + def store_device_error(self, logger, device, key, message, exception=None, code=None): + # Mirrors AMDSMIHelpers.store_device_error: show the error and note it so + # the run resolves to a non-zero exit code. + self.recorded_errors.append(exception if exception is not None else code) + logger.store_output(device, key, message) + _ClkLimit = collections.namedtuple("_ClkLimit", ["clk_type", "lim_type", "val"]) @@ -297,14 +274,15 @@ class TestSetGpuClkLimitCallSite(unittest.TestCase): ``N/A`` opposing-bound skip that keeps the requested limit settable. """ - _SAVED_MODULE_NAMES = TestSnapClkLimitToDpm._SAVED_MODULE_NAMES - @classmethod def setUpClass(cls): - if not SET_VALUE_PATH: - raise unittest.SkipTest("amd-smi CLI set_value.py not found (source or installed)") - cls._saved_modules = {name: sys.modules.get(name) for name in cls._SAVED_MODULE_NAMES} - cls.interface = _install_fake_amdsmi() + if not SET_VALUE_PATH or not os.path.isfile(SET_VALUE_PATH): + raise unittest.SkipTest( + f"amd-smi CLI set_value.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _build_fake_amdsmi() + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] # Extra surface the set_gpu clk-limit branch touches beyond the snap # helper: the clock-info entry point, the set entry point, the clk-type # enum, and the BDF lookup used only to build error strings. @@ -313,14 +291,6 @@ def setUpClass(cls): cls.interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM = 4 cls.module = _load_set_value_module() - @classmethod - def tearDownClass(cls): - for name, saved in cls._saved_modules.items(): - if saved is None: - sys.modules.pop(name, None) - else: - sys.modules[name] = saved - def _run_clk_limit( self, clk_type, lim_type, val, min_clk, max_clk, dpm_hz=(), freq_raises=False ): @@ -497,7 +467,3 @@ def test_min_above_max_is_rejected(self): set_calls, message = self._run_clk_limit("fclk", "min", 2500, min_clk=1000, max_clk=2000) self.assertEqual(set_calls, []) self.assertIn("greater than max", message) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_set_node_power_limit.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_set_node_power_limit.py new file mode 100644 index 000000000000..76586e50b691 --- /dev/null +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_set_node_power_limit.py @@ -0,0 +1,708 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Mock-based unit tests for ``amd-smi set --node-power-limit``. + +Two layers are covered here: + +* ``TestCliSetNodePowerLimit`` drives ``SetValueCommands.set_value()``'s + node-wide (not per-GPU) early-dispatch block, modeled on the existing + ``--gtt`` block: + + * ``--node-power-limit`` combined with ``--gpu`` is rejected before any + library call, exiting with status 2 and a stderr message (mirrors the + ``--gtt``/``--gpu`` argparse-level guard's runtime-level counterpart). + * ``self.node_handle is None`` (no NPM-capable node resolved at startup, + e.g. the pre-existing ``oam_id != 0`` gate on current mock hardware) + short-circuits to a NOT_SUPPORTED-style message without calling the API. + * A successful set reports "Successfully set node power limit to W." + * ``AMDSMI_STATUS_NO_PERM`` from the library is translated into a raised + ``PermissionError`` (so the CLI's elevation-prompt wrapper can catch it), + not swallowed into an output message. + * Any other library exception (e.g. NOT_SUPPORTED bubbling up from + ``rsmi_dev_npm_limit_set()`` for a missing sysfs file) is caught and + reported as an output message referencing the requested value, without + raising. + + ``self.helpers`` is a minimal stub (mirroring ``test_cli_set_clk_limit.py``'s + ``_StubHelpers`` pattern) that reproduces + ``validate_and_set_node_power_limit()``'s call/status-mapping contract, so + these tests can focus on ``set_value.py``'s own dispatch behavior without + re-deriving the validation logic tested in depth below. + +* ``TestValidateAndSetNodePowerLimit`` drives the real + ``amdsmi_helpers.AMDSMIHelpers.validate_and_set_node_power_limit()`` + directly (added for the original F-1 fix, and updated for the F-2 fix + below), against a real import of ``amdsmi_helpers`` with only the C library + faked -- this is the validation logic itself, previously exercised nowhere + in the tree: + + * a request above the platform max (``amdsmi_get_npm_info()``'s + ``max_node_power_limit``) is rejected before any write. + * a request of ``0`` is rejected. + * an in-range request succeeds and calls ``amdsmi_set_npm_limit()`` with + the requested value verbatim. + * an unreadable platform max (``max_node_power_limit == "N/A"``) now fails + *closed*: the request is rejected, not allowed through on a bare + positivity check. (Previously this class of degraded-driver scenario + was fail-open; see the F-2 handoff finding this test locks in.) + +* ``TestNodePowerLimitGuestRegistration`` drives ``AMDSMIParser`` directly + (loaded the same way as ``test_output_file_stdin.py``): commit 567bed2f89d + moved ``-n``/``--node-power-limit`` argparse registration out of the + ``is_baremetal()``-only block so it is gated only by + ``is_amdgpu_initialized()`` (same pattern as ``-o``/``--power-cap``), + since NPM is exposed to 1VF guests. Asserts the argument is still + registered and parseable under a non-baremetal (guest) helpers stub. + +* ``TestGetMaxNodePowerLimit`` drives the real + ``amdsmi_helpers.AMDSMIHelpers.get_max_node_power_limit()`` (added by + ece891b6787 to size --node-power-limit's help-text bound), against a real + import of ``amdsmi_helpers`` with only the C library faked -- mirroring + ``TestValidateAndSetNodePowerLimit``'s pattern: + + * a first-device success returns immediately without probing further + devices. + * an exception on every device falls back to "N/A". + * a device that succeeds but reports "N/A" as the value itself (not an + exception) is a distinct code path from the exception branch; the loop + must still continue to the next device. +""" + +import argparse +import contextlib +import importlib.util +import io +import os +import sys +import types +import unittest +from unittest import mock + +from common.common import cli_search_order, fake_module, find_cli_dir, stub_modules + +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +SET_VALUE_PATH = os.path.join(_CLI_DIR, "subcommands", "set_value.py") if _CLI_DIR else "" + +_STATUS_NOT_SUPPORTED = 2 +_STATUS_NO_PERM = 10 +_STATUS_INVAL = 5 +_STATUS_UNEXPECTED_SIZE = 42 +_STATUS_UNEXPECTED_DATA = 43 +_NPM_STATUS_DISABLED = 0 +_NPM_STATUS_ENABLED = 1 + + +class _FakeLibraryException(Exception): + def __init__(self, err_code=_STATUS_NOT_SUPPORTED, message="mock error"): + super().__init__(message) + self._err_code = err_code + self._message = message + + def get_error_code(self): + return self._err_code + + def get_error_info(self, detailed=True): + return self._message if detailed else self._message.split(" - ")[0] + + +def _install_fake_amdsmi(): + wrapper = fake_module( + "amdsmi.amdsmi_wrapper", + AMDSMI_STATUS_NOT_SUPPORTED=_STATUS_NOT_SUPPORTED, + AMDSMI_STATUS_NO_PERM=_STATUS_NO_PERM, + AMDSMI_STATUS_INVAL=_STATUS_INVAL, + # Unused by this file's own tests, but amdsmi_helpers.AMDSMIHelpers's + # CPER_DECODE_MESSAGES class body reads these off amdsmi_wrapper at + # import time, so `import amdsmi_helpers` under this stub needs them + # present or it dies with an AttributeError before any test runs. + AMDSMI_STATUS_UNEXPECTED_SIZE=_STATUS_UNEXPECTED_SIZE, + AMDSMI_STATUS_UNEXPECTED_DATA=_STATUS_UNEXPECTED_DATA, + AMDSMI_NPM_STATUS_DISABLED=_NPM_STATUS_DISABLED, + AMDSMI_NPM_STATUS_ENABLED=_NPM_STATUS_ENABLED, + ) + interface = fake_module( + "amdsmi.amdsmi_interface", + AMDSMI_MAX_PPT_LIMIT=0, + AMDSMI_MAX_UTIL=100, + amdsmi_wrapper=wrapper, + # Overwritten per-test. + amdsmi_set_npm_limit=lambda _handle, _limit: None, + amdsmi_get_npm_info=lambda _handle: { + "max_node_power_limit": "N/A", + "status": _NPM_STATUS_ENABLED, + }, + ) + exception = fake_module("amdsmi.amdsmi_exception", AmdSmiLibraryException=_FakeLibraryException) + amdsmi_pkg = fake_module("amdsmi", amdsmi_interface=interface, amdsmi_exception=exception) + + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + "amdsmi.amdsmi_wrapper": wrapper, + } + + +def _load_set_value_module(): + if _CLI_DIR and _CLI_DIR not in sys.path: + sys.path.insert(0, _CLI_DIR) + # Some sibling test files (e.g. test_cli_cache_labels.py, + # test_cli_vram_type_lpddr5.py) install a stub top-level + # "amdsmi_cli_exceptions" module into sys.modules to satisfy a different + # CLI submodule's import needs, and don't always restore the original + # entry afterwards. set_value.py does `from amdsmi_cli_exceptions import + # AmdSmiRequiredCommandException` at module scope; if a stale stub + # lacking that class is already cached, the import fails even though the + # real module (importable via _CLI_DIR on sys.path) has it. Drop any + # cached entry so the real module is freely re-resolved here, regardless + # of what ran earlier in the same pytest session. + sys.modules.pop("amdsmi_cli_exceptions", None) + spec = importlib.util.spec_from_file_location("set_value_under_test_npm", SET_VALUE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _FakeLogger: + def __init__(self): + self.output = {} + + def print_output(self, *args, **kwargs): + pass + + def is_json_format(self): + return False + + def is_csv_format(self): + return False + + +class _StubHelpers: + """Minimal ``self.helpers`` stub for the node-power-limit dispatch block. + + Mirrors ``test_cli_set_clk_limit.py``'s ``_StubHelpers`` pattern: just + enough of ``validate_and_set_node_power_limit()``'s call/status-mapping + contract (call the C entry point, map ``AMDSMI_STATUS_NO_PERM`` to a + raised ``PermissionError``, otherwise format success/error strings) for + ``set_value.py``'s dispatch block to exercise, without pulling in the real + max-bound validation -- that logic is exercised directly and in depth by + ``TestValidateAndSetNodePowerLimit`` below, against the real + ``amdsmi_helpers.AMDSMIHelpers`` implementation. + """ + + def validate_and_set_node_power_limit(self, node_handle, requested_limit, logger): + # Resolved lazily (not at module scope) because the fake "amdsmi" + # package is only installed into sys.modules once a test class's + # setUpClass() runs, which is after this class body executes. + amdsmi_pkg = sys.modules["amdsmi"] + interface = amdsmi_pkg.amdsmi_interface + exception = amdsmi_pkg.amdsmi_exception + try: + interface.amdsmi_set_npm_limit(node_handle, requested_limit) + return f"Successfully set node power limit to {requested_limit} W." + except exception.AmdSmiLibraryException as e: + if e.get_error_code() == interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: + raise PermissionError("Command requires elevation") from e + return ( + f"[{e.get_error_info(detailed=False)}] " + f"Unable to set node power limit to {requested_limit} W" + ) + + +class TestCliSetNodePowerLimit(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not SET_VALUE_PATH: + raise unittest.SkipTest("amd-smi CLI set_value.py not found (source or installed)") + modules = _install_fake_amdsmi() + modules["amdsmi_cli_exceptions"] = None + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] + cls.module = _load_set_value_module() + + def _make_command(self, node_handle): + cmd = self.module.SetValueCommands() + cmd.logger = _FakeLogger() + cmd.node_handle = node_handle + cmd.helpers = _StubHelpers() + return cmd + + def _make_args(self, node_power_limit, gpu=None): + return types.SimpleNamespace(gpu=gpu, node_power_limit=node_power_limit) + + def test_gpu_conflict_exits_with_status_2(self): + cmd = self._make_command(node_handle=object()) + args = self._make_args(node_power_limit=250, gpu="gpu0") + + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + with self.assertRaises(SystemExit) as ctx: + cmd.set_value(args) + + self.assertEqual(ctx.exception.code, 2) + self.assertIn("--node-power-limit", stderr.getvalue()) + self.assertIn("--gpu", stderr.getvalue()) + + def test_no_node_handle_reports_not_supported_without_calling_api(self): + calls = [] + self.interface.amdsmi_set_npm_limit = lambda h, l: calls.append((h, l)) + cmd = self._make_command(node_handle=None) + args = self._make_args(node_power_limit=250) + + cmd.set_value(args) + + self.assertEqual(calls, [], "API must not be called when no node handle is available") + message = cmd.logger.output["set_node_power_limit"] + self.assertIn("NOT_SUPPORTED", message) + self.assertIn("no NPM-capable node found", message) + + def test_success_reports_value_in_watts(self): + calls = [] + self.interface.amdsmi_set_npm_limit = lambda h, l: calls.append((h, l)) + node_handle = object() + cmd = self._make_command(node_handle=node_handle) + args = self._make_args(node_power_limit=6000) + + cmd.set_value(args) + + self.assertEqual(calls, [(node_handle, 6000)]) + message = cmd.logger.output["set_node_power_limit"] + self.assertIn("Successfully set node power limit to 6000 W", message) + + def test_no_perm_raises_permission_error(self): + def _raise(_handle, _limit): + raise _FakeLibraryException( + _STATUS_NO_PERM, "AMDSMI_STATUS_NO_PERM - Permission Denied" + ) + + self.interface.amdsmi_set_npm_limit = _raise + cmd = self._make_command(node_handle=object()) + args = self._make_args(node_power_limit=250) + + with self.assertRaises(PermissionError): + cmd.set_value(args) + + def test_not_supported_from_api_reports_error_without_raising(self): + def _raise(_handle, _limit): + raise _FakeLibraryException( + _STATUS_NOT_SUPPORTED, "AMDSMI_STATUS_NOT_SUPPORTED - Feature not supported" + ) + + self.interface.amdsmi_set_npm_limit = _raise + cmd = self._make_command(node_handle=object()) + args = self._make_args(node_power_limit=9999) + + cmd.set_value(args) # must not raise + + message = cmd.logger.output["set_node_power_limit"] + self.assertIn("AMDSMI_STATUS_NOT_SUPPORTED", message) + self.assertIn("Unable to set node power limit to 9999 W", message) + + +# --------------------------------------------------------------------------- +# Direct tests for amdsmi_helpers.AMDSMIHelpers.validate_and_set_node_power_limit() +# --------------------------------------------------------------------------- + + +class _FakeInitFlags: + INIT_ALL_PROCESSORS = 0xFFFFFFFF + INIT_AMD_GPUS = 1 + INIT_AMD_CPUS = 2 + INIT_AMD_NICS = 4 + + +class _FakeParameterException(Exception): + pass + + +def _install_fake_amdsmi_for_helpers(): + """Like ``_install_fake_amdsmi()``, plus the extra surface + ``amdsmi_helpers.py``'s module-level ``from amdsmi_init import *`` needs to + import cleanly: ``amdsmi_init.py`` calls ``amdsmi_cli_init()`` at import + time, which needs ``AmdSmiInitFlags``, a callable ``amdsmi_init()``, and + (for its ``isinstance()`` check on the init result) ``AmdSmiLibraryException`` + / ``AmdSmiParameterException`` directly on the ``amdsmi_interface`` module + (as opposed to the ``amdsmi_exception`` submodule). + """ + modules = _install_fake_amdsmi() + interface = modules["amdsmi.amdsmi_interface"] + interface.AmdSmiInitFlags = _FakeInitFlags + interface.amdsmi_init = lambda _flag: None + interface.amdsmi_shut_down = lambda: None + interface.AmdSmiLibraryException = _FakeLibraryException + interface.AmdSmiParameterException = _FakeParameterException + return modules + + +class TestValidateAndSetNodePowerLimit(unittest.TestCase): + _EXTRA_STUBBED_NAMES = ("amdsmi_init", "amdsmi_helpers", "amdsmi_cli_exceptions", "BDF") + + @classmethod + def setUpClass(cls): + if not _CLI_DIR: + raise unittest.SkipTest("amd-smi CLI source not found") + modules = _install_fake_amdsmi_for_helpers() + modules.update({name: None for name in cls._EXTRA_STUBBED_NAMES}) + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] + + cls._path_added = _CLI_DIR not in sys.path + if cls._path_added: + sys.path.insert(0, _CLI_DIR) + + import amdsmi_helpers as amdsmi_helpers_module + + # staticmethod() wrapping prevents `self.validate` from auto-binding + # this TestCase instance as the method's `self` argument (functions + # assigned as class attributes behave as descriptors; plain instance + # attribute assignment doesn't trigger this, but a classmethod-set + # attribute inherited by instances does). + cls.validate = staticmethod( + amdsmi_helpers_module.AMDSMIHelpers.validate_and_set_node_power_limit + ) + cls.exceptions_module = sys.modules["amdsmi_cli_exceptions"] + + @classmethod + def tearDownClass(cls): + if getattr(cls, "_path_added", False) and _CLI_DIR in sys.path: + sys.path.remove(_CLI_DIR) + + def setUp(self): + self.calls = [] + self.interface.amdsmi_set_npm_limit = lambda h, l: self.calls.append((h, l)) + + def _validate(self, requested_limit, max_node_power_limit, npm_status=_NPM_STATUS_ENABLED): + self.interface.amdsmi_get_npm_info = lambda _h: { + "max_node_power_limit": max_node_power_limit, + "status": npm_status, + } + # A lightweight duck-typed ``self`` -- exercises the real, unbound + # ``validate_and_set_node_power_limit`` method body without paying for + # ``AMDSMIHelpers.__init__``'s platform/hypervisor probing, which is + # irrelevant to this validation logic. + fake_self = types.SimpleNamespace(get_output_format=lambda: "human") + logger = _FakeLogger() + return self.validate(fake_self, object(), requested_limit, logger) + + def test_over_max_rejects(self): + with self.assertRaises(self.exceptions_module.AmdSmiInvalidParameterValueException) as ctx: + self._validate(requested_limit=6401, max_node_power_limit=6400) + self.assertIn("must be between 1W and 6400W", str(ctx.exception)) + self.assertEqual(self.calls, [], "API must not be called on a rejected request") + + def test_zero_rejects(self): + with self.assertRaises(self.exceptions_module.AmdSmiInvalidParameterValueException) as ctx: + self._validate(requested_limit=0, max_node_power_limit=6400) + self.assertIn("must be between 1W and 6400W", str(ctx.exception)) + self.assertEqual(self.calls, []) + + def test_in_range_succeeds(self): + result = self._validate(requested_limit=6000, max_node_power_limit=6400) + + self.assertIn("Successfully set node power limit to 6000 W", result) + self.assertEqual(len(self.calls), 1) + self.assertEqual(self.calls[0][1], 6000) + + def test_npm_disabled_rejects(self): + # NPM disabled on this node: reject before even looking at the max + # bound (writing cur_node_power_limit while NPM is disabled has no + # defined effect). + with self.assertRaises(self.exceptions_module.AmdSmiInvalidParameterValueException) as ctx: + self._validate( + requested_limit=250, max_node_power_limit=6400, npm_status=_NPM_STATUS_DISABLED + ) + self.assertIn("NPM is disabled", str(ctx.exception)) + self.assertEqual(self.calls, [], "API must not be called when NPM is disabled") + + def test_na_max_now_rejects(self): + # F-2 fail-closed fix: an unreadable platform max ("N/A") must reject + # the request outright, not fall back to a bare "> 0" check that would + # let an unbounded value through in a degraded-driver scenario. + with self.assertRaises(self.exceptions_module.AmdSmiInvalidParameterValueException) as ctx: + self._validate(requested_limit=250, max_node_power_limit="N/A") + self.assertIn("platform maximum is unavailable", str(ctx.exception)) + self.assertEqual(self.calls, [], "API must not be called when the platform max is unknown") + + def test_library_inval_raises_instead_of_returning(self): + # amdsmi_set_npm_limit() itself can still reject a request that passed + # the CLI-side pre-check (e.g. a race against a shrinking platform + # max). AMDSMI_STATUS_INVAL from the library must raise, exiting + # non-zero, not be swallowed into a returned error string/dict. + self.interface.amdsmi_set_npm_limit = lambda h, l: (_ for _ in ()).throw( + _FakeLibraryException(_STATUS_INVAL, "AMDSMI_STATUS_INVAL - Invalid parameters") + ) + with self.assertRaises(self.exceptions_module.AmdSmiInvalidParameterValueException): + self._validate(requested_limit=250, max_node_power_limit=6400) + + +# --------------------------------------------------------------------------- +# Direct tests for amdsmi_helpers.AMDSMIHelpers.get_max_node_power_limit() +# --------------------------------------------------------------------------- + + +class TestGetMaxNodePowerLimit(unittest.TestCase): + """Covers get_max_node_power_limit() (added by ece891b6787), which mirrors + get_power_caps()'s per-device probe loop: it queries devices in order and + returns the first successful reading, falling back to "N/A" only if every + device fails or reports "N/A" itself. + """ + + _EXTRA_STUBBED_NAMES = TestValidateAndSetNodePowerLimit._EXTRA_STUBBED_NAMES + + @classmethod + def setUpClass(cls): + if not _CLI_DIR: + raise unittest.SkipTest("amd-smi CLI source not found") + modules = _install_fake_amdsmi_for_helpers() + modules.update({name: None for name in cls._EXTRA_STUBBED_NAMES}) + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] + + cls._path_added = _CLI_DIR not in sys.path + if cls._path_added: + sys.path.insert(0, _CLI_DIR) + + import amdsmi_helpers as amdsmi_helpers_module + + # staticmethod() wrapping avoids auto-binding this TestCase instance + # as `self`; see TestValidateAndSetNodePowerLimit's setUpClass. + cls.get_max = staticmethod(amdsmi_helpers_module.AMDSMIHelpers.get_max_node_power_limit) + + @classmethod + def tearDownClass(cls): + if getattr(cls, "_path_added", False) and _CLI_DIR in sys.path: + sys.path.remove(_CLI_DIR) + + def _fake_self(self, gpu_handles): + # get_max_node_power_limit() only calls self.get_gpu_handles(); a + # duck-typed stub avoids paying for real device enumeration. + return types.SimpleNamespace(get_gpu_handles=lambda: gpu_handles) + + def test_first_device_success_short_circuits(self): + calls = [] + + def node_handle(dev): + calls.append(dev) + return f"node_{dev}" + + self.interface.amdsmi_get_node_handle = node_handle + self.interface.amdsmi_get_npm_info = lambda _h: {"max_node_power_limit": 6000} + + result = self.get_max(self._fake_self(["dev0", "dev1"])) + + self.assertEqual(result, "6000 W") + self.assertEqual(calls, ["dev0"], "second device must not be probed after a success") + + def test_all_devices_exception_falls_back_to_na(self): + def _raise(_dev): + raise _FakeLibraryException("npm info unavailable") + + self.interface.amdsmi_get_node_handle = _raise + + result = self.get_max(self._fake_self(["dev0", "dev1"])) + + self.assertEqual(result, "N/A") + + def test_na_value_on_first_device_continues_to_next(self): + # Distinct from the exception path: the device call succeeds but + # reports "N/A" as the value itself, so the loop must fall through to + # the next device rather than returning "N/A" early. + def node_handle(dev): + return f"node_{dev}" + + def npm_info(node_handle_): + if node_handle_ == "node_dev0": + return {"max_node_power_limit": "N/A"} + return {"max_node_power_limit": 6400} + + self.interface.amdsmi_get_node_handle = node_handle + self.interface.amdsmi_get_npm_info = npm_info + + result = self.get_max(self._fake_self(["dev0", "dev1"])) + + self.assertEqual(result, "6400 W") + + +# --------------------------------------------------------------------------- +# --node-power-limit registration on a non-baremetal (1VF guest) platform +# --------------------------------------------------------------------------- + +PARSER_PATH = os.path.join(_CLI_DIR, "amdsmi_parser.py") if _CLI_DIR else "" + + +def _build_fake_modules_for_parser(): + """Minimal stub surface for importing amdsmi_parser.py. + + Mirrors test_output_file_stdin.py's _install_stubs(): the parser module + only needs these names to bind at import time, not to be functional. + """ + interface = fake_module("amdsmi.amdsmi_interface") + amdsmi_pkg = fake_module("amdsmi", amdsmi_interface=interface) + version_mod = fake_module("_version", __version__="0.0.0-test") + helpers_mod = fake_module("amdsmi_helpers", AMDSMIHelpers=type("AMDSMIHelpers", (), {})) + + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "_version": version_mod, + "amdsmi_helpers": helpers_mod, + } + + +def _load_parser_module(): + if _CLI_DIR and _CLI_DIR not in sys.path: + sys.path.insert(0, _CLI_DIR) + # See _load_set_value_module()'s comment: sibling test files can leave a + # stale/incomplete "amdsmi_cli_exceptions" stub cached in sys.modules. + # amdsmi_parser.py imports it at module scope, so drop any cached entry + # first to force re-resolution of the real module. + sys.modules.pop("amdsmi_cli_exceptions", None) + spec = importlib.util.spec_from_file_location("amdsmi_parser_under_test_npm", PARSER_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _FakeHelpersGuest: + """Non-baremetal (1VF guest) helpers stub for ``_add_set_value_parser``. + + ``is_baremetal()`` is False, so the baremetal-only set arguments (fan, + perf-level, ...) are skipped, but ``-o``/``--power-cap`` and + ``-n``/``--node-power-limit`` are gated only by + ``is_amdgpu_initialized()`` and must still register. + """ + + def is_linux(self): + return True + + def is_amdgpu_initialized(self): + return True + + def is_baremetal(self): + return False + + def is_hypervisor(self): + return False + + def is_amd_hsmp_initialized(self): + return False + + def is_ainic_initialized(self): + return False + + def is_brcm_nic_initialized(self): + return False + + def is_brcm_switch_initialized(self): + return False + + def get_power_caps(self): + return ("0 W", "550 W", "0 W", "550 W") + + def get_max_node_power_limit(self): + return "6000 W" + + def get_output_format(self): + return "human" + + def get_device_handles_from_gpu_selections(self, **_kwargs): + return 1, True, [object()] + + +class TestNodePowerLimitGuestRegistration(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not PARSER_PATH or not os.path.isfile(PARSER_PATH): + raise unittest.SkipTest("amd-smi CLI amdsmi_parser.py not found (source or installed)") + modules = _build_fake_modules_for_parser() + modules["amdsmi_cli_exceptions"] = None + stub_modules(cls, modules) + cls.parser_mod = _load_parser_module() + + def _build_set_parser(self): + # A minimal fake ``self`` -- exercises the real, unbound + # _add_set_value_parser method body without paying for + # AMDSMIParser.__init__'s full platform probing. + fake_self = object.__new__(self.parser_mod.AMDSMIParser) + fake_self.helpers = _FakeHelpersGuest() + fake_self.description = "test" + fake_self.gpu_choices = {} + fake_self.gpu_choices_str = "" + fake_self.cpu_choices_str = "" + fake_self.nic_choices_str = "" + fake_self.core_choices_str = "" + fake_self.switch_choices_str = "" + fake_self.vf_choices = [] + + top_parser = argparse.ArgumentParser() + subparsers = top_parser.add_subparsers() + fake_self._add_set_value_parser(subparsers, func=lambda args: None) + return subparsers.choices["set"] + + def test_node_power_limit_registered_on_guest(self): + set_parser = self._build_set_parser() + + args = set_parser.parse_args(["--node-power-limit", "100"]) + + self.assertEqual(args.node_power_limit, 100) + + def test_unrelated_gpu_error_keeps_argparse_error(self): + parser = argparse.ArgumentParser() + self.parser_mod.AMDSMIParser._guard_gtt_gpu_conflict( + parser, + gtt_flags=("--node-power-limit", "-n"), + reason="--node-power-limit is a node-wide setting, not per-GPU", + ) + argv = ["amd-smi", "set", "--node-power-limit", "100", "--gpu", "0"] + + with mock.patch.object(sys, "argv", argv), contextlib.redirect_stderr( + io.StringIO() + ) as stderr, self.assertRaises(SystemExit) as raised: + parser.error("argument --cpu: not allowed with argument --gpu/-g") + + self.assertEqual(raised.exception.code, 2) + self.assertIn("argument --cpu", stderr.getvalue()) + self.assertNotIn( + "--node-power-limit/-n: not allowed with argument --gpu/-g", stderr.getvalue() + ) + + def test_missing_gpu_value_gets_node_power_limit_error(self): + parser = argparse.ArgumentParser() + self.parser_mod.AMDSMIParser._guard_gtt_gpu_conflict( + parser, + gtt_flags=("--node-power-limit", "-n"), + reason="--node-power-limit is a node-wide setting, not per-GPU", + ) + argv = ["amd-smi", "set", "--node-power-limit", "100", "--gpu"] + + with mock.patch.object(sys, "argv", argv), contextlib.redirect_stderr( + io.StringIO() + ) as stderr, self.assertRaises(SystemExit) as raised: + parser.error("argument --gpu/-g: expected at least one argument") + + self.assertEqual(raised.exception.code, 2) + self.assertIn( + "--node-power-limit/-n: not allowed with argument --gpu/-g", stderr.getvalue() + ) + + def test_missing_unrelated_value_keeps_argparse_error(self): + parser = argparse.ArgumentParser() + self.parser_mod.AMDSMIParser._guard_gtt_gpu_conflict( + parser, + gtt_flags=("--node-power-limit", "-n"), + reason="--node-power-limit is a node-wide setting, not per-GPU", + ) + argv = ["amd-smi", "set", "--node-power-limit", "100", "--gpu", "0", "--power-cap"] + + with mock.patch.object(sys, "argv", argv), contextlib.redirect_stderr( + io.StringIO() + ) as stderr, self.assertRaises(SystemExit) as raised: + parser.error("argument -o/--power-cap: expected at least one argument") + + self.assertEqual(raised.exception.code, 2) + self.assertIn("argument -o/--power-cap", stderr.getvalue()) + self.assertNotIn( + "--node-power-limit/-n: not allowed with argument --gpu/-g", stderr.getvalue() + ) diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_static_bus_pcie.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_static_bus_pcie.py index e6ab4d5fa8bc..3189f672dc94 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_cli_static_bus_pcie.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_static_bus_pcie.py @@ -16,15 +16,20 @@ """ import argparse -import importlib.util import os -import sys -import types import unittest -_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -_REPO_ROOT = os.path.abspath(os.path.join(_THIS_DIR, "..", "..", "..", "..")) -STATIC_PATH = os.path.join(_REPO_ROOT, "amdsmi_cli", "subcommands", "static.py") +from common.common import ( + amdsmi_path, + cli_search_order, + fake_module, + find_cli_dir, + load_cli_module, + stub_modules, +) + +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +STATIC_PATH = os.path.join(_CLI_DIR, "subcommands", "static.py") if _CLI_DIR else None class _FakeLibraryException(Exception): @@ -32,53 +37,47 @@ def get_error_info(self): return str(self) -def _install_fake_modules(pcie_static): - """Register a stub ``amdsmi`` package plus the sibling CLI modules. +def _fake_modules(pcie_static): + """Build the stub ``amdsmi`` package plus the sibling CLI modules. + Returns the name -> module mapping for ``common.stub_modules``; ``pcie_static`` is returned verbatim from ``amdsmi_get_pcie_info``. """ - amdsmi_pkg = types.ModuleType("amdsmi") - interface = types.ModuleType("amdsmi.amdsmi_interface") - exception = types.ModuleType("amdsmi.amdsmi_exception") - - interface.amdsmi_get_gpu_device_bdf = lambda _handle: "0000:00:00.0" - interface.amdsmi_get_pcie_info = lambda _handle: {"pcie_static": pcie_static} - exception.AmdSmiLibraryException = _FakeLibraryException + exception = fake_module("amdsmi.amdsmi_exception", AmdSmiLibraryException=_FakeLibraryException) def _raise_lib_exc(_handle): raise exception.AmdSmiLibraryException("mock: pci bandwidth unavailable") - # Bus info also fetches pci bandwidth for pcie_levels; keep it out of scope - # for these PCIe-speed/version-focused tests by having it degrade to N/A - # the same way the real library does when unsupported. - interface.amdsmi_get_gpu_pci_bandwidth = _raise_lib_exc - - amdsmi_pkg.amdsmi_interface = interface - amdsmi_pkg.amdsmi_exception = exception - sys.modules["amdsmi"] = amdsmi_pkg - sys.modules["amdsmi.amdsmi_interface"] = interface - sys.modules["amdsmi.amdsmi_exception"] = exception + interface = fake_module( + "amdsmi.amdsmi_interface", + amdsmi_get_gpu_device_bdf=lambda _handle: "0000:00:00.0", + amdsmi_get_pcie_info=lambda _handle: {"pcie_static": pcie_static}, + # Bus info also fetches pci bandwidth for pcie_levels; keep it out of scope + # for these PCIe-speed/version-focused tests by having it degrade to N/A + # the same way the real library does when unsupported. + amdsmi_get_gpu_pci_bandwidth=_raise_lib_exc, + ) + amdsmi_pkg = fake_module("amdsmi", amdsmi_interface=interface, amdsmi_exception=exception) # ``static.py`` imports these sibling names at load time; the bus path # never instantiates them (the test injects a fake helpers object). - helpers_mod = types.ModuleType("amdsmi_helpers") - helpers_mod.AMDSMIHelpers = object - sys.modules["amdsmi_helpers"] = helpers_mod - - exceptions_mod = types.ModuleType("amdsmi_cli_exceptions") - exceptions_mod.AmdSmiInvalidParameterException = type( - "AmdSmiInvalidParameterException", (Exception,), {} + helpers_mod = fake_module("amdsmi_helpers", AMDSMIHelpers=object) + exceptions_mod = fake_module( + "amdsmi_cli_exceptions", + AmdSmiInvalidParameterException=type("AmdSmiInvalidParameterException", (Exception,), {}), ) - sys.modules["amdsmi_cli_exceptions"] = exceptions_mod - return interface + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + "amdsmi_helpers": helpers_mod, + "amdsmi_cli_exceptions": exceptions_mod, + } def _load_static_module(): - spec = importlib.util.spec_from_file_location("static_under_test", STATIC_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("static_under_test", STATIC_PATH) class _FakeLogger: @@ -181,9 +180,11 @@ class TestCliStaticBusPcieNA(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(STATIC_PATH): - raise unittest.SkipTest(f"amd-smi CLI static.py not found at {STATIC_PATH}") - cls.interface = _install_fake_modules( + if not STATIC_PATH or not os.path.isfile(STATIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI static.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _fake_modules( { "max_pcie_width": "N/A", "max_pcie_speed": "N/A", @@ -191,6 +192,8 @@ def setUpClass(cls): "slot_type": "N/A", } ) + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] cls.static_module = _load_static_module() def _run_bus(self, fmt): @@ -229,9 +232,11 @@ class TestCliStaticBusPcieValid(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(STATIC_PATH): - raise unittest.SkipTest(f"amd-smi CLI static.py not found at {STATIC_PATH}") - cls.interface = _install_fake_modules( + if not STATIC_PATH or not os.path.isfile(STATIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI static.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _fake_modules( { "max_pcie_width": 16, "max_pcie_speed": 16000, @@ -239,6 +244,8 @@ def setUpClass(cls): "slot_type": "OAM", } ) + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] cls.static_module = _load_static_module() def _run_bus(self, fmt): @@ -266,3 +273,38 @@ def test_json_formats_speed_as_value_unit_dict(self): bus_info = self._run_bus("json") self.assertEqual(bus_info["max_pcie_speed"], {"value": 16, "unit": "GT/s"}) self.assertEqual(bus_info["pcie_interface_version"], "Gen 4") + + +class TestCliStaticDriverVersions(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + if not STATIC_PATH or not os.path.isfile(STATIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI static.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _fake_modules({}) + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] + cls.interface.amdsmi_get_gpu_driver_info = lambda _handle: { + "driver_name": "amdgpu", + "driver_kernel_version": "6.19.14", + "driver_version": "6.19.14.31400000", + "driver_build_version": "2370381", + "driver_full_version": "6.19.14.31400000-2370381", + } + cls.static_module = _load_static_module() + + def test_json_reports_amdgpu_version(self) -> None: + commands = object.__new__(self.static_module.StaticCommands) + commands.logger = _FakeLogger("json") + commands.helpers = _FakeHelpers() + commands.group_check_printed = True + args = _build_args() + args.bus = False + args.driver = True + + commands.static_gpu(args) + + driver_info = commands.logger.store_gpu_json_output[-1]["driver"] + self.assertEqual(set(driver_info), {"name", "version", "os_kernel_version"}) + self.assertEqual(driver_info["version"], "6.19.14.31400000-2370381") diff --git a/projects/amdsmi/tests/python/unit/gpu/test_cli_vram_type_lpddr5.py b/projects/amdsmi/tests/python/unit/gpu/test_cli_vram_type_lpddr5.py index 496ed5de238c..635a8e68b99c 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_cli_vram_type_lpddr5.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_cli_vram_type_lpddr5.py @@ -11,21 +11,28 @@ ``LPDDR5``; it was previously mislabelled ``GDDR7``, which surfaced on gfx117x APUs whose unified memory reports as LPDDR5. -``static.py`` is loaded from the source tree so the test exercises the code -under development rather than a possibly-stale installed copy. +``static.py`` is loaded through cli_search_order, so a plain checkout +exercises the code under development; an explicit AMDSMI_PATH pins that install. """ import argparse import copy -import importlib.util import os -import sys -import types import unittest -_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -_REPO_ROOT = os.path.abspath(os.path.join(_THIS_DIR, "..", "..", "..", "..")) -STATIC_PATH = os.path.join(_REPO_ROOT, "amdsmi_cli", "subcommands", "static.py") +from common.common import ( + amdsmi_path, + cli_search_order, + fake_module, + find_cli_dir, + load_cli_module, + stub_modules, +) + +# Locate the CLI dir; cli_search_order() decides whether the install or this +# checkout wins. None -> setUpClass skips. +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +STATIC_PATH = os.path.join(_CLI_DIR, "subcommands", "static.py") if _CLI_DIR else None # Mirrors amdsmi_wrapper.amdsmi_vram_type_t__enumvalues: LPDDR5 and __MAX share # value 31, and the later duplicate key (__MAX) wins in the dict literal, so a @@ -54,52 +61,47 @@ def get_error_info(self): return str(self) -def _install_fake_modules(holder): - """Register a stub ``amdsmi`` package plus the sibling CLI modules. +def _fake_modules(holder): + """Build a stub ``amdsmi`` package plus the sibling CLI modules. + Returns the name -> module mapping for ``common.stub_modules``. ``holder["info"]`` is the vram payload returned to ``static.py`` so each test can swap the reported ``vram_type`` without reloading the module. """ - amdsmi_pkg = types.ModuleType("amdsmi") - interface = types.ModuleType("amdsmi.amdsmi_interface") - exception = types.ModuleType("amdsmi.amdsmi_exception") def _get_vram_info(_handle): return copy.deepcopy(holder["info"]) - interface.amdsmi_get_gpu_vram_info = _get_vram_info - - wrapper = types.ModuleType("amdsmi.amdsmi_interface.amdsmi_wrapper") - wrapper.AMDSMI_VRAM_TYPE__MAX = _VRAM_TYPE__MAX - wrapper.amdsmi_vram_type_t__enumvalues = dict(_ENUMVALUES) - interface.amdsmi_wrapper = wrapper - - exception.AmdSmiLibraryException = _FakeLibraryException - - amdsmi_pkg.amdsmi_interface = interface - amdsmi_pkg.amdsmi_exception = exception - sys.modules["amdsmi"] = amdsmi_pkg - sys.modules["amdsmi.amdsmi_interface"] = interface - sys.modules["amdsmi.amdsmi_exception"] = exception + wrapper = fake_module( + "amdsmi.amdsmi_interface.amdsmi_wrapper", + AMDSMI_VRAM_TYPE__MAX=_VRAM_TYPE__MAX, + amdsmi_vram_type_t__enumvalues=dict(_ENUMVALUES), + ) + interface = fake_module( + "amdsmi.amdsmi_interface", amdsmi_get_gpu_vram_info=_get_vram_info, amdsmi_wrapper=wrapper + ) + exception = fake_module("amdsmi.amdsmi_exception", AmdSmiLibraryException=_FakeLibraryException) + amdsmi_pkg = fake_module("amdsmi", amdsmi_interface=interface, amdsmi_exception=exception) # ``static.py`` imports these sibling names at load time; the vram path # never instantiates them (the test injects a fake helpers object). - helpers_mod = types.ModuleType("amdsmi_helpers") - helpers_mod.AMDSMIHelpers = object - sys.modules["amdsmi_helpers"] = helpers_mod - - exceptions_mod = types.ModuleType("amdsmi_cli_exceptions") - exceptions_mod.AmdSmiInvalidParameterException = type( - "AmdSmiInvalidParameterException", (Exception,), {} + helpers_mod = fake_module("amdsmi_helpers", AMDSMIHelpers=object) + exceptions_mod = fake_module( + "amdsmi_cli_exceptions", + AmdSmiInvalidParameterException=type("AmdSmiInvalidParameterException", (Exception,), {}), ) - sys.modules["amdsmi_cli_exceptions"] = exceptions_mod + + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + "amdsmi_helpers": helpers_mod, + "amdsmi_cli_exceptions": exceptions_mod, + } def _load_static_module(): - spec = importlib.util.spec_from_file_location("static_under_test", STATIC_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("static_under_test", STATIC_PATH) class _FakeLogger: @@ -178,33 +180,16 @@ def _build_args(): class TestCliVramTypeLpddr5(unittest.TestCase): - _SAVED_MODULE_NAMES = ( - "amdsmi", - "amdsmi.amdsmi_interface", - "amdsmi.amdsmi_exception", - "amdsmi_helpers", - "amdsmi_cli_exceptions", - ) - @classmethod def setUpClass(cls): - if not os.path.isfile(STATIC_PATH): - raise unittest.SkipTest(f"amd-smi CLI static.py not found at {STATIC_PATH}") - # Snapshot any real amdsmi already loaded so the stub does not leak into - # sibling suites sharing the interpreter; restored in tearDownClass. - cls._saved_modules = {name: sys.modules.get(name) for name in cls._SAVED_MODULE_NAMES} + if not STATIC_PATH or not os.path.isfile(STATIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI static.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) cls.holder = {"info": copy.deepcopy(_BASE_VRAM_INFO)} - _install_fake_modules(cls.holder) + stub_modules(cls, _fake_modules(cls.holder)) cls.static_module = _load_static_module() - @classmethod - def tearDownClass(cls): - for name, saved in cls._saved_modules.items(): - if saved is None: - sys.modules.pop(name, None) - else: - sys.modules[name] = saved - def _run_vram(self, vram_type_value, fmt="human"): info = copy.deepcopy(_BASE_VRAM_INFO) info["vram_type"] = vram_type_value @@ -238,7 +223,3 @@ def test_lpddr4_still_labelled(self): def test_non_max_type_unaffected(self): self.assertEqual(self._run_vram(22, "human"), "GDDR6") - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_kfd_process_gpus.py b/projects/amdsmi/tests/python/unit/gpu/test_kfd_process_gpus.py index b629036e8d80..3eb86d002cc7 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_kfd_process_gpus.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_kfd_process_gpus.py @@ -114,7 +114,3 @@ def test_unreadable_vram_file_is_skipped(self): os.remove(target) os.makedirs(target) self.assertEqual(_resolve_process_gpus(proc_dir), set()) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_npm_info_current_node_power.py b/projects/amdsmi/tests/python/unit/gpu/test_npm_info_current_node_power.py new file mode 100644 index 000000000000..fecf711fc904 --- /dev/null +++ b/projects/amdsmi/tests/python/unit/gpu/test_npm_info_current_node_power.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Unit tests for the "current_node_power" key in amdsmi_get_npm_info()'s dict. + +current_node_power moved from amdsmi_power_info_t (per-GPU, amd-smi metric +--power) to amdsmi_npm_info_t (per-node, amd-smi node -p / +amdsmi_get_npm_info()) -- see the "Set npm power limit" design doc. This file +replaces the previous test_power_info_node_power.py, which exercised the +now-removed amdsmi_power_info_t.node_power field. + +Uses the real compiled amdsmi package (same resolution as test_check_res.py / +test_apu_metrics.py, via `from common.common import amdsmi`) but mocks the +underlying ctypes call (`amdsmi_wrapper.amdsmi_get_npm_info`) to populate an +`amdsmi_npm_info_t` struct directly -- no real hardware/root access needed. + +Per the frozen contract, `amdsmi_npm_info_t` gained a new `current_node_power` +uint32 field (right after `max_node_power_limit`), and the Python dict +returned by `amdsmi_get_npm_info()` gained a matching `"current_node_power"` +key that follows the same UINT32_MAX -> "N/A" sentinel convention already +used for `"ubb_power_threshold"`/`"max_node_power_limit"` (see +`_validate_if_max_uint` usage in `amdsmi_get_npm_info()`, amdsmi_interface.py). +""" + +from __future__ import annotations + +import ctypes +import unittest +from unittest import mock + +from common.common import amdsmi + + +def _make_node_handle() -> "amdsmi.amdsmi_wrapper.amdsmi_node_handle": + return amdsmi.amdsmi_wrapper.amdsmi_node_handle() + + +def _make_npm_info( + current_node_power: int, + limit: int = 0, + status: int = 0, + ubb_power_threshold: int = 0, + max_node_power_limit: int = 0, +) -> "amdsmi.amdsmi_wrapper.amdsmi_npm_info_t": + info = amdsmi.amdsmi_wrapper.amdsmi_npm_info_t() + info.status = status + info.limit = limit + info.ubb_power_threshold = ubb_power_threshold + info.max_node_power_limit = max_node_power_limit + info.current_node_power = current_node_power + return info + + +class TestNpmInfoCurrentNodePowerKey(unittest.TestCase): + def _get_npm_info_with_mocked_c_call( + self, current_node_power_raw: int, max_node_power_limit_raw: int = 0 + ): + node_handle = _make_node_handle() + + def _fake_amdsmi_get_npm_info(_node_handle, info_ptr): + info_ptr._obj.__init__() # zero-init as the real ctypes.byref target would be + populated = _make_npm_info( + current_node_power=current_node_power_raw, + max_node_power_limit=max_node_power_limit_raw, + ) + ctypes.memmove( + ctypes.addressof(info_ptr._obj), + ctypes.addressof(populated), + ctypes.sizeof(populated), + ) + return amdsmi.amdsmi_wrapper.AMDSMI_STATUS_SUCCESS + + with mock.patch.object( + amdsmi.amdsmi_wrapper, "amdsmi_get_npm_info", side_effect=_fake_amdsmi_get_npm_info + ): + return amdsmi.amdsmi_get_npm_info(node_handle) + + def test_current_node_power_key_present_with_concrete_value(self): + result = self._get_npm_info_with_mocked_c_call(current_node_power_raw=5800) + self.assertIn("current_node_power", result) + self.assertEqual(result["current_node_power"], 5800) + + def test_current_node_power_sentinel_becomes_na(self): + uint32_max = amdsmi.amdsmi_interface.MaxUIntegerTypes.UINT32_T + result = self._get_npm_info_with_mocked_c_call(current_node_power_raw=uint32_max) + self.assertEqual(result["current_node_power"], "N/A") + + def test_current_node_power_independent_of_max_node_power_limit(self): + # current_node_power and max_node_power_limit are adjacent fields in + # the struct; make sure the sentinel conversion is applied per-field, + # not as an all-or-nothing pass keyed off one of them. + uint64_max = amdsmi.amdsmi_interface.MaxUIntegerTypes.UINT64_T + result = self._get_npm_info_with_mocked_c_call( + current_node_power_raw=5800, max_node_power_limit_raw=uint64_max + ) + self.assertEqual(result["current_node_power"], 5800) + self.assertEqual(result["max_node_power_limit"], "N/A") + + def test_current_node_power_zero_is_not_treated_as_na(self): + # 0 W is a legitimate (if unusual) reading and must not be conflated + # with the sentinel/absent case. + result = self._get_npm_info_with_mocked_c_call(current_node_power_raw=0) + self.assertEqual(result["current_node_power"], 0) diff --git a/projects/amdsmi/tests/python/unit/gpu/test_npm_set_limit.py b/projects/amdsmi/tests/python/unit/gpu/test_npm_set_limit.py new file mode 100644 index 000000000000..69322f67e87c --- /dev/null +++ b/projects/amdsmi/tests/python/unit/gpu/test_npm_set_limit.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Unit tests for the amdsmi_set_npm_limit() Python wrapper. + +Uses the real compiled amdsmi package (resolved the same way test_check_res.py +and test_apu_metrics.py do, via `from common.common import amdsmi`), but mocks +the underlying ctypes call (`amdsmi_wrapper.amdsmi_set_npm_limit`) so no real +hardware/root access is required -- these tests exercise only the Python-level +parameter validation and status-to-exception mapping added around the new C +API, per the frozen contract: + + def amdsmi_set_npm_limit(node_handle: processor_handle_t, limit: int) -> None + +which: + * raises AmdSmiParameterException if node_handle is not an + amdsmi_wrapper.amdsmi_node_handle instance + * raises AmdSmiParameterException if limit is not an int + * otherwise calls amdsmi_wrapper.amdsmi_set_npm_limit(node_handle, + ctypes.c_uint64(limit)) and funnels the returned status code through the + standard _check_res() mapping (AmdSmiLibraryException with the propagated + error code on any non-SUCCESS status, e.g. AMDSMI_STATUS_NO_PERM or + AMDSMI_STATUS_NOT_SUPPORTED; no exception on AMDSMI_STATUS_SUCCESS) +""" + +from __future__ import annotations + +import ctypes +import unittest +from unittest import mock + +from common.common import amdsmi + + +def _make_node_handle() -> "amdsmi.amdsmi_wrapper.amdsmi_node_handle": + """A node handle instance satisfying the isinstance() check, independent + of any real device/board path -- amdsmi_set_npm_limit()'s Python-level + validation only checks the ctypes type, not the pointee's validity.""" + return amdsmi.amdsmi_wrapper.amdsmi_node_handle() + + +class TestAmdSmiSetNpmLimitParameterValidation(unittest.TestCase): + def test_rejects_non_node_handle(self): + with self.assertRaises(amdsmi.AmdSmiParameterException): + amdsmi.amdsmi_set_npm_limit("not-a-node-handle", 100) + + def test_rejects_non_int_limit(self): + node_handle = _make_node_handle() + with self.assertRaises(amdsmi.AmdSmiParameterException): + amdsmi.amdsmi_set_npm_limit(node_handle, "100") + + def test_rejects_none_node_handle(self): + with self.assertRaises(amdsmi.AmdSmiParameterException): + amdsmi.amdsmi_set_npm_limit(None, 100) + + def test_rejects_float_limit(self): + node_handle = _make_node_handle() + with self.assertRaises(amdsmi.AmdSmiParameterException): + amdsmi.amdsmi_set_npm_limit(node_handle, 100.5) + + def test_rejects_negative_limit(self): + # A Python int can represent negative values a raw ctypes.c_uint64() + # cannot (it would silently wrap modulo 2**64); the explicit range + # check must reject this before ever reaching ctypes. + node_handle = _make_node_handle() + with self.assertRaises(amdsmi.AmdSmiParameterException): + amdsmi.amdsmi_set_npm_limit(node_handle, -1) + + def test_rejects_limit_over_uint64_max(self): + node_handle = _make_node_handle() + with self.assertRaises(amdsmi.AmdSmiParameterException): + amdsmi.amdsmi_set_npm_limit(node_handle, 0xFFFFFFFFFFFFFFFF + 1) + + +class TestAmdSmiSetNpmLimitStatusMapping(unittest.TestCase): + def test_success_returns_none_and_forwards_args(self): + node_handle = _make_node_handle() + with mock.patch.object( + amdsmi.amdsmi_wrapper, + "amdsmi_set_npm_limit", + return_value=amdsmi.amdsmi_wrapper.AMDSMI_STATUS_SUCCESS, + ) as mocked_call: + result = amdsmi.amdsmi_set_npm_limit(node_handle, 250) + + self.assertIsNone(result) + mocked_call.assert_called_once() + called_node_handle, called_limit = mocked_call.call_args[0] + self.assertIs(called_node_handle, node_handle) + self.assertIsInstance(called_limit, ctypes.c_uint64) + self.assertEqual(called_limit.value, 250) + + def test_no_perm_status_raises_library_exception_with_code(self): + node_handle = _make_node_handle() + with mock.patch.object( + amdsmi.amdsmi_wrapper, + "amdsmi_set_npm_limit", + return_value=amdsmi.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM, + ): + with self.assertRaises(amdsmi.AmdSmiLibraryException) as ctx: + amdsmi.amdsmi_set_npm_limit(node_handle, 250) + self.assertEqual( + ctx.exception.get_error_code(), amdsmi.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM + ) + + def test_not_supported_status_raises_library_exception_with_code(self): + node_handle = _make_node_handle() + with mock.patch.object( + amdsmi.amdsmi_wrapper, + "amdsmi_set_npm_limit", + return_value=amdsmi.amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED, + ): + with self.assertRaises(amdsmi.AmdSmiLibraryException) as ctx: + amdsmi.amdsmi_set_npm_limit(node_handle, 250) + self.assertEqual( + ctx.exception.get_error_code(), amdsmi.amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED + ) + + def test_inval_status_raises_library_exception_with_code(self): + node_handle = _make_node_handle() + with mock.patch.object( + amdsmi.amdsmi_wrapper, + "amdsmi_set_npm_limit", + return_value=amdsmi.amdsmi_wrapper.AMDSMI_STATUS_INVAL, + ): + with self.assertRaises(amdsmi.AmdSmiLibraryException) as ctx: + amdsmi.amdsmi_set_npm_limit(node_handle, 250) + self.assertEqual(ctx.exception.get_error_code(), amdsmi.amdsmi_wrapper.AMDSMI_STATUS_INVAL) diff --git a/projects/amdsmi/tests/python/unit/gpu/test_process_table_format.py b/projects/amdsmi/tests/python/unit/gpu/test_process_table_format.py index b7bedc857ed2..064f4d38f483 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_process_table_format.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_process_table_format.py @@ -10,33 +10,30 @@ aligned the ``CU %``/``SDMA`` columns and dropped the redundant ``%`` suffix. """ -import importlib.util import os -import sys import types import unittest -from common.common import amdsmi_path +from common.common import amdsmi_path, cli_search_order, find_cli_dir, load_cli_module, stub_modules -_ROCM_ROOT = os.path.dirname(os.path.dirname(amdsmi_path)) -LOGGER_PATH = os.path.join(_ROCM_ROOT, "libexec", "amdsmi_cli", "amdsmi_logger.py") +# Locate the CLI dir; cli_search_order() decides whether the install or this +# checkout wins. None -> setUpClass skips. +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +LOGGER_PATH = os.path.join(_CLI_DIR, "amdsmi_logger.py") if _CLI_DIR else None # Fixed inner width of the default-output box (between the two '|' borders). _BOX_INNER_WIDTH = 78 -def _install_fake_helpers(): - """Register a stub ``amdsmi_helpers`` so ``amdsmi_logger`` imports cleanly.""" +def _fake_helpers(): + """Build a stub ``amdsmi_helpers`` so ``amdsmi_logger`` imports cleanly.""" module = types.ModuleType("amdsmi_helpers") module.AMDSMIHelpers = type("AMDSMIHelpers", (), {}) - sys.modules["amdsmi_helpers"] = module + return module def _load_logger_module(): - spec = importlib.util.spec_from_file_location("amdsmi_logger_under_test", LOGGER_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("amdsmi_logger_under_test", LOGGER_PATH) def _process(name="python3", cu=None, sdma="0", gpu="0", pid="12345"): @@ -60,9 +57,11 @@ def _process(name="python3", cu=None, sdma="0", gpu="0", pid="12345"): class TestProcessTableFormat(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(LOGGER_PATH): - raise unittest.SkipTest(f"amdsmi_logger not installed at {LOGGER_PATH}") - _install_fake_helpers() + if not LOGGER_PATH or not os.path.isfile(LOGGER_PATH): + raise unittest.SkipTest( + f"amd-smi CLI amdsmi_logger.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + stub_modules(cls, {"amdsmi_helpers": _fake_helpers()}) cls.logger = _load_logger_module() def _assert_boxed(self, line): @@ -102,7 +101,3 @@ def test_sdma_and_cu_columns_align_with_header(self): self.assertLessEqual(header.index("CU %"), row.index("99.9")) self.assertLessEqual(header.index("SDMA"), row.index("765")) self._assert_boxed(row) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_topology.py b/projects/amdsmi/tests/python/unit/gpu/test_topology.py index 72e9e49248fb..98002668b27d 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_topology.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_topology.py @@ -21,7 +21,3 @@ def test_link_type_rejects_non_handle(self): def test_p2p_status_rejects_non_handle(self): with self.assertRaises(amdsmi.AmdSmiParameterException): amdsmi.amdsmi_topo_get_p2p_status("not_a_handle", "not_a_handle") - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_vcn_busy_navi.py b/projects/amdsmi/tests/python/unit/gpu/test_vcn_busy_navi.py index cea58d9e084b..e1b137ea747a 100644 --- a/projects/amdsmi/tests/python/unit/gpu/test_vcn_busy_navi.py +++ b/projects/amdsmi/tests/python/unit/gpu/test_vcn_busy_navi.py @@ -20,16 +20,16 @@ """ import argparse -import importlib.util import os -import sys import types import unittest -from common.common import amdsmi_path +from common.common import amdsmi_path, cli_search_order, find_cli_dir, load_cli_module, stub_modules -_ROCM_ROOT = os.path.dirname(os.path.dirname(amdsmi_path)) -METRIC_PATH = os.path.join(_ROCM_ROOT, "libexec", "amdsmi_cli", "subcommands", "metric.py") +# Locate the CLI dir; cli_search_order() decides whether the install or this +# checkout wins. None -> setUpClass skips. +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +METRIC_PATH = os.path.join(_CLI_DIR, "subcommands", "metric.py") if _CLI_DIR else None class _FakeClkType: @@ -71,7 +71,7 @@ def _patch(testcase, obj, **overrides): testcase.addCleanup(_restore_attr, obj, name, original) -def _install_fake_amdsmi(): +def _build_fake_amdsmi(): amdsmi_pkg = types.ModuleType("amdsmi") interface = types.ModuleType("amdsmi.amdsmi_interface") exception = types.ModuleType("amdsmi.amdsmi_exception") @@ -91,6 +91,9 @@ def _install_fake_amdsmi(): "jpeg_activity": "N/A", } interface._NA_amdsmi_get_gpu_metrics_info = lambda: {} + # An unmapped version, so unsupported-field filtering suppresses nothing and + # these tests see the vcn_busy behavior alone. + interface.amdsmi_get_gpu_metrics_header_info = lambda _h: {} interface.amdsmi_get_gpu_partition_metrics_info = lambda _h: None interface.amdsmi_get_gpu_activity = lambda _h: {"gfx_activity": 30} interface.amdsmi_get_vcn_busy_percent = lambda _h: 42 @@ -100,17 +103,15 @@ def _install_fake_amdsmi(): amdsmi_pkg.amdsmi_interface = interface amdsmi_pkg.amdsmi_exception = exception - sys.modules["amdsmi"] = amdsmi_pkg - sys.modules["amdsmi.amdsmi_interface"] = interface - sys.modules["amdsmi.amdsmi_exception"] = exception - return interface + return { + "amdsmi": amdsmi_pkg, + "amdsmi.amdsmi_interface": interface, + "amdsmi.amdsmi_exception": exception, + } def _load_metric_module(): - spec = importlib.util.spec_from_file_location("metric_under_test_vcn", METRIC_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("metric_under_test_vcn", METRIC_PATH, sys_path_dir=_CLI_DIR) class _FakeLogger: @@ -237,9 +238,13 @@ class TestVcnBusyNaviFallback(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(METRIC_PATH): - raise unittest.SkipTest(f"amd-smi CLI metric.py not found at {METRIC_PATH}") - cls.interface = _install_fake_amdsmi() + if not METRIC_PATH or not os.path.isfile(METRIC_PATH): + raise unittest.SkipTest( + f"amd-smi CLI metric.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + modules = _build_fake_amdsmi() + stub_modules(cls, modules) + cls.interface = modules["amdsmi.amdsmi_interface"] cls.metric_module = _load_metric_module() def test_navi_vcn_busy_reads_sysfs(self): @@ -331,7 +336,3 @@ def _spy(_h): self.assertIsInstance(vcn, dict) self.assertIn("xcp_0", vcn) self.assertIn("xcp_1", vcn) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/gpu/test_vram_type_hbm4.py b/projects/amdsmi/tests/python/unit/gpu/test_vram_type_hbm4.py new file mode 100644 index 000000000000..8db9f00e6bba --- /dev/null +++ b/projects/amdsmi/tests/python/unit/gpu/test_vram_type_hbm4.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""HBM VRAM type enum values (hardware-free).""" + +import unittest + +from common.common import amdsmi + + +class TestVramTypeHbm(unittest.TestCase): + """Pins the HBM3E/HBM4 values across the wrapper and the Python enum.""" + + def test_hbm3e(self): + self.assertEqual(amdsmi.amdsmi_wrapper.AMDSMI_VRAM_TYPE_HBM3E, 5) + self.assertEqual( + amdsmi.amdsmi_wrapper.amdsmi_vram_type_t__enumvalues[5], "AMDSMI_VRAM_TYPE_HBM3E" + ) + self.assertEqual(amdsmi.amdsmi_interface.AmdSmiVramType.HBM3E, 5) + + def test_hbm4(self): + self.assertEqual(amdsmi.amdsmi_wrapper.AMDSMI_VRAM_TYPE_HBM4, 6) + self.assertEqual( + amdsmi.amdsmi_wrapper.amdsmi_vram_type_t__enumvalues[6], "AMDSMI_VRAM_TYPE_HBM4" + ) + self.assertEqual(amdsmi.amdsmi_interface.AmdSmiVramType.HBM4, 6) diff --git a/projects/amdsmi/tests/python/unit/system/test_cli_default_driver.py b/projects/amdsmi/tests/python/unit/system/test_cli_default_driver.py new file mode 100644 index 000000000000..a89cef7d73e2 --- /dev/null +++ b/projects/amdsmi/tests/python/unit/system/test_cli_default_driver.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Bare amd-smi driver version row tests.""" + +import importlib.util +import io +import types +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +from common.common import stub_modules_at_import + +_DRIVER_INFO = { + "driver_name": "amdgpu", + "driver_date": "2026/09/16 00:00", + "driver_kernel_version": "6.19.14", + "driver_version": "6.19.14.31400000", + "driver_build_version": "2370381", + "driver_full_version": "6.19.14.31400000-2370381", +} + + +class _LibraryError(Exception): + def get_error_info(self) -> str: + return "not supported" + + +def _project_root() -> Path: + return Path(__file__).resolve().parents[4] + + +def _load_module(relative_path: str, module_name: str) -> types.ModuleType: + spec = importlib.util.spec_from_file_location(module_name, _project_root() / relative_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _raise(*_args: object, **_kwargs: object) -> None: + raise _LibraryError() + + +def _load_default_module() -> types.ModuleType: + interface = types.SimpleNamespace( + amdsmi_get_rocm_version=lambda: (True, "10.1.0"), + amdsmi_get_processor_handles=lambda: ["gpu0"], + amdsmi_get_gpu_driver_info=lambda _gpu: _DRIVER_INFO, + amdsmi_get_fw_info=_raise, + amdsmi_get_gpu_vbios_info=_raise, + amdsmi_get_gpu_metrics_info=_raise, + amdsmi_get_gpu_memory_partition=_raise, + amdsmi_get_gpu_accelerator_partition_profile=_raise, + amdsmi_get_gpu_asic_info=_raise, + amdsmi_get_gpu_device_bdf=_raise, + amdsmi_get_gpu_enumeration_info=_raise, + amdsmi_get_power_cap_info=_raise, + amdsmi_get_gpu_memory_usage=_raise, + amdsmi_get_gpu_memory_total=_raise, + amdsmi_get_gpu_total_ecc_count=_raise, + amdsmi_get_gpu_fan_speed=_raise, + amdsmi_get_gpu_fan_speed_max=_raise, + amdsmi_get_gpu_process_list=_raise, + _NA_amdsmi_get_gpu_metrics_info=lambda: "N/A", + ) + amdsmi = types.ModuleType("amdsmi") + amdsmi.amdsmi_interface = interface + amdsmi.amdsmi_exception = types.SimpleNamespace(AmdSmiLibraryException=_LibraryError) + version_metadata = types.ModuleType("_version") + version_metadata.__version__ = "1.0.0" + helpers_module = types.ModuleType("amdsmi_helpers") + helpers_module.AMDSMIHelpers = type("AMDSMIHelpers", (), {}) + + restore = stub_modules_at_import( + {"amdsmi": amdsmi, "_version": version_metadata, "amdsmi_helpers": helpers_module} + ) + try: + return _load_module("amdsmi_cli/subcommands/default.py", "default_driver_under_test") + finally: + restore() + + +def _run_default() -> dict: + module = _load_default_module() + commands = object.__new__(module.DefaultCommands) + + class _Logger: + def is_json_format(self) -> bool: + return True + + def is_csv_format(self) -> bool: + return False + + def print_output(self) -> None: + return None + + commands.logger = _Logger() + commands.helpers = types.SimpleNamespace( + is_amdgpu_initialized=lambda: True, + check_required_groups=lambda: None, + get_gpu_id_from_device_handle=lambda _gpu: 0, + get_apu_memory_type_and_name=lambda _gpu, _gpu_id: (0, "VRAM"), + convert_SI_unit=lambda value, _unit: value, + convert_bytes_to_readable=lambda value: str(value), + unit_format=lambda _logger, value, _unit: str(value), + ) + commands.group_check_printed = True + commands.default(types.SimpleNamespace()) + return commands.logger.output + + +def _load_logger_module() -> types.ModuleType: + helpers_module = types.ModuleType("amdsmi_helpers") + helpers_module.AMDSMIHelpers = type("AMDSMIHelpers", (), {}) + restore = stub_modules_at_import({"amdsmi_helpers": helpers_module}) + try: + return _load_module("amdsmi_cli/amdsmi_logger.py", "logger_driver_under_test") + finally: + restore() + + +def _banner_payload() -> dict: + return { + "version_info": { + "amd-smi": "27.1.0", + "amdgpu version": _DRIVER_INFO, + "kernel version": "6.8.0-124-generic", + "fw pldm version": "N/A", + "vbios version": "N/A", + "rocm version": (True, "10.1.0"), + }, + "gpu_info_list": [], + "processes": [], + } + + +class TestDefaultDriverHeader(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + root = _project_root() + for relative_path in ("amdsmi_cli/subcommands/default.py", "amdsmi_cli/amdsmi_logger.py"): + if not (root / relative_path).is_file(): + raise unittest.SkipTest(f"amd-smi CLI source not found at {root / relative_path}") + + def test_default_payload_uses_driver_info(self) -> None: + output = _run_default() + self.assertEqual(output["version_info"]["amdgpu version"], _DRIVER_INFO) + self.assertNotIn("amdgpu dkms version", output["version_info"]) + + def test_banner_prints_driver_rows_in_order(self) -> None: + module = _load_logger_module() + logger = module.AMDSMILogger( + helpers=types.SimpleNamespace(os_info=lambda: "Linux Baremetal") + ) + stdout = io.StringIO() + with redirect_stdout(stdout): + logger.print_default_output(_banner_payload()) + + lines = stdout.getvalue().splitlines() + amdgpu_index = next( + i for i, line in enumerate(lines) if line.startswith("| amdgpu Version:") + ) + self.assertIn("6.19.14.31400000-2370381", lines[amdgpu_index]) + self.assertTrue(lines[amdgpu_index + 1].startswith("| ROCm Version:")) + self.assertFalse(any(line.startswith("| Kernel Version:") for line in lines)) + self.assertFalse(any(line.startswith("| Driver Version:") for line in lines)) + self.assertFalse(any(line.startswith("| Build Version:") for line in lines)) + + def test_banner_uses_os_kernel_when_driver_info_is_unavailable(self) -> None: + module = _load_logger_module() + logger = module.AMDSMILogger( + helpers=types.SimpleNamespace(os_info=lambda: "Linux Baremetal") + ) + payload = _banner_payload() + payload["version_info"]["amdgpu version"] = "N/A" + stdout = io.StringIO() + + with redirect_stdout(stdout): + logger.print_default_output(payload) + + lines = stdout.getvalue().splitlines() + os_kernel_line = next(line for line in lines if line.startswith("| OS kernel Version:")) + self.assertIn("6.8.0-124-generic", os_kernel_line) + self.assertFalse(any(line.startswith("| amdgpu Version:") for line in lines)) diff --git a/projects/amdsmi/tests/python/unit/system/test_cli_empty_section_na.py b/projects/amdsmi/tests/python/unit/system/test_cli_empty_section_na.py index 9e471a809c0e..7e898a58b32d 100644 --- a/projects/amdsmi/tests/python/unit/system/test_cli_empty_section_na.py +++ b/projects/amdsmi/tests/python/unit/system/test_cli_empty_section_na.py @@ -13,53 +13,42 @@ ``AMDSMI_SPACING_REMOVAL`` marker and strips it afterwards by literal match, so the marker must never itself become an empty dict. -``amdsmi_logger.py`` is loaded from the source tree so the test exercises the code -under development rather than a possibly-stale installed copy. +``amdsmi_logger.py`` is loaded through cli_search_order, so a plain checkout +exercises the code under development; an explicit AMDSMI_PATH pins that install. """ -import importlib.util import os -import sys -import types import unittest -_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -_REPO_ROOT = os.path.abspath(os.path.join(_THIS_DIR, "..", "..", "..", "..")) -LOGGER_PATH = os.path.join(_REPO_ROOT, "amdsmi_cli", "amdsmi_logger.py") - - -def _install_fake_helpers(): - """Register a stub ``amdsmi_helpers`` so ``amdsmi_logger`` imports cleanly.""" - module = types.ModuleType("amdsmi_helpers") - module.AMDSMIHelpers = type("AMDSMIHelpers", (), {}) - sys.modules["amdsmi_helpers"] = module - +from common.common import ( + amdsmi_path, + cli_search_order, + fake_module, + find_cli_dir, + load_cli_module, + stub_modules, +) -def _restore_helpers(saved): - """Undo ``_install_fake_helpers`` so the stub does not leak into sibling suites.""" - if saved is None: - sys.modules.pop("amdsmi_helpers", None) - else: - sys.modules["amdsmi_helpers"] = saved +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CLI_DIR = find_cli_dir(*cli_search_order(_THIS_DIR)) +LOGGER_PATH = os.path.join(_CLI_DIR, "amdsmi_logger.py") if _CLI_DIR else None def _load_logger_module(): - spec = importlib.util.spec_from_file_location("amdsmi_logger_under_test", LOGGER_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("amdsmi_logger_under_test", LOGGER_PATH) class TestCliEmptySectionNA(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(LOGGER_PATH): - raise unittest.SkipTest(f"amdsmi_logger not found at {LOGGER_PATH}") + if not LOGGER_PATH or not os.path.isfile(LOGGER_PATH): + raise unittest.SkipTest( + f"amd-smi CLI amdsmi_logger.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) # Registered before the stub is installed, so the restore still runs if # loading the logger raises: unittest skips tearDownClass on that path. - saved = sys.modules.get("amdsmi_helpers") - cls.addClassCleanup(_restore_helpers, saved) - _install_fake_helpers() + helpers = fake_module("amdsmi_helpers", AMDSMIHelpers=type("AMDSMIHelpers", (), {})) + stub_modules(cls, {"amdsmi_helpers": helpers}) module = _load_logger_module() cls.logger = module.AMDSMILogger.__new__(module.AMDSMILogger) @@ -102,7 +91,3 @@ def test_payload_of_only_a_device_key_renders_just_that_key(self): def test_empty_payload_renders_nothing(self): self.assertEqual(self._render({}), "") - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/system/test_cli_exit_codes.py b/projects/amdsmi/tests/python/unit/system/test_cli_exit_codes.py new file mode 100644 index 000000000000..8c66a28fc2ed --- /dev/null +++ b/projects/amdsmi/tests/python/unit/system/test_cli_exit_codes.py @@ -0,0 +1,2319 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Driver-free unit tests for the amd-smi CLI exit-code model. + +Ported from the pre-migration ``tests/python_unittest/unit_tests.py`` monolith +into the ``tests/python/unit/`` tree. Exercises the record-then-finalize error +model (``AmdSmiErrorCollector``, the library-status -> POSIX-byte mapping, severity +split, and the reserved 193-253 CLI-only code band) without needing a +GPU/CPU/Core or elevated permissions. +""" + +import os +import pathlib +import sys +import unittest +from typing import Any + +from common.common import ( + amdsmi, + cli_search_order, + fake_module, + find_cli_dir, + generated_version_stub, + stub_modules_at_import, +) + +# Locate the CLI dir and add it to sys.path. cli_search_order() decides whether +# the install or this checkout wins; None -> setUpModule() skips. +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +if _CLI_DIR and _CLI_DIR not in sys.path: + sys.path.append(_CLI_DIR) + + +def _find_amdsmi_header(*start_dirs): + """Return the path to amdsmi.h, or None. + + Walks up rather than indexing a fixed depth, because the header sits one + level above the CLI in a checkout and two in an install. Feed it + cli_search_order() so install-vs-source stays decided in that one place. + """ + for start in start_dirs: + if not start: + continue + start_path = pathlib.Path(os.path.abspath(start)) + for directory in (start_path, *start_path.parents): + candidate = directory / "include" / "amd_smi" / "amdsmi.h" + if candidate.is_file(): + return str(candidate) + return None + + +# Required for when the amdgpu driver is not loaded. We are required to +# fake the initialization module so the set/reset gpu CLI commands can be ran. +# Installed at import because amdsmi_helpers does ``from amdsmi_init import *`` +# at load; tearDownModule puts the originals back. +_stubs = {} +if "amdsmi_init" not in sys.modules: + from amdsmi import amdsmi_exception as _amdsmi_exception + from amdsmi import amdsmi_interface as _amdsmi_interface + + _stubs["amdsmi_init"] = fake_module( + "amdsmi_init", + AMDSMI_INIT_FLAG=0, + AMDSMI_INITIALIZED=True, + amdsmi_interface=_amdsmi_interface, + amdsmi_exception=_amdsmi_exception, + ) + +_stubs.update(generated_version_stub()) + +_restore_stubs = stub_modules_at_import(_stubs) if _stubs else None + +# CLI absent (rare package-only layout) or too old -> record the reason and skip +# in setUpModule() instead of failing collection. The alias is declared Any +# and bound separately from the import: the None only exists to defer that skip, +# so no test ever sees it, and typing it Any keeps that out of every call site. +cli_exc: Any = None +_CLI_IMPORT_ERROR = None +try: + import amdsmi_cli_exceptions as _cli_exc_module # noqa: E402 + + cli_exc = _cli_exc_module +except ImportError as _cli_import_error: + _CLI_IMPORT_ERROR = _cli_import_error +else: + # A CLI predating exit codes imports cleanly, so guarding ImportError alone + # lets collection reach cli_exc.AmdSmiExitCode and die on AttributeError. + if not hasattr(cli_exc, "AmdSmiExitCode"): + _CLI_IMPORT_ERROR = AttributeError( + "amdsmi_cli_exceptions has no AmdSmiExitCode; this CLI predates exit-code support" + ) + cli_exc = None + +from amdsmi import amdsmi_wrapper # noqa: E402 + +# The set-dispatch and parser tests below drive real CLI subcommand/parser +# modules. Resolve them from the same env-var-discovered CLI dir (find_cli_dir, +# which honors AMDSMI_PATH/ROCM_HOME/ROCM_PATH via common.py) and import them +# guarded, so a missing or unimportable CLI SKIPS those tests with a message -- +# mirroring the metric-partition tests -- instead of erroring mid-test. +_SUBCOMMANDS_DIR = os.path.join(_CLI_DIR, "subcommands") if _CLI_DIR else None +if _SUBCOMMANDS_DIR and _SUBCOMMANDS_DIR not in sys.path: + sys.path.append(_SUBCOMMANDS_DIR) +cli_set_value: Any = None +AMDSMIParser: Any = None +_CLI_DRIVE_SKIP = None +try: + import set_value as _set_value_module # noqa: E402 + from amdsmi_parser import AMDSMIParser as _AMDSMIParser # noqa: E402 + + cli_set_value = _set_value_module + AMDSMIParser = _AMDSMIParser +except ImportError as _cli_drive_import_error: + _CLI_DRIVE_SKIP = ( + f"amd-smi CLI drive modules not importable ({_CLI_DIR}): {_cli_drive_import_error}" + ) + + +def setUpModule(): + """Skip the suite when the amd-smi CLI is missing or too old to carry exit codes.""" + if _CLI_IMPORT_ERROR is not None: + raise unittest.SkipTest(f"amd-smi CLI at {_CLI_DIR} unusable: {_CLI_IMPORT_ERROR}") + + +def tearDownModule(): + if _restore_stubs is not None: + _restore_stubs() + + +class TestAmdSmiCliExitCodes(unittest.TestCase): + """Pure-logic tests for the amd-smi CLI exit-code model. + + Exercises the record-then-finalize error model in amdsmi_cli_exceptions + without needing a GPU/CPU/Core or elevated permissions: + the AmdSmiErrorCollector, the library-status -> POSIX-byte mapping, exit codes surfaced by + AmdSmiLibraryErrorException, the FATAL vs DEVICE severity split, and the + reserved 193-253 CLI-only code band. + """ + + ExitCode: Any = cli_exc.AmdSmiExitCode if cli_exc is not None else None + Severity: Any = cli_exc.AmdSmiErrorSeverity if cli_exc is not None else None + + # ---- AmdSmiErrorCollector / finalize ---- + def test_no_failures_resolve_to_success(self): + collector = cli_exc.AmdSmiErrorCollector() + self.assertEqual(collector.resolve_exit_code(), int(self.ExitCode.SUCCESS)) + self.assertFalse(collector.has_errors) + + def test_all_same_code_resolves_to_that_code(self): + collector = cli_exc.AmdSmiErrorCollector() + collector.record(self.ExitCode.INVALID_PARAMETER_VALUE) + collector.record(self.ExitCode.INVALID_PARAMETER_VALUE) + self.assertTrue(collector.has_errors) + self.assertEqual(collector.resolve_exit_code(), int(self.ExitCode.INVALID_PARAMETER_VALUE)) + + def test_mixed_codes_resolve_to_mixed(self): + collector = cli_exc.AmdSmiErrorCollector() + collector.record(self.ExitCode.INVALID_PARAMETER_VALUE) + collector.record(self.ExitCode.DEVICE_NOT_FOUND) + self.assertEqual(collector.resolve_exit_code(), int(self.ExitCode.MIXED_DEVICE_ERRORS)) + + def test_reset_clears_recorded_codes(self): + collector = cli_exc.AmdSmiErrorCollector() + collector.record(self.ExitCode.DEVICE_NOT_FOUND) + collector.reset() + self.assertFalse(collector.has_errors) + self.assertEqual(collector.resolve_exit_code(), int(self.ExitCode.SUCCESS)) + + def test_record_library_error_uses_status_value(self): + collector = cli_exc.AmdSmiErrorCollector() + collector.record_library_error(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED) + self.assertEqual( + collector.resolve_exit_code(), int(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED) + ) + + # ---- library_code_to_exit_code ---- + def test_real_status_passes_through_as_exit_code(self): + # Real statuses (0-56) fit in a byte, so the exit code is the status. + for status in ( + amdsmi_wrapper.AMDSMI_STATUS_INVAL, + amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED, + amdsmi_wrapper.AMDSMI_STATUS_NO_PERM, + amdsmi_wrapper.AMDSMI_STATUS_CORRUPTED_EEPROM, + ): + self.assertEqual(cli_exc.library_code_to_exit_code(status), int(status)) + + def test_sentinel_statuses_fold_to_low_byte(self): + self.assertEqual( + cli_exc.library_code_to_exit_code(amdsmi_wrapper.AMDSMI_STATUS_UNKNOWN_ERROR), 255 + ) + self.assertEqual( + cli_exc.library_code_to_exit_code(amdsmi_wrapper.AMDSMI_STATUS_MAP_ERROR), 254 + ) + + def test_library_exception_surfaces_status(self): + exc = cli_exc.AmdSmiLibraryErrorException("", amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED) + self.assertEqual(exc.value, int(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED)) + + def test_every_library_status_has_a_friendly_message(self): + """Every AMDSMI_STATUS_* the wrapper defines must have its own entry in + AMDSMI_ERROR_MESSAGES. + + Guard: if a new status is added to the library but the message table + isn't updated, this test fails and names the missing codes. That forces + the author to either map the new code here, give it a CLI exit code, or + deliberately alias it to an existing message -- a "generic"/unrecognized + fallback is never an acceptable resting state for a defined status. + """ + missing = [ + f"{name} ({code})" + for code, name in amdsmi_wrapper.amdsmi_status_t__enumvalues.items() + if abs(code) not in cli_exc.AMDSMI_ERROR_MESSAGES + ] + self.assertEqual( + missing, + [], + f"AMDSMI_ERROR_MESSAGES is missing friendly messages for: {', '.join(missing)}", + ) + + def test_cli_command_not_supported_is_distinct_from_library(self): + # The CLI's parse-time "command not supported" must use its own CLI code, + # NOT the library status AMDSMI_STATUS_NOT_SUPPORTED (2), so callers can + # tell a CLI rejection apart from a device/driver "not supported" result. + exc = cli_exc.AmdSmiCommandNotSupportedException("static", "") + self.assertEqual(exc.value, int(self.ExitCode.COMMAND_NOT_SUPPORTED)) + self.assertNotEqual(exc.value, int(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED)) + + def test_import_failure_exits_with_import_error_code(self): + """A failed ``import amdsmi`` must exit with AmdSmiExitCode.IMPORT_ERROR, + pulled from the enum (no stale hardcoded number). + + Runs in a subprocess because the failure path calls ``sys.exit`` at + import time; a meta_path finder forces ``import amdsmi`` to fail + hermetically without touching the real install. + """ + import subprocess + + cli_dir = os.path.dirname(os.path.abspath(cli_exc.__file__)) + child = ( + "import sys\n" + "class _Block:\n" + " def find_spec(self, name, path=None, target=None):\n" + " if name == 'amdsmi' or name.startswith('amdsmi.'):\n" + " raise ImportError('amdsmi blocked for test')\n" + " return None\n" + "sys.meta_path.insert(0, _Block())\n" + f"sys.path.insert(0, {cli_dir!r})\n" + "import amdsmi_init\n" + ) + result = subprocess.run([sys.executable, "-c", child], capture_output=True, text=True) + self.assertEqual( + result.returncode, + int(self.ExitCode.IMPORT_ERROR), + msg=f"stdout={result.stdout!r} stderr={result.stderr!r}", + ) + + # ---- severity classification ---- + def test_library_error_is_device_severity(self): + # A per-device library failure must be recorded (not abort the command). + self.assertEqual(cli_exc.AmdSmiLibraryErrorException.severity, self.Severity.DEVICE) + + def test_cli_errors_default_to_fatal_severity(self): + # Anything not explicitly per-device stops the whole command. + self.assertEqual(cli_exc.AmdSmiException.severity, self.Severity.FATAL) + self.assertEqual(cli_exc.AmdSmiRequiredCommandException.severity, self.Severity.FATAL) + + # ---- CLI-only exit-code band ---- + def test_cli_only_codes_live_above_library_range(self): + # CLI-invented codes sit above the library status range (max 56) so they + # can never be mistaken for / collide with a real status. The band + # bounds come from amdsmi_cli_exceptions, not hardcoded here. + band_start = cli_exc.CLI_EXIT_CODE_BAND_START + band_end = cli_exc.CLI_EXIT_CODE_BAND_END + for code in self.ExitCode: + if code is self.ExitCode.SUCCESS: + continue + self.assertGreaterEqual( + int(code), band_start, f"{code.name} not in {band_start}-{band_end} band" + ) + self.assertLessEqual( + int(code), band_end, f"{code.name} not in {band_start}-{band_end} band" + ) + + def test_cli_band_sits_clear_of_signal_exit_codes(self): + """A signalled process is reported by the shell as 128 + signal number, + so 129-192 (SIGRTMAX = 64) can never be a CLI code: a caller reading 139 (128 + SIGSEGV(11)) + has to be able to conclude a segfault and nothing else. + """ + import signal + + signal_exit_codes = {128 + int(sig) for sig in signal.Signals} + for code in self.ExitCode: + self.assertNotIn( + int(code), + signal_exit_codes, + f"{code.name} ({int(code)}) collides with a 128+signal exit code", + ) + self.assertGreater(cli_exc.CLI_EXIT_CODE_BAND_START, max(signal_exit_codes)) + + def test_library_status_outside_its_range_reports_unrepresentable(self): + """The library owns 0-128 plus its two 32-bit sentinels. A value outside + that cannot survive being squeezed into a byte -- 256 would report + success and 300 would report status 44, a real but unrelated failure -- + so it reports UNREPRESENTABLE_LIBRARY_STATUS rather than a plausible + wrong answer. + """ + to_exit = cli_exc.library_code_to_exit_code + unrepresentable = int(self.ExitCode.UNREPRESENTABLE_LIBRARY_STATUS) + + self.assertEqual(to_exit(cli_exc.AMDSMI_STATUS_MAP_ERROR), 254) + self.assertEqual(to_exit(cli_exc.AMDSMI_STATUS_UNKNOWN_ERROR), 255) + + # Every real status still passes through as itself; the rule change is + # only visible for values the library has never used. + sentinels = (cli_exc.AMDSMI_STATUS_MAP_ERROR, cli_exc.AMDSMI_STATUS_UNKNOWN_ERROR) + real_statuses = [ + abs(int(status)) + for status in amdsmi_wrapper.amdsmi_status_t__enumvalues + if abs(int(status)) not in sentinels + ] + self.assertTrue(real_statuses, "walked zero library statuses -- the wrapper enum is empty") + for value in real_statuses: + self.assertEqual(to_exit(value), value) + + # The boundary itself is not a defined status, so the loop cannot reach it. + self.assertEqual(to_exit(cli_exc.LIBRARY_STATUS_MAX), cli_exc.LIBRARY_STATUS_MAX) + + for status in ( + cli_exc.LIBRARY_STATUS_MAX + 1, # first value past the library's half of the byte + cli_exc.CLI_EXIT_CODE_BAND_START - 1, # last signal code, just below the CLI band + 256, # folds to 0 -- would report success + 300, # folds to 44 -- a real but unrelated status + 0x10000, # 32-bit, but not one of the known sentinels + ): + with self.subTest(status=status): + self.assertEqual(to_exit(status), unrepresentable) + + def test_unrepresentable_library_status_stays_unreachable(self): + """UNREPRESENTABLE_LIBRARY_STATUS is a guard, not a code to emit. + + Reaching it means the library outgrew the byte: every status past + LIBRARY_STATUS_MAX collapses onto that one value, so callers can no + longer tell those failures apart from the exit code. The guard keeps + that from being silent, and this keeps it from being permanent. + + The passthrough loop above would also fail in that case, but only with a + bare value mismatch. This names the offending statuses and what to do. + """ + sentinels = (cli_exc.AMDSMI_STATUS_MAP_ERROR, cli_exc.AMDSMI_STATUS_UNKNOWN_ERROR) + too_large = sorted( + f"{name} ({abs(int(code))})" + for code, name in amdsmi_wrapper.amdsmi_status_t__enumvalues.items() + if abs(int(code)) not in sentinels and abs(int(code)) > cli_exc.LIBRARY_STATUS_MAX + ) + self.assertEqual( + too_large, + [], + f"these statuses no longer fit the library's 0-{cli_exc.LIBRARY_STATUS_MAX} range " + f"and would all report {int(self.ExitCode.UNREPRESENTABLE_LIBRARY_STATUS)}: " + f"{', '.join(too_large)}. The exit-code map in amdsmi_cli_exceptions needs " + "revisiting -- a byte can no longer carry a status one-to-one.", + ) + + def test_python_wrapper_matches_amdsmi_status_t_in_the_header(self): + """amdsmi.h is the source of truth for AMDSMI_STATUS_*; the Python + wrapper is generated from it. + + Nothing else can catch a stale wrapper: every other test reads the + wrapper, so a wrapper that is missing a status the library already + returns is perfectly self-consistent and therefore invisible. This is + the one place that compares the two, so adding a status in C without + regenerating fails here instead of surfacing later as an unrecognized + code at runtime. + """ + import re + + header = _find_amdsmi_header(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) + if header is None: + self.skipTest("amdsmi.h not found in the source tree or the ROCm install") + + with open(header, encoding="utf-8") as handle: + source = handle.read() + block = re.search(r"typedef enum\s*\{(.*?)\}\s*amdsmi_status_t", source, re.S) + self.assertIsNotNone(block, f"could not locate amdsmi_status_t in {header}") + + in_header = { + name: int(value, 0) + for name, value in re.findall( + r"(AMDSMI_STATUS_\w+)\s*=\s*(0x[0-9A-Fa-f]+|\d+)", block.group(1) + ) + } + self.assertTrue(in_header, f"parsed no statuses out of {header}") + in_wrapper = { + name: abs(int(code)) + for code, name in amdsmi_wrapper.amdsmi_status_t__enumvalues.items() + } + + self.assertEqual( + sorted(set(in_header) - set(in_wrapper)), + [], + "in amdsmi.h but missing from the Python wrapper -- regenerate it", + ) + self.assertEqual( + sorted(set(in_wrapper) - set(in_header)), + [], + "in the Python wrapper but not in amdsmi.h -- the wrapper is ahead of the header", + ) + self.assertEqual( + { + name: (in_header[name], in_wrapper[name]) + for name in set(in_header) & set(in_wrapper) + if in_header[name] != in_wrapper[name] + }, + {}, + "status value differs between amdsmi.h and the Python wrapper (header, wrapper)", + ) + + def test_exit_codes_are_unique(self): + """Two members sharing a value is a silent bug: the second becomes an + alias of the first and vanishes from enum iteration, so iterating + self.ExitCode can never see the duplicate. __members__ keeps alias + names, so it is the only view that can catch this. + """ + by_value = {} + for name, code in self.ExitCode.__members__.items(): + by_value.setdefault(int(code), []).append(name) + collisions = {value: names for value, names in by_value.items() if len(names) > 1} + self.assertEqual( + collisions, + {}, + "exit-code values must be unique: " + + "; ".join( + f"{value} shared by {', '.join(names)}" for value, names in collisions.items() + ), + ) + + def test_cli_codes_never_collide_with_library_exit_codes(self): + """Every exit code must mean exactly one thing: either a library status + or a CLI-invented code, never both. + + A band check alone isn't enough, because the two library sentinels also + land above the CLI's 193-253 band (AMDSMI_STATUS_MAP_ERROR -> 254, + AMDSMI_STATUS_UNKNOWN_ERROR -> 255). So this asserts directly that no CLI + code equals any exit code a library status can produce -- otherwise a + caller couldn't tell a library failure from a CLI one by exit code alone. + SUCCESS (0) is shared with the library's success status by design and is + exempt. + + Because it checks every CLI code, this also covers the normalized init + codes: INIT_TIMEOUT, DRIVERS_NOT_LOADED and DEVICE_NOT_FOUND are proven + distinct from the library statuses they map (TIMEOUT, NOT_INIT, + DRIVER_NOT_LOADED), so no separate per-code test is needed. + """ + library_exit_codes = { + cli_exc.library_code_to_exit_code(code) + for code in amdsmi_wrapper.amdsmi_status_t__enumvalues + } + for code in self.ExitCode: + if code is self.ExitCode.SUCCESS: + continue + self.assertNotIn( + int(code), + library_exit_codes, + f"CLI code {code.name} ({int(code)}) collides with a library status exit code", + ) + + # ---- store_device_error choke point (regression guard) ---- + def test_store_device_error_shows_and_records_failure(self): + """A device failure routed through store_device_error must both display + the message AND record the code, so the process exit stays non-zero. + + This guards the record-then-finalize fix for set/reset: forgetting the + record (the original `set -M` bug) would leave resolve_exit_code() at 0. + """ + from amdsmi_helpers import AMDSMIHelpers + + class _FakeLogger: + def __init__(self): + self.stored = [] + + def store_output(self, device, key, value): + self.stored.append((device, key, value)) + + class _FakeLibErr(Exception): + def get_error_code(self): + return amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED + + helpers = AMDSMIHelpers() + logger = _FakeLogger() + helpers.store_device_error( + logger, + 0, + "memory_partition", + "[AMDSMI_STATUS_NOT_SUPPORTED] ...", + exception=_FakeLibErr(), + ) + # Shown to the user... + self.assertEqual(logger.stored[-1][1], "memory_partition") + # ...and counted, so the exit code reflects the failure. + self.assertTrue(helpers.error_collector.has_errors) + self.assertEqual( + helpers.error_collector.resolve_exit_code(), + int(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED), + ) + + def test_store_device_error_cli_code_path(self): + """A CLI-level (non-library) failure records the given CLI exit code.""" + from amdsmi_helpers import AMDSMIHelpers + + class _FakeLogger: + def store_output(self, device, key, value): + pass + + helpers = AMDSMIHelpers() + helpers.store_device_error( + _FakeLogger(), + 0, + "clk_limit", + "Invalid clock type", + code=int(self.ExitCode.INVALID_PARAMETER_VALUE), + ) + self.assertEqual( + helpers.error_collector.resolve_exit_code(), int(self.ExitCode.INVALID_PARAMETER_VALUE) + ) + + def test_store_device_error_requires_exception_or_code(self): + """Calling store_device_error with neither exception= nor code= must + raise TypeError, not silently show the message and record nothing. + + Guards the guard: showing a per-device error without recording it would + leave resolve_exit_code() at 0 (the original silent exit-0 bug). This + locks the fail-fast behavior in so it can't be quietly removed. + """ + from amdsmi_helpers import AMDSMIHelpers + + class _FakeLogger: + def store_output(self, device, key, value): + pass + + helpers = AMDSMIHelpers() + with self.assertRaises(TypeError): + helpers.store_device_error(_FakeLogger(), 0, "key", "msg") + # Nothing should have been recorded when the guard fires. + self.assertFalse(helpers.error_collector.has_errors) + + # ---- multi-field CPU/core set aggregation ---- + def test_set_core_records_each_failed_field_to_mixed(self): + """Drive the real set_core path: two fields fail with DIFFERENT library + statuses. set_core must record BOTH (not collapse to the first) and the + collector must finalize to MIXED_DEVICE_ERRORS. This exercises the + actual per-field record-then-finalize code in set_core, not just the + collector in isolation. + + handle_cores short-circuits on a non-list handle, and the id lookup and + the two library set calls are replaced with stand-ins that raise the two + errors under test. + """ + import argparse + from unittest import mock + + from amdsmi_helpers import AMDSMIHelpers + + set_value = _load_set_value() + + class _FakeLogger: + format = "human" + + def store_core_output(self, device, key, value): + pass + + def print_output(self, multiple_device_enabled=False): + pass + + def store_multiple_device_output(self): + pass + + # A library exception that needs no driver: subclass the real type (so + # set_core's `except amdsmi_exception.AmdSmiLibraryException` catches it) + # but skip its C-backed __init__. + class _FakeLibErr(amdsmi.AmdSmiLibraryException): + def __init__(self, code): + self._code = code + + def get_error_code(self): + return self._code + + def get_error_info(self, detailed=True): + return f"status {self._code}" + + def _raise(code): + def _inner(*args, **kwargs): + raise _FakeLibErr(code) + + return _inner + + cmd: Any = set_value.SetValueCommands() + cmd.helpers = AMDSMIHelpers() + cmd.logger = _FakeLogger() + # Avoid the driver-backed core-id lookup. + cmd.helpers.get_core_id_from_device_handle = lambda input_device_handle: 0 + + args = argparse.Namespace( + core="fake-core-handle", # non-list, non-None -> handle_cores returns (False, handle) + core_boost_limit=[[100]], + core_floor_limit=[[100]], + core_msr_floor_limit=None, + ) + + patch_boost = mock.patch.object( + set_value.amdsmi_interface, + "amdsmi_set_cpu_core_boostlimit", + _raise(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED), + ) + patch_floor = mock.patch.object( + set_value.amdsmi_interface, + "amdsmi_set_cpu_core_floor_freq_limit", + _raise(amdsmi_wrapper.AMDSMI_STATUS_INVAL), + ) + with patch_boost, patch_floor: + cmd.set_core(args) + + self.assertTrue(cmd.helpers.error_collector.has_errors) + self.assertEqual( + cmd.helpers.error_collector.resolve_exit_code(), int(self.ExitCode.MIXED_DEVICE_ERRORS) + ) + + # ---- multi-DEVICE aggregation (drives handle_cores/run_device_subcommand) ---- + def test_handle_cores_aggregates_multiple_device_failures_to_mixed(self): + """Two cores each fail with a DIFFERENT library status. handle_cores + routes each through run_device_subcommand, which records per device and + keeps going; the collector finalizes to MIXED_DEVICE_ERRORS. Covers the + multi-DEVICE record-then-continue path (the set_core test covers the + multi-FIELD path). + """ + import argparse + + from amdsmi_helpers import AMDSMIHelpers + + class _FakeLogger: + def print_output(self, multiple_device_enabled=False): + pass + + class _FakeLibErr(amdsmi.AmdSmiLibraryException): + def __init__(self, code): + self._code = code + + def get_error_code(self): + return self._code + + codes = { + "core-0": amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED, + "core-1": amdsmi_wrapper.AMDSMI_STATUS_INVAL, + } + + def fake_subcommand(args, multiple_devices=False, core=""): + raise _FakeLibErr(codes[core]) + + helpers = AMDSMIHelpers() + args = argparse.Namespace(core=["core-0", "core-1"]) + handled, _ = helpers.handle_cores(args, _FakeLogger(), fake_subcommand) + self.assertTrue(handled) + self.assertEqual( + helpers.error_collector.resolve_exit_code(), int(self.ExitCode.MIXED_DEVICE_ERRORS) + ) + + def test_handle_cores_and_handle_cpus_return_on_empty_list(self): + """An empty args.core/args.cpu must not fall through to an implicit + None -- every caller does `handled, handle = helpers.handle_cores(...)`, + which raises TypeError trying to unpack None. Only the bool matters + here; callers never read the 2nd value when it's True (see handle_gpus). + """ + import argparse + + from amdsmi_helpers import AMDSMIHelpers + + class _FakeLogger: + def print_output(self, multiple_device_enabled=False): + pass + + helpers = AMDSMIHelpers() + handled, _ = helpers.handle_cores( + argparse.Namespace(core=[]), _FakeLogger(), lambda *a, **k: None + ) + self.assertTrue(handled) + handled, _ = helpers.handle_cpus( + argparse.Namespace(cpu=[]), _FakeLogger(), lambda *a, **k: None + ) + self.assertTrue(handled) + + def test_handle_gpus_aggregates_multiple_device_failures_to_mixed(self): + """`-g all` counterpart of the handle_cores test above. Guards + handle_gpus' own gating arithmetic: only a list of length > 1 enters the + loop, so an off-by-one there would silently skip every device. + """ + import argparse + + from amdsmi_helpers import AMDSMIHelpers + + class _FakeLogger: + def __init__(self): + self.printed_multiple = False + + def print_output(self, multiple_device_enabled=False): + self.printed_multiple = multiple_device_enabled + + class _FakeLibErr(amdsmi.AmdSmiLibraryException): + def __init__(self, code): + self._code = code + + def get_error_code(self): + return self._code + + codes = { + "gpu-0": amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED, + "gpu-1": amdsmi_wrapper.AMDSMI_STATUS_INVAL, + } + visited = [] + + def fake_subcommand(args, multiple_devices=False, gpu=""): + visited.append(gpu) + raise _FakeLibErr(codes[gpu]) + + helpers = AMDSMIHelpers() + logger = _FakeLogger() + args = argparse.Namespace(gpu=["gpu-0", "gpu-1"]) + handled, _ = helpers.handle_gpus(args, logger, fake_subcommand) + self.assertTrue(handled) + self.assertEqual(visited, ["gpu-0", "gpu-1"], "loop did not visit every device") + self.assertTrue(logger.printed_multiple) + self.assertEqual( + helpers.error_collector.resolve_exit_code(), int(self.ExitCode.MIXED_DEVICE_ERRORS) + ) + + def test_nic_and_switch_loops_record_device_failures_like_gpus(self): + """NIC and switch counterpart of the handle_gpus test above. These three + loops called the subcommand directly, so the first device-scoped failure + escaped: later devices were never visited, and the output already + buffered for the healthy ones died with the unreached final print. + """ + import argparse + from unittest import mock + + import amdsmi_helpers + from amdsmi_helpers import AMDSMIHelpers + + class _FakeLogger: + def __init__(self): + self.printed_multiple = False + + def print_output(self, multiple_device_enabled=False): + self.printed_multiple = multiple_device_enabled + + class _FakeLibErr(amdsmi.AmdSmiLibraryException): + def __init__(self, code): + self._code = code + + def get_error_code(self): + return self._code + + interface = amdsmi_helpers.amdsmi_interface + # Two devices with different codes: one is too few to enter the loop at + # all, and differing codes are what resolve to MIXED_DEVICE_ERRORS. + device_failures = { + "dev-0": amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED, + "dev-1": amdsmi_wrapper.AMDSMI_STATUS_INVAL, + } + devices = list(device_failures) + + for handler, device_arg, processor_type in ( + ( + AMDSMIHelpers.handle_switchs, + "switch", + amdsmi_wrapper.AMDSMI_PROCESSOR_TYPE_BRCM_SWITCH, + ), + (AMDSMIHelpers.handle_brcm_nics, "nic", amdsmi_wrapper.AMDSMI_PROCESSOR_TYPE_BRCM_NIC), + (AMDSMIHelpers.handle_ainics, "nic", amdsmi_wrapper.AMDSMI_PROCESSOR_TYPE_AMD_NIC), + ): + with self.subTest(handler=handler.__name__): + calls = [] + + def fake_subcommand(args, multiple_devices=False, **device_kwarg): + (device,) = device_kwarg.values() + calls.append((multiple_devices, dict(device_kwarg))) + raise _FakeLibErr(device_failures[device]) + + helpers = AMDSMIHelpers() + logger = _FakeLogger() + args = argparse.Namespace(**{device_arg: devices}) + type_name = interface.AmdSmiProcessorType(processor_type).name + + with mock.patch.object( + interface, + "amdsmi_get_processor_type", + return_value={"processor_type": type_name}, + ): + handled, _ = handler(helpers, args, logger, fake_subcommand) + + self.assertTrue(handled) + self.assertEqual( + [kwargs for _, kwargs in calls], + [{device_arg: device} for device in devices], + f"loop did not visit every device as {device_arg}=", + ) + self.assertEqual( + [flag for flag, _ in calls], + [True] * len(devices), + "subcommand lost multiple_devices=True", + ) + self.assertTrue( + logger.printed_multiple, + "final print_output was skipped, so buffered device output is dropped", + ) + self.assertEqual( + helpers.error_collector.resolve_exit_code(), + int(self.ExitCode.MIXED_DEVICE_ERRORS), + ) + + def test_device_not_found_call_sites_name_their_device_kind(self): + """The parser builds this exception in five places and none of them runs + in any suite, so a bad signature ships silently: the NIC and switch sites + passed two arguments and raised TypeError instead of exiting + DEVICE_NOT_FOUND. Checked by reading the source, not running it. + """ + import ast + import pathlib + + def dotted_name(node): + """Rebuild "pkg.Class.MEMBER" from an AST attribute chain.""" + parts = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + if isinstance(node, ast.Name): + parts.append(node.id) + return ".".join(reversed(parts)) + + parser_source = pathlib.Path(_CLI_DIR) / "amdsmi_parser.py" + self.assertTrue(parser_source.is_file(), f"cannot read {parser_source}") + + raise_sites = [ + node + for node in ast.walk(ast.parse(parser_source.read_text())) + if isinstance(node, ast.Call) + and dotted_name(node.func).endswith("AmdSmiDeviceNotFoundException") + ] + self.assertTrue(raise_sites, "matched no call sites -- this check would pass vacuously") + + known_kinds = [f"AmdSmiDeviceKind.{name}" for name in cli_exc.AmdSmiDeviceKind.__members__] + for call in raise_sites: + with self.subTest(line=call.lineno): + self.assertEqual( + len(call.args), 3, "expected (handles, output_format, device_kind)" + ) + device_kind = dotted_name(call.args[2]) + self.assertTrue( + any(device_kind.endswith(kind) for kind in known_kinds), + f"device_kind is {device_kind or ast.dump(call.args[2])!r}, " + f"expected one of {known_kinds}", + ) + + def test_power_cap_out_of_range_records_invalid_parameter_value(self): + """validate_and_set_power_cap owns its recording; the set_gpu handler + does none. Out of range must record and return a message rather than + raise, so a multi-GPU loop still tries devices with a different range. + """ + from unittest import mock + + import amdsmi_helpers + + class _FakeLogger: + def is_json_format(self): + return False + + def is_csv_format(self): + return False + + helpers = amdsmi_helpers.AMDSMIHelpers() + helpers.get_gpu_id_from_device_handle = lambda *a, **k: 0 + min_w, max_w, current_w = 200, 300, 250 # _PCAP_REQUEST_W sits below min_w + info = { + "min_power_cap": _watts(min_w), + "max_power_cap": _watts(max_w), + "power_cap": _watts(current_w), + } + with mock.patch.object( + amdsmi_helpers.amdsmi_interface, "amdsmi_get_power_cap_info", lambda *a, **k: info + ): + result = helpers.validate_and_set_power_cap( + "fake-gpu-handle", 0, "PPT0", _PCAP_REQUEST_W, _FakeLogger() + ) + + self.assertIn(f"must be between {min_w}W and {max_w}W", result) + self.assertEqual( + helpers.error_collector.resolve_exit_code(), int(self.ExitCode.INVALID_PARAMETER_VALUE) + ) + + # ---- rocm-smi compat exit-code contract ---- + def test_rocm_smi_compat_exit_codes_stay_binary(self): + """The --rocm-smi shim intentionally follows rocm-smi's BINARY 0/1 exit + convention, not amd-smi's 193+ band. Guards against someone 'upgrading' + it to AmdSmiExitCode values, which would break scripts targeting + rocm-smi's contract. + """ + import amdsmi_rocm_smi_compat as compat + + self.assertEqual(int(compat.RocmSmiCompatExitCode.SUCCESS), 0) + self.assertEqual(int(compat.RocmSmiCompatExitCode.ERROR), 1) + self.assertEqual({int(c) for c in compat.RocmSmiCompatExitCode}, {0, 1}) + + # ---- CLI arg guard (drives real set_value) ---- + def test_gtt_with_gpu_raises_invalid_parameter(self): + """`set --gtt` combined with `--gpu` must raise + AmdSmiInvalidParameterException (CLI code INVALID_PARAMETER), NOT + sys.exit(2) which aliases library NOT_SUPPORTED. Drives the real + set_value guard, which runs before any device dispatch. + """ + import argparse + + from amdsmi_helpers import AMDSMIHelpers + + set_value = _load_set_value() + + cmd: Any = set_value.SetValueCommands() + cmd.helpers = AMDSMIHelpers() + + args = argparse.Namespace(gtt=8.0, gpu=["fake-gpu-handle"]) + with self.assertRaises(cli_exc.AmdSmiInvalidParameterException) as ctx: + cmd.set_value(args) + self.assertEqual(ctx.exception.value, int(self.ExitCode.INVALID_PARAMETER)) + + # ---- interactive confirmation decline (drives confirm_out_of_spec_warning) ---- + def test_confirmation_decline_exits_user_aborted(self): + """Declining an interactive confirmation prompt must exit with + USER_ABORTED, not a raw sys.exit(1)/sys.exit(str) (which aliases library + INVAL). Drives the real confirm_out_of_spec_warning with a 'no' response; + auto_respond bypasses the input() prompt so it doesn't block on a TTY. + """ + import contextlib + import io + + from amdsmi_helpers import AMDSMIHelpers + + helpers = AMDSMIHelpers() + silence_out = contextlib.redirect_stdout(io.StringIO()) + silence_err = contextlib.redirect_stderr(io.StringIO()) + with self.assertRaises(SystemExit) as ctx, silence_out, silence_err: + helpers.confirm_out_of_spec_warning(auto_respond="n") + self.assertEqual(ctx.exception.code, int(self.ExitCode.USER_ABORTED)) + + # ---- NO_PERM is fatal (drives set_core) ---- + def test_set_core_no_perm_raises_and_does_not_record(self): + """A permission error is not a per-device failure -- it means the user + needs sudo, which affects the whole command. So when a set call returns + NO_PERM, set_core should stop immediately by raising PermissionError, + instead of recording the error and continuing to the next field/device. + + This checks both halves: PermissionError is raised, and nothing was + recorded in the collector. Uses the same fakes as the multi-field test + above. + """ + import argparse + from unittest import mock + + from amdsmi_helpers import AMDSMIHelpers + + set_value = _load_set_value() + + class _FakeLogger: + format = "human" + + def store_core_output(self, device, key, value): + pass + + def print_output(self, multiple_device_enabled=False): + pass + + def store_multiple_device_output(self): + pass + + class _FakeLibErr(amdsmi.AmdSmiLibraryException): + def __init__(self, code): + self._code = code + + def get_error_code(self): + return self._code + + def get_error_info(self, detailed=True): + return f"status {self._code}" + + cmd: Any = set_value.SetValueCommands() + cmd.helpers = AMDSMIHelpers() + cmd.logger = _FakeLogger() + cmd.helpers.get_core_id_from_device_handle = lambda input_device_handle: 0 + + args = argparse.Namespace( + core="fake-core-handle", + core_boost_limit=[[100]], + core_floor_limit=None, + core_msr_floor_limit=None, + ) + + def _raise_no_perm(*a, **k): + raise _FakeLibErr(amdsmi_wrapper.AMDSMI_STATUS_NO_PERM) + + with mock.patch.object( + set_value.amdsmi_interface, "amdsmi_set_cpu_core_boostlimit", _raise_no_perm + ): + with self.assertRaises(PermissionError): + cmd.set_core(args) + + # NO_PERM aborts the command; it must NOT be recorded as a device error. + self.assertFalse(cmd.helpers.error_collector.has_errors) + + # ---- combined device-class args guard (drives real set_value) ---- + def test_set_value_combined_device_args_raise_invalid_parameter(self): + """Passing arguments for two device classes at once (e.g. a GPU arg and + a CPU arg) must raise AmdSmiInvalidParameterException (INVALID_PARAMETER + / 194), NOT a bare ValueError (which would exit 1). Drives the real + set_value combined-args guard, which runs before any device dispatch. + """ + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + import argparse + + from amdsmi_helpers import AMDSMIHelpers + + class _NoneArgs(argparse.Namespace): + # argparse populates every dest; mirror that so set_value's attribute + # scans see None (not AttributeError) for options we didn't set. + def __getattr__(self, name): + return None + + class _FakeLogger: + format = "human" + + cmd: Any = cli_set_value.SetValueCommands() + cmd.helpers = AMDSMIHelpers() + cmd.logger = _FakeLogger() + + args = _NoneArgs() + args.power_cap = 100 # a GPU set arg + args.cpu_pwr_limit = [[100]] # a CPU set arg + with self.assertRaises(cli_exc.AmdSmiInvalidParameterException) as ctx: + cmd.set_value(args) + self.assertEqual(ctx.exception.value, int(self.ExitCode.INVALID_PARAMETER)) + + # ---- parser: invalid --clk-level clock type (drives _level_select action) ---- + def test_clk_level_invalid_clock_type_is_invalid_parameter_with_hint(self): + """An invalid --clk-level clock type must raise + AmdSmiInvalidParameterException (INVALID_PARAMETER / 194) and its message + must list the valid options (the hint). Exercises the real parser action + directly, without building the whole driver-backed parser. + """ + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + import argparse + + class _Host: + class helpers: + @staticmethod + def get_output_format(): + return "human" + + # Stand-ins for the parser: _level_select only reads helpers off its host, + # and the action never touches the parser argument on this path. + host: Any = _Host() + parser: Any = None + action_cls = AMDSMIParser._level_select(host) + action = action_cls(option_strings=["-c", "--clk-level"], dest="clk_level") + with self.assertRaises(cli_exc.AmdSmiInvalidParameterException) as ctx: + action(parser, argparse.Namespace(), ["badclk", "0"]) + self.assertEqual(ctx.exception.value, int(self.ExitCode.INVALID_PARAMETER)) + message = str(ctx.exception) + self.assertIn("badclk", message) + for valid in ("sclk", "mclk", "pcie", "fclk", "socclk"): + self.assertIn(valid, message) + + def test_unsupported_value_hint_does_not_precede_the_payload(self): + """The valid-input hint must travel inside the error, not ahead of it. + + It used to be printed straight to stdout before the exception was + raised, so `amd-smi set -C 0 --json` emitted a bare line first and + json.loads failed on it. + """ + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + import contextlib + import io + import json as json_mod + from unittest import mock + + class _Host: + class helpers: + @staticmethod + def get_output_format(): + return "json" + + host: Any = _Host() + hint = "Valid inputs are: SPX, DPX. Use `sudo amd-smi partition --accelerator` to find acceptable values." + printed = io.StringIO() + with mock.patch.object(sys, "argv", ["amd-smi", "set"]): + with contextlib.redirect_stdout(printed): + with self.assertRaises(cli_exc.AmdSmiInvalidParameterValueException) as ctx: + AMDSMIParser._is_command_supported(host, "0", ["SPX", "DPX"], hint=hint) + + self.assertEqual(printed.getvalue(), "", "hint was written to stdout") + payload = json_mod.loads(str(ctx.exception)) + self.assertIn(hint, payload["error"]) + self.assertEqual(payload["code"], int(self.ExitCode.INVALID_PARAMETER_VALUE)) + + # ---- CSV error rows must stay parseable ---- + # A message that breaks naive CSV building three ways: the comma splits the + # row into extra columns, the newline splits it into an extra row, and the + # quote has to be escaped rather than emitted raw. + HOSTILE_MESSAGE = 'comma, quote " and\nnewline' + + def _csv_exceptions(self): + """Every CLI exception, built so HOSTILE_MESSAGE reaches its message.""" + text = self.HOSTILE_MESSAGE + return [ + cli_exc.AmdSmiCommandNotSupportedException(command=text, outputformat="csv"), + cli_exc.AmdSmiDeviceNotFoundException( + command=text, outputformat="csv", device_kind=cli_exc.AmdSmiDeviceKind.GPU + ), + cli_exc.AmdSmiInvalidCommandException(command=text, outputformat="csv"), + cli_exc.AmdSmiInvalidFilePathException(command=text, outputformat="csv"), + cli_exc.AmdSmiInvalidParameterException(command="set", arg=text, outputformat="csv"), + cli_exc.AmdSmiInvalidParameterValueException( + command="set", arg=text, outputformat="csv" + ), + cli_exc.AmdSmiInvalidSubcommandException(command=text, outputformat="csv"), + cli_exc.AmdSmiLibraryErrorException( + outputformat="csv", + error_code=amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED, + detail=text, + ), + cli_exc.AmdSmiMissingParameterValueException(command=text, outputformat="csv"), + cli_exc.AmdSmiPermissionDeniedException(command=text, outputformat="csv"), + cli_exc.AmdSmiRequiredCommandException(command=text, outputformat="csv"), + ] + + def test_csv_error_output_round_trips(self): + """--csv errors must survive a CSV reader unchanged. + + The hint strings list valid options comma-separated, so an unquoted row + splits: the reader then takes a fragment of the message as the exit + code. Reading it back must yield the same message and code the JSON + format reports. + """ + import csv + import io + + for exc in self._csv_exceptions(): + with self.subTest(exception=type(exc).__name__): + payload = str(exc) + parsed = list(csv.DictReader(io.StringIO(payload))) + self.assertEqual(len(parsed), 1, "message split across rows") + self.assertEqual( + len(payload.splitlines()), 2, "layout newlines reached the csv payload" + ) + self.assertEqual( + dict(parsed[0]), + { + "error": exc.json_message["error"], + "code": str(exc.value), + "error_type": exc.json_message["error_type"], + }, + ) + + def test_every_exception_is_covered_by_the_csv_round_trip(self): + """A new exception class must be added to _csv_exceptions.""" + import inspect + + declared = { + obj + for obj in vars(cli_exc).values() + if inspect.isclass(obj) + and issubclass(obj, cli_exc.AmdSmiException) + and obj is not cli_exc.AmdSmiException + } + self.assertEqual(declared, {type(exc) for exc in self._csv_exceptions()}) + + # ---- device-init failure reporting (drives amdsmi_commands._exit_on_init_error) ---- + def test_init_error_reports_in_the_requested_format_without_a_traceback(self): + """Device init runs before argv is parsed, so it cannot reach the + top-level handler. It must still print the formatted library error and + exit with the status code -- not a raw log line plus a traceback. + """ + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + import contextlib + import io + import json as json_mod + + import amdsmi_commands + + code = amdsmi_wrapper.AMDSMI_STATUS_DRIVER_NOT_LOADED + + class _FakeLibErr(Exception): + err_code = code + + for fmt in ("human_readable", "json", "csv"): + with self.subTest(output_format=fmt): + out = io.StringIO() + err = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + with self.assertRaises(SystemExit) as ctx: + amdsmi_commands._exit_on_init_error( + fmt, _FakeLibErr(), cli_exc.AmdSmiDeviceKind.GPU + ) + + printed = out.getvalue().strip() + self.assertEqual(ctx.exception.code, cli_exc.library_code_to_exit_code(code)) + self.assertIn("AMDSMI_STATUS_DRIVER_NOT_LOADED", printed) + self.assertNotIn("Traceback", printed + err.getvalue()) + if fmt == "json": + self.assertEqual( + json_mod.loads(printed)["code"], cli_exc.library_code_to_exit_code(code) + ) + elif fmt == "csv": + self.assertEqual(printed.splitlines()[0], "error,code,error_type") + + # ---- --file overwrite prompt: both ways of declining -> USER_ABORTED ---- + def test_output_file_prompt_declined_exits_user_aborted(self): + """Both exits from the --file overwrite prompt are user aborts: typing + the advertised Cancel (or anything unrecognized), and EOF on stdin. + Neither may reuse INVALID_FILE_PATH or a bare exit 1 -- the path was + valid, the user just said no. + """ + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + import argparse + import builtins + import contextlib + import io + import tempfile + from pathlib import Path + from unittest import mock + + class _Host: + class helpers: + @staticmethod + def get_output_format(): + return "human" + + host: Any = _Host() + action_cls = AMDSMIParser._check_output_file_path(host) + action = action_cls(option_strings=["--file"], dest="file") + + # "" is a bare Enter, which the [o/a/N] default sends down the same branch. + for response in ("n", "no", "", "typo"): + with self.subTest(response=response): + with tempfile.TemporaryDirectory() as folder: + existing = Path(folder) / "out.json" + existing.touch() + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(sys, "stdin", io.StringIO())) + stack.enter_context(mock.patch.object(sys.stdin, "isatty", lambda: True)) + stack.enter_context( + mock.patch.object(builtins, "input", lambda *a: response) + ) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + with self.assertRaises(SystemExit) as ctx: + action(None, argparse.Namespace(), str(existing)) + self.assertEqual(ctx.exception.code, int(self.ExitCode.USER_ABORTED)) + + # EOF at the prompt (Ctrl-D) takes the except branch, not the else. + with tempfile.TemporaryDirectory() as folder: + existing = Path(folder) / "out.json" + existing.touch() + + def _eof(*args): + raise EOFError + + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(sys, "stdin", io.StringIO())) + stack.enter_context(mock.patch.object(sys.stdin, "isatty", lambda: True)) + stack.enter_context(mock.patch.object(builtins, "input", _eof)) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + with self.assertRaises(SystemExit) as ctx: + action(None, argparse.Namespace(), str(existing)) + self.assertEqual(ctx.exception.code, int(self.ExitCode.USER_ABORTED)) + + # ---- warning prompts: declining or closed stdin -> USER_ABORTED ---- + def test_warning_prompts_declined_exit_user_aborted(self): + """The out-of-spec and memory-partition warnings exit USER_ABORTED + whether the user types no or stdin is closed. Before EOF was handled, + piping into one of these commands died on an uncaught EOFError and a + bare exit 1. + """ + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + import builtins + import contextlib + import io + from unittest import mock + + from amdsmi_helpers import AMDSMIHelpers + + def _eof(*args): + raise EOFError + + helpers = AMDSMIHelpers() + prompts = ( + helpers.confirm_out_of_spec_warning, + helpers.confirm_changing_memory_partition_gpu_reload_warning, + ) + + for prompt in prompts: + for stdin_state, responder in (("declined", lambda *a: "n"), ("closed", _eof)): + with self.subTest(prompt=prompt.__name__, stdin=stdin_state): + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(builtins, "input", responder)) + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + with self.assertRaises(SystemExit) as ctx: + prompt() + self.assertEqual(ctx.exception.code, int(self.ExitCode.USER_ABORTED)) + + # auto_respond answers on the user's behalf, so stdin must stay untouched. + for prompt in prompts: + with self.subTest(prompt=prompt.__name__, stdin="auto_respond"): + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(builtins, "input", _eof)) + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + prompt(auto_respond="y") + + # ---- direct-call guards: no target -> REQUIRED_COMMAND ---- + def test_set_cpu_without_target_raises_required_command(self): + """set_cpu with no CPU target raises AmdSmiRequiredCommandException + (REQUIRED_COMMAND / 201). Reachable only via a direct/programmatic call + (the dispatcher fills the target first), so this locks in the guard. + """ + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + import argparse + + class _FakeLogger: + format = "human" + + cmd: Any = cli_set_value.SetValueCommands() + cmd.logger = _FakeLogger() + with self.assertRaises(cli_exc.AmdSmiRequiredCommandException) as ctx: + cmd.set_cpu(argparse.Namespace(cpu=None)) + self.assertEqual(ctx.exception.value, int(self.ExitCode.REQUIRED_COMMAND)) + + def test_set_core_without_target_raises_required_command(self): + """set_core with no core target raises AmdSmiRequiredCommandException + (REQUIRED_COMMAND / 201). Same direct-call rationale as set_cpu. + """ + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + import argparse + + class _FakeLogger: + format = "human" + + cmd: Any = cli_set_value.SetValueCommands() + cmd.logger = _FakeLogger() + with self.assertRaises(cli_exc.AmdSmiRequiredCommandException) as ctx: + cmd.set_core(argparse.Namespace(core=None)) + self.assertEqual(ctx.exception.value, int(self.ExitCode.REQUIRED_COMMAND)) + + +# --------------------------------------------------------------------------- +# Generic "-g all" set-failure guards +# +# Lock in the record-then-finalize contract for the per-device GPU `set` +# handlers: a per-device library failure must be RECORDED and the command must +# keep going -- never an unhandled crash. This is the class of bug that broke +# `amd-smi set -M/-C ... -g all` (a handler raising instead of recording). The +# completeness test forces every GPU `set` option to have a guard here. +# --------------------------------------------------------------------------- + + +class _DriverlessLibErr(amdsmi.AmdSmiLibraryException): + """AmdSmiLibraryException stand-in needing no driver. + + Subclasses the real type so a handler's ``except + amdsmi_exception.AmdSmiLibraryException`` catches it, but skips the C-backed + __init__. + """ + + def __init__(self, code): + self._code = code + # The real type's __str__ reads these, so anything that logs the + # exception (rather than its status) would raise AttributeError. + self.err_code = code + self.err_info = f"status {code}" + + def get_error_code(self): + return self._code + + def get_error_info(self, detailed=True): + return f"status {self._code}" + + +class _FakeGpuLogger: + """Minimal logger covering the calls set_gpu makes on a failure path.""" + + format = "human" + + def __init__(self): + self.stored = [] + + def store_output(self, device, key, value): + self.stored.append((device, key, value)) + + def is_json_format(self): + return self.format == "json" + + def is_csv_format(self): + return self.format == "csv" + + def print_output(self, multiple_device_enabled=False): + pass + + def clear_multiple_devices_output(self): + pass + + def store_multiple_device_output(self): + pass + + +def _load_set_value(): + """Import the set_value subcommand standalone. It uses only absolute imports, + so this triggers no driver-backed package init. + """ + if _CLI_DRIVE_SKIP: + raise unittest.SkipTest(_CLI_DRIVE_SKIP) + sub_dir = os.path.join(os.path.dirname(cli_exc.__file__), "subcommands") + if sub_dir not in sys.path: + sys.path.append(sub_dir) + import set_value + + return set_value + + +def _patch_amdsmi_interface(set_value, **overrides): + """Context manager patching one or more amdsmi_interface functions. + + The driverless tests feed a fake (str) device handle, so every real + amdsmi_interface call that dereferences it as a ``c_void_p`` must be + stubbed -- including the handle-consuming reads set_gpu runs before the + handler (e.g. ``amdsmi_get_gpu_device_bdf``). + """ + import contextlib + from unittest import mock + + stack = contextlib.ExitStack() + for name, fn in overrides.items(): + stack.enter_context(mock.patch.object(set_value.amdsmi_interface, name, fn)) + return stack + + +def _raise_not_supported(*args, **kwargs): + raise _DriverlessLibErr(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED) + + +def _fake_bdf(handle): + return "0000:00:00.0" + + +def _noop(*args, **kwargs): + """Success-path stand-in: a patched amdsmi call that returns cleanly.""" + return None + + +def _watts(value): + """Power caps cross the library boundary in micro-watts.""" + return value * 1_000_000 + + +# Power-cap fixture: the request sits inside the range and differs from the +# current cap, so the handler runs all the way to the set call. +_PCAP_MIN_W = 50 +_PCAP_MAX_W = 300 +_PCAP_CURRENT_W = 200 +_PCAP_REQUEST_W = 100 +_PCAP_DEFAULT_W = 250 + +# Too short to be a CPER record, so the library rejects it rather than decoding. +_JUNK_CPER_BYTES = b"\x00" * 64 + + +def _gpu_set_options(set_value): + """Authoritative GPU ``set`` option dests, sourced from set_gpu's own + required-arg check (the ``getattr(args, "X")`` list). Auto-updates when a + new option is added -- that is what makes the completeness test a real + forcing function. + """ + import inspect + import re + + src = inspect.getsource(set_value.SetValueCommands.set_gpu) + return set(re.findall(r'getattr\(args,\s*"(\w+)"', src)) + + +def _make_set_gpu_cmd(set_value): + from amdsmi_helpers import AMDSMIHelpers + + cmd = set_value.SetValueCommands() + cmd.helpers = AMDSMIHelpers() + cmd.logger = _FakeGpuLogger() + cmd.device_handles = [] + cmd.group_check_printed = True + cmd.helpers.is_baremetal = lambda: True + cmd.helpers.get_gpu_id_from_device_handle = lambda input_device_handle: 0 + return cmd + + +def _gpu_set_args(set_value, **option): + """Namespace with every arg set_gpu references defaulted to None, plus a + single-device gpu handle and the one option under test.""" + import argparse + import inspect + import re + + src = inspect.getsource(set_value.SetValueCommands.set_gpu) + names = set(re.findall(r"args\.(\w+)", src)) + ns = argparse.Namespace(**{n: None for n in names}) + # Deliberately non-list: the single-device path has no run_device_subcommand + # wrapper, so it is the only place a handler that raises is still detectable. + ns.gpu = "fake-gpu-handle" + for key, value in option.items(): + setattr(ns, key, value) + return ns + + +def _build_set_specs(set_value): + """Per-option drivers for the generic GPU ``set`` exercisers. + + Each entry is ``name -> configure(cmd, mode)`` where ``mode`` is ``"fail"`` + or ``"success"``. ``configure`` installs any helper stubs the handler needs + on a driverless box and returns ``(arg_value, amdsmi_patches)``: + + * ``arg_value`` -- value placed on ``args.`` to enter the handler. + * ``amdsmi_patches`` -- ``{amdsmi_interface_fn: replacement}``: + - mode="fail": the earliest handle-consuming call raises + NOT_SUPPORTED (must be *recorded*, not raised past the handler). + - mode="success": every call returns a valid value, so the handler + finishes clean -- nothing recorded, exit 0. + + ``amdsmi_get_gpu_device_bdf`` is patched by the caller for every option + (set_gpu reads it up front to build the error string). + + The registry keys are the single source of truth for the completeness test, + so adding a new GPU ``set`` option without a spec here fails the suite. + """ + import collections + + ai = set_value.amdsmi_interface + + def first_member(enum): + return next(n for n in enum.__members__ if n != "INVALID") + + ClkLevel = collections.namedtuple("ClkLevel", "clk_type perf_levels") + ClkLimit = collections.namedtuple("ClkLimit", "clk_type lim_type val") + PowerCap = collections.namedtuple("PowerCap", "pwr_type watts") + + class _Fmt: # PTL format element: only .name is read on the failure path. + def __init__(self, name): + self.name = name + + def fan(cmd, mode): + cmd.helpers.detect_gpu_od = lambda bdf: (False, None) # legacy hwmon path + setfn = _raise_not_supported if mode == "fail" else _noop + return (50, True), {"amdsmi_set_gpu_fan_speed": setfn} + + def perf_level(cmd, mode): + if mode == "fail": + cmd.helpers.get_perf_levels = lambda: (["AUTO", "LOW"],) + setfn = _raise_not_supported if mode == "fail" else _noop + return first_member(ai.AmdSmiDevPerfLevel), {"amdsmi_set_gpu_perf_level": setfn} + + def profile(cmd, mode): + cmd.helpers.get_power_profile_name_mapping = lambda: {"CUSTOM": 1} + if mode == "fail": + return "CUSTOM", { + "amdsmi_set_gpu_power_profile": _raise_not_supported, + "amdsmi_get_gpu_power_profile_presets": _raise_not_supported, + } + return "CUSTOM", {"amdsmi_set_gpu_power_profile": _noop} + + def perf_determinism(cmd, mode): + setfn = _raise_not_supported if mode == "fail" else _noop + return 800, {"amdsmi_set_gpu_perf_determinism_mode": setfn} + + def compute_partition(cmd, mode): + name = first_member(ai.AmdSmiComputePartitionType) + # Force the "profiles not enumerable" fallback so a valid TYPE name is + # attempted and the driver's status is what surfaces. + cmd.helpers.get_accelerator_choices_types_indices = lambda: ( + [name], + {"profile_types": [], "profile_indices": []}, + ) + setfn = _raise_not_supported if mode == "fail" else _noop + return name, {"amdsmi_set_gpu_compute_partition": setfn} + + def memory_partition(cmd, mode): + cmd.helpers.confirm_changing_memory_partition_gpu_reload_warning = lambda *a, **k: None + arg = first_member(ai.AmdSmiMemoryPartitionType) + if mode == "fail": + return arg, { + "amdsmi_get_gpu_memory_partition_config": _raise_not_supported, + "amdsmi_set_gpu_memory_partition_mode": _raise_not_supported, + } + return arg, { + "amdsmi_get_gpu_memory_partition_config": lambda *a, **k: { + "partition_caps": [arg], + "mp_mode": arg, + }, + "amdsmi_set_gpu_memory_partition_mode": _noop, + } + + def soc_pstate(cmd, mode): + setfn = _raise_not_supported if mode == "fail" else _noop + return 0, {"amdsmi_set_soc_pstate": setfn} + + def xgmi_plpd(cmd, mode): + setfn = _raise_not_supported if mode == "fail" else _noop + return 0, {"amdsmi_set_xgmi_plpd": setfn} + + def clk_level(cmd, mode): + if mode == "fail": + # Fails at the first step (set perf level -> MANUAL), which is recorded. + return ClkLevel("sclk", [0]), {"amdsmi_set_gpu_perf_level": _raise_not_supported} + return ClkLevel("sclk", [0]), { + "amdsmi_set_gpu_perf_level": _noop, + "amdsmi_get_clk_freq": lambda *a, **k: {"num_supported": 8}, + "amdsmi_set_clk_freq": _noop, + } + + def ptl_status(cmd, mode): + setfn = _raise_not_supported if mode == "fail" else _noop + return 1, {"amdsmi_set_gpu_ptl_state": setfn} + + def ptl_format(cmd, mode): + arg = (_Fmt("NC"), _Fmt("NC")) + if mode == "fail": + return arg, {"amdsmi_get_gpu_ptl_formats": _raise_not_supported} + ptl_val = list(ai.AmdSmiPtlData)[0].value + return arg, { + "amdsmi_get_gpu_ptl_formats": lambda *a, **k: (ptl_val, ptl_val), + "amdsmi_set_gpu_ptl_formats": _noop, + } + + def power_cap(cmd, mode): + # get_gpu_id_from_device_handle dereferences the handle as a c_void_p; + # incidental plumbing, not the validation logic under test. + cmd.helpers.get_gpu_id_from_device_handle = lambda *a, **k: 0 + requested = PowerCap("ppt0", _PCAP_REQUEST_W) + if mode == "fail": + return requested, {"amdsmi_get_power_cap_info": _raise_not_supported} + return requested, { + "amdsmi_get_power_cap_info": lambda *a, **k: { + "min_power_cap": _watts(_PCAP_MIN_W), + "max_power_cap": _watts(_PCAP_MAX_W), + "power_cap": _watts(_PCAP_CURRENT_W), + }, + "amdsmi_set_power_cap": _noop, + } + + def clk_limit(cmd, mode): + if mode == "fail": + return ClkLimit("sclk", "min", 100), {"amdsmi_get_clock_info": _raise_not_supported} + return ClkLimit("sclk", "min", 1000), { + "amdsmi_get_clock_info": lambda *a, **k: {"max_clk": 2000, "min_clk": 500}, + "amdsmi_set_gpu_clk_limit": _noop, + } + + def process_isolation(cmd, mode): + if mode == "fail": + return 1, {"amdsmi_get_gpu_process_isolation": _raise_not_supported} + return 1, { + "amdsmi_get_gpu_process_isolation": lambda *a, **k: 0, + "amdsmi_set_gpu_process_isolation": _noop, + } + + def mem_carveout(cmd, mode): + cmd.helpers.prompt_reboot = lambda *a, **k: None + if mode == "fail": + return 0, {"amdsmi_get_gpu_uma_carveout_info": _raise_not_supported} + return 0, { + "amdsmi_get_gpu_uma_carveout_info": lambda *a, **k: { + "options": [{"description": "default"}], + "current_index": -1, + }, + "amdsmi_set_gpu_uma_carveout": _noop, + } + + def compute_partition_mem_alloc_mode(cmd, mode): + setfn = _raise_not_supported if mode == "fail" else _noop + return first_member(ai.AmdSmiAcceleratorPartitionMemAllocModeType), { + "amdsmi_set_gpu_accelerator_partition_mem_alloc_mode": setfn + } + + return { + "fan": fan, + "perf_level": perf_level, + "profile": profile, + "perf_determinism": perf_determinism, + "compute_partition": compute_partition, + "memory_partition": memory_partition, + "soc_pstate": soc_pstate, + "xgmi_plpd": xgmi_plpd, + "clk_level": clk_level, + "ptl_status": ptl_status, + "ptl_format": ptl_format, + "power_cap": power_cap, + "clk_limit": clk_limit, + "process_isolation": process_isolation, + "mem_carveout": mem_carveout, + "compute_partition_mem_alloc_mode": compute_partition_mem_alloc_mode, + } + + +class TestSetGpuGAllFailureGuards(unittest.TestCase): + """A per-device GPU `set` failure must be recorded and must NOT crash.""" + + def test_every_gpu_set_option_is_driven(self): + """Forcing function: every GPU `set` option must have a driver in + _build_set_specs. Adding a new option without a spec fails here, which + is what blocks a future `-g all` regression from shipping untested.""" + set_value = _load_set_value() + options = _gpu_set_options(set_value) + self.assertTrue( + options, + "Test Regex issue: scraped zero GPU set options -- _gpu_set_options regex is stale", + ) + specced = set(_build_set_specs(set_value)) + self.assertEqual( + options, + specced, + "GPU set options and their -g all failure drivers are out of sync.\n" + f" unspecced (add a _build_set_specs entry): {sorted(options - specced)}\n" + f" stale (spec exists but no longer an option): {sorted(specced - options)}", + ) + + def test_every_gpu_set_option_records_on_failure_not_crashes(self): + """Generic exerciser: drive each GPU `set` option through set_gpu with a + per-device library failure injected, and assert the record-then-finalize + contract holds -- the failure is recorded (exit != 0) and the handler + returns instead of raising. This is the single-device (`-g 0`) contract; + `test_handle_gpus_aggregates_multiple_device_failures_to_mixed` covers + the `-g all` loop. Both stem from the bug that broke -M/-C in #862.""" + set_value = _load_set_value() + specs = _build_set_specs(set_value) + for name in sorted(specs): + with self.subTest(option=name): + cmd = _make_set_gpu_cmd(set_value) + arg_value, patches = specs[name](cmd, "fail") + args = _gpu_set_args(set_value, **{name: arg_value}) + with _patch_amdsmi_interface( + set_value, amdsmi_get_gpu_device_bdf=_fake_bdf, **patches + ): + cmd.set_gpu(args) # must NOT raise + self.assertTrue( + cmd.helpers.error_collector.has_errors, + f"{name}: per-device failure was not recorded (would exit 0)", + ) + self.assertNotEqual( + cmd.helpers.error_collector.resolve_exit_code(), + 0, + f"{name}: exit code resolved to 0 despite a recorded failure", + ) + + def test_every_gpu_set_option_succeeds_cleanly(self): + """Mirror contract: a successful per-device `set` records NOTHING and + finalizes to exit 0. Guards the opposite regression from #862 -- a + success path that wrongly records an error, crashes, or resolves to a + non-zero exit code.""" + set_value = _load_set_value() + specs = _build_set_specs(set_value) + for name in sorted(specs): + with self.subTest(option=name): + cmd = _make_set_gpu_cmd(set_value) + arg_value, patches = specs[name](cmd, "success") + args = _gpu_set_args(set_value, **{name: arg_value}) + with _patch_amdsmi_interface( + set_value, amdsmi_get_gpu_device_bdf=_fake_bdf, **patches + ): + cmd.set_gpu(args) # must NOT raise + self.assertFalse( + cmd.helpers.error_collector.has_errors, + f"{name}: success path wrongly recorded an error", + ) + self.assertEqual( + cmd.helpers.error_collector.resolve_exit_code(), + 0, + f"{name}: success path did not resolve to exit 0", + ) + + def test_g_all_memory_partition_failure_records_not_crashes(self): + """`set -M ... -g all`: a per-device NOT_SUPPORTED is recorded and the + handler returns (must not raise/crash).""" + set_value = _load_set_value() + cmd = _make_set_gpu_cmd(set_value) + # memory_partition prompts a reload confirmation on the first set. + cmd.helpers.confirm_changing_memory_partition_gpu_reload_warning = lambda *a, **k: None + mp_name = next( + n + for n in set_value.amdsmi_interface.AmdSmiMemoryPartitionType.__members__ + if n != "INVALID" + ) + args = _gpu_set_args(set_value, memory_partition=mp_name) + + with _patch_amdsmi_interface( + set_value, + amdsmi_get_gpu_device_bdf=_fake_bdf, + amdsmi_get_gpu_memory_partition_config=_raise_not_supported, + amdsmi_set_gpu_memory_partition_mode=_raise_not_supported, + ): + cmd.set_gpu(args) # must NOT raise + + self.assertTrue( + cmd.helpers.error_collector.has_errors, + "memory_partition per-device failure was not recorded (would exit 0)", + ) + self.assertEqual( + cmd.helpers.error_collector.resolve_exit_code(), + int(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED), + "memory_partition failure did not finalize to the library exit code", + ) + messages = [v for (_d, key, v) in cmd.logger.stored if key == "memory_partition"] + self.assertTrue( + messages and "memory partition" in messages[-1].lower(), + f"memory_partition error message not surfaced to the user: {messages}", + ) + + def test_g_all_compute_partition_failure_records_not_crashes(self): + """`set -C ... -g all`: a per-device NOT_SUPPORTED is recorded and the + handler returns (must not raise/crash).""" + set_value = _load_set_value() + cmd = _make_set_gpu_cmd(set_value) + cp_name = next( + n + for n in set_value.amdsmi_interface.AmdSmiComputePartitionType.__members__ + if n != "INVALID" + ) + # Force the "profiles not enumerable" fallback so a valid TYPE name is + # attempted and the driver's status is what surfaces. + cmd.helpers.get_accelerator_choices_types_indices = lambda: ( + [cp_name], + {"profile_types": [], "profile_indices": []}, + ) + args = _gpu_set_args(set_value, compute_partition=cp_name) + + with _patch_amdsmi_interface( + set_value, + amdsmi_get_gpu_device_bdf=_fake_bdf, + amdsmi_set_gpu_compute_partition=_raise_not_supported, + ): + cmd.set_gpu(args) # must NOT raise + + self.assertTrue( + cmd.helpers.error_collector.has_errors, + "compute_partition per-device failure was not recorded (would exit 0)", + ) + self.assertEqual( + cmd.helpers.error_collector.resolve_exit_code(), + int(amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED), + "compute_partition failure did not finalize to the library exit code", + ) + messages = [v for (_d, key, v) in cmd.logger.stored if key == "accelerator_partition"] + self.assertTrue( + messages and "accelerator partition" in messages[-1].lower(), + f"compute_partition error message not surfaced to the user: {messages}", + ) + + +# --------------------------------------------------------------------------- +# Generic "-g all" reset-failure guards +# +# Same record-then-finalize contract as the set guards above, applied to the +# per-device GPU `reset` handlers: a per-device library failure must be +# RECORDED and the command must keep going -- never an unhandled crash. `gtt` +# is intentionally excluded (it is a system-wide reset, rejected with --gpu and +# dispatched before the per-device loop). +# --------------------------------------------------------------------------- + + +def _load_reset(): + if _CLI_DRIVE_SKIP: + raise unittest.SkipTest(_CLI_DRIVE_SKIP) + sub_dir = os.path.join(os.path.dirname(cli_exc.__file__), "subcommands") + if sub_dir not in sys.path: + sys.path.append(sub_dir) + import reset + + return reset + + +def _reset_gpu_options(reset): + """Authoritative per-device GPU ``reset`` option dests, sourced from + reset_gpu's own required-arg ``any([...])`` gate. Auto-updates when a new + option is added -- that is what makes the completeness test a forcing + function. ``gtt``/``mem_carveout`` are not in the gate (system-wide / no + per-device handler) and are correctly excluded. + """ + import inspect + import re + + src = inspect.getsource(reset.ResetCommands.reset) + names = set() + for group in re.findall(r"if not any\(\s*\[(.*?)\]\s*\)", src, re.DOTALL): + names |= set(re.findall(r"args\.(\w+)", group)) + return names + + +def _make_reset_cmd(reset): + from amdsmi_helpers import AMDSMIHelpers + + cmd = reset.ResetCommands() + cmd.helpers = AMDSMIHelpers() + cmd.logger = _FakeGpuLogger() + cmd.device_handles = [] + cmd.group_check_printed = True + cmd.helpers.is_baremetal = lambda: True + cmd.helpers.get_gpu_id_from_device_handle = lambda input_device_handle: 0 + return cmd + + +def _reset_args(reset, **option): + """Namespace with every arg reset references defaulted to None, plus a + single-device gpu handle and the one option under test set truthy.""" + import argparse + import inspect + import re + + src = inspect.getsource(reset.ResetCommands.reset) + names = set(re.findall(r"args\.(\w+)", src)) + ns = argparse.Namespace(**{n: None for n in names}) + # Deliberately non-list: the single-device path has no run_device_subcommand + # wrapper, so it is the only place a handler that raises is still detectable. + ns.gpu = "fake-gpu-handle" + for key, value in option.items(): + setattr(ns, key, value) + return ns + + +_RESET_POWER_CAP_SENSORS = [0, 1] + + +def _reset_power_cap_success_patches(reset, on_set=None): + """Patches letting reset's per-sensor power-cap loop run to completion. + + sensor_inds/sensor_types are parallel lists, as amdsmi_get_supported_power_cap + returns them -- not the empty containers that used to make the loop a no-op. + """ + ai = reset.amdsmi_interface + return { + "amdsmi_get_supported_power_cap": lambda *a, **k: { + "sensor_inds": list(_RESET_POWER_CAP_SENSORS), + "sensor_types": [ai.AmdSmiPowerCapType.PPT0, ai.AmdSmiPowerCapType.PPT1], + }, + "amdsmi_get_power_cap_info": lambda *a, **k: { + "default_power_cap": _watts(_PCAP_DEFAULT_W), + "power_cap": _watts(_PCAP_CURRENT_W), + }, + "amdsmi_set_power_cap": on_set or _noop, + } + + +def _build_reset_specs(reset): + """Per-option drivers for the generic GPU ``reset`` exercisers. + + Each entry is ``name -> configure(cmd, mode)`` where ``mode`` is ``"fail"`` + or ``"success"``. ``configure`` installs any helper stubs the handler needs + and returns the ``{amdsmi_interface_fn: replacement}`` map: + + * mode="fail": the failure path raises NOT_SUPPORTED (must be recorded). + * mode="success": every call returns cleanly (nothing recorded, exit 0). + + Every reset option is a boolean flag, so the exerciser sets + ``args. = True``; there is no per-option value to build. + + The registry keys are the single source of truth for the completeness test, + so adding a new per-device ``reset`` option without a spec here fails the + suite. + """ + + def gpureset(cmd, mode): + cmd.helpers.is_amd_device = lambda gpu: True + fn = _raise_not_supported if mode == "fail" else _noop + return {"amdsmi_reset_gpu": fn} + + def clocks(cmd, mode): + # Overdrive is reset first, then perf level (twice) with the same + # handle; both must be stubbed so a fake handle never reaches hardware. + fn = _raise_not_supported if mode == "fail" else _noop + return {"amdsmi_set_gpu_overdrive_level": fn, "amdsmi_set_gpu_perf_level": fn} + + def fans(cmd, mode): + fn = _raise_not_supported if mode == "fail" else _noop + return {"amdsmi_reset_gpu_fan": fn} + + def profile(cmd, mode): + fn = _raise_not_supported if mode == "fail" else _noop + return {"amdsmi_set_gpu_power_profile": fn} + + def xgmierr(cmd, mode): + fn = _raise_not_supported if mode == "fail" else _noop + return {"amdsmi_reset_gpu_xgmi_error": fn} + + def perf_determinism(cmd, mode): + fn = _raise_not_supported if mode == "fail" else _noop + return {"amdsmi_set_gpu_perf_level": fn} + + def power_cap(cmd, mode): + if mode == "fail": + return {"amdsmi_get_supported_power_cap": _raise_not_supported} + return _reset_power_cap_success_patches(reset) + + def clean_local_data(cmd, mode): + fn = _raise_not_supported if mode == "fail" else _noop + return {"amdsmi_clean_gpu_local_data": fn} + + return { + "gpureset": gpureset, + "clocks": clocks, + "fans": fans, + "profile": profile, + "xgmierr": xgmierr, + "perf_determinism": perf_determinism, + "power_cap": power_cap, + "clean_local_data": clean_local_data, + } + + +class _CperLogger: + """Logger stand-in for dump_cper_entries: only the format probes are used.""" + + def __init__(self, fmt): + self.format = fmt + + def is_json_format(self): + return self.format == "json" + + def is_human_readable_format(self): + return self.format == "human" + + def is_csv_format(self): + return self.format == "csv" + + +class TestRasCperAfidExitCodes(unittest.TestCase): + """A ``ras --cper`` AFID decode failure must record, not just print. + + The --afid paths record the library status; dump_cper_entries caught the same + AmdSmiLibraryException with a bare ``except Exception`` and only logged it, + so the decode failure never reached the exit code. + """ + + ROW = ("2026-01-01 00:00:00", 0, "fatal", "fatal-1.cper", b"\x00\x01") + AFIDS = [11, 22, 33] + + def setUp(self): + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + from amdsmi_helpers import AMDSMIHelpers + + self.helpers = AMDSMIHelpers() + self.printed = [] + self.helpers.cper_print = lambda text, logger=None: self.printed.append(text) + self.helpers._write_cper_files = lambda *a, **k: {"/tmp/fatal-1.cper": self.ROW} + + def _drive(self, fmt, afids=None, fail_code=None): + if fail_code is None: + self.helpers.cper_dump_afids = lambda raw: list(afids or []) + else: + + def _boom(raw): + raise _DriverlessLibErr(fail_code) + + self.helpers.cper_dump_afids = _boom + rows = self.helpers.dump_cper_entries( + "/tmp", {}, [], "gpu-handle", logger=_CperLogger(fmt), emit_inline=False + ) + return rows + + def test_decode_failure_records_the_library_status(self): + """A decode failure must resolve to the library status, not 0.""" + self._drive("json", fail_code=amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_DATA) + self.assertTrue(self.helpers.error_collector.has_errors) + self.assertEqual( + self.helpers.error_collector.resolve_exit_code(), + amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_DATA, + ) + + def test_decode_failure_json_row_carries_the_status_fields(self): + """The row keeps afids/decode_failed and gains status/message/code.""" + rows = self._drive("json", fail_code=amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_DATA) + self.assertEqual(len(rows), 1) + row = rows[0] + self.assertEqual(row["afids"], []) + self.assertTrue(row["decode_failed"]) + self.assertEqual(row["status"], "AMDSMI_STATUS_UNEXPECTED_DATA") + self.assertNotEqual(row["code"], 0) + + def test_successful_decode_keeps_afids_a_list_and_records_nothing(self): + """Control: afids stays a list so len() counts AFIDs, not characters.""" + rows = self._drive("json", afids=self.AFIDS) + self.assertEqual(rows[0]["afids"], self.AFIDS) + self.assertFalse(rows[0]["decode_failed"]) + self.assertFalse(self.helpers.error_collector.has_errors) + + def test_afid_cell_renders_na_for_a_successful_empty_decode(self): + """A successful decode with zero AFIDs shows 'N/A', not '-' or blank.""" + rows = self._drive("json", afids=[]) + self.assertFalse(rows[0]["decode_failed"]) + self.assertEqual(self.helpers.afid_cell(rows[0]), "N/A") + + def test_afid_cell_renders_space_joined_afids(self): + """Control: a non-empty decode renders as space-joined AFIDs.""" + rows = self._drive("json", afids=self.AFIDS) + self.assertEqual(self.helpers.afid_cell(rows[0]), "11 22 33") + + def test_real_decode_failure_through_the_file_write_path(self): + """End-to-end minus the driver read: real files, real library decode. + + The other tests stub _write_cper_files and cper_dump_afids. This one + stubs neither -- it hands dump_cper_entries the entries/bytes the driver + would have returned, lets it write the .cper/.json pair and call the real + amdsmi_get_afids_from_cper, and checks the resulting status reaches the + exit code. Only amdsmi_get_gpu_cper_entries() is unexercised, and that + needs a GPU with logged RAS records. + """ + import tempfile + from pathlib import Path + + from amdsmi_helpers import AMDSMIHelpers + + helpers = AMDSMIHelpers() + raw = _JUNK_CPER_BYTES + entries = {0: {"error_severity": "fatal", "notify_type": "", "timestamp": "2026-01-01"}} + cper_data = [{"bytes": raw, "size": len(raw)}] + with tempfile.TemporaryDirectory() as folder: + rows = helpers.dump_cper_entries( + folder, + entries, + cper_data, + Path(folder), # a Path device handle keeps gpu_id off the driver + logger=_CperLogger("json"), + emit_inline=False, + ) + written = sorted(p.name for p in Path(folder).glob("*")) + # The dump itself succeeded: both files exist even though decoding failed. + self.assertEqual(written, ["fatal-1.cper", "fatal-1.json"]) + self.assertEqual(len(rows), 1) + row = rows[0] + self.assertTrue(row["decode_failed"]) + self.assertEqual(row["afids"], []) + # The exact status depends on how the library rejects these bytes, so + # pin the contract (a real status reached the exit code), not the value. + self.assertTrue(row["status"].startswith("AMDSMI_STATUS_")) + self.assertNotEqual(row["code"], 0) + self.assertTrue(helpers.error_collector.has_errors) + self.assertEqual(helpers.error_collector.resolve_exit_code(), row["code"]) + + def test_human_table_shows_the_status_instead_of_a_generic_message(self): + """Human output names the failure, replacing 'Error fetching AFIDs'.""" + self._drive("human", fail_code=amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_DATA) + self.assertTrue(self.printed) + self.assertIn("[AMDSMI_STATUS_UNEXPECTED_DATA]", self.printed[-1]) + self.assertTrue(self.helpers.error_collector.has_errors) + + +class TestSetGpuFanGpuOdExitCodes(unittest.TestCase): + """The gpu_od ``--fan`` error branches must record, not just print. + + _build_set_specs drives --fan down the legacy hwmon path (detect_gpu_od -> + False), so these two branches are reached nowhere else: both printed an + error and left the exit code at 0, while their hwmon sibling recorded. + """ + + OD_RANGE = (0, 100) # (min, max) percent, as parse_gpu_od_fan_range returns + OD_RANGE_UNREADABLE = (None, None) + IN_RANGE_PERCENT = (50, True) # (value, is_percentage) + ABOVE_RANGE_RAW = (250, False) + + def setUp(self): + if _CLI_DRIVE_SKIP: + self.skipTest(_CLI_DRIVE_SKIP) + self.set_value = _load_set_value() + + def _drive_fan(self, od_range, fan_arg): + cmd = _make_set_gpu_cmd(self.set_value) + cmd.helpers.detect_gpu_od = lambda bdf: (True, "/fake/gpu_od") + cmd.helpers.parse_gpu_od_fan_range = lambda path: od_range + args = _gpu_set_args(self.set_value, fan=fan_arg) + with _patch_amdsmi_interface( + self.set_value, amdsmi_get_gpu_device_bdf=_fake_bdf, amdsmi_set_gpu_fan_speed=_noop + ): + cmd.set_gpu(args) + return cmd.helpers.error_collector + + def test_unreadable_od_range_records_device_interface_unavailable(self): + """OD_RANGE unreadable: parse_gpu_od_fan_range swallows the OSError and + returns (None, None), so there is no library status -- the CLI records + DEVICE_INTERFACE_UNAVAILABLE rather than exiting 0.""" + collector = self._drive_fan(self.OD_RANGE_UNREADABLE, self.IN_RANGE_PERCENT) + self.assertTrue(collector.has_errors) + self.assertEqual( + collector.resolve_exit_code(), int(cli_exc.AmdSmiExitCode.DEVICE_INTERFACE_UNAVAILABLE) + ) + + def test_value_outside_od_range_records_invalid_parameter_value(self): + """Out-of-range value on gpu_od must resolve to the same code its legacy + hwmon sibling records; the fan interface a GPU exposes must not change + the exit code for identical user input.""" + collector = self._drive_fan(self.OD_RANGE, self.ABOVE_RANGE_RAW) + self.assertTrue(collector.has_errors) + self.assertEqual( + collector.resolve_exit_code(), int(cli_exc.AmdSmiExitCode.INVALID_PARAMETER_VALUE) + ) + + def test_gpu_od_success_records_nothing(self): + """Control: a value inside OD_RANGE still exits 0.""" + collector = self._drive_fan(self.OD_RANGE, self.IN_RANGE_PERCENT) + self.assertFalse(collector.has_errors) + + +class TestResetGpuGAllFailureGuards(unittest.TestCase): + """A per-device GPU `reset` failure must be recorded and must NOT crash.""" + + def test_every_reset_option_is_driven(self): + """Forcing function: every per-device GPU `reset` option must have a + driver in _build_reset_specs. Adding a new option without a spec fails + here, blocking a future `-g all` reset regression from shipping.""" + reset = _load_reset() + options = _reset_gpu_options(reset) + self.assertTrue( + options, + "Test Regex issue: scraped zero GPU reset options -- _reset_gpu_options regex is stale", + ) + specced = set(_build_reset_specs(reset)) + self.assertEqual( + options, + specced, + "GPU reset options and their -g all failure drivers are out of sync.\n" + f" unspecced (add a _build_reset_specs entry): {sorted(options - specced)}\n" + f" stale (spec exists but no longer an option): {sorted(specced - options)}", + ) + + def test_every_reset_option_records_on_failure_not_crashes(self): + """Generic exerciser: drive each GPU `reset` option through reset with a + per-device library failure injected, and assert the record-then-finalize + contract holds -- the failure is recorded (exit != 0) and the handler + returns instead of raising.""" + reset = _load_reset() + specs = _build_reset_specs(reset) + for name in sorted(specs): + with self.subTest(option=name): + cmd = _make_reset_cmd(reset) + patches = specs[name](cmd, "fail") + args = _reset_args(reset, **{name: True}) + with _patch_amdsmi_interface(reset, **patches): + cmd.reset(args) # must NOT raise + self.assertTrue( + cmd.helpers.error_collector.has_errors, + f"{name}: per-device failure was not recorded (would exit 0)", + ) + self.assertNotEqual( + cmd.helpers.error_collector.resolve_exit_code(), + 0, + f"{name}: exit code resolved to 0 despite a recorded failure", + ) + + def test_every_reset_option_succeeds_cleanly(self): + """Mirror contract: a successful per-device `reset` records NOTHING and + finalizes to exit 0. Guards against a success path that wrongly records + an error, crashes, or resolves to a non-zero exit code.""" + reset = _load_reset() + specs = _build_reset_specs(reset) + for name in sorted(specs): + with self.subTest(option=name): + cmd = _make_reset_cmd(reset) + patches = specs[name](cmd, "success") + args = _reset_args(reset, **{name: True}) + with _patch_amdsmi_interface(reset, **patches): + cmd.reset(args) # must NOT raise + self.assertFalse( + cmd.helpers.error_collector.has_errors, + f"{name}: success path wrongly recorded an error", + ) + self.assertEqual( + cmd.helpers.error_collector.resolve_exit_code(), + 0, + f"{name}: success path did not resolve to exit 0", + ) + + def test_reset_power_cap_writes_the_default_back_for_every_sensor(self): + """`reset --power_cap` must write the default cap back for every sensor + the GPU reports. The generic success exerciser only checks that nothing + was recorded, which stays true even if the loop never runs. + """ + reset = _load_reset() + cmd = _make_reset_cmd(reset) + set_calls = [] + + def _record_set(handle, sensor, value): + set_calls.append((sensor, value)) + + patches = _reset_power_cap_success_patches(reset, on_set=_record_set) + args = _reset_args(reset, power_cap=True) + with _patch_amdsmi_interface(reset, **patches): + cmd.reset(args) + + self.assertEqual( + [sensor for sensor, _ in set_calls], + [0, 1], + "per-sensor loop did not visit every supported sensor", + ) + for _, value in set_calls: + self.assertEqual(value, _watts(_PCAP_DEFAULT_W), "did not write the default cap back") + self.assertFalse(cmd.helpers.error_collector.has_errors) diff --git a/projects/amdsmi/tests/python/unit/system/test_cli_version_driver.py b/projects/amdsmi/tests/python/unit/system/test_cli_version_driver.py new file mode 100644 index 000000000000..2189c6bc8f6a --- /dev/null +++ b/projects/amdsmi/tests/python/unit/system/test_cli_version_driver.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""amd-smi version driver field tests.""" + +import importlib.util +import io +import types +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +from common.common import stub_modules_at_import + +_DRIVER_INFO = { + "driver_name": "amdgpu", + "driver_date": "2026/09/16 00:00", + "driver_kernel_version": "6.19.14", + "driver_version": "6.19.14.31400000", + "driver_build_version": "2370381", + "driver_full_version": "6.19.14.31400000-2370381", +} + + +class _LibraryError(RuntimeError): + def get_error_info(self) -> str: + return "not supported" + + +class _Logger: + def __init__(self, human_readable: bool = False) -> None: + self.output = {} + self.destination = "stdout" + self._human_readable = human_readable + + def is_human_readable_format(self) -> bool: + return self._human_readable + + def is_json_format(self) -> bool: + return not self._human_readable + + def is_csv_format(self) -> bool: + return False + + def print_output(self) -> None: + return None + + +def _default_interface() -> types.SimpleNamespace: + return types.SimpleNamespace( + amdsmi_get_lib_version=lambda: {"major": 27, "minor": 1, "release": 0}, + amdsmi_get_rocm_version=lambda: (True, "10.1.0"), + amdsmi_get_processor_handles=lambda: ["gpu0"], + amdsmi_get_gpu_driver_info=lambda _gpu: _DRIVER_INFO, + ) + + +def _load_version_module(interface: types.SimpleNamespace) -> types.ModuleType: + amdsmi = types.ModuleType("amdsmi") + amdsmi.amdsmi_interface = interface + amdsmi.amdsmi_exception = types.SimpleNamespace(AmdSmiLibraryException=_LibraryError) + version_metadata = types.ModuleType("_version") + version_metadata.__version__ = "1.0.0" + + restore = stub_modules_at_import({"amdsmi": amdsmi, "_version": version_metadata}) + try: + path = Path(__file__).resolve().parents[4] / "amdsmi_cli/subcommands/version.py" + spec = importlib.util.spec_from_file_location("version_driver_under_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + finally: + restore() + + +def _run_version(gpu_version: bool = True, human_readable: bool = False) -> dict: + return _run_version_with_interface( + _default_interface(), gpu_version=gpu_version, human_readable=human_readable + ) + + +def _run_version_with_interface( + interface: types.SimpleNamespace, gpu_version: bool = True, human_readable: bool = False +) -> dict: + module = _load_version_module(interface) + commands = object.__new__(module.VersionCommands) + commands.logger = _Logger(human_readable=human_readable) + commands.helpers = types.SimpleNamespace() + commands.group_check_printed = True + args = types.SimpleNamespace(gpu_version=gpu_version, cpu_version=False, nic_version=False) + commands.version(args) + return commands.logger.output + + +class TestVersionDriverOutput(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + path = Path(__file__).resolve().parents[4] / "amdsmi_cli/subcommands/version.py" + if not path.is_file(): + raise unittest.SkipTest(f"amd-smi CLI version.py not found at {path}") + + def test_json_reports_driver_fields(self) -> None: + output = _run_version() + self.assertEqual(output["amdgpu_version"], "6.19.14.31400000-2370381") + self.assertNotIn("driver_kernel_version", output) + self.assertNotIn("driver_version", output) + self.assertNotIn("driver_build_version", output) + self.assertNotIn("driver_full_version", output) + self.assertNotIn("amdgpu_dkms_version", output) + + def test_human_readable_reports_driver_fields_in_order(self) -> None: + stdout = io.StringIO() + with redirect_stdout(stdout): + _run_version(human_readable=True) + + line = stdout.getvalue() + self.assertIn("amdgpu version: 6.19.14.31400000-2370381", line) + self.assertNotIn("Kernel version:", line) + self.assertNotIn("Driver version:", line) + self.assertNotIn("Build version:", line) + + def test_gpu_fields_are_na_when_gpu_version_is_disabled(self) -> None: + output = _run_version(gpu_version=False) + self.assertEqual(output["amdgpu_version"], "N/A") + self.assertNotIn("driver_kernel_version", output) + self.assertNotIn("driver_version", output) + self.assertNotIn("driver_build_version", output) + + def test_empty_gpu_list_reports_amdgpu_version_as_na(self) -> None: + interface = _default_interface() + interface.amdsmi_get_processor_handles = lambda: [] + + output = _run_version_with_interface(interface) + + self.assertEqual(output["amdgpu_version"], "N/A") + + def test_driver_info_failure_reports_amdgpu_version_as_na(self) -> None: + interface = _default_interface() + + def raise_driver_info(_gpu: object) -> None: + raise _LibraryError("not supported") + + interface.amdsmi_get_gpu_driver_info = raise_driver_info + + output = _run_version_with_interface(interface) + + self.assertEqual(output["amdgpu_version"], "N/A") diff --git a/projects/amdsmi/tests/python/unit/system/test_driver_versions.py b/projects/amdsmi/tests/python/unit/system/test_driver_versions.py new file mode 100644 index 000000000000..2a2000bd3625 --- /dev/null +++ b/projects/amdsmi/tests/python/unit/system/test_driver_versions.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""GPU driver version field tests.""" + +import ctypes +import importlib +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + + +_PACKAGE_DIR = Path(__file__).resolve().parents[4] / "py-interface" + + +def _load_source_package() -> types.ModuleType: + package = types.ModuleType("amdsmi_driver_versions_under_test") + package.__path__ = [str(_PACKAGE_DIR)] + sys.modules[package.__name__] = package + package.amdsmi_wrapper = importlib.import_module(f"{package.__name__}.amdsmi_wrapper") + package.amdsmi_interface = importlib.import_module(f"{package.__name__}.amdsmi_interface") + return package + + +amdsmi = _load_source_package() if _PACKAGE_DIR.is_dir() else None + + +class TestDriverVersions(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + if amdsmi is None: + raise unittest.SkipTest(f"amdsmi py-interface source not found at {_PACKAGE_DIR}") + + def test_struct_layout_includes_new_fields(self) -> None: + struct_type = amdsmi.amdsmi_wrapper.struct_amdsmi_driver_info_t + self.assertEqual(ctypes.sizeof(struct_type), 1792) + self.assertEqual(struct_type.driver_kernel_version.offset, 768) + self.assertEqual(struct_type.amdgpu_driver_version.offset, 1024) + self.assertEqual(struct_type.driver_build_version.offset, 1280) + self.assertEqual(struct_type.driver_full_version.offset, 1536) + + def test_reports_split_driver_versions(self) -> None: + handle = amdsmi.amdsmi_wrapper.amdsmi_processor_handle() + + def _stub(_handle: object, info_ptr: object) -> int: + info_ptr._obj.driver_name = b"amdgpu" + info_ptr._obj.driver_version = b"6.19.14.31400000" + info_ptr._obj.driver_date = b"2026/09/16 00:00" + info_ptr._obj.driver_kernel_version = b"6.19.14" + info_ptr._obj.amdgpu_driver_version = b"31400000" + info_ptr._obj.driver_build_version = b"2370381" + info_ptr._obj.driver_full_version = b"6.19.14.31400000-2370381" + return amdsmi.amdsmi_wrapper.AMDSMI_STATUS_SUCCESS + + with mock.patch.object(amdsmi.amdsmi_wrapper, "amdsmi_get_gpu_driver_info", _stub): + info = amdsmi.amdsmi_interface.amdsmi_get_gpu_driver_info(handle) + + self.assertEqual(info["driver_kernel_version"], "6.19.14") + self.assertEqual(info["amdgpu_driver_version"], "31400000") + self.assertEqual(info["driver_version"], "6.19.14.31400000") + self.assertEqual(info["driver_build_version"], "2370381") + self.assertEqual(info["driver_full_version"], "6.19.14.31400000-2370381") + + def test_empty_versions_render_na(self) -> None: + handle = amdsmi.amdsmi_wrapper.amdsmi_processor_handle() + + def _stub(_handle: object, info_ptr: object) -> int: + info_ptr._obj.driver_name = b"amdgpu" + info_ptr._obj.driver_date = b"" + info_ptr._obj.driver_kernel_version = b"" + info_ptr._obj.amdgpu_driver_version = b"" + info_ptr._obj.driver_version = b"" + info_ptr._obj.driver_build_version = b"" + info_ptr._obj.driver_full_version = b"" + return amdsmi.amdsmi_wrapper.AMDSMI_STATUS_SUCCESS + + with mock.patch.object(amdsmi.amdsmi_wrapper, "amdsmi_get_gpu_driver_info", _stub): + info = amdsmi.amdsmi_interface.amdsmi_get_gpu_driver_info(handle) + + self.assertEqual(info["driver_kernel_version"], "N/A") + self.assertEqual(info["amdgpu_driver_version"], "N/A") + self.assertEqual(info["driver_version"], "N/A") + self.assertEqual(info["driver_build_version"], "N/A") + self.assertEqual(info["driver_full_version"], "N/A") diff --git a/projects/amdsmi/tests/python/unit/system/test_output_file_stdin.py b/projects/amdsmi/tests/python/unit/system/test_output_file_stdin.py index 5cbf198697f2..95e84960d801 100644 --- a/projects/amdsmi/tests/python/unit/system/test_output_file_stdin.py +++ b/projects/amdsmi/tests/python/unit/system/test_output_file_stdin.py @@ -16,18 +16,25 @@ import argparse import builtins -import importlib.util import os import sys import tempfile import types import unittest -from common.common import amdsmi_path +from common.common import ( + amdsmi_path, + cli_search_order, + find_cli_dir, + generated_version_stub, + load_cli_module, + stub_modules, +) -_ROCM_ROOT = os.path.dirname(os.path.dirname(amdsmi_path)) -_CLI_DIR = os.path.join(_ROCM_ROOT, "libexec", "amdsmi_cli") -PARSER_PATH = os.path.join(_CLI_DIR, "amdsmi_parser.py") +# Locate the CLI dir; cli_search_order() decides whether the install or this +# checkout wins. None -> setUpClass skips. +_CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__)))) +PARSER_PATH = os.path.join(_CLI_DIR, "amdsmi_parser.py") if _CLI_DIR else None class _FakeStdin: @@ -40,46 +47,50 @@ def isatty(self): return self._is_tty -def _install_stubs(): - """Register lightweight stubs for the parser's non-stdlib imports. +def _missing_stubs(): + """Stubs for the parser's non-stdlib imports that are not already loaded. - ``setdefault`` leaves any already-imported real module in place (the CLI is - installed, so ``amdsmi``/``amdsmi_helpers`` may already be loaded by the - shared test harness); only the missing ones fall back to stubs. The parser - references these names at import only to bind them, so stubs are sufficient. + Only the missing names are stubbed: the CLI is installed, so ``amdsmi`` and + ``amdsmi_helpers`` may already be real modules loaded by the shared harness, + and replacing those would hide the thing under test. The parser references + these names at import only to bind them, so stubs are sufficient. ``amdsmi_cli_exceptions`` is intentionally NOT stubbed -- the test asserts on the real exception type raised by the action. + + Hand the result to ``stub_modules`` so every name is restored afterwards; a + stub that outlives the class corrupts later test modules in the same run. """ - amdsmi_mod = types.ModuleType("amdsmi") - amdsmi_mod.amdsmi_interface = types.SimpleNamespace() - sys.modules.setdefault("amdsmi", amdsmi_mod) + stubs = {} + + if "amdsmi" not in sys.modules: + amdsmi_mod = types.ModuleType("amdsmi") + amdsmi_mod.amdsmi_interface = types.SimpleNamespace() + stubs["amdsmi"] = amdsmi_mod + + stubs.update(generated_version_stub()) - version_mod = types.ModuleType("_version") - version_mod.__version__ = "0.0.0-test" - sys.modules.setdefault("_version", version_mod) + if "amdsmi_helpers" not in sys.modules: + helpers_mod = types.ModuleType("amdsmi_helpers") + helpers_mod.AMDSMIHelpers = type("AMDSMIHelpers", (), {}) + stubs["amdsmi_helpers"] = helpers_mod - helpers_mod = types.ModuleType("amdsmi_helpers") - helpers_mod.AMDSMIHelpers = type("AMDSMIHelpers", (), {}) - sys.modules.setdefault("amdsmi_helpers", helpers_mod) + return stubs def _load_parser_module(): # Put the installed CLI dir on sys.path so the parser's own # ``import amdsmi_cli_exceptions`` resolves to the real module. - if _CLI_DIR not in sys.path: - sys.path.insert(0, _CLI_DIR) - spec = importlib.util.spec_from_file_location("amdsmi_parser_under_test", PARSER_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_cli_module("amdsmi_parser_under_test", PARSER_PATH, sys_path_dir=_CLI_DIR) class TestOutputFileStdinGuard(unittest.TestCase): @classmethod def setUpClass(cls): - if not os.path.isfile(PARSER_PATH): - raise unittest.SkipTest(f"amdsmi_parser not installed at {PARSER_PATH}") - _install_stubs() + if not PARSER_PATH or not os.path.isfile(PARSER_PATH): + raise unittest.SkipTest( + f"amd-smi CLI amdsmi_parser.py not found (looked in {_CLI_DIR or amdsmi_path})" + ) + stub_modules(cls, _missing_stubs()) cls.parser_mod = _load_parser_module() import amdsmi_cli_exceptions # resolved from _CLI_DIR above @@ -172,7 +183,3 @@ def test_tty_still_prompts_and_overwrites(self): self.assertEqual(os.path.getsize(tmp), 0) finally: os.unlink(tmp) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/amdsmi/tests/python/unit/system/test_runner_hygiene.py b/projects/amdsmi/tests/python/unit/system/test_runner_hygiene.py new file mode 100644 index 000000000000..b995b95c22e6 --- /dev/null +++ b/projects/amdsmi/tests/python/unit/system/test_runner_hygiene.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Runner hygiene: leaf tests must run only through their suite runner. + +Executing a test file standalone -- via an ``if __name__ == "__main__":`` +``unittest.main()`` block -- bypasses the shared runner (``common.run_test_dir``): +the root-privilege check, the ``sys.path`` / CLI resolution, the GTest-style +summary, and (most importantly) the ``sys.modules`` isolation guard that stops +one test class from polluting another. Every leaf test must instead be run +through its suite runner with a ``-k`` / ``-x`` filter, e.g.:: + + unit_tests.py -k "TestClass" -v # unit/ + cli_unit_test.py -k "test_some_cli_behavior" -v # cli/ + integration_test.py -k "TestDiscovery" -v # functional/ + +This meta-test fails if any ``test_*.py`` module ships a standalone ``__main__`` +runner, and names the runner that owns it. + +It also fails if a module builds a CLI path out of hardcoded ``amdsmi_cli`` / +``libexec`` components instead of calling ``common.find_cli_dir``. Hand-rolled +resolution silently skips the suite in whichever layout it was not written for, +and typically prefers the source tree over ``AMDSMI_PATH``, so an explicit +override is ignored. + +Finally it fails if a module installs a ``sys.modules`` stub without going +through ``common.stub_modules``. The runner's isolation guard already catches +leaks at runtime, but only for stubs a running class installs after the baseline +is taken -- a class that skips, or a stub installed at import time, is invisible +to it. This check covers both. +""" + +import os +import re +import unittest + +# tests/python root: .../unit/system/test_runner_hygiene.py -> .../tests/python +# (or the installed .../python_unittest). os.walk from here covers every suite. +_TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Matches a module-level ``if __name__ == "__main__":`` guard (either quote style). +_MAIN_GUARD = re.compile(r"^\s*if\s+__name__\s*==\s*['\"]__main__['\"]\s*:", re.MULTILINE) + +# A quoted CLI path component -- the building blocks of a hand-rolled resolver. +# The alternation is not itself quote-wrapped, so this file does not self-match. +_CLI_PATH_LITERAL = re.compile(r"""['"](?:amdsmi_cli|libexec)['"]""") + +# A call to the shared resolver, which is what the literals above should be. +_FIND_CLI_DIR_CALL = re.compile(r"\bfind_cli_dir\s*\(") + +# Installing a stub over a known module name -- the direction that leaks. +# Requires a literal key: ``sys.modules[spec.name] = mod`` is the importlib +# idiom for loading a real module from a path, not a stub, so it is not matched. +# Popping a name is cleanup, so it is not matched either. +_SYS_MODULES_ASSIGN = re.compile(r"""^\s*sys\.modules\[\s*['"][^'"]+['"]\s*\]\s*=""", re.MULTILINE) + +# The shared save/restore helpers that every stub install should go through: +# ``stub_modules`` (class-scoped) or ``stub_modules_at_import`` (module-scoped, +# restored from tearDownModule). +_STUB_MODULES_CALL = re.compile(r"\bstub_modules(?:_at_import)?\s*\(") + +# Top-level suite directory -> the runner script that discovers/runs it +# (see the common.run_test_dir callers). +_RUNNERS = {"unit": "unit_tests.py", "cli": "cli_unit_test.py", "functional": "integration_test.py"} + +# TODO(amdsmi_team): most of these tests are packaging/ABI related - they should be under +# their own category/runner (will update later..) +_STANDALONE_ALLOWED = frozenset( + { + "test_abi_compat.py", + "test_dual_copy_guard.py", + "test_cpack_path_guard.py", + "run_amdsmi_python_versions_test.py", + "test_packaging_scriptlets.py", + "test_upgrade_downgrade_guard.py", + } +) + + +def _runner_for(rel_path): + """Return the runner script that owns the suite containing *rel_path* + (a path relative to the tests root).""" + top = rel_path.split(os.sep)[0] + return _RUNNERS.get(top, "the applicable runner") + + +# --- Failure-message pieces (edit these to change the wording/format) --------- +# Header: the "what/why" shown once. Keep it self-contained. +_MESSAGE_HEADER = ( + 'These test modules define a standalone `if __name__ == "__main__"` runner.\n\n' + "Running a test file directly bypasses the suite runner (root check,\n" + "sys.path/CLI resolution, the GTest summary, and the sys.modules isolation\n" + "guard), so it is not allowed. Remove the block and run the suite through its\n" + 'runner with a filter instead: `.py -k "" -v` (or `-x` to exclude).' +) + + +def _offender_block(rel_path, runner_width): + """One indented block per offending file. ``runner_width`` pads the runner + name so the trailing ``(see ...)`` hint lines up across offenders. Edit the + template here to restyle every entry at once.""" + runner = _runner_for(rel_path) + return ( + f" [*] {rel_path}\n" + f' Run via: {runner:<{runner_width}} -k "" -v ' + f"(see `{runner} -h` for more options)\n" + ) + + +def _format_message(offenders): + """Assemble the full assertion message from the header and one block per + offender.""" + # Pad every runner name to the widest one so the "(see ...)" hints align. + runner_width = max((len(_runner_for(rel)) for rel in offenders), default=0) + blocks = "\n".join(_offender_block(rel, runner_width) for rel in offenders) + return f"{_MESSAGE_HEADER}\n\nOffenders:\n{blocks}" + + +_CLI_PATH_MESSAGE_HEADER = ( + "These test modules build a CLI path from hardcoded components instead of\n" + "calling `common.find_cli_dir`.\n\n" + "The CLI lives at `/libexec/amdsmi_cli` when installed and at\n" + "`projects/amdsmi/amdsmi_cli` in a source checkout, so a path written for one\n" + "layout resolves to nothing in the other and the suite skips itself instead of\n" + "failing. A hand-rolled resolver also gets the priority wrong. `cli_search_order`\n" + "owns that rule: the source tree wins by default, so a checkout tests itself\n" + "rather than a stale install, and an explicit AMDSMI_PATH wins over source.\n\n" + "Resolve the directory once, at module scope, and skip only when it is None:\n\n" + " from common.common import cli_search_order, find_cli_dir\n\n" + " _CLI_DIR = find_cli_dir(*cli_search_order(os.path.dirname(os.path.abspath(__file__))))\n" + ' SOME_PATH = os.path.join(_CLI_DIR, "subcommands", "static.py") if _CLI_DIR else None\n\n' + " if not SOME_PATH or not os.path.isfile(SOME_PATH):\n" + " raise unittest.SkipTest(\n" + ' f"amd-smi CLI static.py not found (looked in {_CLI_DIR or amdsmi_path})"\n' + " )" +) + + +def _format_cli_path_message(offenders): + blocks = "\n".join(f" [*] {rel}" for rel in offenders) + return f"{_CLI_PATH_MESSAGE_HEADER}\n\nOffenders:\n{blocks}\n" + + +_STUB_MESSAGE_HEADER = ( + "These test modules assign into `sys.modules` without going through\n" + "`common.stub_modules`.\n\n" + "A stub that outlives its class corrupts every later test module in the same\n" + "interpreter -- an empty AMDSMIHelpers stub, for example, hides handle_gpus\n" + "from test_cli_exit_codes. The runner's isolation guard catches most of these\n" + "at runtime, but not all: a class that skips never installs its stub, and an\n" + "import-time stub is already in the guard's baseline, so neither is reported.\n" + "A static check is what closes those two gaps.\n\n" + "stub_modules snapshots the names, installs the replacements, and restores\n" + "them when the class finishes:\n\n" + " from common.common import stub_modules\n\n" + " @classmethod\n" + " def setUpClass(cls):\n" + " modules = _fake_modules() # name -> module (None removes)\n" + " stub_modules(cls, modules)\n\n" + "If the stub has to exist before this module's own imports run, use\n" + "`stub_modules_at_import(modules)` and call the returned restore callable\n" + "from `tearDownModule`." +) + + +def _format_stub_message(offenders): + blocks = "\n".join(f" [*] {rel}" for rel in offenders) + return f"{_STUB_MESSAGE_HEADER}\n\nOffenders:\n{blocks}\n" + + +def _iter_test_sources(): + """Yield ``(rel_path, source)`` for every ``test_*.py`` under the tests root.""" + for root, _dirs, files in os.walk(_TESTS_ROOT): + for fname in sorted(files): + if not (fname.startswith("test_") and fname.endswith(".py")): + continue + path = os.path.join(root, fname) + try: + with open(path, encoding="utf-8") as handle: + source = handle.read() + except OSError: + continue + yield os.path.relpath(path, _TESTS_ROOT), source + + +class TestRunnerHygiene(unittest.TestCase): + def test_no_test_module_defines_standalone_main(self): + offenders = [] + for rel_path, source in _iter_test_sources(): + if _MAIN_GUARD.search(source) and rel_path not in _STANDALONE_ALLOWED: + offenders.append(rel_path) + + offenders.sort() + self.assertEqual(offenders, [], _format_message(offenders)) + + def test_cli_paths_resolve_through_find_cli_dir(self): + this_file = os.path.basename(__file__) + offenders = [] + for rel_path, source in _iter_test_sources(): + if os.path.basename(rel_path) == this_file: + continue + if _CLI_PATH_LITERAL.search(source) and not _FIND_CLI_DIR_CALL.search(source): + offenders.append(rel_path) + + offenders.sort() + self.assertEqual(offenders, [], _format_cli_path_message(offenders)) + + def test_module_stubs_go_through_stub_modules(self): + this_file = os.path.basename(__file__) + offenders = [] + for rel_path, source in _iter_test_sources(): + if os.path.basename(rel_path) == this_file: + continue + if _SYS_MODULES_ASSIGN.search(source) and not _STUB_MODULES_CALL.search(source): + offenders.append(rel_path) + + offenders.sort() + self.assertEqual(offenders, [], _format_stub_message(offenders)) diff --git a/projects/clr/CHANGELOG.md b/projects/clr/CHANGELOG.md index 275aaf23facd..5f6c061eeb42 100644 --- a/projects/clr/CHANGELOG.md +++ b/projects/clr/CHANGELOG.md @@ -10,6 +10,10 @@ Full documentation for HIP is available at [rocm.docs.amd.com](https://rocm.docs * `hipModuleEnumerateFunctions` returns the function handles defined in a loaded module. * Disable HRR capture feature +### Resolved issues +* A registered `__device__` global that is absent from the loaded code object no longer aborts the process. Symbol lookup now returns `hipErrorInvalidSymbol` from the runtime's variable materialization path (`hipGetSymbolAddress`, `hipLibraryGetGlobal`, and related entry points). `hipModuleGetGlobal` still reports `hipErrorNotFound` for a missing name. +* Fixed `__hip_bfloat162` comparisons that ignored or misread the high lane. `__hbneu2` now returns true only when both lanes are unordered-not-equal, `__hgt2` and `__hisnan2` now return the per-lane result in `.y` instead of always 1.0, and the `<`, `<=`, `>`, `>=` operators now compare `.y` with `.y`. Code that relied on the previous results may see different values. + ## HIP 10.1.0 for ROCm 10.1.0 ### Added @@ -36,6 +40,8 @@ The HIP/HSA runtime now correctly releases allocated signal objects during strea ### Added * New HIP APIs + - GPUDirect RDMA: support for API parity with the corresponding CUDA API. + * `hipDeviceFlushGPUDirectRDMAWrites` blocks until GPUDirect RDMA writes issued by a third-party device, such as an RDMA-capable NIC, are visible to the requested scope. Capability is reported by the new device attributes `hipDeviceAttributeGPUDirectRDMASupported`, `hipDeviceAttributeGPUDirectRDMAFlushWritesOptions` and `hipDeviceAttributeGPUDirectRDMAWritesOrdering`, which are also mirrored in `hipDeviceProp_t`. - Stream Ordered Memory Allocator: support for API parity with corresponding CUDA API. * `hipMemGetDefaultMemPool` returns the default memory pool for the specified location and allocation type - Cooperative Groups scan functions are now supported, providing feature parity with CUDA. diff --git a/projects/clr/CMakeLists.txt b/projects/clr/CMakeLists.txt index 5c39b2aa2066..ed19fe27fb6a 100644 --- a/projects/clr/CMakeLists.txt +++ b/projects/clr/CMakeLists.txt @@ -41,6 +41,7 @@ endif() option(CLR_BUILD_HIP "Build HIP" OFF) option(CLR_BUILD_OCL "Build OCL" OFF) +option(CLR_HRR_CODEGEN_CHECK "Generate and validate HRR generated sources during CLR builds" OFF) option(USE_NEW_HOSTCALL_IMPL "Use the new phase/occupancy-based hostcall implementation" OFF) diff --git a/projects/clr/cmake/HrrCodegen.cmake b/projects/clr/cmake/HrrCodegen.cmake new file mode 100644 index 000000000000..b0ddf7eaf8f7 --- /dev/null +++ b/projects/clr/cmake/HrrCodegen.cmake @@ -0,0 +1,117 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +include_guard(GLOBAL) + +set(_hrr_codegen_module_dir "${CMAKE_CURRENT_LIST_DIR}") + +function(hrr_verify_generated_files) + set(updated_files) + + foreach(kind HEADER CAPTURE PLAYBACK) + set(generated "${GENERATED_${kind}}") + set(expected "${EXPECTED_${kind}}") + + if(NOT EXISTS "${generated}") + message(FATAL_ERROR "HRR code generation did not produce ${generated}") + endif() + if(NOT EXISTS "${expected}") + message(FATAL_ERROR "HRR checked-in generated file is missing: ${expected}") + endif() + + execute_process( + COMMAND "${CMAKE_COMMAND}" -E compare_files --ignore-eol "${generated}" "${expected}" + RESULT_VARIABLE compare_result) + if(NOT compare_result EQUAL 0) + execute_process( + COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${generated}" "${expected}" + RESULT_VARIABLE copy_result) + if(NOT copy_result EQUAL 0) + message(FATAL_ERROR "Failed to update checked-in HRR generated file: ${expected}") + endif() + list(APPEND updated_files "${expected}") + endif() + endforeach() + + if(updated_files) + string(JOIN "\n " updated_files_text ${updated_files}) + message(FATAL_ERROR + "HRR generated files were stale relative to newly generated output and were updated in the working tree:\n" + " ${updated_files_text}\n\n" + "Add the updated files to the change and rerun the build.\n" + "Review whether new or changed HIP APIs require explicit classification in\n" + "projects/hrr/tools/gen_hrr_api_args.py, and then run script again.\n") + endif() +endfunction() + +if(HRR_CODEGEN_VERIFY) + hrr_verify_generated_files() + return() +endif() + +function(hrr_add_codegen_check) + cmake_parse_arguments(ARG + "" + "TARGET;HRR_SOURCE_DIR;CLR_SOURCE_DIR;HIP_SOURCE_DIR;OUTPUT_DIR" + "" + ${ARGN}) + + foreach(required TARGET HRR_SOURCE_DIR CLR_SOURCE_DIR HIP_SOURCE_DIR OUTPUT_DIR) + if(NOT ARG_${required}) + message(FATAL_ERROR "hrr_add_codegen_check requires ${required}") + endif() + endforeach() + + find_package(Python3 COMPONENTS Interpreter REQUIRED) + + set(generator "${ARG_HRR_SOURCE_DIR}/tools/gen_hrr_api_args.py") + set(api_trace "${ARG_CLR_SOURCE_DIR}/hipamd/include/hip/amd_detail/hip_api_trace.hpp") + set(public_header "${ARG_HIP_SOURCE_DIR}/include/hip/hip_runtime_api.h") + set(header "${ARG_OUTPUT_DIR}/include/hrr/hrr_api_args.h") + set(capture "${ARG_OUTPUT_DIR}/hip_capture_generated.cpp") + set(playback "${ARG_OUTPUT_DIR}/hip_playback_generated.cpp") + set(expected_header "${ARG_HRR_SOURCE_DIR}/include/hrr/hrr_api_args.h") + set(expected_capture "${ARG_CLR_SOURCE_DIR}/hipamd/src/hrr/hip_capture_generated.cpp") + set(expected_playback "${ARG_HRR_SOURCE_DIR}/playback/hip_playback_generated.cpp") + set(manual_capture_source "${ARG_CLR_SOURCE_DIR}/hipamd/src/hrr/hip_capture.cpp") + + foreach(input generator api_trace public_header manual_capture_source) + if(NOT EXISTS "${${input}}") + message(FATAL_ERROR "HRR code generation requires ${input}: ${${input}}") + endif() + endforeach() + + add_custom_command( + OUTPUT "${header}" "${capture}" "${playback}" + COMMAND ${Python3_EXECUTABLE} "${generator}" + --input "${api_trace}" + --public-header "${public_header}" + --output-header "${header}" + --output-capture "${capture}" + --output-playback "${playback}" + --check-hrr-coverage + --manual-capture-source "${manual_capture_source}" + --silent + COMMAND ${CMAKE_COMMAND} + "-DGENERATED_HEADER=${header}" + "-DEXPECTED_HEADER=${expected_header}" + "-DGENERATED_CAPTURE=${capture}" + "-DEXPECTED_CAPTURE=${expected_capture}" + "-DGENERATED_PLAYBACK=${playback}" + "-DEXPECTED_PLAYBACK=${expected_playback}" + -DHRR_CODEGEN_VERIFY=ON + -P "${_hrr_codegen_module_dir}/HrrCodegen.cmake" + DEPENDS + "${generator}" + "${api_trace}" + "${public_header}" + "${manual_capture_source}" + "${expected_header}" + "${expected_capture}" + "${expected_playback}" + COMMENT "Generating and verifying HRR API sources" + VERBATIM) + set_source_files_properties("${header}" "${capture}" "${playback}" PROPERTIES GENERATED TRUE) + add_custom_target(${ARG_TARGET} DEPENDS "${header}" "${capture}" "${playback}") +endfunction() diff --git a/projects/clr/hipamd/CMakeLists.txt b/projects/clr/hipamd/CMakeLists.txt index bb823a31a0a7..d0eac64c7abf 100755 --- a/projects/clr/hipamd/CMakeLists.txt +++ b/projects/clr/hipamd/CMakeLists.txt @@ -27,7 +27,7 @@ list(APPEND CMAKE_MODULE_PATH ${HIP_COMMON_DIR}/cmake) ############################# # Options ############################# -option(__HIP_ENABLE_PCH "Enable/Disable pre-compiled hip headers" ON) +option(__HIP_ENABLE_PCH "Enable/Disable pre-compiled hip headers" OFF) option(BUILD_SHARED_LIBS "Build the shared library" ON) option(THEROCK_FLAG_STAMP_LIBRARY_GIT_VERSIONS "Stamp git revisions into generated library version metadata" diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_atomic.h b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_atomic.h index abac83c8999a..b350f48d71b2 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_atomic.h +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_atomic.h @@ -21,14 +21,13 @@ #define __HIP_ATOMIC_BACKWARD_COMPAT_MEMORY #endif -template struct Cond_t; - -template struct Cond_t { - using type = T; -}; -template struct Cond_t { - using type = F; -}; +// Annoyingly, unrecognized attributes are a warning and the scoped +// atomic builtins predate clang_atomic_attributes. +#if defined(__has_extension) && __has_extension(clang_atomic_attributes) +#define __HIP_ATOMIC_ATTR(...) [[clang::atomic(__VA_ARGS__)]] +#else +#define __HIP_ATOMIC_ATTR(...) +#endif #if !__HIP_DEVICE_COMPILE__ // TODO: Remove this after compiler pre-defines the following Macros. @@ -559,15 +558,15 @@ __device__ inline double atomicMax_system(double* addr, double val) { } __device__ inline unsigned int atomicInc(unsigned int* address, unsigned int val) { - if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_atomic_inc32)) - return __builtin_amdgcn_atomic_inc32(address, val, __ATOMIC_RELAXED, "agent"); - return 0; + __HIP_ATOMIC_ATTR(remote_memory, no_fine_grained_memory) { + return __scoped_atomic_fetch_uinc(address, val, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + } } __device__ inline unsigned int atomicDec(unsigned int* address, unsigned int val) { - if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_atomic_dec32)) - return __builtin_amdgcn_atomic_dec32(address, val, __ATOMIC_RELAXED, "agent"); - return 0; + __HIP_ATOMIC_ATTR(remote_memory, no_fine_grained_memory) { + return __scoped_atomic_fetch_udec(address, val, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + } } __device__ inline int atomicAnd(int* address, int val) { diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_bf16.h b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_bf16.h index 8ddd98dae3dd..640b5073c7e7 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_bf16.h +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_bf16.h @@ -1457,10 +1457,10 @@ __BF16_HOST_DEVICE_STATIC__ bool __hbne2(const __hip_bfloat162 a, const __hip_bf /** * \ingroup HIP_INTRINSIC_BFLOAT162_COMP - * \brief Check for a != b + * \brief Check for a != b - unordered */ __BF16_HOST_DEVICE_STATIC__ bool __hbneu2(const __hip_bfloat162 a, const __hip_bfloat162 b) { - return __hneu(a.x, b.x) || __hneu(a.y, b.y); + return __hneu(a.x, b.x) && __hneu(a.y, b.y); } /** @@ -1490,7 +1490,7 @@ __BF16_HOST_DEVICE_STATIC__ __hip_bfloat162 __hge2(const __hip_bfloat162 a, __BF16_HOST_DEVICE_STATIC__ __hip_bfloat162 __hgt2(const __hip_bfloat162 a, const __hip_bfloat162 b) { return __hip_bfloat162{{__hgt(a.x, b.x) ? HIPRT_ONE_BF16 : HIPRT_ZERO_BF16}, - {__hgt(a.y, b.y) ? HIPRT_ONE_BF16 : HIPRT_ONE_BF16}}; + {__hgt(a.y, b.y) ? HIPRT_ONE_BF16 : HIPRT_ZERO_BF16}}; } /** @@ -1499,7 +1499,7 @@ __BF16_HOST_DEVICE_STATIC__ __hip_bfloat162 __hgt2(const __hip_bfloat162 a, */ __BF16_HOST_DEVICE_STATIC__ __hip_bfloat162 __hisnan2(const __hip_bfloat162 a) { return __hip_bfloat162{{__hisnan(a.x) ? HIPRT_ONE_BF16 : HIPRT_ZERO_BF16}, - {__hisnan(a.y) ? HIPRT_ONE_BF16 : HIPRT_ONE_BF16}}; + {__hisnan(a.y) ? HIPRT_ONE_BF16 : HIPRT_ZERO_BF16}}; } /** @@ -1620,8 +1620,7 @@ __BF16_HOST_DEVICE_STATIC__ bool operator!=(const __hip_bfloat162& l, const __hi * \brief Operator to perform a less than on two __hip_bfloat16 numbers */ __BF16_HOST_DEVICE_STATIC__ bool operator<(const __hip_bfloat162& l, const __hip_bfloat162& r) { - float2 fl = l, fr = r; - return fl.x < fr.x && fl.x < fr.y; + return __hblt2(l, r); } /** @@ -1629,8 +1628,7 @@ __BF16_HOST_DEVICE_STATIC__ bool operator<(const __hip_bfloat162& l, const __hip * \brief Operator to perform a less than equal on two __hip_bfloat16 numbers */ __BF16_HOST_DEVICE_STATIC__ bool operator<=(const __hip_bfloat162& l, const __hip_bfloat162& r) { - float2 fl = l, fr = r; - return fl.x <= fr.x && fl.x <= fr.y; + return __hble2(l, r); } /** @@ -1638,8 +1636,7 @@ __BF16_HOST_DEVICE_STATIC__ bool operator<=(const __hip_bfloat162& l, const __hi * \brief Operator to perform a greater than on two __hip_bfloat16 numbers */ __BF16_HOST_DEVICE_STATIC__ bool operator>(const __hip_bfloat162& l, const __hip_bfloat162& r) { - float2 fl = l, fr = r; - return fl.x > fr.x && fl.x > fr.y; + return __hbgt2(l, r); } /** @@ -1647,8 +1644,7 @@ __BF16_HOST_DEVICE_STATIC__ bool operator>(const __hip_bfloat162& l, const __hip * \brief Operator to perform a greater than equal on two __hip_bfloat16 numbers */ __BF16_HOST_DEVICE_STATIC__ bool operator>=(const __hip_bfloat162& l, const __hip_bfloat162& r) { - float2 fl = l, fr = r; - return fl.x >= fr.x && fl.x >= fr.y; + return __hbge2(l, r); } #if defined(__clang__) && defined(__HIP__) diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h index f75e7b9259e2..841ffa960763 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h @@ -1431,13 +1431,37 @@ struct CGIdentity> { } }; +template <> +struct CGIdentity> { + __CG_QUALIFIER__ bool operator()() + { + return true; + } +}; + +template <> +struct CGIdentity> { + __CG_QUALIFIER__ bool operator()() + { + return false; + } +}; + template struct CGIdentity> { __CG_QUALIFIER__ T operator()() { // CUDA would return 0 in this case. But in our case we mimic what __ockl_wfscan_* // would do - return __hip_internal::NumericLimits::maximum(); + return __hip_internal::ExclusiveScanIdentity::maximum(); + } +}; + +template <> +struct CGIdentity> { + __CG_QUALIFIER__ bool operator()() + { + return true; } }; @@ -1445,7 +1469,15 @@ template struct CGIdentity> { __CG_QUALIFIER__ T operator()() { - return __hip_internal::NumericLimits::minimum(); + return __hip_internal::ExclusiveScanIdentity::minimum(); + } +}; + +template <> +struct CGIdentity> { + __CG_QUALIFIER__ bool operator()() + { + return false; } }; @@ -1472,13 +1504,15 @@ __CG_QUALIFIER__ unsigned long long groupMask(const TyGroup& group) // for up to 32 bytes (__hip_ds_bpermute() can only do 4 bytes at a time, // this function calls it (or the floating point version) multiple times to // implement it for bigger sizes -template ::type = 0> +template ::type = 0> __CG_QUALIFIER__ void bPermute(T&, T, int from) { } // trivial case: the type fits within the permute size -template ::type = 0> +template ::type = 0> __CG_QUALIFIER__ void bPermute(T& permuteResult, T result, int from) { auto backwardPermute = [](int index, T arg) { @@ -1490,11 +1524,11 @@ __CG_QUALIFIER__ void bPermute(T& permuteResult, T result, int from) } }; - if constexpr (sizeof(T) == 2) { + if constexpr (sizeof(T) <= 2) { union { int i; T f; - } tmp; + } tmp = {}; tmp.f = result; tmp.i = __hip_ds_bpermute(from << 2, tmp.i); diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups_scan.h b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups_scan.h index 3a9c2eabdcd0..5241c8af9090 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups_scan.h +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups_scan.h @@ -67,10 +67,13 @@ namespace impl { }; // all the arithmetic operations accept the same types, so we just based it on the overload being - // present for scan_add + // present for scan_add. We require the overload picked for T to return T exactly (rather than just + // being callable with a T), otherwise an unsupported type could sneak in via an implicit promotion/ + // conversion to one of the supported types (e.g. bool -> int) and silently produce the wrong result. template struct has_arithmetic_scan(T {}))> + typename __hip_internal::enable_if<__hip_internal::is_same< + decltype(scan_add(__hip_internal::declval())), T>::value>::type > : __hip_internal::true_type { }; @@ -79,10 +82,11 @@ namespace impl { }; // all the arithmetic operations accept the same types, so we just based it on the overload being - // present for scan_and + // present for scan_and. See has_arithmetic_scan above for why we require an exact return type match. template struct has_boolean_scan(T {}))> + typename __hip_internal::enable_if<__hip_internal::is_same< + decltype(scan_and(__hip_internal::declval())), T>::value>::type > : __hip_internal::true_type { }; @@ -114,13 +118,14 @@ namespace impl { using Op = typename __hip_internal::remove_cvref::type; using Val = typename __hip_internal::remove_cvref::type; - constexpr bool isPrimitiveType = impl::has_arithmetic_scan::value; - using permuteType = typename __hip_internal::conditional::type; - - // the number of backward permutes will be: size(Val) / 4 rounded up + static constexpr bool isPrimitiveType = impl::has_arithmetic_scan::value || + __hip_internal::is_same::value; + // the amount of backward permutes a type requires to be transferred from one lane to another: + // size(Val) / 4 rounded up static constexpr int kNumOfPermutes = (sizeof(Val) <= 4)? 1 : (sizeof(Val) + sizeof(unsigned int) - 1) / sizeof(unsigned int); + using permuteType = typename __hip_internal::conditional::type; static_assert(cooperative_groups::impl::is_param_type_same::value, "Operator input and output types differ"); static_assert(__hip_internal::is_trivially_copyable::value, "val must be trivially copyable"); static_assert(sizeof(Val) <= 32, "scan only operate on values of size up to 32 bytes"); @@ -170,13 +175,13 @@ namespace impl { // unsigned int[N] is used in some cases, e.g. when T is wider than 32-bit using ResultType = typename __hip_internal::conditional< - isPrimitiveType && (sizeof(Val) == 4 || sizeof(Val) == 2), permuteType, + isPrimitiveType && kNumOfPermutes == 1, permuteType, permuteType[kNumOfPermutes]>::type; static constexpr int alignment = alignof(Val) <= 4? 4 : alignof(Val); - alignas(alignment) ResultType result; + alignas(alignment) ResultType result = {}; alignas(alignment) ResultType permuteResult; - if constexpr (isPrimitiveType && (sizeof(Val) == 2 || sizeof(Val) == 4)) { + if constexpr (isPrimitiveType && kNumOfPermutes == 1) { result = val; } else { __builtin_memcpy(result, &val, sizeof(Val)); @@ -225,7 +230,7 @@ namespace impl { Val toReturn; toReturn = op(*reinterpret_cast(permuteResult), *reinterpret_cast(result)); __builtin_memcpy(result, &toReturn, sizeof(Val)); - } else if constexpr (sizeof(Val) == 4 || sizeof(Val) == 2) { + } else if constexpr (kNumOfPermutes == 1) { result = op(permuteResult, result); } else if constexpr (sizeof(Val) == 8) { Val tmp; diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp16.h b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp16.h index 9e498b88f21d..939362efd2e3 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp16.h +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp16.h @@ -142,10 +142,10 @@ struct __half { } #endif -// MANIPULATORS - DEVICE ONLY +// MANIPULATORS - INTEGRAL #if !defined(__HIP_NO_HALF_CONVERSIONS__) template {}>* = nullptr> - __device__ __half& operator=(T x) { + __HOST_DEVICE__ __half& operator=(T x) { data = static_cast<_Float16>(x); return *this; } @@ -964,17 +964,18 @@ inline __device__ __half unsafeAtomicAdd(__half* address, __half value) { namespace __hip_internal { template <> -struct NumericLimits<__half> { - static constexpr __half maximum() { - __half_raw raw { .x = 0x7C00U }; - return __half(raw); - } - static constexpr __half minimum() { - __half_raw raw { .x = 0xFC00U }; - return __half(raw); - } +struct ExclusiveScanIdentity<__half> { + static constexpr __half maximum() { + __half_raw raw { .x = 0x7C00U }; + return __half(raw); + } + static constexpr __half minimum() { + __half_raw raw { .x = 0xFC00U }; + return __half(raw); + } }; } // namespace __hip_internal + #endif // defined(__clang__) && defined(__HIP__) // Math functions diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_ocp_host.hpp b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_ocp_host.hpp index 210b3777c173..d16818a06777 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_ocp_host.hpp +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_ocp_host.hpp @@ -68,8 +68,8 @@ struct Float { bool HasInf; }; -static const float ieee754_nan = __hip_internal::NumericLimits::quiet_NaN(); -static const float ieee754_inf = __hip_internal::NumericLimits::infinity(); +static const float ieee754_nan = __hip_internal::numeric_limits::quiet_NaN(); +static const float ieee754_inf = __hip_internal::numeric_limits::infinity(); __OCP_FP_HOST_DEVICE_STATIC__ __hip_uint32_t U32(float f) { static_assert(sizeof(__hip_uint32_t) == sizeof(float), ""); diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_unsafe_atomics.h b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_unsafe_atomics.h index 3ccf5a709787..ba1b07b827e5 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_unsafe_atomics.h +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_unsafe_atomics.h @@ -8,11 +8,13 @@ #ifdef __cplusplus -#pragma push_macro("__HIP_ATOMICS_IGNORE_DENORMAL_MODE") +// Pass all options to one __HIP_ATOMICS_ATTR(...); separate [[clang::atomic]] +// attributes do not combine (the last one wins). +#pragma push_macro("__HIP_ATOMICS_ATTR") #if defined(__has_extension) && __has_extension(clang_atomic_attributes) -#define __HIP_ATOMICS_IGNORE_DENORMAL_MODE [[clang::atomic(ignore_denormal_mode)]] +#define __HIP_ATOMICS_ATTR(...) [[clang::atomic(__VA_ARGS__)]] #else -#define __HIP_ATOMICS_IGNORE_DENORMAL_MODE +#define __HIP_ATOMICS_ATTR(...) #endif /** @@ -42,30 +44,9 @@ * @return Original value contained in \p addr. */ __device__ inline float unsafeAtomicAdd(float* addr, float value) { -#if defined(__gfx90a__) && __has_builtin(__builtin_amdgcn_is_shared) && \ - __has_builtin(__builtin_amdgcn_is_private) && \ - __has_builtin(__builtin_amdgcn_ds_atomic_fadd_f32) && \ - __has_builtin(__builtin_amdgcn_global_atomic_fadd_f32) - if (__builtin_amdgcn_is_shared((const __attribute__((address_space(0))) void*)addr)) { - if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_ds_atomic_fadd_f32)) - return __builtin_amdgcn_ds_atomic_fadd_f32(addr, value); - return 0.0f; - } else if (__builtin_amdgcn_is_private((const __attribute__((address_space(0))) void*)addr)) { - float temp = *addr; - *addr = temp + value; - return temp; - } else { - if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_global_atomic_fadd_f32)) - return __builtin_amdgcn_global_atomic_fadd_f32(addr, value); - return 0.0f; - } -#elif __has_builtin(__hip_atomic_fetch_add) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { - return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + __HIP_ATOMICS_ATTR(remote_memory, no_fine_grained_memory, ignore_denormal_mode) { + return __scoped_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); } -#else - return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED); -#endif } /** @@ -84,7 +65,7 @@ __device__ inline float unsafeAtomicAdd(float* addr, float value) { */ __device__ inline float unsafeAtomicMax(float* addr, float val) { #if __has_builtin(__hip_atomic_load) && __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); bool done = false; while (!done && value < val) { @@ -121,7 +102,7 @@ __device__ inline float unsafeAtomicMax(float* addr, float val) { */ __device__ inline float unsafeAtomicMin(float* addr, float val) { #if __has_builtin(__hip_atomic_load) && __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); bool done = false; while (!done && value > val) { @@ -169,17 +150,9 @@ __device__ inline float unsafeAtomicMin(float* addr, float val) { * @return Original value contained in \p addr. */ __device__ inline double unsafeAtomicAdd(double* addr, double value) { -#if defined(__gfx90a__) && __has_builtin(__builtin_amdgcn_flat_atomic_fadd_f64) - if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_flat_atomic_fadd_f64)) - return __builtin_amdgcn_flat_atomic_fadd_f64(addr, value); - return 0.0; -#elif __has_builtin(__hip_atomic_fetch_add) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { - return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + __HIP_ATOMICS_ATTR(remote_memory, no_fine_grained_memory, ignore_denormal_mode) { + return __scoped_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); } -#else - return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED); -#endif } /** @@ -209,33 +182,9 @@ __device__ inline double unsafeAtomicAdd(double* addr, double value) { * @return Original value contained at \p addr. */ __device__ inline double unsafeAtomicMax(double* addr, double val) { -#if (defined(__gfx90a__) || defined(__gfx94plus_clr__)) && \ - __has_builtin(__builtin_amdgcn_flat_atomic_fmax_f64) - if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_flat_atomic_fmax_f64)) - return __builtin_amdgcn_flat_atomic_fmax_f64(addr, val); - return 0.0; -#else -#if __has_builtin(__hip_atomic_load) && __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { - double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); - bool done = false; - while (!done && value < val) { - done = __hip_atomic_compare_exchange_strong(addr, &value, val, __ATOMIC_RELAXED, - __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); - } - return value; - } -#else - unsigned long long* uaddr = (unsigned long long*)addr; - unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED); - bool done = false; - while (!done && __longlong_as_double(value) < val) { - done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false, - __ATOMIC_RELAXED, __ATOMIC_RELAXED); + __HIP_ATOMICS_ATTR(remote_memory, no_fine_grained_memory, ignore_denormal_mode) { + return __scoped_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); } - return __longlong_as_double(value); -#endif -#endif } /** @@ -265,33 +214,9 @@ __device__ inline double unsafeAtomicMax(double* addr, double val) { * @return Original value contained at \p addr. */ __device__ inline double unsafeAtomicMin(double* addr, double val) { -#if (defined(__gfx90a__) || defined(__gfx94plus_clr__)) && \ - __has_builtin(__builtin_amdgcn_flat_atomic_fmin_f64) - if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_flat_atomic_fmin_f64)) - return __builtin_amdgcn_flat_atomic_fmin_f64(addr, val); - return 0.0; -#else -#if __has_builtin(__hip_atomic_load) && __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { - double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); - bool done = false; - while (!done && value > val) { - done = __hip_atomic_compare_exchange_strong(addr, &value, val, __ATOMIC_RELAXED, - __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); - } - return value; - } -#else - unsigned long long* uaddr = (unsigned long long*)addr; - unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED); - bool done = false; - while (!done && __longlong_as_double(value) > val) { - done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false, - __ATOMIC_RELAXED, __ATOMIC_RELAXED); + __HIP_ATOMICS_ATTR(remote_memory, no_fine_grained_memory, ignore_denormal_mode) { + return __scoped_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); } - return __longlong_as_double(value); -#endif -#endif } /** @@ -317,7 +242,7 @@ __device__ inline float safeAtomicAdd(float* addr, float value) { // need to force a CAS loop here. float old_val; #if __has_builtin(__hip_atomic_load) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { old_val = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } #else // !__has_builtin(__hip_atomic_load) @@ -328,7 +253,7 @@ __device__ inline float safeAtomicAdd(float* addr, float value) { do { temp = expected = old_val; #if __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { __hip_atomic_compare_exchange_strong(addr, &expected, old_val + value, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } @@ -344,11 +269,11 @@ __device__ inline float safeAtomicAdd(float* addr, float value) { // atomics will produce safe CAS loops, but are otherwise not different than // agent-scope atomics. This logic is only applicable for gfx90a, and should // not be assumed on other architectures. - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); } #elif __has_builtin(__hip_atomic_fetch_add) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } #else @@ -372,7 +297,7 @@ __device__ inline float safeAtomicAdd(float* addr, float value) { */ __device__ inline float safeAtomicMax(float* addr, float val) { #if __has_builtin(__hip_atomic_load) && __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); bool done = false; while (!done && value < val) { @@ -409,7 +334,7 @@ __device__ inline float safeAtomicMax(float* addr, float val) { */ __device__ inline float safeAtomicMin(float* addr, float val) { #if __has_builtin(__hip_atomic_load) && __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); bool done = false; while (!done && value > val) { @@ -450,7 +375,7 @@ __device__ inline double safeAtomicAdd(double* addr, double value) { // atomics will produce safe CAS loops, but are otherwise not different than // agent-scope atomics. This logic is only applicable for gfx90a, and should // not be assumed on other architectures. - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); } #elif defined(__gfx90a__) @@ -458,7 +383,7 @@ __device__ inline double safeAtomicAdd(double* addr, double value) { // force a CAS loop here. double old_val; #if __has_builtin(__hip_atomic_load) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { old_val = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } #else // !__has_builtin(__hip_atomic_load) @@ -469,7 +394,7 @@ __device__ inline double safeAtomicAdd(double* addr, double value) { do { temp = expected = old_val; #if __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { __hip_atomic_compare_exchange_strong(addr, &expected, old_val + value, __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } @@ -482,7 +407,7 @@ __device__ inline double safeAtomicAdd(double* addr, double value) { return old_val; #else // !defined(__gfx90a__) #if __has_builtin(__hip_atomic_fetch_add) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } #else // !__has_builtin(__hip_atomic_fetch_add) @@ -514,7 +439,7 @@ __device__ inline double safeAtomicMax(double* addr, double val) { } else { #endif #if __has_builtin(__hip_atomic_load) && __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); bool done = false; while (!done && value < val) { @@ -561,7 +486,7 @@ __device__ inline double safeAtomicMin(double* addr, double val) { } else { #endif #if __has_builtin(__hip_atomic_load) && __has_builtin(__hip_atomic_compare_exchange_strong) - __HIP_ATOMICS_IGNORE_DENORMAL_MODE { + __HIP_ATOMICS_ATTR(ignore_denormal_mode) { double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); bool done = false; while (!done && value > val) { @@ -585,6 +510,6 @@ __device__ inline double safeAtomicMin(double* addr, double val) { #endif } -#pragma pop_macro("__HIP_ATOMICS_IGNORE_DENORMAL_MODE") +#pragma pop_macro("__HIP_ATOMICS_ATTR") #endif diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_warp_sync_functions.h b/projects/clr/hipamd/include/hip/amd_detail/amd_warp_sync_functions.h index 51d07f6e3529..64696d9c4682 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_warp_sync_functions.h +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_warp_sync_functions.h @@ -417,7 +417,7 @@ __device__ inline T __reduce_op_sync(MaskT mask, T val, BinaryOp op, WfReduce wf union { int i; T f; - } tmp; + } tmp = {}; tmp.f = result; tmp.i = __hip_ds_bpermute(nextBit << 2, tmp.i); @@ -466,7 +466,7 @@ __device__ inline T __reduce_op_sync(MaskT mask, T val, BinaryOp op, WfReduce wf union { int i; T f; - } tmp; + } tmp = {}; tmp.f = result; tmp.i = __hip_ds_bpermute(firstLane << 2, tmp.i); return tmp.f; diff --git a/projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp b/projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp index 83a7a6ba77db..9c1e05da8ca5 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp +++ b/projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp @@ -48,7 +48,7 @@ #define HIP_API_TABLE_STEP_VERSION 0 #define HIP_COMPILER_API_TABLE_STEP_VERSION 0 #define HIP_TOOLS_API_TABLE_STEP_VERSION 1 -#define HIP_RUNTIME_API_TABLE_STEP_VERSION 34 +#define HIP_RUNTIME_API_TABLE_STEP_VERSION 35 // HIP API interface // HIP compiler dispatch functions @@ -1185,6 +1185,10 @@ typedef hipError_t (*t_hipMemGetDefaultMemPool)(hipMemPool_t* memPool, hipMemLoc hipMemAllocationType type); typedef hipError_t (*t_hipModuleEnumerateFunctions)(hipFunction_t* functions, unsigned int numFunctions, hipModule_t module); +// The enum keyword is required: this header is reached from C translation units (e.g. via +// rocprofiler-sdk/hip/api_args.h), and these enums are declared without a typedef. +typedef hipError_t (*t_hipDeviceFlushGPUDirectRDMAWrites)( + enum hipFlushGPUDirectRDMAWritesTarget target, enum hipFlushGPUDirectRDMAWritesScope scope); // HIP Compiler dispatch table struct HipCompilerDispatchTable { // HIP_COMPILER_API_TABLE_STEP_VERSION == 0 @@ -1841,8 +1845,11 @@ struct HipDispatchTable { // HIP_RUNTIME_API_TABLE_STEP_VERSION == 34 t_hipModuleEnumerateFunctions hipModuleEnumerateFunctions_fn; - // DO NOT EDIT ABOVE! + // HIP_RUNTIME_API_TABLE_STEP_VERSION == 35 + t_hipDeviceFlushGPUDirectRDMAWrites hipDeviceFlushGPUDirectRDMAWrites_fn; + // DO NOT EDIT ABOVE! + // HIP_RUNTIME_API_TABLE_STEP_VERSION == 36 // ******************************************************************************************* // // diff --git a/projects/clr/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h b/projects/clr/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h index 8e4705f1aeb1..f654d8dc2406 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h +++ b/projects/clr/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h @@ -437,9 +437,10 @@ template __CG_STATIC_QUALIFIER__ T* map_shared_rank(T* in, int rank __CG_STATIC_QUALIFIER__ unsigned int query_shared_rank(const void* in) { #if __has_builtin(__builtin_amdgcn_query_shared_rank) - if (!__builtin_amdgcn_is_invocable(__builtin_amdgcn_query_shared_rank)) __builtin_trap(); - return static_cast( - __builtin_amdgcn_query_shared_rank((__attribute__((address_space(11))) const void*)in)); + if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_query_shared_rank)) + return static_cast( + __builtin_amdgcn_query_shared_rank((__attribute__((address_space(11))) const void*)in)); + __builtin_trap(); #else return 0; #endif diff --git a/projects/clr/hipamd/include/hip/amd_detail/hip_prof_str.h b/projects/clr/hipamd/include/hip/amd_detail/hip_prof_str.h index 22cf07317456..3ee6ac9bc68b 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/hip_prof_str.h +++ b/projects/clr/hipamd/include/hip/amd_detail/hip_prof_str.h @@ -507,7 +507,8 @@ enum hip_api_id_t { HIP_API_ID_hipDeviceGetLuid = 485, HIP_API_ID_hipInitDevice = 486, HIP_API_ID_hipModuleEnumerateFunctions = 487, - HIP_API_ID_LAST = 487, + HIP_API_ID_hipDeviceFlushGPUDirectRDMAWrites = 488, + HIP_API_ID_LAST = 488, HIP_API_ID_hipBindTexture = HIP_API_ID_NONE, @@ -578,6 +579,7 @@ static inline const char* hip_api_name(const uint32_t id) { case HIP_API_ID_hipDeviceComputeCapability: return "hipDeviceComputeCapability"; case HIP_API_ID_hipDeviceDisablePeerAccess: return "hipDeviceDisablePeerAccess"; case HIP_API_ID_hipDeviceEnablePeerAccess: return "hipDeviceEnablePeerAccess"; + case HIP_API_ID_hipDeviceFlushGPUDirectRDMAWrites: return "hipDeviceFlushGPUDirectRDMAWrites"; case HIP_API_ID_hipDeviceGet: return "hipDeviceGet"; case HIP_API_ID_hipDeviceGetAttribute: return "hipDeviceGetAttribute"; case HIP_API_ID_hipDeviceGetByPCIBusId: return "hipDeviceGetByPCIBusId"; @@ -1059,6 +1061,7 @@ static inline uint32_t hipApiIdByName(const char* name) { if (strcmp("hipDeviceComputeCapability", name) == 0) return HIP_API_ID_hipDeviceComputeCapability; if (strcmp("hipDeviceDisablePeerAccess", name) == 0) return HIP_API_ID_hipDeviceDisablePeerAccess; if (strcmp("hipDeviceEnablePeerAccess", name) == 0) return HIP_API_ID_hipDeviceEnablePeerAccess; + if (strcmp("hipDeviceFlushGPUDirectRDMAWrites", name) == 0) return HIP_API_ID_hipDeviceFlushGPUDirectRDMAWrites; if (strcmp("hipDeviceGet", name) == 0) return HIP_API_ID_hipDeviceGet; if (strcmp("hipDeviceGetAttribute", name) == 0) return HIP_API_ID_hipDeviceGetAttribute; if (strcmp("hipDeviceGetByPCIBusId", name) == 0) return HIP_API_ID_hipDeviceGetByPCIBusId; @@ -1694,6 +1697,10 @@ typedef struct hip_api_data_s { int peerDeviceId; unsigned int flags; } hipDeviceEnablePeerAccess; + struct { + enum hipFlushGPUDirectRDMAWritesTarget target; + enum hipFlushGPUDirectRDMAWritesScope scope; + } hipDeviceFlushGPUDirectRDMAWrites; struct { hipDevice_t* device; hipDevice_t device__val; @@ -4611,6 +4618,11 @@ typedef struct hip_api_data_s { cb_data.args.hipDeviceEnablePeerAccess.peerDeviceId = (int)peerDeviceId; \ cb_data.args.hipDeviceEnablePeerAccess.flags = (unsigned int)flags; \ }; +// hipDeviceFlushGPUDirectRDMAWrites[('hipFlushGPUDirectRDMAWritesTarget', 'target'), ('hipFlushGPUDirectRDMAWritesScope', 'scope')] +#define INIT_hipDeviceFlushGPUDirectRDMAWrites_CB_ARGS_DATA(cb_data) { \ + cb_data.args.hipDeviceFlushGPUDirectRDMAWrites.target = (hipFlushGPUDirectRDMAWritesTarget)target; \ + cb_data.args.hipDeviceFlushGPUDirectRDMAWrites.scope = (hipFlushGPUDirectRDMAWritesScope)scope; \ +}; // hipDeviceGet[('hipDevice_t*', 'device'), ('int', 'ordinal')] #define INIT_hipDeviceGet_CB_ARGS_DATA(cb_data) { \ cb_data.args.hipDeviceGet.device = (hipDevice_t*)device; \ @@ -7562,6 +7574,9 @@ static inline void hipApiArgsInit(hip_api_id_t id, hip_api_data_t* data) { // hipDeviceEnablePeerAccess[('int', 'peerDeviceId'), ('unsigned int', 'flags')] case HIP_API_ID_hipDeviceEnablePeerAccess: break; +// hipDeviceFlushGPUDirectRDMAWrites[('hipFlushGPUDirectRDMAWritesTarget', 'target'), ('hipFlushGPUDirectRDMAWritesScope', 'scope')] + case HIP_API_ID_hipDeviceFlushGPUDirectRDMAWrites: + break; // hipDeviceGet[('hipDevice_t*', 'device'), ('int', 'ordinal')] case HIP_API_ID_hipDeviceGet: if (data->args.hipDeviceGet.device) data->args.hipDeviceGet.device__val = *(data->args.hipDeviceGet.device); @@ -9587,6 +9602,12 @@ static inline const char* hipApiString(hip_api_id_t id, const hip_api_data_t* da oss << ", flags="; roctracer::hip_support::detail::operator<<(oss, data->args.hipDeviceEnablePeerAccess.flags); oss << ")"; break; + case HIP_API_ID_hipDeviceFlushGPUDirectRDMAWrites: + oss << "hipDeviceFlushGPUDirectRDMAWrites("; + oss << "target="; roctracer::hip_support::detail::operator<<(oss, data->args.hipDeviceFlushGPUDirectRDMAWrites.target); + oss << ", scope="; roctracer::hip_support::detail::operator<<(oss, data->args.hipDeviceFlushGPUDirectRDMAWrites.scope); + oss << ")"; + break; case HIP_API_ID_hipDeviceGet: oss << "hipDeviceGet("; if (data->args.hipDeviceGet.device == NULL) oss << "device=NULL"; diff --git a/projects/clr/hipamd/include/hip/amd_detail/host_defines.h b/projects/clr/hipamd/include/hip/amd_detail/host_defines.h index aa343848460a..562e4e67cc7a 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/host_defines.h +++ b/projects/clr/hipamd/include/hip/amd_detail/host_defines.h @@ -193,6 +193,20 @@ template <> struct numeric_limits { static constexpr char lowest() noexcept { return min(); } }; +template <> struct numeric_limits { + static constexpr bool is_specialized = true; + static constexpr bool is_signed = true; + static constexpr float min() noexcept { + return __builtin_bit_cast(float, 0x00800000); + } + static constexpr float lowest() noexcept { return -max(); }; + static constexpr float max() noexcept { + return __builtin_bit_cast(float, 0x7F7FFFFF); + } + static constexpr float quiet_NaN() noexcept { return __builtin_nanf(""); } + static constexpr float infinity() noexcept { return __builtin_huge_valf(); } +}; + #if defined(_WIN32) #pragma pop_macro("max") #pragma pop_macro("min") @@ -310,49 +324,24 @@ constexpr index_sequence make_index_sequence_value(index_sequence::max() and lowest(). Note that the -// class name and methods have been changed intentionally to reflect the fact that is not -// a one-to-one replacement of std::numeric_limits and also to avoid a name collision with -// the Win32 max() macro +// Allows to calculate the max() and lowest() of each type. Uses __hip_internal::numeric_limits() +// most of the time, but gets specialized for some floating point types template -struct NumericLimits; - -template <> -struct NumericLimits { - static constexpr int maximum() { return 0x7FFFFFFF; } - static constexpr int minimum() { return ~0x7FFFFFFF; } -}; - -template <> -struct NumericLimits { - static constexpr unsigned int maximum() { return 0xFFFFFFFFu; } - static constexpr unsigned int minimum() { return 0u; } -}; - -template <> -struct NumericLimits { - static constexpr long long maximum() { return 0x7FFFFFFFFFFFFFFFLL; } - static constexpr long long minimum() { return ~0x7FFFFFFFFFFFFFFFLL; } -}; - -template <> -struct NumericLimits { - static constexpr unsigned long long maximum() { return 0xFFFFFFFFFFFFFFFFull; } - static constexpr unsigned long long minimum() { return 0ull; } +struct ExclusiveScanIdentity { + static constexpr T maximum() { return (__hip_internal::numeric_limits::max)(); } + static constexpr T minimum() { return (__hip_internal::numeric_limits::min)(); } }; template <> -struct NumericLimits { - static constexpr float maximum() { return __builtin_bit_cast(float, 0x7f800000); } - static constexpr float minimum() { return -maximum(); } - static constexpr float quiet_NaN() { return __builtin_nanf(""); } - static constexpr float infinity() { return __builtin_huge_valf(); } +struct ExclusiveScanIdentity { + static constexpr float maximum() { return __builtin_bit_cast(float, 0x7f800000); } + static constexpr float minimum() { return -maximum(); } }; template <> -struct NumericLimits { - static constexpr double maximum() { return __builtin_bit_cast(double, 0x7FF0000000000000LL); } - static constexpr double minimum() { return -maximum(); } +struct ExclusiveScanIdentity { + static constexpr double maximum() { return __builtin_bit_cast(double, 0x7FF0000000000000LL); } + static constexpr double minimum() { return -maximum(); } }; #if defined(_MSC_VER) && !defined(__clang__) diff --git a/projects/clr/hipamd/src/amdhip.def.in b/projects/clr/hipamd/src/amdhip.def.in index ae497257f321..9e58bba8c62c 100644 --- a/projects/clr/hipamd/src/amdhip.def.in +++ b/projects/clr/hipamd/src/amdhip.def.in @@ -564,4 +564,5 @@ hipExecutionCtxWaitEvent hipMemGetDefaultMemPool hipKernelNameRefByPtr hipModuleEnumerateFunctions +hipDeviceFlushGPUDirectRDMAWrites @HSA_EXPORTS@ diff --git a/projects/clr/hipamd/src/hip_api_trace.cpp b/projects/clr/hipamd/src/hip_api_trace.cpp index bbf8939cef49..65f6627f3ee2 100644 --- a/projects/clr/hipamd/src/hip_api_trace.cpp +++ b/projects/clr/hipamd/src/hip_api_trace.cpp @@ -941,6 +941,8 @@ hipError_t hipMemGetDefaultMemPool(hipMemPool_t* memPool, hipMemLocation* locati hipMemAllocationType type); hipError_t hipModuleEnumerateFunctions(hipFunction_t* functions, unsigned int numFunctions, hipModule_t mod); +hipError_t hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTarget target, + hipFlushGPUDirectRDMAWritesScope scope); } // namespace hip namespace hip { @@ -1528,6 +1530,7 @@ void UpdateDispatchTable(HipDispatchTable* ptrDispatchTable) { ptrDispatchTable->hipMemGetDefaultMemPool_fn = hip::hipMemGetDefaultMemPool; ptrDispatchTable->hipInitDevice_fn = hip::hipInitDevice; ptrDispatchTable->hipModuleEnumerateFunctions_fn = hip::hipModuleEnumerateFunctions; + ptrDispatchTable->hipDeviceFlushGPUDirectRDMAWrites_fn = hip::hipDeviceFlushGPUDirectRDMAWrites; } #if HIP_ROCPROFILER_REGISTER > 0 @@ -2269,15 +2272,17 @@ HIP_ENFORCE_ABI(HipDispatchTable, hipDeviceGetLuid_fn, 542); HIP_ENFORCE_ABI(HipDispatchTable, hipInitDevice_fn, 543); // HIP_RUNTIME_API_TABLE_STEP_VERSION == 34 HIP_ENFORCE_ABI(HipDispatchTable, hipModuleEnumerateFunctions_fn, 544); +// HIP_RUNTIME_API_TABLE_STEP_VERSION == 35 +HIP_ENFORCE_ABI(HipDispatchTable, hipDeviceFlushGPUDirectRDMAWrites_fn, 545); // if HIP_ENFORCE_ABI entries are added for each new function pointer in the table, the number below // will be +1 of the number in the last HIP_ENFORCE_ABI line. E.g.: // // HIP_ENFORCE_ABI(, , 8) // // HIP_ENFORCE_ABI_VERSIONING(
, 9) <- 8 + 1 = 9 -HIP_ENFORCE_ABI_VERSIONING(HipDispatchTable, 545) +HIP_ENFORCE_ABI_VERSIONING(HipDispatchTable, 546) -static_assert(HIP_RUNTIME_API_TABLE_MAJOR_VERSION == 0 && HIP_RUNTIME_API_TABLE_STEP_VERSION == 34, +static_assert(HIP_RUNTIME_API_TABLE_MAJOR_VERSION == 0 && HIP_RUNTIME_API_TABLE_STEP_VERSION == 35, "If you get this error, add new HIP_ENFORCE_ABI(...) code for the new function " "pointers and then update this check so it is true"); #endif diff --git a/projects/clr/hipamd/src/hip_code_object.cpp b/projects/clr/hipamd/src/hip_code_object.cpp index 4b6536ca78bd..b0784fb5b18d 100644 --- a/projects/clr/hipamd/src/hip_code_object.cpp +++ b/projects/clr/hipamd/src/hip_code_object.cpp @@ -708,4 +708,31 @@ void StatCO::ForEachFatBinaryBlob(void (*cb)(const void*)) const { cb(data); } } + +// ================================================================================================ +void StatCO::ForEachGlobalVar(void (*cb)(const void*, const char*, size_t, + const void*)) { + std::scoped_lock lock(sclock_); + // Resolved here rather than by the caller: this sweep runs from HRR's + // capture init, which is inside hip::init(), and re-entering the runtime + // through hipGetSymbolAddress from there deadlocks on the init once-flag. + // No device is current that early, and GetGlobalVar asserts rather than + // erroring on a negative device id, so the address is reported as + // unresolved instead. + // No device is current this early in init, and GetGlobalVar asserts rather + // than erroring on a negative id, so fall back to device 0 — which is the + // device a recording resolves its globals on anyway. + int device_id = ihipGetDevice(); + if (device_id < 0) device_id = 0; + const bool resolvable = static_cast(device_id) < g_devices.size(); + for (const auto& [host_var, var] : vars_) { + if (var == nullptr) continue; + hipDeviceptr_t dev_ptr = nullptr; + size_t sym_size = 0; + if (resolvable && + GetGlobalVar(host_var, device_id, &dev_ptr, &sym_size) != hipSuccess) + dev_ptr = nullptr; + cb(host_var, var->GetName().c_str(), var->GetSize(), dev_ptr); + } +} } // namespace hip diff --git a/projects/clr/hipamd/src/hip_code_object.hpp b/projects/clr/hipamd/src/hip_code_object.hpp index a98402f0d9ff..362bbb9322f6 100644 --- a/projects/clr/hipamd/src/hip_code_object.hpp +++ b/projects/clr/hipamd/src/hip_code_object.hpp @@ -193,6 +193,12 @@ class StatCO : public CodeObject { // Iterate all registered fat binary data pointers — for HRR capture post-registration sweep. void ForEachFatBinaryBlob(void (*cb)(const void*)) const; + // Iterate all registered __device__ globals as (host shadow address, symbol + // name, size, device address) — the same post-registration sweep for HRR, + // which otherwise never sees __hipRegisterVar because it fires at + // static-init time. The device address is null if it cannot be resolved yet. + void ForEachGlobalVar(void (*cb)(const void*, const char*, size_t, const void*)); + private: mutable std::recursive_mutex sclock_; //!< Guards Static Code object const PlatformState& owner_; //!< Reference to owning PlatformState diff --git a/projects/clr/hipamd/src/hip_device.cpp b/projects/clr/hipamd/src/hip_device.cpp index ea82c7be48e0..49f563faac17 100644 --- a/projects/clr/hipamd/src/hip_device.cpp +++ b/projects/clr/hipamd/src/hip_device.cpp @@ -805,9 +805,11 @@ hipError_t ihipGetDeviceProperties(hipDeviceProp_t* props, int device) { // Mapping HIP array deviceProps.deferredMappingHipArraySupported = 0; // RDMA options - deviceProps.gpuDirectRDMASupported = 0; - deviceProps.gpuDirectRDMAFlushWritesOptions = 0; - deviceProps.gpuDirectRDMAWritesOrdering = 0; + deviceProps.gpuDirectRDMASupported = (info.dmabufSupported_ && info.largeBar_) ? 1 : 0; + // Set option if we have a HDP register + deviceProps.gpuDirectRDMAFlushWritesOptions = + hip::ihipRdmaFlushWritesOptions(info.hdpMemFlushCntl); + deviceProps.gpuDirectRDMAWritesOrdering = hip::ihipRdmaWritesOrdering(); // The LUID is a Windows/DXGI adapter concept; the backend reports a zero LUID // and zero node mask on platforms without a WDDM adapter. *reinterpret_cast(&deviceProps.luid[0]) = info.luidLowPart_; diff --git a/projects/clr/hipamd/src/hip_device_runtime.cpp b/projects/clr/hipamd/src/hip_device_runtime.cpp index bdcf7e0dad0d..f77ad1d360de 100644 --- a/projects/clr/hipamd/src/hip_device_runtime.cpp +++ b/projects/clr/hipamd/src/hip_device_runtime.cpp @@ -450,6 +450,15 @@ hipError_t hipDeviceGetAttribute(int* pi, hipDeviceAttribute_t attr, int device) *pi = static_cast( g_devices[device]->devices()[0]->info().gpuDirectRdmaWithHipVmmSupported_); break; + case hipDeviceAttributeGPUDirectRDMASupported: + *pi = prop.gpuDirectRDMASupported; + break; + case hipDeviceAttributeGPUDirectRDMAFlushWritesOptions: + *pi = static_cast(prop.gpuDirectRDMAFlushWritesOptions); + break; + case hipDeviceAttributeGPUDirectRDMAWritesOrdering: + *pi = prop.gpuDirectRDMAWritesOrdering; + break; case hipDeviceAttributeHandleTypeFabricSupported: *pi = static_cast(g_devices[device]->devices()[0]->info().fabric_handle_); break; @@ -507,6 +516,37 @@ hipError_t hipDeviceGetByPCIBusId(int* device, const char* pciBusIdstr) { HIP_RETURN(hipSuccess); } +hipError_t hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTarget target, + hipFlushGPUDirectRDMAWritesScope scope) { + HIP_INIT_API(hipDeviceFlushGPUDirectRDMAWrites, target, scope); + + if ((target != hipFlushGPUDirectRDMAWritesTargetCurrentDevice) || + (scope != hipFlushGPUDirectRDMAWritesToOwner && + scope != hipFlushGPUDirectRDMAWritesToAllDevices)) { + HIP_RETURN(hipErrorInvalidValue); + } + + const auto& info = hip::getCurrentDevice()->devices()[0]->info(); + + if ((hip::ihipRdmaFlushWritesOptions(info.hdpMemFlushCntl) & + hipFlushGPUDirectRDMAWritesOptionHost) == 0) { + HIP_RETURN(hipErrorNotSupported); + } + + // Already ordered by hardware at the requested scope: nothing to do. + // In our case it will be false all the time. + if (hip::ihipRdmaWritesOrdering() >= static_cast(scope)) { + HIP_RETURN(hipSuccess); + } + + // volatile, to protect it from optimization passes + volatile uint32_t* hdpMemFlushCntl = info.hdpMemFlushCntl; + *hdpMemFlushCntl = 1u; + static_cast(*hdpMemFlushCntl); + + HIP_RETURN(hipSuccess); +} + hipError_t hipDeviceGetCacheConfig(hipFuncCache_t* cacheConfig) { HIP_INIT_API(hipDeviceGetCacheConfig, cacheConfig); diff --git a/projects/clr/hipamd/src/hip_embed_pch.sh b/projects/clr/hipamd/src/hip_embed_pch.sh index b3756143d2c4..b450d0c4b561 100755 --- a/projects/clr/hipamd/src/hip_embed_pch.sh +++ b/projects/clr/hipamd/src/hip_embed_pch.sh @@ -106,8 +106,7 @@ EOF fi cat << EOF >> $tmp/hip_pch.mcin - .section .hip_pch_wave32,"aMS",@progbits,1 - .data + .section .hip_pch_wave32,"a",@progbits .globl __hip_pch_wave32 .globl __hip_pch_wave32_size .p2align 3 @@ -116,8 +115,7 @@ __hip_pch_wave32: __hip_pch_wave32_size: .long __hip_pch_wave32_size - __hip_pch_wave32 .type __hip_pch_wave64,@object - .section .hip_pch_wave64,"aMS",@progbits,1 - .data + .section .hip_pch_wave64,"a",@progbits .globl __hip_pch_wave64 .globl __hip_pch_wave64_size .p2align 3 diff --git a/projects/clr/hipamd/src/hip_event.cpp b/projects/clr/hipamd/src/hip_event.cpp index 353bf79b9c3c..203608f1a8b9 100644 --- a/projects/clr/hipamd/src/hip_event.cpp +++ b/projects/clr/hipamd/src/hip_event.cpp @@ -513,9 +513,10 @@ static hipError_t checkEventCaptureRestrictions(hipEvent_t event) { return hipSuccess; } // Case 1: Event was recorded during an active stream capture and is part of an active capture. + // The whole sequence is invalidated, not just the stream the query names. auto* s = reinterpret_cast(hip_stream); if (s->GetCaptureStatus() == hipStreamCaptureStatusActive && s->IsEventCaptured(event)) { - s->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + s->InvalidateCapture(); return hipErrorCapturedEvent; } // Case 2: The event was recorded on a stream that is neither actively capturing nor part of an @@ -531,7 +532,7 @@ static hipError_t checkEventCaptureRestrictions(hipEvent_t event) { amd::ScopedLock lock(g_captureStreamsLock); if (!g_captureStreams.empty()) { for (auto stream : g_captureStreams) { - stream->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + stream->InvalidateCapture(); } return hipErrorStreamCaptureUnsupported; } @@ -540,7 +541,7 @@ static hipError_t checkEventCaptureRestrictions(hipEvent_t event) { // Block if calling thread itself is capturing (both GLOBAL and THREAD_LOCAL) if (!hip::tls.capture_streams_.empty()) { for (auto stream : hip::tls.capture_streams_) { - stream->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + stream->InvalidateCapture(); } return hipErrorStreamCaptureUnsupported; } diff --git a/projects/clr/hipamd/src/hip_global.cpp b/projects/clr/hipamd/src/hip_global.cpp index 0ff63140c374..c476c7dfdb68 100644 --- a/projects/clr/hipamd/src/hip_global.cpp +++ b/projects/clr/hipamd/src/hip_global.cpp @@ -24,7 +24,10 @@ extern const char __hip_pch_wave32[]; extern const char __hip_pch_wave64[]; extern unsigned __hip_pch_wave32_size; extern unsigned __hip_pch_wave64_size; +#endif + void __hipGetPCH(const char** pch, unsigned int* size) { +#ifdef __HIP_ENABLE_PCH hipDeviceProp_t deviceProp; int deviceId; hipError_t error = hipGetDevice(&deviceId); @@ -36,8 +39,11 @@ void __hipGetPCH(const char** pch, unsigned int* size) { *pch = __hip_pch_wave64; *size = __hip_pch_wave64_size; } -} +#else + *pch = nullptr; + *size = 0; #endif +} namespace hip { // forward declaration of methods required for managed variables diff --git a/projects/clr/hipamd/src/hip_graph.cpp b/projects/clr/hipamd/src/hip_graph.cpp index 44424e75047d..0e24c2014c36 100644 --- a/projects/clr/hipamd/src/hip_graph.cpp +++ b/projects/clr/hipamd/src/hip_graph.cpp @@ -457,7 +457,7 @@ hipError_t capturehipDrvLaunchKernelEx(hipStream_t& stream, const HIP_LAUNCH_CON if (attr.value.dynDataPrefetch == nullptr) { return hipErrorInvalidValue; } - s->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + s->InvalidateCapture(); return hipErrorStreamCaptureUnsupported; default: LogPrintfError("Attribute %u not supported", attr.id); @@ -1240,18 +1240,16 @@ hipError_t hipStreamBeginCapture_common(hipStream_t stream, hipStreamCaptureMode return hipErrorInvalidValue; } hip::Stream* s = reinterpret_cast(stream); - // It can be initiated if the stream is not already in capture mode - if (s->GetCaptureStatus() == hipStreamCaptureStatusActive) { + // A capture can only be started on a stream that is not already part of one. An + // invalidated capture still has to be ended on its origin before any of its streams can be + // reused, so an invalidated stream is not eligible either. + if (s->GetCaptureStatus() != hipStreamCaptureStatusNone) { return hipErrorIllegalState; } - if (graph == nullptr) { - s->SetCaptureGraph(new hip::Graph(s->GetDevice())); - } else { - s->SetCaptureGraph(reinterpret_cast(graph)); - } - s->SetCaptureID(); - s->SetCaptureMode(mode); - s->SetOriginStream(); + std::unique_ptr captureGraph( + (graph == nullptr) ? new hip::Graph(s->GetDevice()) : reinterpret_cast(graph)); + s->StartCapture(std::move(captureGraph), mode); + if (mode != hipStreamCaptureModeRelaxed) { hip::tls.capture_streams_.push_back(s); } @@ -1349,11 +1347,9 @@ hipError_t hipStreamEndCapture_common(hipStream_t stream, hip::Graph** pGraph) { } // If capture was invalidated, due to a violation of the rules of stream capture if (s->GetCaptureStatus() == hipStreamCaptureStatusInvalidated) { - *pGraph = nullptr; - // When capture is invalidated, graph should be deleted, otherwise it leaks - s->ReleaseCaptureGraph(); - // Reset capture state to None so the stream is usable after a failed capture - (void)s->EndCapture(); + // Reset capture state to None so the stream is usable after a failed capture. An + // invalidated capture yields no graph. + *pGraph = s->EndCapture(); return hipErrorStreamCaptureInvalidated; } @@ -1389,8 +1385,11 @@ hipError_t hipStreamEndCapture_common(hipStream_t stream, hip::Graph** pGraph) { s->GetCaptureGraph()->RemoveNode(pGraphNode); s->GetCaptureGraph()->RemoveManualNodesDuringCapture(); if (leafNodes.size() > 1 && foundInRemovedDep == false) { - // Release created graph as it can't be retrieved anymore - s->ReleaseCaptureGraph(); + // The unjoined work leaves a graph that can no longer be retrieved. Marking the capture + // invalidated has EndCapture discard it and return every stream in it to the + // non-capturing state. + s->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + (void)s->EndCapture(); return hipErrorStreamCaptureUnjoined; } } else { @@ -1398,14 +1397,15 @@ hipError_t hipStreamEndCapture_common(hipStream_t stream, hip::Graph** pGraph) { s->GetCaptureGraph()->RemoveNode(pGraphNode); } - *pGraph = s->GetCaptureGraph(); - // end capture on all streams/events part of graph capture - return s->EndCapture(); + // end capture on all streams/events part of graph capture, which yields the graph the caller + // now owns + *pGraph = s->EndCapture(); + return hipSuccess; } hipError_t hipStreamEndCapture(hipStream_t stream, hipGraph_t* pGraph) { HIP_INIT_API(hipStreamEndCapture, stream, pGraph); - hip::Graph* graph; + hip::Graph* graph = nullptr; hipError_t status = hipStreamEndCapture_common(stream, &graph); if (pGraph != nullptr) { *pGraph = reinterpret_cast(graph); @@ -1416,7 +1416,7 @@ hipError_t hipStreamEndCapture(hipStream_t stream, hipGraph_t* pGraph) { hipError_t hipStreamEndCapture_spt(hipStream_t stream, hipGraph_t* pGraph) { HIP_INIT_API(hipStreamEndCapture, stream, pGraph); PER_THREAD_DEFAULT_STREAM(stream); - hip::Graph* graph; + hip::Graph* graph = nullptr; hipError_t status = hipStreamEndCapture_common(stream, &graph); if (pGraph != nullptr) { *pGraph = reinterpret_cast(graph); @@ -1439,9 +1439,8 @@ hipError_t hipGraphDestroy(hipGraph_t graph) { HIP_RETURN(hipErrorInvalidValue); } hip::Graph* g = reinterpret_cast(graph); - // if graph is not valid its destroyed already if (!hip::Graph::isGraphValid(g)) { - HIP_RETURN(hipErrorIllegalState); + HIP_RETURN(hipErrorInvalidValue); } delete g; HIP_RETURN(hipSuccess); @@ -1673,6 +1672,8 @@ hipError_t ihipGraphInstantiate(hip::GraphExecBase** pGraphExec, hip::Graph* gra // DEBUG_HIP_GRAPH_CLASSIC_PATH=1 forces this path on Linux for testing without // enabling the full PAL backend. if (GPU_ENABLE_PAL != 0 || DEBUG_HIP_GRAPH_CLASSIC_PATH) { + // hipGraphInstantiateFlagUseNodePriority is a scheduling hint; CUDA never + // fails instantiation for it, so accept and ignore it here too. auto* classicExec = new hip::GraphExecClassic(flags); graph->clone(classicExec, true); hipError_t initStatus = validateClonedKernelNodes(static_cast(classicExec)); @@ -1727,6 +1728,13 @@ hipError_t hipGraphInstantiate(hipGraphExec_t* pGraphExec, hipGraph_t graph, HIP_RETURN(status, ReturnPtrValue(pGraphExec)); } +// Accepted flag bitmasks per entry point; intentionally narrower than all declared flags. +static constexpr unsigned long long kValidInstantiateFlags = + hipGraphInstantiateFlagAutoFreeOnLaunch | hipGraphInstantiateFlagUseNodePriority; +static constexpr unsigned long long kValidInstantiateParamsFlags = + hipGraphInstantiateFlagAutoFreeOnLaunch | hipGraphInstantiateFlagUpload | + hipGraphInstantiateFlagDeviceLaunch | hipGraphInstantiateFlagUseNodePriority; + hipError_t hipGraphInstantiateWithFlags(hipGraphExec_t* pGraphExec, hipGraph_t graph, unsigned long long flags = 0) { HIP_INIT_API(hipGraphInstantiateWithFlags, pGraphExec, graph, flags); @@ -1734,9 +1742,7 @@ hipError_t hipGraphInstantiateWithFlags(hipGraphExec_t* pGraphExec, hipGraph_t g HIP_RETURN(hipErrorInvalidValue); } - // invalid flag check - if (flags != 0 && flags != hipGraphInstantiateFlagAutoFreeOnLaunch && - flags != hipGraphInstantiateFlagUseNodePriority) { + if ((flags & ~kValidInstantiateFlags) != 0) { HIP_RETURN(hipErrorInvalidValue); } @@ -1755,9 +1761,7 @@ hipError_t hipGraphInstantiateWithParams(hipGraphExec_t* pGraphExec, hipGraph_t unsigned long long flags = instantiateParams->flags; - if (flags != 0 && flags != hipGraphInstantiateFlagAutoFreeOnLaunch && - flags != hipGraphInstantiateFlagUpload && flags != hipGraphInstantiateFlagDeviceLaunch && - flags != hipGraphInstantiateFlagUseNodePriority) { + if ((flags & ~kValidInstantiateParamsFlags) != 0) { instantiateParams->result_out = hipGraphInstantiateError; HIP_RETURN(hipErrorInvalidValue); } @@ -1773,7 +1777,7 @@ hipError_t hipGraphInstantiateWithParams(hipGraphExec_t* pGraphExec, hipGraph_t instantiateParams->result_out = hipGraphInstantiateSuccess; instantiateParams->errNode_out = nullptr; - if (flags == hipGraphInstantiateFlagUpload) { + if ((flags & hipGraphInstantiateFlagUpload) != 0) { hipError_t status = ihipGraphUpload(*pGraphExec, instantiateParams->uploadStream); HIP_RETURN(status); } @@ -2306,6 +2310,9 @@ hipError_t ihipStreamUpdateCaptureDependencies(hipStream_t stream, hipGraphNode_ if (s->GetCaptureStatus() == hipStreamCaptureStatusNone) { HIP_RETURN(hipErrorIllegalState); } + if (s->GetCaptureStatus() == hipStreamCaptureStatusInvalidated) { + HIP_RETURN(hipErrorStreamCaptureInvalidated); + } if ((s->GetCaptureGraph()->GetNodeCount() < numDependencies) || (numDependencies > 0 && deps == nullptr) || (flags != 0 && flags != hipStreamAddCaptureDependencies && @@ -2321,13 +2328,14 @@ hipError_t ihipStreamUpdateCaptureDependencies(hipStream_t stream, hipGraphNode_ } depNodes.push_back(deps[i]); } + hipError_t status = hipSuccess; if (flags == hipStreamAddCaptureDependencies) { - s->AddCrossCapturedNode(depNodes); + status = s->AddCrossCapturedNode(depNodes); } else if (flags == hipStreamSetCaptureDependencies) { bool replace = true; - s->AddCrossCapturedNode(depNodes, replace); + status = s->AddCrossCapturedNode(depNodes, replace); } - HIP_RETURN(hipSuccess); + HIP_RETURN(status); } hipError_t hipStreamUpdateCaptureDependencies(hipStream_t stream, hipGraphNode_t* dependencies, diff --git a/projects/clr/hipamd/src/hip_graph_internal.cpp b/projects/clr/hipamd/src/hip_graph_internal.cpp index 88856a0c0b24..5aec9bd639ae 100644 --- a/projects/clr/hipamd/src/hip_graph_internal.cpp +++ b/projects/clr/hipamd/src/hip_graph_internal.cpp @@ -903,7 +903,19 @@ hipError_t Graph::CreateSegmentsFromPaths( segments_.clear(); node_to_segment_id_.clear(); - // Create a segment for each execution path at this level + // Process child graphs first so their segment priorities are cached before + // parent segments read them. + for (size_t i = 0; i < exec_paths.child_graph_paths.size(); ++i) { + const auto& child_paths = exec_paths.child_graph_paths[i]; + if (child_paths.graph_ptr != nullptr) { + hipError_t status = child_paths.graph_ptr->CreateSegmentsFromPaths(child_paths); + if (status != hipSuccess) { + return status; + } + } + } + + // Create a segment for each execution path at this level. int segment_id = 0; for (size_t i = 0; i < exec_paths.paths.size(); ++i) { const auto& h_path = exec_paths.paths[i]; @@ -916,16 +928,42 @@ hipError_t Graph::CreateSegmentsFromPaths( segment.first_node = h_path.nodes.front(); segment.last_node = h_path.nodes.back(); - // Preserve child graph information from hierarchical path + // Preserve child graph information from hierarchical path. if (h_path.child_graph_node != nullptr && h_path.child_graph_paths_index >= 0) { - // Get direct pointer to child graph from the node auto childGraphNode = reinterpret_cast(h_path.child_graph_node); segment.child_graph_ptr = childGraphNode->GetChildGraph(); } + // Cache declared priority from kernel nodes and child graph segments. + // Accumulate from Low upward so all-Low segments stay Low; unset kernel + // nodes return Normal (0) which pulls the accumulator up via std::min. + // Non-kernel nodes (memcpy, event, …) are skipped — if no kernel or + // child-graph node is found the segment keeps its Normal default. + int seg_prio = hip::Stream::Priority::Low; + bool has_priority_node = false; + for (const auto& node : segment.nodes) { + if (node == nullptr) continue; + int node_priority; + if (node->GetType() == hipGraphNodeTypeKernel) { + node_priority = static_cast(node)->GetDeclaredPriority(); + } else if (node->GetType() == hipGraphNodeTypeGraph) { + Graph* child = node->GetChildGraph(); + if (child == nullptr || child->segments_.empty()) continue; + node_priority = hip::Stream::Priority::Low; + for (const auto& child_seg : child->segments_) + node_priority = std::min(node_priority, child_seg.declared_priority); + } else { + continue; + } + seg_prio = std::min(seg_prio, node_priority); + has_priority_node = true; + } + if (has_priority_node) + segment.declared_priority = seg_prio; + segments_.push_back(segment); - // Map each node in this segment to the segment ID (local to this graph) + // Map each node in this segment to the segment ID (local to this graph). for (const auto& node : segment.nodes) { node_to_segment_id_[node] = segment_id; node->segment_id_ = segment_id; @@ -934,18 +972,6 @@ hipError_t Graph::CreateSegmentsFromPaths( segment_id++; } - // Recursively process child graphs - for (size_t i = 0; i < exec_paths.child_graph_paths.size(); ++i) { - const auto& child_paths = exec_paths.child_graph_paths[i]; - - if (child_paths.graph_ptr != nullptr) { - // Let the child graph create its own segments - hipError_t status = child_paths.graph_ptr->CreateSegmentsFromPaths(child_paths); - if (status != hipSuccess) { - return status; - } - } - } return hipSuccess; } @@ -1294,6 +1320,16 @@ void GraphExecSegmented::RoundRobinStreamAssignment() { ? static_cast(it->second) : 1; }; + // Sort segments by declared priority so high-priority segments get slot 0 (the launch stream). + // Only reorder when the caller opts in via hipGraphInstantiateFlagUseNodePriority; recorded + // priorities remain available for the flagged path regardless. + const bool priority_declared = (flags_ & hipGraphInstantiateFlagUseNodePriority) != 0; + auto priority_of = [&](int seg_id) -> int { + if (seg_id >= 0 && seg_id < static_cast(segments_.size())) + return segments_[seg_id].declared_priority; + return static_cast(hip::Stream::Priority::Normal); + }; + for (int level = 0; level <= max_dependency_level_; ++level) { auto it = segments_per_level_.find(level); if (it == segments_per_level_.end()) continue; @@ -1302,6 +1338,11 @@ void GraphExecSegmented::RoundRobinStreamAssignment() { // the same device spread evenly across that device's stream pool. std::unordered_map dev_idx; + if (priority_declared) { + std::stable_sort(it->second.begin(), it->second.end(), + [&](int lhs, int rhs) { return priority_of(lhs) < priority_of(rhs); }); + } + for (int seg_id : it->second) { if (seg_id >= 0 && seg_id < static_cast(segments_.size())) { auto& seg = segments_[seg_id]; diff --git a/projects/clr/hipamd/src/hip_graph_internal.hpp b/projects/clr/hipamd/src/hip_graph_internal.hpp index 16444c2a65df..e4965a6577b9 100644 --- a/projects/clr/hipamd/src/hip_graph_internal.hpp +++ b/projects/clr/hipamd/src/hip_graph_internal.hpp @@ -487,6 +487,7 @@ class GraphNode : public hipGraphNodeDOTAttribute { // Default false: conservatively forces SetParams for node types without an override. virtual bool HasSameParams(const GraphNode* node) const { return false; } virtual GraphMemcpyNodeKind GetMemcpyNodeKind() const { return GraphMemcpyNodeKind::None; } + virtual void SetCapturedPriority(int priority) {} virtual void GenerateDOT(std::ostream& fout, hipGraphDebugDotFlags flag) {} virtual void GenerateDOTNode(size_t graphId, std::ostream& fout, hipGraphDebugDotFlags flag) { fout << "\n"; @@ -1027,6 +1028,9 @@ class Graph { Graph* child_graph_ptr = nullptr; // Direct pointer to child graph for quick access bool needs_completion_signal = false; // True if any downstream segment is on a different stream/device, or this is a leaf + + // Most urgent declared priority across this segment's nodes. + int declared_priority = hip::Stream::Priority::Normal; }; //! Segment information for batch scheduling @@ -1596,8 +1600,10 @@ class ChildGraphNode : public GraphNode, public GraphExecSegmented { class GraphKernelNode : public GraphNode { hipKernelNodeParams kernelParams_{}; //!< Kernel node parameters unsigned int numParams_ = 0; //!< No. of kernel params as part of signature - hipKernelNodeAttrValue kernelAttr_; //!< Kernel node attributes - unsigned int kernelAttrInUse_; //!< Kernel attributes in use + //! Each attribute has its own slot; independent of each other. + hipAccessPolicyWindow accessPolicyWindow_; //!< hipKernelNodeAttributeAccessPolicyWindow + int cooperative_; //!< hipKernelNodeAttributeCooperative + int priority_; //!< hipLaunchAttributePriority ihipExtKernelEvents kernelEvents_; //!< Events for Ext launch kernel bool hasHiddenHeap_ = false; //!< Kernel has hidden heap(device side allocation) int coopKernel_; //!< Launch cooperative kernel @@ -1629,14 +1635,20 @@ class GraphKernelNode : public GraphNode { kernelParams_ = {}; ClPrint(amd::LOG_ERROR, amd::LOG_CODE, "[hipGraph] Failed to allocate memory to copy params"); } - memset(&kernelAttr_, 0, sizeof(kernelAttr_)); - kernelAttrInUse_ = 0; + ResetAttrs(); status = CopyAttr(&rhs); if (status != hipSuccess) { ClPrint(amd::LOG_ERROR, amd::LOG_CODE, "[hipGraph] Failed to during copy attrs"); } } + //! Put every attribute slot back in its "never written" state. + void ResetAttrs() { + memset(&accessPolicyWindow_, 0, sizeof(accessPolicyWindow_)); + cooperative_ = 0; + priority_ = hip::Stream::Priority::Normal; + } + public: bool HasHiddenHeap() const { return hasHiddenHeap_; } @@ -1698,10 +1710,9 @@ class GraphKernelNode : public GraphNode { kernelParams_.blockDim.y, kernelParams_.blockDim.z, globalWorkSizeX_remainder_, globalWorkSizeY_remainder_, globalWorkSizeZ_remainder_, kernelParams_.sharedMemBytes, this, kernelParams_.func, - kernelAttr_.accessPolicyWindow.base_ptr, kernelAttr_.accessPolicyWindow.num_bytes, - kernelAttr_.accessPolicyWindow.hitRatio, kernelAttr_.accessPolicyWindow.hitProp, - kernelAttr_.accessPolicyWindow.missProp, kernelAttr_.cooperative, - kernelAttr_.priority); + accessPolicyWindow_.base_ptr, accessPolicyWindow_.num_bytes, + accessPolicyWindow_.hitRatio, accessPolicyWindow_.hitProp, + accessPolicyWindow_.missProp, cooperative_, priority_); label = buffer; } else if (flag == hipGraphDebugDotFlagsKernelNodeAttributes) { sprintf(buffer, @@ -1709,10 +1720,9 @@ class GraphKernelNode : public GraphNode { "| {accessPolicyWindow | {base_ptr | num_bytes | " "hitRatio | hitProp | missProp} |\n| {%p | %zu | %f | %d | %d}}\n| {cooperative | " "%u}\n| {priority | %d}\n}", - label_, GetID(), demangledName.c_str(), kernelAttr_.accessPolicyWindow.base_ptr, - kernelAttr_.accessPolicyWindow.num_bytes, kernelAttr_.accessPolicyWindow.hitRatio, - kernelAttr_.accessPolicyWindow.hitProp, kernelAttr_.accessPolicyWindow.missProp, - kernelAttr_.cooperative, kernelAttr_.priority); + label_, GetID(), demangledName.c_str(), accessPolicyWindow_.base_ptr, + accessPolicyWindow_.num_bytes, accessPolicyWindow_.hitRatio, + accessPolicyWindow_.hitProp, accessPolicyWindow_.missProp, cooperative_, priority_); label = buffer; } else if (flag == hipGraphDebugDotFlagsKernelNodeParams) { @@ -1875,8 +1885,8 @@ class GraphKernelNode : public GraphNode { if (paramCopyStatus_ != hipSuccess) { ClPrint(amd::LOG_ERROR, amd::LOG_CODE, "[hipGraph] Failed to copy params"); } - memset(&kernelAttr_, 0, sizeof(kernelAttr_)); - kernelAttrInUse_ = 0; + ResetAttrs(); + hasHiddenHeap_ = false; coopKernel_ = coopKernel; globalWorkSizeX_remainder_ = globalWorkSizeX_remainder; globalWorkSizeY_remainder_ = globalWorkSizeY_remainder; @@ -1922,8 +1932,9 @@ class GraphKernelNode : public GraphNode { globalWorkSizeZ_remainder_ != other->globalWorkSizeZ_remainder_ || clusterDim_.x != other->clusterDim_.x || clusterDim_.y != other->clusterDim_.y || clusterDim_.z != other->clusterDim_.z || launchFlags_ != other->launchFlags_ || - kernelAttrInUse_ != other->kernelAttrInUse_ || - std::memcmp(&kernelAttr_, &other->kernelAttr_, sizeof(kernelAttr_)) != 0 || + std::memcmp(&accessPolicyWindow_, &other->accessPolicyWindow_, sizeof(accessPolicyWindow_)) != 0 || + cooperative_ != other->cooperative_ || + priority_ != other->priority_ || std::memcmp(&kernelEvents_, &other->kernelEvents_, sizeof(kernelEvents_)) != 0) { return false; } @@ -1959,6 +1970,13 @@ class GraphKernelNode : public GraphNode { return true; } + int GetDeclaredPriority() const { return priority_; } + + void SetCapturedPriority(int priority) override { + priority_ = std::clamp(priority, static_cast(hip::Stream::Priority::High), + static_cast(hip::Stream::Priority::Low)); + } + hipError_t CreateCommand(hip::Stream* stream) override { // Clear commands_ first, even if node is disabled hipError_t status = GraphNode::CreateCommand(stream); @@ -1999,6 +2017,9 @@ class GraphKernelNode : public GraphNode { status = ihipLaunchKernelCommand( command, func, launch_params, stream, kernelParams_.kernelParams, kernelParams_.extra, kernelEvents_.startEvent_, kernelEvents_.stopEvent_, flags, coopKernel_, 0, 0, 0, 0, 0); + if (status != hipSuccess) { + return status; + } if (signal_is_required_) { // Optimize the barriers by adding a signal into the dispatch packet directly command->SetProfiling(); @@ -2068,19 +2089,21 @@ class GraphKernelNode : public GraphNode { return hipErrorInvalidValue; } - kernelAttr_.accessPolicyWindow.base_ptr = params->accessPolicyWindow.base_ptr; - kernelAttr_.accessPolicyWindow.hitProp = params->accessPolicyWindow.hitProp; - kernelAttr_.accessPolicyWindow.hitRatio = params->accessPolicyWindow.hitRatio; - kernelAttr_.accessPolicyWindow.missProp = params->accessPolicyWindow.missProp; - kernelAttr_.accessPolicyWindow.num_bytes = params->accessPolicyWindow.num_bytes; + accessPolicyWindow_.base_ptr = params->accessPolicyWindow.base_ptr; + accessPolicyWindow_.hitProp = params->accessPolicyWindow.hitProp; + accessPolicyWindow_.hitRatio = params->accessPolicyWindow.hitRatio; + accessPolicyWindow_.missProp = params->accessPolicyWindow.missProp; + accessPolicyWindow_.num_bytes = params->accessPolicyWindow.num_bytes; } else if (attr == hipKernelNodeAttributeCooperative) { - kernelAttr_.cooperative = params->cooperative; + cooperative_ = params->cooperative; } else if (attr == hipLaunchAttributePriority) { - if (params->priority < hip::Stream::Priority::Low || - params->priority > hip::Stream::Priority::High) { + // Priority::High is the numerically smallest value and Priority::Low the + // largest, so the valid range is [High, Low] and not [Low, High]. + if (params->priority < hip::Stream::Priority::High || + params->priority > hip::Stream::Priority::Low) { return hipErrorInvalidValue; } - kernelAttr_.priority = params->priority; + priority_ = params->priority; } else if (attr == hipLaunchAttributeClusterDimension) { dim3 clusterDim = {params->clusterDim.x, params->clusterDim.y, params->clusterDim.z}; if (clusterDim.x == 0 || clusterDim.y == 0 || clusterDim.z == 0) { @@ -2099,61 +2122,39 @@ class GraphKernelNode : public GraphNode { } clusterDim_ = clusterDim; return hipSuccess; + } else { + return hipErrorInvalidValue; } - kernelAttrInUse_ = attr; return hipSuccess; } hipError_t GetAttrParams(hipKernelNodeAttrID attr, hipKernelNodeAttrValue* params) { - // Get kernel attr params - if (attr != hipLaunchAttributeClusterDimension && - kernelAttrInUse_ != 0 && kernelAttrInUse_ != attr) { - return hipErrorInvalidValue; - } + // An unset attribute reads back as its default. if (attr == hipKernelNodeAttributeAccessPolicyWindow) { - params->accessPolicyWindow.base_ptr = kernelAttr_.accessPolicyWindow.base_ptr; - params->accessPolicyWindow.hitProp = kernelAttr_.accessPolicyWindow.hitProp; - params->accessPolicyWindow.hitRatio = kernelAttr_.accessPolicyWindow.hitRatio; - params->accessPolicyWindow.missProp = kernelAttr_.accessPolicyWindow.missProp; - params->accessPolicyWindow.num_bytes = kernelAttr_.accessPolicyWindow.num_bytes; + params->accessPolicyWindow.base_ptr = accessPolicyWindow_.base_ptr; + params->accessPolicyWindow.hitProp = accessPolicyWindow_.hitProp; + params->accessPolicyWindow.hitRatio = accessPolicyWindow_.hitRatio; + params->accessPolicyWindow.missProp = accessPolicyWindow_.missProp; + params->accessPolicyWindow.num_bytes = accessPolicyWindow_.num_bytes; } else if (attr == hipKernelNodeAttributeCooperative) { - params->cooperative = kernelAttr_.cooperative; + params->cooperative = cooperative_; } else if (attr == hipLaunchAttributePriority) { - params->priority = kernelAttr_.priority; + params->priority = priority_; } else if (attr == hipLaunchAttributeClusterDimension) { params->clusterDim.x = clusterDim_.x; params->clusterDim.y = clusterDim_.y; params->clusterDim.z = clusterDim_.z; + } else { + return hipErrorInvalidValue; } return hipSuccess; } + //! Copy all attribute slots from srcNode, mirroring its exact state. hipError_t CopyAttr(const GraphKernelNode* srcNode) { - if (kernelAttrInUse_ != 0 && srcNode->kernelAttrInUse_ != kernelAttrInUse_) { - return hipErrorInvalidContext; - } clusterDim_ = srcNode->clusterDim_; - if (kernelAttrInUse_ == 0 && srcNode->kernelAttrInUse_ == 0) { - return hipSuccess; - } - kernelAttrInUse_ = srcNode->kernelAttrInUse_; - switch (srcNode->kernelAttrInUse_) { - case hipKernelNodeAttributeAccessPolicyWindow: - kernelAttr_.accessPolicyWindow.base_ptr = srcNode->kernelAttr_.accessPolicyWindow.base_ptr; - kernelAttr_.accessPolicyWindow.hitProp = srcNode->kernelAttr_.accessPolicyWindow.hitProp; - kernelAttr_.accessPolicyWindow.hitRatio = srcNode->kernelAttr_.accessPolicyWindow.hitRatio; - kernelAttr_.accessPolicyWindow.missProp = srcNode->kernelAttr_.accessPolicyWindow.missProp; - kernelAttr_.accessPolicyWindow.num_bytes = - srcNode->kernelAttr_.accessPolicyWindow.num_bytes; - break; - case hipKernelNodeAttributeCooperative: - kernelAttr_.cooperative = srcNode->kernelAttr_.cooperative; - break; - case hipLaunchAttributePriority: - kernelAttr_.priority = srcNode->kernelAttr_.priority; - break; - default: - return hipErrorInvalidValue; - } + accessPolicyWindow_ = srcNode->accessPolicyWindow_; + cooperative_ = srcNode->cooperative_; + priority_ = srcNode->priority_; return hipSuccess; } @@ -2180,8 +2181,7 @@ class GraphKernelNode : public GraphNode { } launchFlags_ = kernelNode->launchFlags_; kernelEvents_ = kernelNode->kernelEvents_; - kernelAttr_ = kernelNode->kernelAttr_; - kernelAttrInUse_ = kernelNode->kernelAttrInUse_; + CopyAttr(kernelNode); return status; } @@ -4114,4 +4114,20 @@ class hipGraphBatchMemOpNode : public GraphNode { } }; + +// Defined here so hip::GraphNode is complete. +inline void hip::Stream::SetLastCapturedNode(hip::GraphNode* graphNode) { + if (graphNode == nullptr) { + return; + } + lastCapturedNodes_ = {graphNode}; + // Every caller passes a node freshly created on this stream during capture, + // so this is the point where a captured kernel inherits the stream priority. + // Fork/join propagation goes through AddCrossCapturedNode, which never + // re-stamps an existing node. + if (graphNode->GetType() == hipGraphNodeTypeKernel) { + graphNode->SetCapturedPriority(priority_); + } +} + } // namespace hip diff --git a/projects/clr/hipamd/src/hip_hcc.map.in b/projects/clr/hipamd/src/hip_hcc.map.in index d5de32e3dad7..7299b28945d3 100644 --- a/projects/clr/hipamd/src/hip_hcc.map.in +++ b/projects/clr/hipamd/src/hip_hcc.map.in @@ -701,3 +701,10 @@ global: local: *; } hip_7.16; + +hip_7.18 { +global: + hipDeviceFlushGPUDirectRDMAWrites; +local: + *; +} hip_7.17; diff --git a/projects/clr/hipamd/src/hip_internal.hpp b/projects/clr/hipamd/src/hip_internal.hpp index 9cd1747402a7..46bf03ef81c8 100644 --- a/projects/clr/hipamd/src/hip_internal.hpp +++ b/projects/clr/hipamd/src/hip_internal.hpp @@ -14,6 +14,7 @@ #include "hip_graph_capture.hpp" #include +#include #include #include #include @@ -256,7 +257,7 @@ HIP_PUBLIC_API void __hipOnError(const void *err_info); hip::tls.stream_capture_mode_ == hipStreamCaptureModeGlobal) { \ if (!hip::tls.capture_streams_.empty()) { \ for (auto stream : hip::tls.capture_streams_) { \ - stream->SetCaptureStatus(hipStreamCaptureStatusInvalidated); \ + stream->InvalidateCapture(); \ } \ HIP_RETURN(hipErrorStreamCaptureUnsupported); \ } \ @@ -264,7 +265,7 @@ HIP_PUBLIC_API void __hipOnError(const void *err_info); if (hip::tls.stream_capture_mode_ == hipStreamCaptureModeGlobal && \ !g_captureStreams.empty()) { \ for (auto stream : g_captureStreams) { \ - stream->SetCaptureStatus(hipStreamCaptureStatusInvalidated); \ + stream->InvalidateCapture(); \ } \ HIP_RETURN(hipErrorStreamCaptureUnsupported); \ } @@ -273,7 +274,7 @@ HIP_PUBLIC_API void __hipOnError(const void *err_info); #define INVALIDATE_ALL_CAPTURING_AND_RETURN(err) \ if (!g_allCapturingStreams.empty()) { \ for (auto stream : g_allCapturingStreams) { \ - stream->SetCaptureStatus(hipStreamCaptureStatusInvalidated); \ + stream->InvalidateCapture(); \ } \ return err; \ } @@ -372,8 +373,7 @@ namespace hip { enum Priority : int { High = -1, Normal = 0, Low = 1 }; Stream(Device* dev, Priority p = Priority::Normal, unsigned int f = 0, bool null_stream = false, - const std::vector& cuMask = {}, - hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone); + const std::vector& cuMask = {}); // --- Core stream operations --- @@ -418,56 +418,48 @@ namespace hip { static bool StreamCaptureBlocking(); /// Check Stream Capture status to make sure it is done static bool StreamCaptureOngoing(hipStream_t hStream); - /// Generate ID for stream capture unique over the lifetime of the process - static uint64_t GenerateCaptureID() { - static std::atomic uid(0); - return ++uid; - } - /// Returns capture status of the current stream - hipStreamCaptureStatus GetCaptureStatus() const { return captureStatus_; } + hipStreamCaptureStatus GetCaptureStatus() const { + return captureStatus_.load(std::memory_order_acquire); + } /// Returns capture mode of the current stream hipStreamCaptureMode GetCaptureMode() const { return captureMode_; } /// Returns if stream is origin stream bool IsOriginStream() const { return originStream_; } - /// Mark this stream as the origin of a capture - void SetOriginStream() { originStream_ = true; } - /// Returns captured graph - hip::Graph* GetCaptureGraph() const { return pCaptureGraph_; } + /// Returns the graph being recorded by the capture this stream takes part in, or null when + /// it is not capturing. The graph lives on the origin. + hip::Graph* GetCaptureGraph() const { + return captureOwner_ != nullptr ? captureOwner_->captureGraph_.get() : nullptr; + } /// Returns last captured graph node const std::vector& GetLastCapturedNodes() const { return lastCapturedNodes_; } - /// Set last captured graph node - void SetLastCapturedNode(hip::GraphNode* graphNode) { lastCapturedNodes_ = {graphNode}; } + /// Set last captured graph node; copies stream priority onto kernel nodes. + void SetLastCapturedNode(hip::GraphNode* graphNode); /// Returns dependencies removed during capture const std::vector& GetRemovedDependencies() const { return removedDependencies_; } /// Append captured node via the wait event cross stream - void AddCrossCapturedNode(const std::vector& graphNodes, - bool replace = false); - /// Set graph that is being captured - void SetCaptureGraph(hip::Graph* pGraph) { - pCaptureGraph_ = pGraph; - captureStatus_ = hipStreamCaptureStatusActive; - } - /// Release graph when capture is invalidated - void ReleaseCaptureGraph(); - /// Drop the capture-graph pointer without freeing it (forks alias the origin's graph). - void ClearCaptureGraph() { pCaptureGraph_ = nullptr; } - /// Generate and assign a new capture ID (used at BeginCapture) - void SetCaptureID() { captureID_ = GenerateCaptureID(); } - /// Inherit capture ID from the parent stream - void SetCaptureID(uint64_t captureId) { captureID_ = captureId; } + hipError_t AddCrossCapturedNode(const std::vector& graphNodes, + bool replace = false); + /// Begin a capture with this stream as its origin, and its only member until others join. + /// The origin owns the graph for the life of the capture. + void StartCapture(std::unique_ptr graph, hipStreamCaptureMode mode); + /// Enroll this stream in the capture that `member` takes part in. Membership is tracked on + /// the capture's origin, so this publishes into the origin's set, not into `member`'s. + void JoinCapture(hip::Stream* member); /// Reset capture parameters, optionally keeping an invalidated status observable. - hipError_t EndCapture(bool preserveInvalidated = false); + /// The single entry point for capture teardown, on the origin and on participants alike, + /// and the only place the captured graph is disposed of. + /// + /// Returns the graph, which the caller then owns, when the origin ends a capture that is + /// still valid. Returns null for a participant, whose departure leaves the capture running, + /// and for an invalidated capture, whose graph is destroyed here instead. + [[nodiscard]] hip::Graph* EndCapture(bool preserveInvalidated = false); /// Set capture status - void SetCaptureStatus(hipStreamCaptureStatus captureStatus) { captureStatus_ = captureStatus; } - /// Set capture mode - void SetCaptureMode(hipStreamCaptureMode captureMode) { captureMode_ = captureMode; } - /// Set parent stream - void SetParentStream(hipStream_t parentStream) { parentStream_ = parentStream; } - /// Get parent stream - hipStream_t GetParentStream() const { return parentStream_; } + void SetCaptureStatus(hipStreamCaptureStatus captureStatus) { + captureStatus_.store(captureStatus, std::memory_order_release); + } /// Get capture ID uint64_t GetCaptureID() const { return captureID_; } /// Associate an event with the current capture @@ -485,10 +477,10 @@ namespace hip { std::scoped_lock lock(lock_); captureEvents_.erase(e); } - /// Register a parallel (forked) capture stream - void SetParallelCaptureStream(hipStream_t s) { parallelCaptureStreams_.insert(s); } - /// Remove a parallel capture stream - void EraseParallelCaptureStream(hipStream_t s) { parallelCaptureStreams_.erase(s); } + /// Mark the whole capture that this stream belongs to as invalidated: the origin and + /// every stream enrolled in it. Callable from the origin or from any participant, and a + /// no-op on a stream that is not taking part in a capture. + void InvalidateCapture(); // --- Execution context (green context) lifecycle --- /// Marks the stream as detached: its owning ExecutionCtx has been @@ -506,9 +498,19 @@ namespace hip { } private: - ~Stream() = default; + ~Stream(); - mutable std::recursive_mutex lock_; //!< Guards captureEvents_ bookkeeping + /// Generate ID for stream capture unique over the lifetime of the process + static uint64_t GenerateCaptureID() { + static std::atomic uid(0); + return ++uid; + } + + /// Return this stream's capture fields to defaults. Requires lock_ to be held, since + /// EndCapture applies it to a participant while holding the origin's lock too. + void ResetCaptureStateLocked(bool preserveInvalidated); + + mutable std::recursive_mutex lock_; //!< Guards captureEvents_ and captureStreams_ Device* device_; //!< Device that owns this stream Priority priority_; //!< Scheduling priority (High / Normal / Low) unsigned int flags_; //!< Creation flags (e.g. hipStreamNonBlocking) @@ -520,14 +522,20 @@ namespace hip { std::atomic async_error_{hipSuccess}; // ----- Stream capture state ----- - hipStreamCaptureStatus captureStatus_{hipStreamCaptureStatusNone}; //!< Current capture status - hip::Graph* pCaptureGraph_ = nullptr; //!< Graph being constructed by capture + //!< Current capture status. Atomic because every capture API reads it without taking lock_. + std::atomic captureStatus_{hipStreamCaptureStatusNone}; + /// Graph being constructed by the capture this stream started, and set on no other + /// stream. EndCapture hands it to the caller, or destroys it if the capture was + /// invalidated. + std::unique_ptr captureGraph_; hipStreamCaptureMode captureMode_{hipStreamCaptureModeGlobal}; //!< API restriction mode bool originStream_ = false; //!< True if this stream started capture - hipStream_t parentStream_ = nullptr; //!< Parent stream (null for origin) + hip::Stream* captureOwner_ = nullptr; //!< Origin owning this capture; the + //!< origin points at itself std::vector lastCapturedNodes_; //!< Last graph node(s) captured std::vector removedDependencies_; //!< Deps removed via UpdateCaptureDeps - std::unordered_set parallelCaptureStreams_; //!< Forked parallel capture branches + std::unordered_set captureStreams_; //!< Streams enrolled in this capture, + //!< excluding the origin. Origin only. std::unordered_set captureEvents_; //!< Events tied to this capture uint64_t captureID_ = 0; //!< Unique ID for this capture sequence @@ -755,6 +763,18 @@ namespace hip { extern void setCurrentDevice(unsigned int index); + /// If we have the flush register set, we return the option + inline unsigned int ihipRdmaFlushWritesOptions(const uint32_t* hdpMemFlushCntl) { + return (hdpMemFlushCntl != nullptr) + ? static_cast(hipFlushGPUDirectRDMAWritesOptionHost) + : 0u; + } + + /// The hardware does not make the writes visible on its own. Hence it's none here + inline int ihipRdmaWritesOrdering() { + return static_cast(hipGPUDirectRDMAWritesOrderingNone); // i.e. 0 + } + /// Get ROCclr queue associated with hipStream /// Note: This follows the CUDA spec to sync with default streams /// and Blocking streams diff --git a/projects/clr/hipamd/src/hip_module.cpp b/projects/clr/hipamd/src/hip_module.cpp index 1cb30cf84c02..a4e52d1373e2 100644 --- a/projects/clr/hipamd/src/hip_module.cpp +++ b/projects/clr/hipamd/src/hip_module.cpp @@ -407,14 +407,44 @@ hipError_t ihipLaunchKernel_validate(hipFunction_t f, const amd::LaunchParams& l } // ================================================================================================= -bool UpdateNumClustersFromKernel(const hip::Stream* stream, const amd::Kernel* kernel, - amd::LaunchParams& launch_params) { +hipError_t UpdateNumClustersFromKernel(const hip::Stream* stream, const amd::Kernel* kernel, + amd::LaunchParams& launch_params) { const amd::Device& device = stream->vdev()->device(); amd::device::Kernel* devKernel = const_cast(kernel->getDeviceKernel(device)); + // All-zero dims mean "no cluster" and emit no metadata, so a zero here is malformed. The + // kernel's block index is computed from it, so no dispatch of such a kernel is correct. + if (devKernel->hasClusterAttr() && + (devKernel->getClusterSize(0) == 0 || devKernel->getClusterSize(1) == 0 || + devKernel->getClusterSize(2) == 0)) { + LogPrintfError("Kernel %s was compiled for cluster dimensions (%zu, %zu, %zu) " + "with a zero dimension", + kernel->name().c_str(), devKernel->getClusterSize(0), + devKernel->getClusterSize(1), devKernel->getClusterSize(2)); + return hipErrorInvalidConfiguration; + } // If cluster size from device kernel is > 1, then we need to update the cluster params. if (devKernel->getClusterSize(0) > 1 || devKernel->getClusterSize(1) > 1 || devKernel->getClusterSize(2) > 1) { + // Code-object dims bypass the hipLaunchKernelExC() checks. An oversized cluster is dropped + // by the SPI without signalling completion, which hangs the host, so bound it here. + if (device.info().clusterMaxSize_ == 0) { + LogPrintfError("Kernel %s requires a multi-workgroup cluster, " + "but this device does not support clusters", + kernel->name().c_str()); + return hipErrorInvalidClusterSize; + } + const size_t requestedClusterSize = devKernel->getClusterSize(0) * + devKernel->getClusterSize(1) * + devKernel->getClusterSize(2); + if (requestedClusterSize > device.info().clusterMaxSize_) { + LogPrintfError("Kernel %s was compiled for a cluster of %zu workgroups (%zu, %zu, %zu), " + "but this device supports at most %zu", + kernel->name().c_str(), requestedClusterSize, devKernel->getClusterSize(0), + devKernel->getClusterSize(1), devKernel->getClusterSize(2), + device.info().clusterMaxSize_); + return hipErrorInvalidClusterSize; + } if (!launch_params.UpdateClusterLaunchParams(devKernel->getClusterSize(0), devKernel->getClusterSize(1), devKernel->getClusterSize(2))) { @@ -426,10 +456,10 @@ bool UpdateNumClustersFromKernel(const hip::Stream* stream, const amd::Kernel* k launch_params.local_[1], launch_params.local_[2], devKernel->getClusterSize(0), devKernel->getClusterSize(1), devKernel->getClusterSize(2)); - return false; + return hipErrorInvalidValue; } } - return true; + return hipSuccess; } // ================================================================================================= @@ -443,8 +473,9 @@ hipError_t ihipLaunchKernelCommand(amd::Command*& command, hipFunction_t f, amd::Kernel* kernel = hip::asKernel(f); // Check if the kernel metadata has cluster info we need to act on. - if (!UpdateNumClustersFromKernel(stream, kernel, launch_params)) { - return hipErrorInvalidValue; + hipError_t clusterStatus = UpdateNumClustersFromKernel(stream, kernel, launch_params); + if (clusterStatus != hipSuccess) { + return clusterStatus; } size_t globalWorkOffset[3] = {0}; @@ -1032,7 +1063,7 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL // Not supported while stream is capturing hip::Stream* s = reinterpret_cast(launch.stream); if (s->GetCaptureStatus() == hipStreamCaptureStatusActive) { - s->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + s->InvalidateCapture(); return hipErrorStreamCaptureUnsupported; } if (s->GetCaptureStatus() == hipStreamCaptureStatusInvalidated) { diff --git a/projects/clr/hipamd/src/hip_prof_gen.py b/projects/clr/hipamd/src/hip_prof_gen.py index 11b80d19b9b6..ad556294d769 100755 --- a/projects/clr/hipamd/src/hip_prof_gen.py +++ b/projects/clr/hipamd/src/hip_prof_gen.py @@ -561,6 +561,8 @@ def generate_prof_header(f, api_map, callback_ids, opts_map): arg_name = arg_tuple[1] # Checking for enum type if arg_type == "hipLimit_t": arg_type = 'enum ' + arg_type + if arg_type == "hipFlushGPUDirectRDMAWritesTarget": arg_type = 'enum ' + arg_type + if arg_type == "hipFlushGPUDirectRDMAWritesScope": arg_type = 'enum ' + arg_type # Structuer field code f.write(' ' + arg_type + ' ' + arg_name + ';\n') if ptr_type != '': diff --git a/projects/clr/hipamd/src/hip_stream.cpp b/projects/clr/hipamd/src/hip_stream.cpp index 26487a515f83..99c94a2e9833 100644 --- a/projects/clr/hipamd/src/hip_stream.cpp +++ b/projects/clr/hipamd/src/hip_stream.cpp @@ -39,7 +39,7 @@ void EraseCaptureTracking(Stream* stream) { // ================================================================================================ Stream::Stream(hip::Device* dev, Priority p, unsigned int f, bool null_stream, - const std::vector& cuMask, hipStreamCaptureStatus captureStatus) + const std::vector& cuMask) : amd::HostQueue(*dev->asContext(), *dev->devices()[0], 0, amd::CommandQueue::RealTimeDisabled, convertToQueuePriority(p), cuMask, null_stream), device_(dev), @@ -47,38 +47,111 @@ Stream::Stream(hip::Device* dev, Priority p, unsigned int f, bool null_stream, flags_(f), null_(null_stream), cuMask_(cuMask), - stream_id_(GenerateStreamId()), - captureStatus_(captureStatus) { + stream_id_(GenerateStreamId()) { device_->AddStream(this); } // ================================================================================================ -hipError_t Stream::EndCapture(bool preserveInvalidated) { - // Detach all captured events from this stream. - { - std::scoped_lock lock(lock_); - for (auto event : captureEvents_) { - reinterpret_cast(event)->SetCaptureStream(nullptr); - } - captureEvents_.clear(); +// Out of line so that captureGraph_'s deleter is instantiated where hip::Graph is complete. +Stream::~Stream() = default; + +// ================================================================================================ +void Stream::InvalidateCapture() { + hip::Stream* owner = captureOwner_; + if (owner == nullptr) { + return; } - // Recursively end capture on all parallel (forked) streams. - for (auto stream : parallelCaptureStreams_) { - [[maybe_unused]] const auto err = - reinterpret_cast(stream)->EndCapture(preserveInvalidated); - assert(err == hipSuccess); + + owner->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + std::scoped_lock lock(owner->lock_); + for (auto* participant : owner->captureStreams_) { + participant->SetCaptureStatus(hipStreamCaptureStatusInvalidated); } +} - // Reset all capture state to defaults. - captureStatus_ = preserveInvalidated ? hipStreamCaptureStatusInvalidated - : hipStreamCaptureStatusNone; - pCaptureGraph_ = nullptr; +// ================================================================================================ +void Stream::ResetCaptureStateLocked(bool preserveInvalidated) { + for (auto event : captureEvents_) { + reinterpret_cast(event)->SetCaptureStream(nullptr); + } + captureEvents_.clear(); + captureStreams_.clear(); originStream_ = false; - parentStream_ = nullptr; + captureOwner_ = nullptr; lastCapturedNodes_.clear(); - parallelCaptureStreams_.clear(); + removedDependencies_.clear(); - return hipSuccess; + SetCaptureStatus(preserveInvalidated ? hipStreamCaptureStatusInvalidated + : hipStreamCaptureStatusNone); +} + +// ================================================================================================ +void Stream::StartCapture(std::unique_ptr graph, hipStreamCaptureMode mode) { + assert(GetCaptureStatus() == hipStreamCaptureStatusNone && + "a stream already in a capture cannot start another"); + + std::scoped_lock lock(lock_); + captureOwner_ = this; + captureGraph_ = std::move(graph); + captureMode_ = mode; + captureID_ = GenerateCaptureID(); + originStream_ = true; + SetCaptureStatus(hipStreamCaptureStatusActive); +} + +// ================================================================================================ +void Stream::JoinCapture(hip::Stream* member) { + assert(GetCaptureStatus() == hipStreamCaptureStatusNone && + "a stream already in a capture cannot join another"); + + // Enroll against the capture's origin, not the stream that happened to record the event, so + // every participant is a direct member of one flat set. + auto* owner = member->captureOwner_; + assert(owner != nullptr && "a stream mid-capture always has a capture owner"); + + std::scoped_lock lock(owner->lock_, lock_); + owner->captureStreams_.insert(this); + captureOwner_ = owner; + captureMode_ = owner->captureMode_; + captureID_ = owner->captureID_; + + // Inherits the capture's status instead of assuming Active. This protects against the + // scenario where InvalidateCapture() runs concurrently on the owner. + SetCaptureStatus(owner->GetCaptureStatus()); +} + +// ================================================================================================ +hip::Graph* Stream::EndCapture(bool preserveInvalidated) { + if (!originStream_ && captureOwner_ != nullptr) { + // A participant leaving the capture on its own, via hipStreamDestroy or Detach. + std::scoped_lock ownerLock(captureOwner_->lock_); + captureOwner_->captureStreams_.erase(this); + } + + // Only the origin holds the graph, so only the origin disposes of it. An invalidated capture + // can no longer produce a graph, so it is destroyed instead of handed back. + hip::Graph* graph = nullptr; + if (originStream_) { + if (GetCaptureStatus() == hipStreamCaptureStatusInvalidated) { + captureGraph_.reset(); + } else { + graph = captureGraph_.release(); + } + } + + { + // This lock keeps captureStreams_ stable, because both JoinCapture and a participant's own + // departure take the origin's lock to touch that set. + std::scoped_lock lock(lock_); + if (originStream_) { + for (auto* participant : captureStreams_) { + std::scoped_lock participantLock(participant->lock_); + participant->ResetCaptureStateLocked(preserveInvalidated); + } + } + ResetCaptureStateLocked(preserveInvalidated); + } + return graph; } // ================================================================================================ @@ -97,23 +170,17 @@ void Stream::Detach() { // Invoked by ~ExecutionCtx() on every stream the destroyed ctx still owns. // If this stream still participates in a capture, hipStreamEndCapture's // invalidated cleanup is about to become unreachable through the API. - if (captureStatus_ == hipStreamCaptureStatusActive || - captureStatus_ == hipStreamCaptureStatusInvalidated) { - captureStatus_ = hipStreamCaptureStatusInvalidated; - - if (parentStream_ != nullptr) { - reinterpret_cast(parentStream_)->EraseParallelCaptureStream( - reinterpret_cast(this)); - ClearCaptureGraph(); - } + const hipStreamCaptureStatus captureStatus = GetCaptureStatus(); + if (captureStatus == hipStreamCaptureStatusActive || + captureStatus == hipStreamCaptureStatusInvalidated) { + SetCaptureStatus(hipStreamCaptureStatusInvalidated); if (originStream_) { EraseCaptureTracking(this); - ReleaseCaptureGraph(); } - // Keep the invalidated status visible on the detached origin and every - // fork while clearing all graph, event, and parent/child bookkeeping. + // Keep the invalidated status visible on the detached origin and every participant while + // clearing all graph, event and membership bookkeeping. (void)EndCapture(/*preserveInvalidated=*/true); } detached_.store(true, std::memory_order_release); @@ -141,13 +208,16 @@ bool isValid(hipStream_t& stream) { } // ================================================================================================ -void Stream::ReleaseCaptureGraph() { - delete pCaptureGraph_; - pCaptureGraph_ = nullptr; -} +hipError_t Stream::AddCrossCapturedNode(const std::vector& graphNodes, + bool replace) { + std::scoped_lock lock(lock_); + const auto captureStatus = GetCaptureStatus(); + if (captureStatus != hipStreamCaptureStatusActive) { + return captureStatus == hipStreamCaptureStatusInvalidated + ? hipErrorStreamCaptureInvalidated + : hipErrorIllegalState; + } -// ================================================================================================ -void Stream::AddCrossCapturedNode(const std::vector& graphNodes, bool replace) { // Replace dependencies as per flag hipStreamSetCaptureDependencies. if (replace) { removedDependencies_.insert(removedDependencies_.end(), @@ -160,6 +230,7 @@ void Stream::AddCrossCapturedNode(const std::vector& graphNodes lastCapturedNodes_.push_back(node); } } + return hipSuccess; } // ================================================================================================ @@ -194,7 +265,7 @@ bool Stream::StreamCaptureOngoing(hipStream_t hStream) { const auto captureStatus = s->GetCaptureStatus(); if (captureStatus == hipStreamCaptureStatusActive) { - s->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + s->InvalidateCapture(); return true; } if (captureStatus == hipStreamCaptureStatusInvalidated) { @@ -214,7 +285,7 @@ bool Stream::StreamCaptureOngoing(hipStream_t hStream) { amd::ScopedLock lock(g_captureStreamsLock); if (!g_captureStreams.empty()) { for (auto stream : hip::g_captureStreams) { - stream->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + stream->InvalidateCapture(); } return true; } @@ -222,7 +293,7 @@ bool Stream::StreamCaptureOngoing(hipStream_t hStream) { // ThreadLocal mode — invalidate all capturing streams in current thread. if (!hip::tls.capture_streams_.empty()) { for (auto stream : hip::tls.capture_streams_) { - stream->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + stream->InvalidateCapture(); } return true; } @@ -456,10 +527,11 @@ hipError_t hipStreamDestroy(hipStream_t stream) { } hip::Stream* s = reinterpret_cast(stream); if (s->GetCaptureStatus() != hipStreamCaptureStatusNone) { - if (s->GetParentStream() != nullptr) { - reinterpret_cast(s->GetParentStream())->EraseParallelCaptureStream(stream); + if (s->IsOriginStream()) { + s->SetCaptureStatus(hipStreamCaptureStatusInvalidated); } - [[maybe_unused]] auto error = s->EndCapture(); + // EndCapture removes a participant from its owner's set itself. + (void)s->EndCapture(); } s->GetDevice()->RemoveStreamFromPools(s); @@ -527,18 +599,30 @@ hipError_t hipStreamWaitEvent_common(hipStream_t stream, hipEvent_t event, unsig if (waitStream == nullptr) { return hipErrorInvalidHandle; } - // Don't set when a stream waits on its own event, or when a forked stream joins back to - // the parent. - if (waitStream != eventStream && !waitStream->IsOriginStream() && - waitStream != reinterpret_cast(eventStream->GetParentStream())) { - waitStream->SetCaptureGraph(eventStream->GetCaptureGraph()); - waitStream->SetCaptureID(eventStream->GetCaptureID()); - waitStream->SetCaptureMode(eventStream->GetCaptureMode()); - waitStream->SetParentStream(reinterpret_cast(eventStream)); - eventStream->SetParallelCaptureStream(stream); + + if (waitStream->GetCaptureStatus() == hipStreamCaptureStatusInvalidated) { + return hipErrorStreamCaptureInvalidated; } - waitStream->AddCrossCapturedNode(e->GetNodesPrevToRecorded()); - return hipSuccess; + + // Waiting on an event recorded in a different capture would splice two independent + // graphs together. Reject it rather than merging them. + if (waitStream->GetCaptureStatus() == hipStreamCaptureStatusActive && + waitStream->GetCaptureID() != eventStream->GetCaptureID()) { + waitStream->InvalidateCapture(); + if (eventStream->GetCaptureStatus() != hipStreamCaptureStatusInvalidated) { + eventStream->InvalidateCapture(); + } + return hipErrorStreamCaptureMerge; + } + + if (eventStream->GetCaptureStatus() == hipStreamCaptureStatusInvalidated) { + return hipErrorInvalidValue; + } + + if (waitStream->GetCaptureStatus() == hipStreamCaptureStatusNone) { + waitStream->JoinCapture(eventStream); + } + return waitStream->AddCrossCapturedNode(e->GetNodesPrevToRecorded()); } // Non-capture path: validate isolation, register safe-stream, and wait. @@ -683,7 +767,7 @@ hipError_t hipStreamAddCallback_common(hipStream_t stream, hipStreamCallback_t c if (stream != nullptr && stream != hipStreamLegacy && hip::isValid(stream)) { hip::Stream* s = reinterpret_cast(stream); if (s->GetCaptureStatus() != hipStreamCaptureStatusNone) { - s->SetCaptureStatus(hipStreamCaptureStatusInvalidated); + s->InvalidateCapture(); return hipErrorStreamCaptureUnsupported; } } else if (Stream::StreamCaptureBlocking()) { diff --git a/projects/clr/hipamd/src/hip_table_interface.cpp b/projects/clr/hipamd/src/hip_table_interface.cpp index f0df800d146e..ac322a03a4da 100644 --- a/projects/clr/hipamd/src/hip_table_interface.cpp +++ b/projects/clr/hipamd/src/hip_table_interface.cpp @@ -3365,4 +3365,11 @@ hipError_t hipModuleEnumerateFunctions(hipFunction_t* functions, unsigned int nu TRY; return hip::GetHipDispatchTable()->hipModuleEnumerateFunctions_fn(functions, numFunctions, mod); CATCH; -} \ No newline at end of file +} + +hipError_t hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTarget target, + hipFlushGPUDirectRDMAWritesScope scope) { + TRY; + return hip::GetHipDispatchTable()->hipDeviceFlushGPUDirectRDMAWrites_fn(target, scope); + CATCH; +} diff --git a/projects/clr/hipamd/src/hrr/CMakeLists.txt b/projects/clr/hipamd/src/hrr/CMakeLists.txt index 5b5784e411f5..6cb64bf620f9 100644 --- a/projects/clr/hipamd/src/hrr/CMakeLists.txt +++ b/projects/clr/hipamd/src/hrr/CMakeLists.txt @@ -1,6 +1,22 @@ # HRR In-Tree Capture Layer # Adds capture source files to the amdhip64 target. # No separate library — capture is compiled into amdhip64 directly. +if(CLR_HRR_CODEGEN_CHECK) + # Regenerate HRR sources in the binary tree and verify the checked-in files. + # The checked-in generated files remain the compilation inputs. + set(HRR_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../hrr") + set(HRR_CLR_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../..") + set(HRR_HIP_SOURCE_DIR "${HIP_COMMON_DIR}") + include("${HRR_CLR_SOURCE_DIR}/cmake/HrrCodegen.cmake") + set(HRR_CODEGEN_CHECK_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated/hrr") + hrr_add_codegen_check( + TARGET hrr_codegen_check + HRR_SOURCE_DIR "${HRR_SOURCE_DIR}" + CLR_SOURCE_DIR "${HRR_CLR_SOURCE_DIR}" + HIP_SOURCE_DIR "${HRR_HIP_SOURCE_DIR}" + OUTPUT_DIR "${HRR_CODEGEN_CHECK_DIR}") + add_dependencies(amdhip64 hrr_codegen_check) +endif() target_sources(amdhip64 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/hip_capture.cpp diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index 8c39978b60c6..146d62da59fc 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -11,6 +11,7 @@ * - Memcpy H2D variants with blob snapshotting (hipMemcpy, hipMemcpyAsync, * hipMemcpyHtoD, hipMemcpyHtoDAsync, hipMemcpyWithStream) * - Module load with code object snapshotting (hipModuleLoad*) + * - Module unload (hipModuleUnload) to drop the program hash cache entry * - Kernel launch with arg introspection via kernel->signature() * - Fat binary registration (__hipRegisterFatBinary) * - Host memory registration (hipHostRegister / hipHostUnregister) @@ -53,6 +54,7 @@ #include #include +#include #include #include #include @@ -97,11 +99,27 @@ static thread_local hipStream_t g_pushed_stream{}; // Helpers // --------------------------------------------------------------------------- -bool hip_capture_enabled() { - return !flagIsDefault(HIP_HRR_CAPTURE_OUTPUT) && - HIP_HRR_CAPTURE_OUTPUT[0] != '\0'; +// The kernel sets AT_SECURE for a set-user-ID, set-group-ID or file-capability +// exec, and for an LSM transition. HIP_HRR_CAPTURE_OUTPUT comes from whoever +// launched the process, so such a process must not write where it says, nor +// record its memory for them. +static bool hrr_secure_exec() { + static const bool secure = hrr_cap::metadata::secure_exec(); + return secure; } +// CLR's flag parser stores an exported empty variable as a single space, so a +// value that is only blanks counts as unset rather than as a directory named " ". +static bool hrr_capture_requested() { + if (flagIsDefault(HIP_HRR_CAPTURE_OUTPUT)) return false; + for (const char* p = HIP_HRR_CAPTURE_OUTPUT; *p != '\0'; ++p) { + if (!std::isspace(static_cast(*p))) return true; + } + return false; +} + +bool hip_capture_enabled() { return hrr_capture_requested() && !hrr_secure_exec(); } + const char* hip_capture_output_dir() { return HIP_HRR_CAPTURE_OUTPUT; } @@ -121,6 +139,14 @@ static void hrr_trace_h2d(const char* api, const void* dst, size_t sz) { sz); } +// Wait for an async D2H copy before its host buffer is snapshotted. Once the +// writer has closed, for instance after capture stopped for lack of space, the +// snapshot would be dropped, so the copy is left asynchronous as the app asked. +static hipError_t sync_for_d2h_snapshot(hipStream_t stream) { + if (!hrr_cap::writer::is_open()) return hipSuccess; + return g_real_table.hipStreamSynchronize_fn(stream); +} + // Parse the extra[] sentinel format for packed kernarg buffers. static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_size) { if (!extra) return false; @@ -276,6 +302,15 @@ static void scan_embedded_ptr_offsets(const void* func_key, uint32_t arg_idx, hip::tls.last_error_ = saved_err; } +// api_id selects the event type the launch is written under. It is +// HRR_API_HIPMODULELAUNCHKERNEL for the plain launches, and the API's own id +// for the extensible spellings, whose events must appear under their own name +// so that a replay of them is attributable to the API the program called. +// +// attrs / num_attrs carry the launch-attribute list of an extensible launch. +// It is a second pointer hanging off the descriptor, so recording the +// descriptor alone would still leave replay with a capture-time address; the +// list is appended to the payload instead. static void serialize_kernel_launch( const char* kernel_name, const void* func_key, @@ -287,7 +322,10 @@ static void serialize_kernel_launch( void** kernel_params, const void* kbuf, size_t ksz, - hrr_cap::Hash128 co_hash) + hrr_cap::Hash128 co_hash, + uint16_t api_id = HRR_API_HIPMODULELAUNCHKERNEL, + const hipLaunchAttribute* attrs = nullptr, + uint32_t num_attrs = 0) { // Reserve space for hrr_event_header at front; payload body follows. std::vector payload(sizeof(hrr_event_header), 0); @@ -311,7 +349,11 @@ static void serialize_kernel_launch( // raw stream handle as first payload field after header (for replay stream routing) push_u64(reinterpret_cast(stream)); - uint16_t name_len = static_cast(std::strlen(kernel_name)); + // name_len is a uint16_t on the wire; a longer name is dropped loudly below + // rather than recorded truncated. + const size_t kernel_name_len = std::strlen(kernel_name); + const bool name_oversized = kernel_name_len > UINT16_MAX; + uint16_t name_len = name_oversized ? 0 : static_cast(kernel_name_len); push_u16(name_len); push_bytes(kernel_name, name_len); push_u64(co_hash.lo); push_u64(co_hash.hi); // code object identity (0 = unknown) @@ -419,37 +461,162 @@ static void serialize_kernel_launch( } } + // Launch-attribute tail. Written for every launch so the payload has one + // shape: a plain launch records a count of zero. The stride lets replay + // reject a recording made against a different hipLaunchAttribute layout + // rather than reinterpreting its bytes. + push_u32(num_attrs); + push_u32(num_attrs ? static_cast(sizeof(hipLaunchAttribute)) : 0u); + if (attrs && num_attrs) + push_bytes(attrs, static_cast(num_attrs) * sizeof(hipLaunchAttribute)); + // payload_length is a uint32_t (wire v4), so the practical ceiling is ~4 GiB — // a real launch never approaches it. A per-arg size that exceeds the uint16_t // arg-size field (arg_oversized) is the genuine remaining limit. In either // case the launch cannot be recorded faithfully: drop it LOUDLY and mark the // whole archive incomplete so replay/validation can never silently treat a // capture missing a GPU launch (and its downstream writes) as faithful. - if (arg_oversized || payload.size() > UINT32_MAX) { - LogPrintfError("[HRR capture] Kernel launch for '%s' cannot be serialized " - "(payload=%zu bytes, oversized_by_value_arg=%s) — dropping the " - "event and marking the capture INCOMPLETE. Replay of this " - "archive will be unfaithful (this launch and its effects are " - "absent).", - kernel_name, payload.size(), arg_oversized ? "yes" : "no"); - hrr_cap::writer::mark_incomplete( - "kernel launch payload exceeds wire-format limits"); + if (arg_oversized || name_oversized || payload.size() > UINT32_MAX) { + // log_printf truncates at 4 KiB, so the name goes last and is capped. + LogPrintfError( + "[HRR capture] Kernel launch cannot be serialized " + "(name_len=%zu, payload=%zu bytes, oversized_by_value_arg=%s): dropping " + "the event and marking the capture INCOMPLETE. Replay of this " + "archive will be unfaithful (this launch and its effects are " + "absent). Kernel: '%.1024s'", + kernel_name_len, payload.size(), arg_oversized ? "yes" : "no", kernel_name); + const char* reason = name_oversized ? "kernel name exceeds the uint16 wire-format length" + : "kernel launch payload exceeds wire-format limits"; + hrr_cap::writer::mark_incomplete(reason); return; } - hrr_cap::writer::write_event_raw(HRR_API_HIPMODULELAUNCHKERNEL, + hrr_cap::writer::write_event_raw(api_id, reinterpret_cast(payload.data()), static_cast(payload.size())); } static hrr_cap::Hash128 kernel_code_object_hash(amd::Kernel* kernel); +// --------------------------------------------------------------------------- +// Graph kernel nodes +// +// hipKernelNodeParams names its kernel by a host function address and points +// at a void** argument array — exactly the two things a launch records by +// name and by value instead. So a kernel node event is its ordinary +// hrr_args_* struct followed by this tail, which is the launch encoding minus +// the parts a node does not have (no stream, and grid/block/shared are +// already fields of the struct). +// +// Tail layout: u16 name_len, name bytes, u64 co_hash_lo, u64 co_hash_hi, +// u16 num_args, then per argument: u8 kind, u16 size, size bytes, and for +// kind 3 a u16 count of embedded-pointer offsets followed by that many u16. +// Kinds match the launch encoding (0 scalar, 1 whole-arg pointer, 3 scalar +// with embedded pointers) so replay can decode both with one decoder. +// --------------------------------------------------------------------------- +static void append_kernel_node_tail(std::vector& payload, + const void* host_func, + void** kernel_params, void** extra) { + auto push_u8 = [&](uint8_t v) { payload.push_back(v); }; + auto push_u16 = [&](uint16_t v) { + payload.push_back(static_cast(v)); + payload.push_back(static_cast(v >> 8)); + }; + auto push_u64 = [&](uint64_t v) { + for (int i = 0; i < 8; i++) payload.push_back(static_cast(v >> (i*8))); + }; + auto push_bytes = [&](const void* d, size_t n) { + const auto* q = static_cast(d); + payload.insert(payload.end(), q, q + n); + }; + + // A node names its kernel the way hipLaunchKernel does — a host stub — so + // it needs the same resolution to reach the signature and the real name. + hipFunction_t f = nullptr; + amd::Kernel* kernel = nullptr; + if (host_func && g_real_table.hipGetFuncBySymbol_fn && + g_real_table.hipGetFuncBySymbol_fn(&f, host_func) == hipSuccess && f) + kernel = hip::asKernel(f); + + if (!kernel) { + // Nothing to name the kernel by. Write an empty tail; replay reports the + // node as unreconstructable and marks its graph incomplete rather than + // building a node that points at nothing. + push_u16(0); + push_u64(0); push_u64(0); + push_u16(0); + return; + } + + const amd::KernelSignature& sig = kernel->signature(); + const std::string& name = kernel->name(); + push_u16(static_cast(name.size())); + push_bytes(name.data(), name.size()); + const hrr_cap::Hash128 co = kernel_code_object_hash(kernel); + push_u64(co.lo); push_u64(co.hi); + + const void* kbuf = nullptr; + size_t ksz = 0; + if (!kernel_params && extra) parse_kernel_extra(extra, kbuf, ksz); + + const uint32_t n_all = sig.numParametersAll(); + uint16_t num_args = 0; + if (kbuf || kernel_params) + for (uint32_t i = 0; i < n_all; i++) + if (!sig.at(i).info_.hidden_) num_args++; + push_u16(num_args); + + auto emit_arg = [&](uint32_t idx, bool is_ptr, const uint8_t* bytes, + uint16_t sz) { + if (is_ptr) { + push_u8(1); push_u16(sz); + if (bytes) push_bytes(bytes, sz); + else for (uint16_t j = 0; j < sz; j++) push_u8(0); + return; + } + std::vector offs; + scan_embedded_ptr_offsets(kernel, idx, bytes, sz, offs); + push_u8(offs.empty() ? 0 : 3); push_u16(sz); + if (bytes) push_bytes(bytes, sz); + else for (uint16_t j = 0; j < sz; j++) push_u8(0); + if (!offs.empty()) { + push_u16(static_cast(offs.size())); + for (uint16_t o : offs) push_u16(o); + } + }; + + if (kbuf && ksz > 0) { + const auto* buf_bytes = static_cast(kbuf); + for (uint32_t i = 0; i < n_all; i++) { + const auto& desc = sig.at(i); + if (desc.info_.hidden_ || desc.size_ > UINT16_MAX) continue; + uint16_t sz = static_cast(desc.size_); + const uint8_t* bytes = + (desc.offset_ + sz <= ksz) ? buf_bytes + desc.offset_ : nullptr; + emit_arg(i, desc.type_ == T_POINTER, bytes, sz); + } + } else if (kernel_params) { + uint32_t param_idx = 0; + for (uint32_t i = 0; i < n_all; i++) { + const auto& desc = sig.at(i); + if (desc.info_.hidden_ || desc.size_ > UINT16_MAX) continue; + emit_arg(i, desc.type_ == T_POINTER, + static_cast(kernel_params[param_idx]), + static_cast(desc.size_)); + param_idx++; + } + } +} + static void record_launch( hipFunction_t f, unsigned gx, unsigned gy, unsigned gz, unsigned bx, unsigned by, unsigned bz, unsigned shared_mem, hipStream_t stream, - void** kernel_params, void** extra) + void** kernel_params, void** extra, + uint16_t api_id = HRR_API_HIPMODULELAUNCHKERNEL, + const hipLaunchAttribute* attrs = nullptr, + uint32_t num_attrs = 0) { amd::Kernel* kernel = hip::asKernel(f); if (!kernel) return; @@ -467,7 +634,8 @@ static void record_launch( gx, gy, gz, bx, by, bz, static_cast(shared_mem), stream, sig, kernel_params, kbuf, ksz, - kernel_code_object_hash(kernel)); + kernel_code_object_hash(kernel), + api_id, attrs, num_attrs); } // --------------------------------------------------------------------------- @@ -515,7 +683,7 @@ hipError_t capture_hipMemcpyAsync(void* dst, const void* src, hrr_trace_h2d("hipMemcpyAsync", dst, sizeBytes); } else if (kind == hipMemcpyDeviceToHost && dst && sizeBytes > 0) { // Sync the stream so host dst is valid before we snapshot it. - hipError_t sync_r = g_real_table.hipStreamSynchronize_fn(stream); + hipError_t sync_r = sync_for_d2h_snapshot(stream); if (sync_r == hipSuccess) h = hrr_cap::writer::write_blob(dst, sizeBytes); else @@ -599,7 +767,7 @@ hipError_t capture_hipMemcpyDtoHAsync(void* dst, hipDeviceptr_t src, hrr_cap::Hash128 h{0, 0}; if (dst && sizeBytes > 0) { // Sync the stream so host dst is valid before we snapshot it. - hipError_t sync_r = g_real_table.hipStreamSynchronize_fn(stream); + hipError_t sync_r = sync_for_d2h_snapshot(stream); if (sync_r == hipSuccess) h = hrr_cap::writer::write_blob(dst, sizeBytes); else @@ -655,55 +823,173 @@ hipError_t capture_hipMemcpyWithStream(void* dst, const void* src, // Module shims // --------------------------------------------------------------------------- -// Helper: get the actual device binary from a successfully loaded hipModule_t. -// The caller passes an in-memory image (which may be a fat binary bundle, not -// a raw ELF). The runtime unbundles/extracts the device ELF internally and -// stores it in the amd::Program. We read it back from there so we always -// capture the processed ELF, not the raw (possibly bundled) input image. -static hrr_cap::Hash128 write_module_code_object(hipModule_t module) { - amd::Program* prog = as_amd(reinterpret_cast(module)); - if (!prog) return {0, 0}; +// Is this buffer something hipModuleLoadData can load at replay? +// +// amd::Program::binary() hands back a bare pointer and length with no promise +// the memory is still the program's own. A code object whose buffer the +// runtime has already released reads back as whatever the allocator put there +// next, at exactly the recorded length, and nothing downstream notices: the +// bytes hash cleanly, land in code_objects/, and only fail at replay with +// "device kernel image is invalid" (HIP 200). Checking the magic turns that +// into a capture-time warning and a launch that resolves by name instead. +static bool image_looks_loadable(const void* data, size_t size) { + if (!data || size < 8) return false; + const auto* p = static_cast(data); + if (std::memcmp(p, "\x7f" "ELF", 4) == 0) return true; + if (size >= hip::symbols::kOffloadBundleCompressedMagicStrSize - 1 && + std::memcmp(p, hip::symbols::kOffloadBundleCompressedMagicStr, + hip::symbols::kOffloadBundleCompressedMagicStrSize - 1) == 0) + return true; + if (size >= hip::symbols::kOffloadBundleUncompressedMagicStrSize - 1 && + std::memcmp(p, hip::symbols::kOffloadBundleUncompressedMagicStr, + hip::symbols::kOffloadBundleUncompressedMagicStrSize - 1) == 0) + return true; + return false; +} + +// Hash of the code object behind an amd::Program, recorded once and reused. +// +// Keyed by program because a launch has to name the same bytes the module load +// recorded: Triton/inductor emits the generic entry symbol "triton_" from +// dozens of distinct code objects, so name-only resolution at replay binds +// every "triton_" launch to one arbitrary kernel and faults (HIP error 719). +// +// The cache is also what keeps the read *timely*. Populating it lazily at first +// launch meant re-reading binary() long after the module was built, which is +// where the stale-buffer problem above showed up in practice — an inductor +// module's 6.5 KB code object came back as Triton MLIR source text. Module load +// seeds the cache now, so the launch path only re-reads for programs that were +// never loaded through a module shim (fat binaries registered at static init). +// +// An entry records which image it hashed, not just the hash, because an +// amd::Program is freed when its module is unloaded and the next one can land +// on the same address — inductor churns through modules, so this happens. A +// hash keyed on the address alone then hands the later program the earlier +// one's code object, and its launches resolve to nothing at replay ("not found +// in any loaded module"). Comparing the image pointer and length is a second +// check, but both are recyclable; capture_hipModuleUnload erases the entry so +// a recycled (program, buffer, length) triple cannot hit a stale hash. +struct ProgramCodeObject { + const void* image; + size_t size; + hrr_cap::Hash128 hash; +}; +// g_prog_hash_mu guards g_prog_hash *and* every call HRR makes into +// amd::Program::binary(). The latter is `return binary_[&device];` — a +// non-const, inserting std::unordered_map::operator[] that the runtime does +// not lock. Reading a span is therefore a write as far as binary_ is +// concerned: the first read for a device the program was not built for +// inserts an empty entry and can rehash the bucket array. Two capture threads +// launching kernels from the same program would otherwise do that +// concurrently. Serializing HRR's reads here keeps us off that path. +static std::mutex g_prog_hash_mu; +static std::unordered_map g_prog_hash; +static std::atomic g_span_no_device_warned{false}; + +// Fetch a program's device binary as (pointer, length) without dereferencing +// the bytes (the call itself still goes through binary_'s inserting +// operator[] — see above). +// Caller must hold g_prog_hash_mu. +// Returns false when there is no current device (or the index is out of +// range): that is not "the program has no binary", so callers must not seed +// g_prog_hash with a {0,0} / (nullptr,0) entry. +static bool program_binary_span_locked(amd::Program* prog, const uint8_t*& data, size_t& size) { + data = nullptr; + size = 0; + if (!prog) return false; const int dev = hip::ihipGetDevice(); + if (dev < 0 || static_cast(dev) >= hip::g_devices.size()) { + if (!g_span_no_device_warned.exchange(true)) { + LogPrintfWarning("[HRR capture] program %p: no current HIP device " + "(ihipGetDevice()=%d, g_devices=%zu); not hashing a code object", + (const void*)prog, dev, hip::g_devices.size()); + } + return false; + } const amd::Device* device = hip::g_devices[dev]->devices()[0]; const auto& bin = prog->binary(*device); - const uint8_t* data = std::get<0>(bin); - const size_t size = std::get<1>(bin).first; + data = std::get<0>(bin); + size = std::get<1>(bin).first; + return true; +} + +static bool program_binary_span(amd::Program* prog, const uint8_t*& data, size_t& size) { + std::lock_guard lk(g_prog_hash_mu); + return program_binary_span_locked(prog, data, size); +} + +static void remember_program_hash(const amd::Program* prog, const void* image, size_t size, + hrr_cap::Hash128 h) { + if (!prog || !image || size == 0) return; + std::lock_guard lk(g_prog_hash_mu); + g_prog_hash[prog] = ProgramCodeObject{image, size, h}; +} + +static void forget_program_hash(const amd::Program* prog) { + if (!prog) return; + std::lock_guard lk(g_prog_hash_mu); + g_prog_hash.erase(prog); +} + +// Hash an already-read device binary span, refusing anything that does not +// look like a loadable image. Returns {0,0} when there is nothing safe to +// record. Takes the span rather than re-reading it so callers that already +// have one do not go back through amd::Program::binary(); must not be called +// with g_prog_hash_mu held (write_code_object takes the writer locks). +static hrr_cap::Hash128 hash_program_image(const amd::Program* prog, const uint8_t* data, + size_t size) { if (!data || !size) return {0, 0}; + if (!image_looks_loadable(data, size)) { + LogPrintfWarning("[HRR capture] program %p: %llu bytes at %p are not a loadable image" + " (no ELF/bundle magic) — not recording a code object for it;" + " its module event (if any) will not replay, and launches fall back" + " to name resolution", + (const void*)prog, (unsigned long long)size, (const void*)data); + return {0, 0}; + } return hrr_cap::writer::write_code_object(data, size); } -// Code-object hash for a launched kernel, derived from its owning amd::Program's -// device binary — the SAME bytes hashed by write_module_code_object() at module -// load — so replay can resolve the launch to the exact recorded code object. -// -// This is the fix for non-unique kernel names: Triton/inductor emits the generic -// entry symbol "triton_" from dozens of distinct code objects, so name-only -// resolution at replay binds every "triton_" launch to one arbitrary kernel and -// faults (HIP error 719). Recording the hash disambiguates them. -// -// Hashing is per-program (cached), so we pay it once per code object rather than -// on every one of millions of launches. -static std::mutex g_prog_hash_mu; -static std::unordered_map g_prog_hash; +// Helper: get the actual device binary from a successfully loaded hipModule_t. +// The module may come from an in-memory image or from a file (possibly a +// bundle); either way the runtime stores the extracted device ELF in the +// amd::Program, and we read it back from there. +// It also seeds the launch-path hash cache (g_prog_hash) for this program. +static hrr_cap::Hash128 write_module_code_object(hipModule_t module) { + amd::Program* prog = as_amd(reinterpret_cast(module)); + if (!prog) return {0, 0}; + const uint8_t* image = nullptr; + size_t size = 0; + if (!program_binary_span(prog, image, size)) return {0, 0}; + hrr_cap::Hash128 h = hash_program_image(prog, image, size); + remember_program_hash(prog, image, size, h); + return h; +} static hrr_cap::Hash128 kernel_code_object_hash(amd::Kernel* kernel) { if (!kernel) return {0, 0}; amd::Program* prog = &kernel->program(); if (!prog) return {0, 0}; + const uint8_t* image = nullptr; + size_t size = 0; + bool got_span = false; + // One critical section for the span read and the lookup: the span read is + // the part that must not race (see g_prog_hash_mu above), and folding the + // lookup in costs nothing since we hold the lock anyway. { std::lock_guard lk(g_prog_hash_mu); - auto it = g_prog_hash.find(prog); - if (it != g_prog_hash.end()) return it->second; + got_span = program_binary_span_locked(prog, image, size); + if (got_span) { + auto it = g_prog_hash.find(prog); + if (it != g_prog_hash.end() && it->second.image == image && it->second.size == size) + return it->second.hash; + } } - hrr_cap::Hash128 h{0, 0}; - const int dev = hip::ihipGetDevice(); - const amd::Device* device = hip::g_devices[dev]->devices()[0]; - const auto& bin = prog->binary(*device); - const uint8_t* data = std::get<0>(bin); - const size_t size = std::get<1>(bin).first; - if (data && size) h = hrr_cap::writer::write_code_object(data, size); - std::lock_guard lk(g_prog_hash_mu); - g_prog_hash[prog] = h; + if (!got_span) return {0, 0}; + // Miss. Hash outside the lock — write_code_object takes the writer's own + // locks, and hashing the same image twice is harmless (content-addressed). + hrr_cap::Hash128 h = hash_program_image(prog, image, size); + remember_program_hash(prog, image, size, h); return h; } @@ -749,27 +1035,11 @@ hipError_t capture_hipModuleLoadDataEx(hipModule_t* module, const void* image, hipError_t capture_hipModuleLoad(hipModule_t* module, const char* fname) { hipError_t r = g_real_table.hipModuleLoad_fn(module, fname); if (r == hipSuccess) { - hrr_cap::Hash128 h{0, 0}; - // Read the file from disk and snapshot it as a code object so the replay - // can load it by hash. Without this, fname is a capture-time address and - // is useless at replay time. - if (fname) { - if (FILE* fh = fopen(fname, "rb")) { - fseek(fh, 0, SEEK_END); - long sz = ftell(fh); - rewind(fh); - if (sz > 0) { - std::vector buf(static_cast(sz)); - if (fread(buf.data(), 1, buf.size(), fh) == buf.size()) - h = hrr_cap::writer::write_code_object(buf.data(), buf.size()); - } - fclose(fh); - } - } + hrr_cap::Hash128 h = write_module_code_object(*module); hrr_args_hipModuleLoad a{}; a.ret = static_cast(r); a.module = reinterpret_cast(*module); - a.fname = 0; // not a valid cross-process address; hash identifies the file + a.fname = 0; // not a valid cross-process address; hash identifies the code object a.co_hash_lo = h.lo; a.co_hash_hi = h.hi; a.module_id = 0; @@ -778,6 +1048,20 @@ hipError_t capture_hipModuleLoad(hipModule_t* module, const char* fname) { return r; } +hipError_t capture_hipModuleUnload(hipModule_t module) { + // Take the program pointer before the real unload frees it. + amd::Program* prog = module ? as_amd(reinterpret_cast(module)) : nullptr; + hipError_t r = g_real_table.hipModuleUnload_fn(module); + if (r == hipSuccess) { + forget_program_hash(prog); + hrr_args_hipModuleUnload a{}; + a.ret = static_cast(r); + a.module = reinterpret_cast(module); + hrr_cap::writer::write_event_raw(HRR_API_HIPMODULEUNLOAD, &a.hdr, sizeof(a)); + } + return r; +} + // --------------------------------------------------------------------------- // Kernel launch shims — variable-length binary payload (not hrr_args_*) // --------------------------------------------------------------------------- @@ -800,6 +1084,28 @@ hipError_t capture_hipModuleLaunchKernel( return r; } +// A cooperative launch carries the same payload as any other module launch — +// the only thing that makes it cooperative is which entry point runs it, and +// that is what the event id records. +hipError_t capture_hipModuleLaunchCooperativeKernel( + hipFunction_t f, + unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, + unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, + unsigned int sharedMemBytes, hipStream_t stream, + void** kernelParams) { + hipError_t r = g_real_table.hipModuleLaunchCooperativeKernel_fn( + f, gridDimX, gridDimY, gridDimZ, + blockDimX, blockDimY, blockDimZ, + sharedMemBytes, stream, kernelParams); + if (r == hipSuccess) { + record_launch(f, gridDimX, gridDimY, gridDimZ, + blockDimX, blockDimY, blockDimZ, + sharedMemBytes, stream, kernelParams, nullptr, + HRR_API_HIPMODULELAUNCHCOOPERATIVEKERNEL); + } + return r; +} + hipError_t capture_hipExtModuleLaunchKernel( hipFunction_t f, uint32_t globalWorkSizeX, uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, @@ -852,6 +1158,123 @@ hipError_t capture_hipLaunchKernel(const void* function_address, return r; } +// The stream-per-thread spelling of the same launch. Any translation unit +// compiled with -fgpu-default-stream=per-thread calls this instead of +// hipLaunchKernel, and the generated shim recorded the host stub address, which +// replay then handed to the runtime — a segfault, not an error. It gets the +// same encoding as the plain spelling under its own event id, so a recording +// still says which entry point the program used. +hipError_t capture_hipLaunchKernel_spt(const void* function_address, + dim3 numBlocks, dim3 dimBlocks, + void** args, size_t sharedMemBytes, + hipStream_t stream) { + hipError_t r = g_real_table.hipLaunchKernel_spt_fn( + function_address, numBlocks, dimBlocks, args, sharedMemBytes, stream); + if (r == hipSuccess) { + hipFunction_t f = nullptr; + if (g_real_table.hipGetFuncBySymbol_fn && + g_real_table.hipGetFuncBySymbol_fn(&f, function_address) == hipSuccess && f) { + record_launch(f, + numBlocks.x, numBlocks.y, numBlocks.z, + dimBlocks.x, dimBlocks.y, dimBlocks.z, + static_cast(sharedMemBytes), stream, args, nullptr, + HRR_API_HIPLAUNCHKERNEL_SPT); + } + } + return r; +} + +// The cooperative launches by host stub, plain and stream-per-thread. Both +// carry the module-launch payload; the event id is what tells replay to go +// back through the cooperative entry point rather than the ordinary one. +hipError_t capture_hipLaunchCooperativeKernel(const void* f, + dim3 gridDim, dim3 blockDimX, + void** kernelParams, + unsigned int sharedMemBytes, + hipStream_t stream) { + hipError_t r = g_real_table.hipLaunchCooperativeKernel_fn( + f, gridDim, blockDimX, kernelParams, sharedMemBytes, stream); + if (r == hipSuccess) { + hipFunction_t fn = nullptr; + if (g_real_table.hipGetFuncBySymbol_fn && + g_real_table.hipGetFuncBySymbol_fn(&fn, f) == hipSuccess && fn) { + record_launch(fn, + gridDim.x, gridDim.y, gridDim.z, + blockDimX.x, blockDimX.y, blockDimX.z, + sharedMemBytes, stream, kernelParams, nullptr, + HRR_API_HIPLAUNCHCOOPERATIVEKERNEL); + } + } + return r; +} + +hipError_t capture_hipLaunchCooperativeKernel_spt(const void* f, + dim3 gridDim, dim3 blockDim, + void** kernelParams, + uint32_t sharedMemBytes, + hipStream_t hStream) { + hipError_t r = g_real_table.hipLaunchCooperativeKernel_spt_fn( + f, gridDim, blockDim, kernelParams, sharedMemBytes, hStream); + if (r == hipSuccess) { + hipFunction_t fn = nullptr; + if (g_real_table.hipGetFuncBySymbol_fn && + g_real_table.hipGetFuncBySymbol_fn(&fn, f) == hipSuccess && fn) { + record_launch(fn, + gridDim.x, gridDim.y, gridDim.z, + blockDim.x, blockDim.y, blockDim.z, + sharedMemBytes, hStream, kernelParams, nullptr, + HRR_API_HIPLAUNCHCOOPERATIVEKERNEL_SPT); + } + } + return r; +} + +// --------------------------------------------------------------------------- +// Extensible launches (hipDrvLaunchKernelEx / hipLaunchKernelExC) +// +// The descriptor is a const struct pointer and its attribute list is a second +// pointer hanging off it, so the generated shim recorded two capture-time host +// addresses and nothing else — Triton's and CK's default launch path replayed +// as an invalid resource handle. Both spellings record the same +// variable-length launch payload as every other launch, under their own event +// id, with the attribute list appended. +// --------------------------------------------------------------------------- + +hipError_t capture_hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, + hipFunction_t f, void** params, + void** extra) { + hipError_t r = g_real_table.hipDrvLaunchKernelEx_fn(config, f, params, extra); + if (r == hipSuccess && config) { + record_launch(f, + config->gridDimX, config->gridDimY, config->gridDimZ, + config->blockDimX, config->blockDimY, config->blockDimZ, + config->sharedMemBytes, config->hStream, params, extra, + HRR_API_HIPDRVLAUNCHKERNELEX, config->attrs, config->numAttrs); + } + return r; +} + +hipError_t capture_hipLaunchKernelExC(const hipLaunchConfig_t* config, + const void* fPtr, void** args) { + hipError_t r = g_real_table.hipLaunchKernelExC_fn(config, fPtr, args); + if (r == hipSuccess && config) { + // fPtr is a host stub address, not a hipFunction_t: the same resolution + // hipLaunchKernel needs, and the same reason replay cannot use the + // recorded value directly. + hipFunction_t f = nullptr; + if (g_real_table.hipGetFuncBySymbol_fn && + g_real_table.hipGetFuncBySymbol_fn(&f, fPtr) == hipSuccess && f) { + record_launch(f, + config->gridDim.x, config->gridDim.y, config->gridDim.z, + config->blockDim.x, config->blockDim.y, config->blockDim.z, + static_cast(config->dynamicSmemBytes), + config->stream, args, nullptr, + HRR_API_HIPLAUNCHKERNELEXC, config->attrs, config->numAttrs); + } + } + return r; +} + hipError_t capture___hipPushCallConfiguration(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream) { hipError_t r = g_real_compiler_table.__hipPushCallConfiguration_fn( @@ -1079,19 +1502,24 @@ hipError_t capture_hipHostUnregister(void* hostPtr) { } // --------------------------------------------------------------------------- -// hipMemPoolSetAttribute — value is void* to a scalar; copy 8 bytes inline. +// hipMemPoolSetAttribute: value is void* to a scalar, stored inline in the low +// bytes of value_u64 (an int32_t for the three reuse policies, uint64_t otherwise). // --------------------------------------------------------------------------- hipError_t capture_hipMemPoolSetAttribute(hipMemPool_t mem_pool, hipMemPoolAttr attr, void* value) { hipError_t r = g_real_table.hipMemPoolSetAttribute_fn(mem_pool, attr, value); + if (r != hipSuccess) return r; hrr_args_hipMemPoolSetAttribute a{}; a.ret = static_cast(r); a.mem_pool = reinterpret_cast(mem_pool); a.attr = static_cast(attr); a.value = reinterpret_cast(value); - if (value) std::memcpy(&a.value_u64, value, sizeof(a.value_u64)); + const bool int_valued = attr == hipMemPoolReuseFollowEventDependencies || + attr == hipMemPoolReuseAllowOpportunistic || + attr == hipMemPoolReuseAllowInternalDependencies; + if (value) std::memcpy(&a.value_u64, value, int_valued ? sizeof(int32_t) : sizeof(a.value_u64)); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMPOOLSETATTRIBUTE, &a.hdr, sizeof(a)); return r; } @@ -1103,9 +1531,10 @@ hipError_t capture_hipMemPoolSetAttribute(hipMemPool_t mem_pool, hipError_t capture_hipMemPoolCreate(hipMemPool_t* mem_pool, const hipMemPoolProps* pool_props) { hipError_t r = g_real_table.hipMemPoolCreate_fn(mem_pool, pool_props); + if (r != hipSuccess) return r; hrr_args_hipMemPoolCreate a{}; a.ret = static_cast(r); - a.mem_pool = reinterpret_cast(r == hipSuccess ? *mem_pool : nullptr); + a.mem_pool = reinterpret_cast(*mem_pool); if (pool_props) std::memcpy(a.pool_props_bytes, pool_props, sizeof(a.pool_props_bytes)); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMPOOLCREATE, &a.hdr, sizeof(a)); @@ -1116,9 +1545,15 @@ hipError_t capture_hipMemPoolCreate(hipMemPool_t* mem_pool, // hipMemcpy3D / hipMemcpy3DAsync — inline parms + H2D blob + D2H expected blob // --------------------------------------------------------------------------- -// Helper: compute byte count from 3D extent -static size_t memcpy3d_byte_count(const struct hipMemcpy3DParms* p) { - return p->extent.width * p->extent.height * p->extent.depth; +// Helper: compute byte count from 3D extent into *bytes (0 for a zero extent). +// Returns false when the product overflows size_t. +static bool memcpy3d_byte_count(const struct hipMemcpy3DParms* p, size_t* bytes) { + const size_t w = p->extent.width, h = p->extent.height, d = p->extent.depth; + *bytes = 0; + if (w == 0 || h == 0 || d == 0) return true; + if (h > SIZE_MAX / w || d > SIZE_MAX / (w * h)) return false; + *bytes = w * h * d; + return true; } // Helper shared by all four 3D variants. @@ -1133,7 +1568,20 @@ static void capture_memcpy3d_impl( return; } std::memcpy(a.parms_bytes, p, sizeof(a.parms_bytes)); - size_t byte_count = memcpy3d_byte_count(p); + size_t byte_count = 0; + const bool sized = memcpy3d_byte_count(p, &byte_count); + const bool host_side = (p->kind == hipMemcpyHostToDevice && p->srcPtr.ptr) || + (p->kind == hipMemcpyDeviceToHost && p->dstPtr.ptr); + // The runtime validates with the same wrapping product, so it can accept an + // extent that overflows. Without a blob, replay cannot perform a host-side copy. + if (!sized && host_side) { + LogPrintfError( + "[HRR capture] 3D copy extent %zux%zux%zu overflows size_t, so no blob can be " + "sized for it: dropping the event and marking the capture INCOMPLETE.", + p->extent.width, p->extent.height, p->extent.depth); + hrr_cap::writer::mark_incomplete("3D copy extent overflows size_t"); + return; + } if (p->kind == hipMemcpyHostToDevice && p->srcPtr.ptr && byte_count > 0) { // H2D: host source is valid at call time — no stream sync needed. @@ -1144,7 +1592,7 @@ static void capture_memcpy3d_impl( // D2H: real call already completed (sync API) or stream sync done below; // host buffer now holds GPU result — capture it as the expected output. if (is_async && stream) { - hipError_t sync_r = g_real_table.hipStreamSynchronize_fn(stream); + hipError_t sync_r = sync_for_d2h_snapshot(stream); if (sync_r != hipSuccess) { LogPrintfWarning("[HRR capture] hipStreamSynchronize failed (%d) — D2H 3D blob skipped", sync_r); @@ -1159,8 +1607,12 @@ static void capture_memcpy3d_impl( hrr_cap::writer::write_event_raw(api_id, &a.hdr, sizeof(a)); } +// Success-gated like the 2D and driver-style copies below. A copy the runtime +// rejected has an extent nothing validated, so blobbing w*h*d bytes of its host +// side would read past the caller's buffer. hipError_t capture_hipMemcpy3D(const struct hipMemcpy3DParms* p) { hipError_t r = g_real_table.hipMemcpy3D_fn(p); + if (r != hipSuccess) return r; hrr_args_hipMemcpy3D a{}; a.ret = static_cast(r); capture_memcpy3d_impl(a, HRR_API_HIPMEMCPY3D, p, nullptr, false); @@ -1169,6 +1621,7 @@ hipError_t capture_hipMemcpy3D(const struct hipMemcpy3DParms* p) { hipError_t capture_hipMemcpy3DAsync(const struct hipMemcpy3DParms* p, hipStream_t stream) { hipError_t r = g_real_table.hipMemcpy3DAsync_fn(p, stream); + if (r != hipSuccess) return r; hrr_args_hipMemcpy3DAsync a{}; a.ret = static_cast(r); a.stream = reinterpret_cast(stream); @@ -1178,6 +1631,7 @@ hipError_t capture_hipMemcpy3DAsync(const struct hipMemcpy3DParms* p, hipStream_ hipError_t capture_hipMemcpy3D_spt(const struct hipMemcpy3DParms* p) { hipError_t r = g_real_table.hipMemcpy3D_spt_fn(p); + if (r != hipSuccess) return r; hrr_args_hipMemcpy3D_spt a{}; a.ret = static_cast(r); capture_memcpy3d_impl(a, HRR_API_HIPMEMCPY3D_SPT, p, nullptr, false); @@ -1186,6 +1640,7 @@ hipError_t capture_hipMemcpy3D_spt(const struct hipMemcpy3DParms* p) { hipError_t capture_hipMemcpy3DAsync_spt(const struct hipMemcpy3DParms* p, hipStream_t stream) { hipError_t r = g_real_table.hipMemcpy3DAsync_spt_fn(p, stream); + if (r != hipSuccess) return r; hrr_args_hipMemcpy3DAsync_spt a{}; a.ret = static_cast(r); a.stream = reinterpret_cast(stream); @@ -1258,7 +1713,7 @@ static void capture_drvmemcpy3d_impl(T& a, hrr_api_id_t api_id, // is valid and waits on the blocking streams, which is what the app itself // would have to do before reading dstHost. if (is_async) { - hipError_t sync_r = g_real_table.hipStreamSynchronize_fn(stream); + hipError_t sync_r = sync_for_d2h_snapshot(stream); if (sync_r != hipSuccess) { LogPrintfWarning("[HRR capture] hipStreamSynchronize failed (%d): D2H drv blob skipped", sync_r); @@ -1333,6 +1788,32 @@ hipError_t capture_hipDrvMemcpy2DUnaligned(const hip_Memcpy2D* pCopy) { return r; } +// The same descriptor under the runtime-API spellings. Both were recording the +// pointer and nothing behind it, which is why they were no-ops at replay. +hipError_t capture_hipMemcpyParam2D(const hip_Memcpy2D* pCopy) { + hipError_t r = g_real_table.hipMemcpyParam2D_fn(pCopy); + if (r != hipSuccess) return r; + hrr_args_hipMemcpyParam2D a{}; + a.ret = static_cast(r); + capture_drvmemcpy2d_impl(a, HRR_API_HIPMEMCPYPARAM2D, pCopy); + return r; +} + +hipError_t capture_hipMemcpyParam2DAsync(const hip_Memcpy2D* pCopy, + hipStream_t stream) { + hipError_t r = g_real_table.hipMemcpyParam2DAsync_fn(pCopy, stream); + if (r != hipSuccess) return r; + hrr_args_hipMemcpyParam2DAsync a{}; + a.ret = static_cast(r); + a.stream = reinterpret_cast(stream); + // A D2H blob taken before the copy lands would record a stale expected + // output, the same reason the async 3D spelling synchronises first. + if (pCopy && pCopy->dstMemoryType == hipMemoryTypeHost && stream) + (void)sync_for_d2h_snapshot(stream); + capture_drvmemcpy2d_impl(a, HRR_API_HIPMEMCPYPARAM2DASYNC, pCopy); + return r; +} + // --------------------------------------------------------------------------- // hipMemcpy2D / hipMemcpy2DAsync — pitched H2D blob + D2H expected blob // @@ -1368,7 +1849,7 @@ static void capture_memcpy2d_impl( size_t n = memcpy2d_host_byte_count(dpitch, width, height); if (n > 0) { if (is_async && stream) { - hipError_t sync_r = g_real_table.hipStreamSynchronize_fn(stream); + hipError_t sync_r = sync_for_d2h_snapshot(stream); if (sync_r != hipSuccess) { LogPrintfWarning("[HRR capture] hipStreamSynchronize failed (%d) — D2H 2D blob skipped", sync_r); @@ -1431,9 +1912,10 @@ hipError_t capture_hipMemcpy2DAsync(void* dst, size_t dpitch, const void* src, hipError_t capture_hipArrayCreate(hipArray_t* pHandle, const HIP_ARRAY_DESCRIPTOR* pAllocateArray) { hipError_t r = g_real_table.hipArrayCreate_fn(pHandle, pAllocateArray); + if (r != hipSuccess) return r; hrr_args_hipArrayCreate a{}; a.ret = static_cast(r); - a.pHandle = reinterpret_cast(r == hipSuccess ? *pHandle : nullptr); + a.pHandle = reinterpret_cast(*pHandle); if (pAllocateArray) std::memcpy(a.array_desc_bytes, pAllocateArray, sizeof(a.array_desc_bytes)); hrr_cap::writer::write_event_raw(HRR_API_HIPARRAYCREATE, &a.hdr, sizeof(a)); @@ -1443,9 +1925,10 @@ hipError_t capture_hipArrayCreate(hipArray_t* pHandle, hipError_t capture_hipArray3DCreate(hipArray_t* array, const HIP_ARRAY3D_DESCRIPTOR* pAllocateArray) { hipError_t r = g_real_table.hipArray3DCreate_fn(array, pAllocateArray); + if (r != hipSuccess) return r; hrr_args_hipArray3DCreate a{}; a.ret = static_cast(r); - a.array = reinterpret_cast(r == hipSuccess ? *array : nullptr); + a.array = reinterpret_cast(*array); if (pAllocateArray) std::memcpy(a.array3d_desc_bytes, pAllocateArray, sizeof(a.array3d_desc_bytes)); hrr_cap::writer::write_event_raw(HRR_API_HIPARRAY3DCREATE, &a.hdr, sizeof(a)); @@ -1459,6 +1942,7 @@ hipError_t capture_hipArray3DCreate(hipArray_t* array, hipError_t capture_hipStreamSetAttribute(hipStream_t stream, hipStreamAttrID attr, const hipStreamAttrValue* value) { hipError_t r = g_real_table.hipStreamSetAttribute_fn(stream, attr, value); + if (r != hipSuccess) return r; hrr_args_hipStreamSetAttribute a{}; a.ret = static_cast(r); a.stream = reinterpret_cast(stream); @@ -1476,6 +1960,7 @@ hipError_t capture_hipMemGetAllocationGranularity(size_t* granularity, const hipMemAllocationProp* prop, hipMemAllocationGranularity_flags option) { hipError_t r = g_real_table.hipMemGetAllocationGranularity_fn(granularity, prop, option); + if (r != hipSuccess) return r; hrr_args_hipMemGetAllocationGranularity a{}; a.ret = static_cast(r); a.granularity = reinterpret_cast(granularity); @@ -1492,6 +1977,7 @@ hipError_t capture_hipMemGetAllocationGranularity(size_t* granularity, hipError_t capture_hipMemPoolSetAccess(hipMemPool_t mem_pool, const hipMemAccessDesc* desc_list, size_t count) { hipError_t r = g_real_table.hipMemPoolSetAccess_fn(mem_pool, desc_list, count); + if (r != hipSuccess) return r; hrr_args_hipMemPoolSetAccess a{}; a.ret = static_cast(r); a.mem_pool = reinterpret_cast(mem_pool); @@ -1505,6 +1991,7 @@ hipError_t capture_hipMemPoolSetAccess(hipMemPool_t mem_pool, hipError_t capture_hipMemSetAccess(void* ptr, size_t size, const hipMemAccessDesc* desc, size_t count) { hipError_t r = g_real_table.hipMemSetAccess_fn(ptr, size, desc, count); + if (r != hipSuccess) return r; hrr_args_hipMemSetAccess a{}; a.ret = static_cast(r); a.ptr = reinterpret_cast(ptr); @@ -1516,6 +2003,365 @@ hipError_t capture_hipMemSetAccess(void* ptr, size_t size, return r; } +// --------------------------------------------------------------------------- +// Graph kernel nodes — struct fields plus the kernel tail +// +// The generated shim already carries the dependency array and the 64 bytes of +// hipKernelNodeParams (DEREF_FIELDS), which is enough for the dimensions and +// the shared-memory size. What it cannot carry is `func` (a host address) and +// `kernelParams` (a host array of pointers to argument values); those are the +// tail, written the same way a launch writes them. +// --------------------------------------------------------------------------- + +template +static void write_kernel_node_event(A& a, hrr_api_id_t api_id, + const hipKernelNodeParams* p) { + std::vector payload(sizeof(A)); + std::memcpy(payload.data(), &a, sizeof(A)); + append_kernel_node_tail(payload, p ? p->func : nullptr, + p ? p->kernelParams : nullptr, p ? p->extra : nullptr); + hrr_cap::writer::write_event_raw( + api_id, reinterpret_cast(payload.data()), + static_cast(payload.size())); +} + +hipError_t capture_hipGraphAddKernelNode(hipGraphNode_t* pGraphNode, + hipGraph_t graph, + const hipGraphNode_t* pDependencies, + size_t numDependencies, + const hipKernelNodeParams* pNodeParams) { + hipError_t r = g_real_table.hipGraphAddKernelNode_fn( + pGraphNode, graph, pDependencies, numDependencies, pNodeParams); + hrr_args_hipGraphAddKernelNode a{}; + a.ret = static_cast(r); + a.graph = reinterpret_cast(graph); + a.numDependencies = static_cast(numDependencies); + a.pGraphNode = reinterpret_cast( + (r == hipSuccess && pGraphNode) ? *pGraphNode : nullptr); + if (pDependencies && numDependencies > 0) { + uint32_t n = static_cast(numDependencies); + const uint32_t cap = sizeof(a.pDependencies_bytes) / sizeof(hipGraphNode_t); + if (n > cap) n = cap; + std::memcpy(a.pDependencies_bytes, pDependencies, n * sizeof(hipGraphNode_t)); + a.pDependencies_n = n; + a.pDependencies_present = 1; + } + if (pNodeParams) { + std::memcpy(a.pNodeParams_bytes, pNodeParams, sizeof(hipKernelNodeParams)); + a.pNodeParams_present = 1; + } + write_kernel_node_event(a, HRR_API_HIPGRAPHADDKERNELNODE, pNodeParams); + return r; +} + +// A copy naming a __device__ global. The symbol is a host shadow address the +// replay cannot use, but the symbol sweep records its name, so all this shim +// has to add is the host buffer on the other side of the copy — which exists +// only when the copy is to the symbol from host memory. +static bool memcpy_src_is_host(const void* src, hipMemcpyKind kind) { + if (kind == hipMemcpyHostToDevice || kind == hipMemcpyHostToHost) return true; + if (kind != hipMemcpyDefault) return false; + if (!src || !g_real_table.hipPointerGetAttributes_fn) return false; + hipPointerAttribute_t attr{}; + if (g_real_table.hipPointerGetAttributes_fn(&attr, src) != hipSuccess) + return true; // unregistered memory is ordinary host memory + return attr.type != hipMemoryTypeDevice && attr.type != hipMemoryTypeUnified; +} + +template +static void fill_node_deps(A& a, const hipGraphNode_t* deps, size_t n_deps) { + if (!deps || n_deps == 0) return; + uint32_t n = static_cast(n_deps); + const uint32_t cap = sizeof(a.pDependencies_bytes) / sizeof(hipGraphNode_t); + if (n > cap) n = cap; + std::memcpy(a.pDependencies_bytes, deps, n * sizeof(hipGraphNode_t)); + a.pDependencies_n = n; + a.pDependencies_present = 1; +} + +hipError_t capture_hipGraphAddMemcpyNodeToSymbol( + hipGraphNode_t* pGraphNode, hipGraph_t graph, + const hipGraphNode_t* pDependencies, size_t numDependencies, + const void* symbol, const void* src, size_t count, size_t offset, + hipMemcpyKind kind) { + hipError_t r = g_real_table.hipGraphAddMemcpyNodeToSymbol_fn( + pGraphNode, graph, pDependencies, numDependencies, symbol, src, count, + offset, kind); + hrr_args_hipGraphAddMemcpyNodeToSymbol a{}; + a.ret = static_cast(r); + a.graph = reinterpret_cast(graph); + a.numDependencies = static_cast(numDependencies); + a.symbol = reinterpret_cast(symbol); + a.src = reinterpret_cast(src); + a.count = static_cast(count); + a.offset = static_cast(offset); + a.kind = static_cast(kind); + a.pGraphNode = reinterpret_cast( + (r == hipSuccess && pGraphNode) ? *pGraphNode : nullptr); + fill_node_deps(a, pDependencies, numDependencies); + if (src && count && memcpy_src_is_host(src, kind)) { + auto h = hrr_cap::writer::write_blob(src, count); + a.blob_hash_lo = h.lo; + a.blob_hash_hi = h.hi; + } + hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMCPYNODETOSYMBOL, + &a.hdr, sizeof(a)); + return r; +} + +hipError_t capture_hipGraphAddMemcpyNodeFromSymbol( + hipGraphNode_t* pGraphNode, hipGraph_t graph, + const hipGraphNode_t* pDependencies, size_t numDependencies, void* dst, + const void* symbol, size_t count, size_t offset, hipMemcpyKind kind) { + hipError_t r = g_real_table.hipGraphAddMemcpyNodeFromSymbol_fn( + pGraphNode, graph, pDependencies, numDependencies, dst, symbol, count, + offset, kind); + hrr_args_hipGraphAddMemcpyNodeFromSymbol a{}; + a.ret = static_cast(r); + a.graph = reinterpret_cast(graph); + a.numDependencies = static_cast(numDependencies); + a.dst = reinterpret_cast(dst); + a.symbol = reinterpret_cast(symbol); + a.count = static_cast(count); + a.offset = static_cast(offset); + a.kind = static_cast(kind); + a.pGraphNode = reinterpret_cast( + (r == hipSuccess && pGraphNode) ? *pGraphNode : nullptr); + fill_node_deps(a, pDependencies, numDependencies); + hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMCPYNODEFROMSYMBOL, + &a.hdr, sizeof(a)); + return r; +} + +// A JIT linker input. The image is the whole point of the call and was +// recorded as an address, so replay had nothing to link even once the linker +// state itself was tracked. +hipError_t capture_hipLinkAddData(hipLinkState_t state, hipJitInputType type, + void* data, size_t size, const char* name, + unsigned int numOptions, + hipJitOption* options, void** optionValues) { + hipError_t r = g_real_table.hipLinkAddData_fn(state, type, data, size, name, + numOptions, options, + optionValues); + hrr_args_hipLinkAddData a{}; + a.ret = static_cast(r); + a.state = reinterpret_cast(state); + a.type = static_cast(type); + a.data = reinterpret_cast(data); + a.size = static_cast(size); + a.name = reinterpret_cast(name); + a.numOptions = static_cast(numOptions); + a.options = reinterpret_cast(options); + a.optionValues = reinterpret_cast(optionValues); + if (name) { + size_t n = std::strlen(name); + if (n > 255u) n = 255u; + std::memcpy(a.name_bytes, name, n); + a.name_bytes[n] = '\0'; + a.name_present = 1; + } + if (options && numOptions) { + uint32_t n = numOptions > 32u ? 32u : numOptions; + std::memcpy(a.options_bytes, options, n * sizeof(hipJitOption)); + a.options_n = n; + a.options_present = 1; + } + if (optionValues && numOptions) { + uint32_t n = numOptions > 32u ? 32u : numOptions; + std::memcpy(a.optionValues_bytes, optionValues, n * sizeof(void*)); + a.optionValues_n = n; + a.optionValues_present = 1; + } + if (data && size) { + auto h = hrr_cap::writer::write_blob(data, size); + a.blob_hash_lo = h.lo; + a.blob_hash_hi = h.hi; + } + hrr_cap::writer::write_event_raw(HRR_API_HIPLINKADDDATA, &a.hdr, sizeof(a)); + return r; +} + +// The mutation spellings of the same copy, node-level and exec-level. Only the +// to-symbol direction has a host buffer worth recording. +hipError_t capture_hipGraphMemcpyNodeSetParamsToSymbol( + hipGraphNode_t node, const void* symbol, const void* src, size_t count, + size_t offset, hipMemcpyKind kind) { + hipError_t r = g_real_table.hipGraphMemcpyNodeSetParamsToSymbol_fn( + node, symbol, src, count, offset, kind); + hrr_args_hipGraphMemcpyNodeSetParamsToSymbol a{}; + a.ret = static_cast(r); + a.node = reinterpret_cast(node); + a.symbol = reinterpret_cast(symbol); + a.src = reinterpret_cast(src); + a.count = static_cast(count); + a.offset = static_cast(offset); + a.kind = static_cast(kind); + if (src && count && memcpy_src_is_host(src, kind)) { + auto h = hrr_cap::writer::write_blob(src, count); + a.blob_hash_lo = h.lo; + a.blob_hash_hi = h.hi; + } + hrr_cap::writer::write_event_raw( + HRR_API_HIPGRAPHMEMCPYNODESETPARAMSTOSYMBOL, &a.hdr, sizeof(a)); + return r; +} + +hipError_t capture_hipGraphExecMemcpyNodeSetParamsToSymbol( + hipGraphExec_t hGraphExec, hipGraphNode_t node, const void* symbol, + const void* src, size_t count, size_t offset, hipMemcpyKind kind) { + hipError_t r = g_real_table.hipGraphExecMemcpyNodeSetParamsToSymbol_fn( + hGraphExec, node, symbol, src, count, offset, kind); + hrr_args_hipGraphExecMemcpyNodeSetParamsToSymbol a{}; + a.ret = static_cast(r); + a.hGraphExec = reinterpret_cast(hGraphExec); + a.node = reinterpret_cast(node); + a.symbol = reinterpret_cast(symbol); + a.src = reinterpret_cast(src); + a.count = static_cast(count); + a.offset = static_cast(offset); + a.kind = static_cast(kind); + if (src && count && memcpy_src_is_host(src, kind)) { + auto h = hrr_cap::writer::write_blob(src, count); + a.blob_hash_lo = h.lo; + a.blob_hash_hi = h.hi; + } + hrr_cap::writer::write_event_raw( + HRR_API_HIPGRAPHEXECMEMCPYNODESETPARAMSTOSYMBOL, &a.hdr, sizeof(a)); + return r; +} + +hipError_t capture_hipGraphKernelNodeSetParams( + hipGraphNode_t node, const hipKernelNodeParams* pNodeParams) { + hipError_t r = g_real_table.hipGraphKernelNodeSetParams_fn(node, pNodeParams); + hrr_args_hipGraphKernelNodeSetParams a{}; + a.ret = static_cast(r); + a.node = reinterpret_cast(node); + if (pNodeParams) { + std::memcpy(a.pNodeParams_bytes, pNodeParams, sizeof(hipKernelNodeParams)); + a.pNodeParams_present = 1; + } + write_kernel_node_event(a, HRR_API_HIPGRAPHKERNELNODESETPARAMS, pNodeParams); + return r; +} + +hipError_t capture_hipGraphExecKernelNodeSetParams( + hipGraphExec_t hGraphExec, hipGraphNode_t node, + const hipKernelNodeParams* pNodeParams) { + hipError_t r = g_real_table.hipGraphExecKernelNodeSetParams_fn( + hGraphExec, node, pNodeParams); + hrr_args_hipGraphExecKernelNodeSetParams a{}; + a.ret = static_cast(r); + a.hGraphExec = reinterpret_cast(hGraphExec); + a.node = reinterpret_cast(node); + if (pNodeParams) { + std::memcpy(a.pNodeParams_bytes, pNodeParams, sizeof(hipKernelNodeParams)); + a.pNodeParams_present = 1; + } + write_kernel_node_event(a, HRR_API_HIPGRAPHEXECKERNELNODESETPARAMS, + pNodeParams); + return r; +} + +// --------------------------------------------------------------------------- +// Graph batch-memory-operation nodes — struct fields plus the op-array tail +// +// hipBatchMemOpNodeParams is {ctx, count, paramArray, flags}: the operations +// are a second pointer hop, so the 32 bytes DEREF_FIELDS carries name an array +// in the capturing process and nothing else. The array follows the struct as +// a tail: u32 count, u32 stride, then count entries. +// --------------------------------------------------------------------------- + +template +static void write_batch_memop_node_event(A& a, hrr_api_id_t api_id, + const hipBatchMemOpNodeParams* p) { + std::vector payload(sizeof(A)); + std::memcpy(payload.data(), &a, sizeof(A)); + const uint32_t n = (p && p->paramArray) ? p->count : 0u; + const uint32_t stride = + n ? static_cast(sizeof(hipStreamBatchMemOpParams)) : 0u; + const auto push_u32 = [&](uint32_t v) { + for (int i = 0; i < 4; i++) payload.push_back(static_cast(v >> (i*8))); + }; + push_u32(n); + push_u32(stride); + if (n) { + const auto* q = reinterpret_cast(p->paramArray); + payload.insert(payload.end(), q, q + static_cast(n) * stride); + } + hrr_cap::writer::write_event_raw( + api_id, reinterpret_cast(payload.data()), + static_cast(payload.size())); +} + +// The dependency array is the same shape in every hipGraphAdd*Node, so the +// hand-written shims fill it the same way the generated ones do. +template +static void fill_node_dependencies(A& a, const hipGraphNode_t* deps, + size_t num_deps) { + if (!deps || num_deps == 0) return; + uint32_t n = static_cast(num_deps); + const uint32_t cap = sizeof(a.dependencies_bytes) / sizeof(hipGraphNode_t); + if (n > cap) n = cap; + std::memcpy(a.dependencies_bytes, deps, n * sizeof(hipGraphNode_t)); + a.dependencies_n = n; + a.dependencies_present = 1; +} + +hipError_t capture_hipGraphAddBatchMemOpNode( + hipGraphNode_t* phGraphNode, hipGraph_t hGraph, + const hipGraphNode_t* dependencies, size_t numDependencies, + const hipBatchMemOpNodeParams* nodeParams) { + hipError_t r = g_real_table.hipGraphAddBatchMemOpNode_fn( + phGraphNode, hGraph, dependencies, numDependencies, nodeParams); + hrr_args_hipGraphAddBatchMemOpNode a{}; + a.ret = static_cast(r); + a.hGraph = reinterpret_cast(hGraph); + a.numDependencies = static_cast(numDependencies); + a.phGraphNode = reinterpret_cast( + (r == hipSuccess && phGraphNode) ? *phGraphNode : nullptr); + fill_node_dependencies(a, dependencies, numDependencies); + if (nodeParams) { + std::memcpy(a.nodeParams_bytes, nodeParams, sizeof(hipBatchMemOpNodeParams)); + a.nodeParams_present = 1; + } + write_batch_memop_node_event(a, HRR_API_HIPGRAPHADDBATCHMEMOPNODE, nodeParams); + return r; +} + +hipError_t capture_hipGraphBatchMemOpNodeSetParams( + hipGraphNode_t hNode, hipBatchMemOpNodeParams* nodeParams) { + hipError_t r = + g_real_table.hipGraphBatchMemOpNodeSetParams_fn(hNode, nodeParams); + hrr_args_hipGraphBatchMemOpNodeSetParams a{}; + a.ret = static_cast(r); + a.hNode = reinterpret_cast(hNode); + if (nodeParams) { + std::memcpy(a.nodeParams_bytes, nodeParams, sizeof(hipBatchMemOpNodeParams)); + a.nodeParams_present = 1; + } + write_batch_memop_node_event( + a, HRR_API_HIPGRAPHBATCHMEMOPNODESETPARAMS, nodeParams); + return r; +} + +hipError_t capture_hipGraphExecBatchMemOpNodeSetParams( + hipGraphExec_t hGraphExec, hipGraphNode_t hNode, + const hipBatchMemOpNodeParams* nodeParams) { + hipError_t r = g_real_table.hipGraphExecBatchMemOpNodeSetParams_fn( + hGraphExec, hNode, nodeParams); + hrr_args_hipGraphExecBatchMemOpNodeSetParams a{}; + a.ret = static_cast(r); + a.hGraphExec = reinterpret_cast(hGraphExec); + a.hNode = reinterpret_cast(hNode); + if (nodeParams) { + std::memcpy(a.nodeParams_bytes, nodeParams, sizeof(hipBatchMemOpNodeParams)); + a.nodeParams_present = 1; + } + write_batch_memop_node_event( + a, HRR_API_HIPGRAPHEXECBATCHMEMOPNODESETPARAMS, nodeParams); + return r; +} + // --------------------------------------------------------------------------- // Install / uninstall (build_table functions live in hip_capture_generated.cpp) // --------------------------------------------------------------------------- @@ -1552,6 +2398,34 @@ static void record_fat_binary_blob(const void* blob_ptr) { hrr_cap::writer::write_event_raw(HRR_API_HIPREGISTERFATBINARY, &a.hdr, sizeof(a)); } +// Record one registered __device__ global as a HRR_API_HIPREGISTERVAR event. +// +// The name is what survives the process boundary; the host shadow address and +// the capture-time device address are recorded beside it so replay can map +// both onto the global it resolves by name. Without the device address a +// recorded hipMemcpy reading the symbol has a source that translates to +// nothing, which is what made hipMemcpyFromSymbolAsync fail as an inner copy. +static void record_registered_var(const void* host_var, const char* name, + size_t size, const void* dev_ptr) { + if (!host_var || !name || !*name) return; + + hrr_args___hipRegisterVar a{}; + a.var = reinterpret_cast(host_var); + a.hostVar = reinterpret_cast(host_var); + a.deviceVar = reinterpret_cast(name); + a.size = static_cast(size); + a.global = 1; + a.dev_addr = reinterpret_cast(dev_ptr); + + size_t n = std::strlen(name); + if (n > 255u) n = 255u; + std::memcpy(a.deviceVar_bytes, name, n); + a.deviceVar_bytes[n] = '\0'; + a.deviceVar_present = 1; + + hrr_cap::writer::write_event_raw(HRR_API_HIPREGISTERVAR, &a.hdr, sizeof(a)); +} + // Runtime dispatch-table capture is installed only from hip_capture_init() // (after hip::init() has built the live HipDispatchTable). We intentionally // do NOT hook at libamdhip64 static-init time when HIP_HRR_CAPTURE_OUTPUT is set: @@ -1593,7 +2467,14 @@ void hrr_install_clr_exception_handler() { void hip_capture_init() { #if defined(HIP_HRR_CAPTURE_ENABLED) - if (!hip_capture_enabled()) return; + if (!hip_capture_enabled()) { + if (hrr_secure_exec() && hrr_capture_requested()) + fprintf(stderr, + "[HRR capture] HIP_HRR_CAPTURE_OUTPUT ignored: the program was started in " + "secure-execution mode (set-user-ID, set-group-ID, file capabilities or an " + "LSM transition).\n"); + return; + } // HIP_HRR_DEBUG_ARGS traces are emitted via LogPrintfInfo (amd::LOG_INFO). // ClPrint filters anything above AMD_LOG_LEVEL, so a user who set the trace @@ -1616,7 +2497,12 @@ void hip_capture_init() { } // Open the events writer now — Flag::init() has run so output_dir is valid. - if (!hrr_cap::writer::open(hip_capture_output_dir())) return; + // A refused open leaves capture off, so take the shims out of the dispatch + // table too rather than leave every call going through them for nothing. + if (!hrr_cap::writer::open(hip_capture_output_dir())) { + hip_capture_uninstall(); + return; + } hrr_cap::writer::set_capture_metadata_json( hrr_cap::metadata::collect_json()); @@ -1631,11 +2517,27 @@ void hip_capture_init() { // __hipRegisterFatBinary fires at app static-init, before hip_capture_init(). hip::PlatformState::Instance().StatCO().ForEachFatBinaryBlob(record_fat_binary_blob); + // And the __device__ globals registered against them, for the same reason + // and in this order: the fat binaries have to be in the archive before the + // symbols that live inside them. + hip::PlatformState::Instance().StatCO().ForEachGlobalVar(record_registered_var); + std::call_once(g_hrr_atexit_once, [] { std::atexit(hip_capture_shutdown); }); + + // Deliberately not gated on AMD_LOG_LEVEL: whoever runs the process must be + // able to see that it is being recorded. The path must match the pid- + // directory writer::open() creates. A fork() child records to a sibling + // pid- directory without printing, because hip::init() does not run again. + fprintf(stderr, + "[HRR capture] Recording this process's HIP calls, with their host buffers, kernel " + "arguments and code objects, to %s/pid-%d (child processes record to their own " + "pid-* directories in %s)\n", + hip_capture_output_dir(), amd::Os::getProcessId(), hip_capture_output_dir()); #else // HRR capture is disabled // below to avoid -Wunused-function / -Wunused-variable (void)&record_fat_binary_blob; + (void)&record_registered_var; (void)&hrr_install_clr_exception_handler; (void)&g_hrr_atexit_once; #endif @@ -1646,8 +2548,8 @@ void hip_capture_shutdown() { hrr_cap::writer::flush(hip_capture_output_dir()); hrr_cap::writer::close(); - LogPrintfInfo("[HRR capture] Wrote %llu events, %llu blobs to: %s", - static_cast(hrr_cap::writer::event_count()), - static_cast(hrr_cap::writer::blob_count()), - hip_capture_output_dir()); + LogPrintfInfo("[HRR capture] Wrote %llu events, %llu blobs to: %s/pid-%d", + static_cast(hrr_cap::writer::event_count()), + static_cast(hrr_cap::writer::blob_count()), + hip_capture_output_dir(), amd::Os::getProcessId()); } diff --git a/projects/clr/hipamd/src/hrr/hip_capture.h b/projects/clr/hipamd/src/hrr/hip_capture.h index 8de8f08e503d..2913050a7793 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.h +++ b/projects/clr/hipamd/src/hrr/hip_capture.h @@ -33,7 +33,8 @@ struct Hash128 { // Public API — called from hip_context.cpp and hip_capture.cpp // --------------------------------------------------------------------------- -// Check if capture is enabled (HIP_HRR_CAPTURE_OUTPUT env var set and non-empty) +// Check if capture is enabled (HIP_HRR_CAPTURE_OUTPUT env var set and not empty or blank, +// and the process not started in secure-execution mode) bool hip_capture_enabled(); // Return the output directory from the env var diff --git a/projects/clr/hipamd/src/hrr/hip_capture_generated.cpp b/projects/clr/hipamd/src/hrr/hip_capture_generated.cpp index b6028683e527..5ab1e7017207 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_generated.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture_generated.cpp @@ -36,9 +36,18 @@ #include "hrr/hrr_api_args.h" #include "hip/amd_detail/hip_api_trace.hpp" +// LogPrintfWarning — capture diagnostics go through amd's log-level machinery, +// never raw stderr. debug.hpp's ClPrint macro reads the AMD_LOG_LEVEL and +// AMD_LOG_MASK flag variables but does not declare them, so flags.hpp must come +// with it, and flags.hpp spells its defaults with top.hpp's unit suffixes (Mi) +// and typedefs (uint). All three, in this order. +#include "top.hpp" +#include "utils/flags.hpp" +#include "utils/debug.hpp" #include #include +#include #include // These global tables are defined (non-static) in hip_capture.cpp @@ -117,8 +126,8 @@ static hipError_t capture_hipArrayGetInfo(hipChannelFormatDesc* desc, hipExtent* if (r == hipSuccess) { hrr_args_hipArrayGetInfo a{}; a.ret = static_cast(r); - a.desc = 0; // non-castable type skipped - a.extent = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); + a.extent = reinterpret_cast(extent); a.flags = reinterpret_cast(flags); a.array = reinterpret_cast(array); hrr_cap::writer::write_event_raw(HRR_API_HIPARRAYGETINFO, &a.hdr, sizeof(a)); @@ -135,7 +144,7 @@ static hipError_t capture_hipBindTexture(size_t* offset, const textureReference* a.offset = reinterpret_cast(offset); a.tex = reinterpret_cast(tex); a.devPtr = reinterpret_cast(devPtr); - a.desc = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); a.size = static_cast(size); hrr_cap::writer::write_event_raw(HRR_API_HIPBINDTEXTURE, &a.hdr, sizeof(a)); } @@ -151,7 +160,7 @@ static hipError_t capture_hipBindTexture2D(size_t* offset, const textureReferenc a.offset = reinterpret_cast(offset); a.tex = reinterpret_cast(tex); a.devPtr = reinterpret_cast(devPtr); - a.desc = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); a.width = static_cast(width); a.height = static_cast(height); a.pitch = static_cast(pitch); @@ -168,7 +177,7 @@ static hipError_t capture_hipBindTextureToArray(const textureReference* tex, hip a.ret = static_cast(r); a.tex = reinterpret_cast(tex); a.array = reinterpret_cast(array); - a.desc = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); hrr_cap::writer::write_event_raw(HRR_API_HIPBINDTEXTURETOARRAY, &a.hdr, sizeof(a)); } return r; @@ -182,7 +191,7 @@ static hipError_t capture_hipBindTextureToMipmappedArray(const textureReference* a.ret = static_cast(r); a.tex = reinterpret_cast(tex); a.mipmappedArray = reinterpret_cast(mipmappedArray); - a.desc = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); hrr_cap::writer::write_event_raw(HRR_API_HIPBINDTEXTURETOMIPMAPPEDARRAY, &a.hdr, sizeof(a)); } return r; @@ -196,6 +205,10 @@ static hipError_t capture_hipChooseDevice(int* device, const hipDeviceProp_tR060 a.ret = static_cast(r); a.device = reinterpret_cast(device); a.prop = reinterpret_cast(prop); + if (prop) { + std::memcpy(a.prop_bytes, prop, sizeof(hipDeviceProp_t)); + a.prop_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPCHOOSEDEVICE, &a.hdr, sizeof(a)); } return r; @@ -209,6 +222,10 @@ static hipError_t capture_hipChooseDeviceR0000(int* device, const hipDeviceProp_ a.ret = static_cast(r); a.device = reinterpret_cast(device); a.properties = reinterpret_cast(properties); + if (properties) { + std::memcpy(a.properties_bytes, properties, sizeof(hipDeviceProp_tR0000)); + a.properties_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPCHOOSEDEVICER0000, &a.hdr, sizeof(a)); } return r; @@ -963,11 +980,35 @@ static hipError_t capture_hipDrvGraphAddMemcpyNode(hipGraphNode_t* phGraphNode, hrr_args_hipDrvGraphAddMemcpyNode a{}; a.ret = static_cast(r); a.hGraph = reinterpret_cast(hGraph); + a.dependencies = reinterpret_cast(dependencies); a.numDependencies = static_cast(numDependencies); a.copyParams = reinterpret_cast(copyParams); a.ctx = reinterpret_cast(ctx); if (phGraphNode) a.phGraphNode = reinterpret_cast(*phGraphNode); - if (dependencies) a.dependencies = reinterpret_cast(*dependencies); + if (dependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_dependencies = false; + if (!warned_dependencies) { + warned_dependencies = true; + LogPrintfWarning( + "[HRR] hipDrvGraphAddMemcpyNode: recording only the first 16 " + "of %u dependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipDrvGraphAddMemcpyNode", + "only the first 16 dependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.dependencies_bytes, dependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.dependencies_n = _n; + a.dependencies_present = 1; + } + if (copyParams) { + std::memcpy(a.copyParams_bytes, copyParams, sizeof(HIP_MEMCPY3D)); + a.copyParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPDRVGRAPHADDMEMCPYNODE, &a.hdr, sizeof(a)); } return r; @@ -1127,6 +1168,8 @@ static hipError_t capture_hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* la a.launchParamsList = reinterpret_cast(launchParamsList); a.numDevices = static_cast(numDevices); a.flags = static_cast(flags); + hrr_cap::writer::note_unreplayable("hipExtLaunchMultiKernelMultiDevice", + "each hipLaunchParams entry names its kernel by a host function address in the capturing process, and a cooperative multi-device launch cannot be decomposed into per-device launches without breaking the grid-wide barrier it exists for"); hrr_cap::writer::write_event_raw(HRR_API_HIPEXTLAUNCHMULTIKERNELMULTIDEVICE, &a.hdr, sizeof(a)); } return r; @@ -1337,7 +1380,7 @@ static hipError_t capture_hipGetChannelDesc(hipChannelFormatDesc* desc, hipArray if (r == hipSuccess) { hrr_args_hipGetChannelDesc a{}; a.ret = static_cast(r); - a.desc = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); a.array = reinterpret_cast(array); hrr_cap::writer::write_event_raw(HRR_API_HIPGETCHANNELDESC, &a.hdr, sizeof(a)); } @@ -1553,10 +1596,30 @@ static hipError_t capture_hipGraphAddChildGraphNode(hipGraphNode_t* pGraphNode, hrr_args_hipGraphAddChildGraphNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.childGraph = reinterpret_cast(childGraph); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddChildGraphNode: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddChildGraphNode", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDCHILDGRAPHNODE, &a.hdr, sizeof(a)); } return r; @@ -1569,9 +1632,49 @@ static hipError_t capture_hipGraphAddDependencies(hipGraph_t graph, const hipGra hrr_args_hipGraphAddDependencies a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.from = reinterpret_cast(from); + a.to = reinterpret_cast(to); a.numDependencies = static_cast(numDependencies); - if (from) a.from = reinterpret_cast(*from); - if (to) a.to = reinterpret_cast(*to); + if (from && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_from = false; + if (!warned_from) { + warned_from = true; + LogPrintfWarning( + "[HRR] hipGraphAddDependencies: recording only the first 16 " + "of %u from entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddDependencies", + "only the first 16 from entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.from_bytes, from, static_cast(_n) * sizeof(hipGraphNode_t)); + a.from_n = _n; + a.from_present = 1; + } + if (to && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_to = false; + if (!warned_to) { + warned_to = true; + LogPrintfWarning( + "[HRR] hipGraphAddDependencies: recording only the first 16 " + "of %u to entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddDependencies", + "only the first 16 to entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.to_bytes, to, static_cast(_n) * sizeof(hipGraphNode_t)); + a.to_n = _n; + a.to_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDDEPENDENCIES, &a.hdr, sizeof(a)); } return r; @@ -1584,9 +1687,29 @@ static hipError_t capture_hipGraphAddEmptyNode(hipGraphNode_t* pGraphNode, hipGr hrr_args_hipGraphAddEmptyNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddEmptyNode: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddEmptyNode", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDEMPTYNODE, &a.hdr, sizeof(a)); } return r; @@ -1599,10 +1722,30 @@ static hipError_t capture_hipGraphAddEventRecordNode(hipGraphNode_t* pGraphNode, hrr_args_hipGraphAddEventRecordNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.event = reinterpret_cast(event); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddEventRecordNode: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddEventRecordNode", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDEVENTRECORDNODE, &a.hdr, sizeof(a)); } return r; @@ -1615,10 +1758,30 @@ static hipError_t capture_hipGraphAddEventWaitNode(hipGraphNode_t* pGraphNode, h hrr_args_hipGraphAddEventWaitNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.event = reinterpret_cast(event); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddEventWaitNode: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddEventWaitNode", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDEVENTWAITNODE, &a.hdr, sizeof(a)); } return r; @@ -1631,30 +1794,17 @@ static hipError_t capture_hipGraphAddHostNode(hipGraphNode_t* pGraphNode, hipGra hrr_args_hipGraphAddHostNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.pNodeParams = reinterpret_cast(pNodeParams); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + hrr_cap::writer::note_unreplayable("hipGraphAddHostNode", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDHOSTNODE, &a.hdr, sizeof(a)); } return r; } -// Generated shim -static hipError_t capture_hipGraphAddKernelNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const hipKernelNodeParams* pNodeParams) { - hipError_t r = g_real_table.hipGraphAddKernelNode_fn(pGraphNode, graph, pDependencies, numDependencies, pNodeParams); - if (r == hipSuccess) { - hrr_args_hipGraphAddKernelNode a{}; - a.ret = static_cast(r); - a.graph = reinterpret_cast(graph); - a.numDependencies = static_cast(numDependencies); - a.pNodeParams = reinterpret_cast(pNodeParams); - if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDKERNELNODE, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipGraphAddMemAllocNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, hipMemAllocNodeParams* pNodeParams) { @@ -1663,10 +1813,34 @@ static hipError_t capture_hipGraphAddMemAllocNode(hipGraphNode_t* pGraphNode, hi hrr_args_hipGraphAddMemAllocNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.pNodeParams = reinterpret_cast(pNodeParams); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddMemAllocNode: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddMemAllocNode", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } + if (pNodeParams) { + std::memcpy(a.pNodeParams_bytes, pNodeParams, sizeof(hipMemAllocNodeParams)); + a.pNodeParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMALLOCNODE, &a.hdr, sizeof(a)); } return r; @@ -1679,10 +1853,30 @@ static hipError_t capture_hipGraphAddMemFreeNode(hipGraphNode_t* pGraphNode, hip hrr_args_hipGraphAddMemFreeNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.dev_ptr = reinterpret_cast(dev_ptr); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddMemFreeNode: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddMemFreeNode", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMFREENODE, &a.hdr, sizeof(a)); } return r; @@ -1695,10 +1889,34 @@ static hipError_t capture_hipGraphAddMemcpyNode(hipGraphNode_t* pGraphNode, hipG hrr_args_hipGraphAddMemcpyNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.pCopyParams = reinterpret_cast(pCopyParams); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddMemcpyNode: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddMemcpyNode", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } + if (pCopyParams) { + std::memcpy(a.pCopyParams_bytes, pCopyParams, sizeof(hipMemcpy3DParms)); + a.pCopyParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMCPYNODE, &a.hdr, sizeof(a)); } return r; @@ -1711,57 +1929,39 @@ static hipError_t capture_hipGraphAddMemcpyNode1D(hipGraphNode_t* pGraphNode, hi hrr_args_hipGraphAddMemcpyNode1D a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.dst = reinterpret_cast(dst); a.src = reinterpret_cast(src); a.count = static_cast(count); a.kind = static_cast(kind); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddMemcpyNode1D: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddMemcpyNode1D", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMCPYNODE1D, &a.hdr, sizeof(a)); } return r; } -// Generated shim -static hipError_t capture_hipGraphAddMemcpyNodeFromSymbol(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* symbol, size_t count, size_t offset, hipMemcpyKind kind) { - hipError_t r = g_real_table.hipGraphAddMemcpyNodeFromSymbol_fn(pGraphNode, graph, pDependencies, numDependencies, dst, symbol, count, offset, kind); - if (r == hipSuccess) { - hrr_args_hipGraphAddMemcpyNodeFromSymbol a{}; - a.ret = static_cast(r); - a.graph = reinterpret_cast(graph); - a.numDependencies = static_cast(numDependencies); - a.dst = reinterpret_cast(dst); - a.symbol = reinterpret_cast(symbol); - a.count = static_cast(count); - a.offset = static_cast(offset); - a.kind = static_cast(kind); - if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMCPYNODEFROMSYMBOL, &a.hdr, sizeof(a)); - } - return r; -} -// Generated shim -static hipError_t capture_hipGraphAddMemcpyNodeToSymbol(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind) { - hipError_t r = g_real_table.hipGraphAddMemcpyNodeToSymbol_fn(pGraphNode, graph, pDependencies, numDependencies, symbol, src, count, offset, kind); - if (r == hipSuccess) { - hrr_args_hipGraphAddMemcpyNodeToSymbol a{}; - a.ret = static_cast(r); - a.graph = reinterpret_cast(graph); - a.numDependencies = static_cast(numDependencies); - a.symbol = reinterpret_cast(symbol); - a.src = reinterpret_cast(src); - a.count = static_cast(count); - a.offset = static_cast(offset); - a.kind = static_cast(kind); - if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMCPYNODETOSYMBOL, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipGraphAddMemsetNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const hipMemsetParams* pMemsetParams) { @@ -1770,10 +1970,34 @@ static hipError_t capture_hipGraphAddMemsetNode(hipGraphNode_t* pGraphNode, hipG hrr_args_hipGraphAddMemsetNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.pMemsetParams = reinterpret_cast(pMemsetParams); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); + if (pDependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_pDependencies = false; + if (!warned_pDependencies) { + warned_pDependencies = true; + LogPrintfWarning( + "[HRR] hipGraphAddMemsetNode: recording only the first 16 " + "of %u pDependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipGraphAddMemsetNode", + "only the first 16 pDependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.pDependencies_bytes, pDependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.pDependencies_n = _n; + a.pDependencies_present = 1; + } + if (pMemsetParams) { + std::memcpy(a.pMemsetParams_bytes, pMemsetParams, sizeof(hipMemsetParams)); + a.pMemsetParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDMEMSETNODE, &a.hdr, sizeof(a)); } return r; @@ -1971,24 +2195,13 @@ static hipError_t capture_hipGraphExecHostNodeSetParams(hipGraphExec_t hGraphExe a.hGraphExec = reinterpret_cast(hGraphExec); a.node = reinterpret_cast(node); a.pNodeParams = reinterpret_cast(pNodeParams); + hrr_cap::writer::note_unreplayable("hipGraphExecHostNodeSetParams", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHEXECHOSTNODESETPARAMS, &a.hdr, sizeof(a)); } return r; } -// Generated shim -static hipError_t capture_hipGraphExecKernelNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t node, const hipKernelNodeParams* pNodeParams) { - hipError_t r = g_real_table.hipGraphExecKernelNodeSetParams_fn(hGraphExec, node, pNodeParams); - if (r == hipSuccess) { - hrr_args_hipGraphExecKernelNodeSetParams a{}; - a.ret = static_cast(r); - a.hGraphExec = reinterpret_cast(hGraphExec); - a.node = reinterpret_cast(node); - a.pNodeParams = reinterpret_cast(pNodeParams); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHEXECKERNELNODESETPARAMS, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipGraphExecMemcpyNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t node, hipMemcpy3DParms* pNodeParams) { @@ -1999,6 +2212,10 @@ static hipError_t capture_hipGraphExecMemcpyNodeSetParams(hipGraphExec_t hGraphE a.hGraphExec = reinterpret_cast(hGraphExec); a.node = reinterpret_cast(node); a.pNodeParams = reinterpret_cast(pNodeParams); + if (pNodeParams) { + std::memcpy(a.pNodeParams_bytes, pNodeParams, sizeof(hipMemcpy3DParms)); + a.pNodeParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHEXECMEMCPYNODESETPARAMS, &a.hdr, sizeof(a)); } return r; @@ -2039,23 +2256,6 @@ static hipError_t capture_hipGraphExecMemcpyNodeSetParamsFromSymbol(hipGraphExec return r; } -// Generated shim -static hipError_t capture_hipGraphExecMemcpyNodeSetParamsToSymbol(hipGraphExec_t hGraphExec, hipGraphNode_t node, const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind) { - hipError_t r = g_real_table.hipGraphExecMemcpyNodeSetParamsToSymbol_fn(hGraphExec, node, symbol, src, count, offset, kind); - if (r == hipSuccess) { - hrr_args_hipGraphExecMemcpyNodeSetParamsToSymbol a{}; - a.ret = static_cast(r); - a.hGraphExec = reinterpret_cast(hGraphExec); - a.node = reinterpret_cast(node); - a.symbol = reinterpret_cast(symbol); - a.src = reinterpret_cast(src); - a.count = static_cast(count); - a.offset = static_cast(offset); - a.kind = static_cast(kind); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHEXECMEMCPYNODESETPARAMSTOSYMBOL, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipGraphExecMemsetNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t node, const hipMemsetParams* pNodeParams) { @@ -2066,6 +2266,10 @@ static hipError_t capture_hipGraphExecMemsetNodeSetParams(hipGraphExec_t hGraphE a.hGraphExec = reinterpret_cast(hGraphExec); a.node = reinterpret_cast(node); a.pNodeParams = reinterpret_cast(pNodeParams); + if (pNodeParams) { + std::memcpy(a.pNodeParams_bytes, pNodeParams, sizeof(hipMemsetParams)); + a.pNodeParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHEXECMEMSETNODESETPARAMS, &a.hdr, sizeof(a)); } return r; @@ -2150,6 +2354,8 @@ static hipError_t capture_hipGraphHostNodeSetParams(hipGraphNode_t node, const h a.ret = static_cast(r); a.node = reinterpret_cast(node); a.pNodeParams = reinterpret_cast(pNodeParams); + hrr_cap::writer::note_unreplayable("hipGraphHostNodeSetParams", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHHOSTNODESETPARAMS, &a.hdr, sizeof(a)); } return r; @@ -2234,23 +2440,15 @@ static hipError_t capture_hipGraphKernelNodeSetAttribute(hipGraphNode_t hNode, h a.hNode = reinterpret_cast(hNode); a.attr = static_cast(attr); a.value = reinterpret_cast(value); + if (value) { + std::memcpy(a.value_bytes, value, sizeof(hipKernelNodeAttrValue)); + a.value_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHKERNELNODESETATTRIBUTE, &a.hdr, sizeof(a)); } return r; } -// Generated shim -static hipError_t capture_hipGraphKernelNodeSetParams(hipGraphNode_t node, const hipKernelNodeParams* pNodeParams) { - hipError_t r = g_real_table.hipGraphKernelNodeSetParams_fn(node, pNodeParams); - if (r == hipSuccess) { - hrr_args_hipGraphKernelNodeSetParams a{}; - a.ret = static_cast(r); - a.node = reinterpret_cast(node); - a.pNodeParams = reinterpret_cast(pNodeParams); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHKERNELNODESETPARAMS, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipGraphLaunch(hipGraphExec_t graphExec, hipStream_t stream) { @@ -2312,6 +2510,10 @@ static hipError_t capture_hipGraphMemcpyNodeSetParams(hipGraphNode_t node, const a.ret = static_cast(r); a.node = reinterpret_cast(node); a.pNodeParams = reinterpret_cast(pNodeParams); + if (pNodeParams) { + std::memcpy(a.pNodeParams_bytes, pNodeParams, sizeof(hipMemcpy3DParms)); + a.pNodeParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHMEMCPYNODESETPARAMS, &a.hdr, sizeof(a)); } return r; @@ -2350,22 +2552,6 @@ static hipError_t capture_hipGraphMemcpyNodeSetParamsFromSymbol(hipGraphNode_t n return r; } -// Generated shim -static hipError_t capture_hipGraphMemcpyNodeSetParamsToSymbol(hipGraphNode_t node, const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind) { - hipError_t r = g_real_table.hipGraphMemcpyNodeSetParamsToSymbol_fn(node, symbol, src, count, offset, kind); - if (r == hipSuccess) { - hrr_args_hipGraphMemcpyNodeSetParamsToSymbol a{}; - a.ret = static_cast(r); - a.node = reinterpret_cast(node); - a.symbol = reinterpret_cast(symbol); - a.src = reinterpret_cast(src); - a.count = static_cast(count); - a.offset = static_cast(offset); - a.kind = static_cast(kind); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHMEMCPYNODESETPARAMSTOSYMBOL, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipGraphMemsetNodeGetParams(hipGraphNode_t node, hipMemsetParams* pNodeParams) { @@ -2388,6 +2574,10 @@ static hipError_t capture_hipGraphMemsetNodeSetParams(hipGraphNode_t node, const a.ret = static_cast(r); a.node = reinterpret_cast(node); a.pNodeParams = reinterpret_cast(pNodeParams); + if (pNodeParams) { + std::memcpy(a.pNodeParams_bytes, pNodeParams, sizeof(hipMemsetParams)); + a.pNodeParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHMEMSETNODESETPARAMS, &a.hdr, sizeof(a)); } return r; @@ -2497,9 +2687,9 @@ static hipError_t capture_hipGraphRemoveDependencies(hipGraph_t graph, const hip hrr_args_hipGraphRemoveDependencies a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.from = reinterpret_cast(from); + a.to = reinterpret_cast(to); a.numDependencies = static_cast(numDependencies); - if (from) a.from = reinterpret_cast(*from); - if (to) a.to = reinterpret_cast(*to); hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHREMOVEDEPENDENCIES, &a.hdr, sizeof(a)); } return r; @@ -2756,7 +2946,7 @@ static hipError_t capture_hipIpcGetEventHandle(hipIpcEventHandle_t* handle, hipE if (r == hipSuccess) { hrr_args_hipIpcGetEventHandle a{}; a.ret = static_cast(r); - a.handle = 0; // non-castable type skipped + a.handle = reinterpret_cast(handle); a.event = reinterpret_cast(event); hrr_cap::writer::write_event_raw(HRR_API_HIPIPCGETEVENTHANDLE, &a.hdr, sizeof(a)); } @@ -2769,8 +2959,12 @@ static hipError_t capture_hipIpcGetMemHandle(hipIpcMemHandle_t* handle, void* de if (r == hipSuccess) { hrr_args_hipIpcGetMemHandle a{}; a.ret = static_cast(r); - a.handle = 0; // non-castable type skipped + a.handle = reinterpret_cast(handle); a.devPtr = reinterpret_cast(devPtr); + if (handle) { + std::memcpy(a.handle_bytes, handle, sizeof(hipIpcMemHandle_t)); + a.handle_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPIPCGETMEMHANDLE, &a.hdr, sizeof(a)); } return r; @@ -2782,7 +2976,7 @@ static hipError_t capture_hipIpcOpenEventHandle(hipEvent_t* event, hipIpcEventHa if (r == hipSuccess) { hrr_args_hipIpcOpenEventHandle a{}; a.ret = static_cast(r); - a.handle = 0; // non-castable type skipped + std::memcpy(a.handle_bytes, &handle, sizeof(handle)); if (event) a.event = reinterpret_cast(*event); hrr_cap::writer::write_event_raw(HRR_API_HIPIPCOPENEVENTHANDLE, &a.hdr, sizeof(a)); } @@ -2795,7 +2989,7 @@ static hipError_t capture_hipIpcOpenMemHandle(void** devPtr, hipIpcMemHandle_t h if (r == hipSuccess) { hrr_args_hipIpcOpenMemHandle a{}; a.ret = static_cast(r); - a.handle = 0; // non-castable type skipped + std::memcpy(a.handle_bytes, &handle, sizeof(handle)); a.flags = static_cast(flags); if (devPtr) a.devPtr = reinterpret_cast(*devPtr); hrr_cap::writer::write_event_raw(HRR_API_HIPIPCOPENMEMHANDLE, &a.hdr, sizeof(a)); @@ -2829,26 +3023,6 @@ static const char* capture_hipKernelNameRefByPtr(const void* hostFunction, hipSt } -// Generated shim -static hipError_t capture_hipLaunchCooperativeKernel(const void* f, dim3 gridDim, dim3 blockDimX, void** kernelParams, unsigned int sharedMemBytes, hipStream_t stream) { - hipError_t r = g_real_table.hipLaunchCooperativeKernel_fn(f, gridDim, blockDimX, kernelParams, sharedMemBytes, stream); - if (r == hipSuccess) { - hrr_args_hipLaunchCooperativeKernel a{}; - a.ret = static_cast(r); - a.f = reinterpret_cast(f); - a.gridDim_x = gridDim.x; - a.gridDim_y = gridDim.y; - a.gridDim_z = gridDim.z; - a.blockDimX_x = blockDimX.x; - a.blockDimX_y = blockDimX.y; - a.blockDimX_z = blockDimX.z; - a.sharedMemBytes = static_cast(sharedMemBytes); - a.stream = reinterpret_cast(stream); - if (kernelParams) a.kernelParams = reinterpret_cast(*kernelParams); - hrr_cap::writer::write_event_raw(HRR_API_HIPLAUNCHCOOPERATIVEKERNEL, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList, int numDevices, unsigned int flags) { @@ -2859,6 +3033,8 @@ static hipError_t capture_hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* a.launchParamsList = reinterpret_cast(launchParamsList); a.numDevices = static_cast(numDevices); a.flags = static_cast(flags); + hrr_cap::writer::note_unreplayable("hipLaunchCooperativeKernelMultiDevice", + "each hipLaunchParams entry names its kernel by a host function address in the capturing process, and a cooperative multi-device launch cannot be decomposed into per-device launches without breaking the grid-wide barrier it exists for"); hrr_cap::writer::write_event_raw(HRR_API_HIPLAUNCHCOOPERATIVEKERNELMULTIDEVICE, &a.hdr, sizeof(a)); } return r; @@ -2873,6 +3049,8 @@ static hipError_t capture_hipLaunchHostFunc(hipStream_t stream, hipHostFn_t fn, a.stream = reinterpret_cast(stream); a.fn = 0; // non-castable type skipped a.userData = reinterpret_cast(userData); + hrr_cap::writer::note_unreplayable("hipLaunchHostFunc", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); hrr_cap::writer::write_event_raw(HRR_API_HIPLAUNCHHOSTFUNC, &a.hdr, sizeof(a)); } return r; @@ -2898,8 +3076,8 @@ static hipError_t capture_hipMalloc3D(hipPitchedPtr* pitchedDevPtr, hipExtent ex if (r == hipSuccess) { hrr_args_hipMalloc3D a{}; a.ret = static_cast(r); - a.pitchedDevPtr = 0; // non-castable type skipped - a.extent = 0; // non-castable type skipped + a.pitchedDevPtr = reinterpret_cast(pitchedDevPtr); + std::memcpy(a.extent_bytes, &extent, sizeof(extent)); hrr_cap::writer::write_event_raw(HRR_API_HIPMALLOC3D, &a.hdr, sizeof(a)); } return r; @@ -2911,8 +3089,8 @@ static hipError_t capture_hipMalloc3DArray(hipArray_t* array, const struct hipCh if (r == hipSuccess) { hrr_args_hipMalloc3DArray a{}; a.ret = static_cast(r); - a.desc = 0; // non-castable type skipped - a.extent = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); + std::memcpy(a.extent_bytes, &extent, sizeof(extent)); a.flags = static_cast(flags); if (array) a.array = reinterpret_cast(*array); hrr_cap::writer::write_event_raw(HRR_API_HIPMALLOC3DARRAY, &a.hdr, sizeof(a)); @@ -2926,7 +3104,7 @@ static hipError_t capture_hipMallocArray(hipArray_t* array, const hipChannelForm if (r == hipSuccess) { hrr_args_hipMallocArray a{}; a.ret = static_cast(r); - a.desc = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); a.width = static_cast(width); a.height = static_cast(height); a.flags = static_cast(flags); @@ -2998,8 +3176,8 @@ static hipError_t capture_hipMallocMipmappedArray(hipMipmappedArray_t* mipmapped if (r == hipSuccess) { hrr_args_hipMallocMipmappedArray a{}; a.ret = static_cast(r); - a.desc = 0; // non-castable type skipped - a.extent = 0; // non-castable type skipped + a.desc = reinterpret_cast(desc); + std::memcpy(a.extent_bytes, &extent, sizeof(extent)); a.numLevels = static_cast(numLevels); a.flags = static_cast(flags); if (mipmappedArray) a.mipmappedArray = reinterpret_cast(*mipmappedArray); @@ -3106,6 +3284,10 @@ static hipError_t capture_hipMemCreate(hipMemGenericAllocationHandle_t* handle, a.prop = reinterpret_cast(prop); a.flags = static_cast(flags); if (handle) a.handle = reinterpret_cast(*handle); + if (prop) { + std::memcpy(a.prop_bytes, prop, sizeof(hipMemAllocationProp)); + a.prop_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPMEMCREATE, &a.hdr, sizeof(a)); } return r; @@ -3133,8 +3315,16 @@ static hipError_t capture_hipMemGetAccess(unsigned long long* flags, const hipMe hrr_args_hipMemGetAccess a{}; a.ret = static_cast(r); a.flags = reinterpret_cast(flags); - a.location = 0; // non-castable type skipped + a.location = reinterpret_cast(location); a.ptr = reinterpret_cast(ptr); + if (location) { + std::memcpy(a.location_bytes, location, sizeof(hipMemLocation)); + a.location_present = 1; + } + if (flags) { + std::memcpy(a.flags_bytes, flags, sizeof(unsigned long long)); + a.flags_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPMEMGETACCESS, &a.hdr, sizeof(a)); } return r; @@ -3190,6 +3380,8 @@ static hipError_t capture_hipMemImportFromShareableHandle(hipMemGenericAllocatio a.osHandle = reinterpret_cast(osHandle); a.shHandleType = static_cast(shHandleType); if (handle) a.handle = reinterpret_cast(*handle); + hrr_cap::writer::note_unreplayable("hipMemImportFromShareableHandle", + "the recorded argument is an OS handle (a POSIX fd or a Win32 HANDLE) belonging to the process that exported it, and the same number in the replaying process names a different object or nothing at all"); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMIMPORTFROMSHAREABLEHANDLE, &a.hdr, sizeof(a)); } return r; @@ -3274,7 +3466,7 @@ static hipError_t capture_hipMemPoolGetAccess(hipMemAccessFlags* flags, hipMemPo a.ret = static_cast(r); a.flags = reinterpret_cast(flags); a.mem_pool = reinterpret_cast(mem_pool); - a.location = 0; // non-castable type skipped + a.location = reinterpret_cast(location); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMPOOLGETACCESS, &a.hdr, sizeof(a)); } return r; @@ -3304,6 +3496,8 @@ static hipError_t capture_hipMemPoolImportFromShareableHandle(hipMemPool_t* mem_ a.handle_type = static_cast(handle_type); a.flags = static_cast(flags); if (mem_pool) a.mem_pool = reinterpret_cast(*mem_pool); + hrr_cap::writer::note_unreplayable("hipMemPoolImportFromShareableHandle", + "the recorded argument is an OS handle (a POSIX fd or a Win32 HANDLE) belonging to the process that exported it, and the same number in the replaying process names a different object or nothing at all"); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMPOOLIMPORTFROMSHAREABLEHANDLE, &a.hdr, sizeof(a)); } return r; @@ -3635,30 +3829,7 @@ static hipError_t capture_hipMemcpyHtoA(hipArray_t dstArray, size_t dstOffset, c -// Generated shim -static hipError_t capture_hipMemcpyParam2D(const hip_Memcpy2D* pCopy) { - hipError_t r = g_real_table.hipMemcpyParam2D_fn(pCopy); - if (r == hipSuccess) { - hrr_args_hipMemcpyParam2D a{}; - a.ret = static_cast(r); - a.pCopy = reinterpret_cast(pCopy); - hrr_cap::writer::write_event_raw(HRR_API_HIPMEMCPYPARAM2D, &a.hdr, sizeof(a)); - } - return r; -} -// Generated shim -static hipError_t capture_hipMemcpyParam2DAsync(const hip_Memcpy2D* pCopy, hipStream_t stream) { - hipError_t r = g_real_table.hipMemcpyParam2DAsync_fn(pCopy, stream); - if (r == hipSuccess) { - hrr_args_hipMemcpyParam2DAsync a{}; - a.ret = static_cast(r); - a.pCopy = reinterpret_cast(pCopy); - a.stream = reinterpret_cast(stream); - hrr_cap::writer::write_event_raw(HRR_API_HIPMEMCPYPARAM2DASYNC, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipMemcpyPeer(void* dst, int dstDeviceId, const void* src, int srcDeviceId, size_t sizeBytes) { @@ -3797,9 +3968,9 @@ static hipError_t capture_hipMemset3D(hipPitchedPtr pitchedDevPtr, int value, hi if (r == hipSuccess) { hrr_args_hipMemset3D a{}; a.ret = static_cast(r); - a.pitchedDevPtr = 0; // non-castable type skipped + std::memcpy(a.pitchedDevPtr_bytes, &pitchedDevPtr, sizeof(pitchedDevPtr)); a.value = static_cast(value); - a.extent = 0; // non-castable type skipped + std::memcpy(a.extent_bytes, &extent, sizeof(extent)); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMSET3D, &a.hdr, sizeof(a)); } return r; @@ -3811,9 +3982,9 @@ static hipError_t capture_hipMemset3DAsync(hipPitchedPtr pitchedDevPtr, int valu if (r == hipSuccess) { hrr_args_hipMemset3DAsync a{}; a.ret = static_cast(r); - a.pitchedDevPtr = 0; // non-castable type skipped + std::memcpy(a.pitchedDevPtr_bytes, &pitchedDevPtr, sizeof(pitchedDevPtr)); a.value = static_cast(value); - a.extent = 0; // non-castable type skipped + std::memcpy(a.extent_bytes, &extent, sizeof(extent)); a.stream = reinterpret_cast(stream); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMSET3DASYNC, &a.hdr, sizeof(a)); } @@ -3997,26 +4168,6 @@ static hipError_t capture_hipModuleGetTexRef(textureReference** texRef, hipModul return r; } -// Generated shim -static hipError_t capture_hipModuleLaunchCooperativeKernel(hipFunction_t f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, hipStream_t stream, void** kernelParams) { - hipError_t r = g_real_table.hipModuleLaunchCooperativeKernel_fn(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, stream, kernelParams); - if (r == hipSuccess) { - hrr_args_hipModuleLaunchCooperativeKernel a{}; - a.ret = static_cast(r); - a.f = reinterpret_cast(f); - a.gridDimX = static_cast(gridDimX); - a.gridDimY = static_cast(gridDimY); - a.gridDimZ = static_cast(gridDimZ); - a.blockDimX = static_cast(blockDimX); - a.blockDimY = static_cast(blockDimY); - a.blockDimZ = static_cast(blockDimZ); - a.sharedMemBytes = static_cast(sharedMemBytes); - a.stream = reinterpret_cast(stream); - if (kernelParams) a.kernelParams = reinterpret_cast(*kernelParams); - hrr_cap::writer::write_event_raw(HRR_API_HIPMODULELAUNCHCOOPERATIVEKERNEL, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipModuleLaunchCooperativeKernelMultiDevice(hipFunctionLaunchParams* launchParamsList, unsigned int numDevices, unsigned int flags) { @@ -4100,17 +4251,6 @@ static hipError_t capture_hipModuleOccupancyMaxPotentialBlockSizeWithFlags(int* return r; } -// Generated shim -static hipError_t capture_hipModuleUnload(hipModule_t module) { - hipError_t r = g_real_table.hipModuleUnload_fn(module); - if (r == hipSuccess) { - hrr_args_hipModuleUnload a{}; - a.ret = static_cast(r); - a.module = reinterpret_cast(module); - hrr_cap::writer::write_event_raw(HRR_API_HIPMODULEUNLOAD, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* f, int blockSize, size_t dynSharedMemPerBlk) { @@ -4278,6 +4418,26 @@ static hipError_t capture_hipSetupArgument(const void* arg, size_t size, size_t a.arg = reinterpret_cast(arg); a.size = static_cast(size); a.offset = static_cast(offset); + if (arg && size > 0) { + uint32_t _n = static_cast(size); + if (_n > 256u) { + static bool warned_arg = false; + if (!warned_arg) { + warned_arg = true; + LogPrintfWarning( + "[HRR] hipSetupArgument: recording only the first 256 " + "of %u arg entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipSetupArgument", + "only the first 256 arg entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 256u; + } + std::memcpy(a.arg_bytes, arg, static_cast(_n) * sizeof(unsigned char)); + a.arg_n = _n; + a.arg_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPSETUPARGUMENT, &a.hdr, sizeof(a)); } return r; @@ -4308,6 +4468,8 @@ static hipError_t capture_hipStreamAddCallback(hipStream_t stream, hipStreamCall a.callback = 0; // non-castable type skipped a.userData = reinterpret_cast(userData); a.flags = static_cast(flags); + hrr_cap::writer::note_unreplayable("hipStreamAddCallback", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); hrr_cap::writer::write_event_raw(HRR_API_HIPSTREAMADDCALLBACK, &a.hdr, sizeof(a)); } return r; @@ -5016,6 +5178,8 @@ static hipError_t capture_hipUserObjectCreate(hipUserObject_t* object_out, void* a.initialRefcount = static_cast(initialRefcount); a.flags = static_cast(flags); if (object_out) a.object_out = reinterpret_cast(*object_out); + hrr_cap::writer::note_unreplayable("hipUserObjectCreate", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); hrr_cap::writer::write_event_raw(HRR_API_HIPUSEROBJECTCREATE, &a.hdr, sizeof(a)); } return r; @@ -5245,9 +5409,9 @@ static hipError_t capture_hipMemset3DAsync_spt(hipPitchedPtr pitchedDevPtr, int if (r == hipSuccess) { hrr_args_hipMemset3DAsync_spt a{}; a.ret = static_cast(r); - a.pitchedDevPtr = 0; // non-castable type skipped + std::memcpy(a.pitchedDevPtr_bytes, &pitchedDevPtr, sizeof(pitchedDevPtr)); a.value = static_cast(value); - a.extent = 0; // non-castable type skipped + std::memcpy(a.extent_bytes, &extent, sizeof(extent)); a.stream = reinterpret_cast(stream); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMSET3DASYNC_SPT, &a.hdr, sizeof(a)); } @@ -5260,9 +5424,9 @@ static hipError_t capture_hipMemset3D_spt(hipPitchedPtr pitchedDevPtr, int value if (r == hipSuccess) { hrr_args_hipMemset3D_spt a{}; a.ret = static_cast(r); - a.pitchedDevPtr = 0; // non-castable type skipped + std::memcpy(a.pitchedDevPtr_bytes, &pitchedDevPtr, sizeof(pitchedDevPtr)); a.value = static_cast(value); - a.extent = 0; // non-castable type skipped + std::memcpy(a.extent_bytes, &extent, sizeof(extent)); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMSET3D_SPT, &a.hdr, sizeof(a)); } return r; @@ -5488,6 +5652,8 @@ static hipError_t capture_hipStreamAddCallback_spt(hipStream_t stream, hipStream a.callback = 0; // non-castable type skipped a.userData = reinterpret_cast(userData); a.flags = static_cast(flags); + hrr_cap::writer::note_unreplayable("hipStreamAddCallback_spt", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); hrr_cap::writer::write_event_raw(HRR_API_HIPSTREAMADDCALLBACK_SPT, &a.hdr, sizeof(a)); } return r; @@ -5506,47 +5672,7 @@ static hipError_t capture_hipEventRecord_spt(hipEvent_t event, hipStream_t strea return r; } -// Generated shim -static hipError_t capture_hipLaunchCooperativeKernel_spt(const void* f, dim3 gridDim, dim3 blockDim, void** kernelParams, uint32_t sharedMemBytes, hipStream_t hStream) { - hipError_t r = g_real_table.hipLaunchCooperativeKernel_spt_fn(f, gridDim, blockDim, kernelParams, sharedMemBytes, hStream); - if (r == hipSuccess) { - hrr_args_hipLaunchCooperativeKernel_spt a{}; - a.ret = static_cast(r); - a.f = reinterpret_cast(f); - a.gridDim_x = gridDim.x; - a.gridDim_y = gridDim.y; - a.gridDim_z = gridDim.z; - a.blockDim_x = blockDim.x; - a.blockDim_y = blockDim.y; - a.blockDim_z = blockDim.z; - a.sharedMemBytes = static_cast(sharedMemBytes); - a.hStream = reinterpret_cast(hStream); - if (kernelParams) a.kernelParams = reinterpret_cast(*kernelParams); - hrr_cap::writer::write_event_raw(HRR_API_HIPLAUNCHCOOPERATIVEKERNEL_SPT, &a.hdr, sizeof(a)); - } - return r; -} -// Generated shim -static hipError_t capture_hipLaunchKernel_spt(const void* function_address, dim3 numBlocks, dim3 dimBlocks, void** args, size_t sharedMemBytes, hipStream_t stream) { - hipError_t r = g_real_table.hipLaunchKernel_spt_fn(function_address, numBlocks, dimBlocks, args, sharedMemBytes, stream); - if (r == hipSuccess) { - hrr_args_hipLaunchKernel_spt a{}; - a.ret = static_cast(r); - a.function_address = reinterpret_cast(function_address); - a.numBlocks_x = numBlocks.x; - a.numBlocks_y = numBlocks.y; - a.numBlocks_z = numBlocks.z; - a.dimBlocks_x = dimBlocks.x; - a.dimBlocks_y = dimBlocks.y; - a.dimBlocks_z = dimBlocks.z; - a.sharedMemBytes = static_cast(sharedMemBytes); - a.stream = reinterpret_cast(stream); - if (args) a.args = reinterpret_cast(*args); - hrr_cap::writer::write_event_raw(HRR_API_HIPLAUNCHKERNEL_SPT, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipGraphLaunch_spt(hipGraphExec_t graphExec, hipStream_t stream) { @@ -5640,6 +5766,8 @@ static hipError_t capture_hipLaunchHostFunc_spt(hipStream_t stream, hipHostFn_t a.stream = reinterpret_cast(stream); a.fn = 0; // non-castable type skipped a.userData = reinterpret_cast(userData); + hrr_cap::writer::note_unreplayable("hipLaunchHostFunc_spt", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); hrr_cap::writer::write_event_raw(HRR_API_HIPLAUNCHHOSTFUNC_SPT, &a.hdr, sizeof(a)); } return r; @@ -5657,11 +5785,35 @@ static hipError_t capture_hipDrvGraphAddMemsetNode(hipGraphNode_t* phGraphNode, hrr_args_hipDrvGraphAddMemsetNode a{}; a.ret = static_cast(r); a.hGraph = reinterpret_cast(hGraph); + a.dependencies = reinterpret_cast(dependencies); a.numDependencies = static_cast(numDependencies); a.memsetParams = reinterpret_cast(memsetParams); a.ctx = reinterpret_cast(ctx); if (phGraphNode) a.phGraphNode = reinterpret_cast(*phGraphNode); - if (dependencies) a.dependencies = reinterpret_cast(*dependencies); + if (dependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_dependencies = false; + if (!warned_dependencies) { + warned_dependencies = true; + LogPrintfWarning( + "[HRR] hipDrvGraphAddMemsetNode: recording only the first 16 " + "of %u dependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipDrvGraphAddMemsetNode", + "only the first 16 dependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.dependencies_bytes, dependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.dependencies_n = _n; + a.dependencies_present = 1; + } + if (memsetParams) { + std::memcpy(a.memsetParams_bytes, memsetParams, sizeof(hipMemsetParams)); + a.memsetParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPDRVGRAPHADDMEMSETNODE, &a.hdr, sizeof(a)); } return r; @@ -5674,10 +5826,10 @@ static hipError_t capture_hipGraphAddExternalSemaphoresWaitNode(hipGraphNode_t* hrr_args_hipGraphAddExternalSemaphoresWaitNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.nodeParams = reinterpret_cast(nodeParams); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDEXTERNALSEMAPHORESWAITNODE, &a.hdr, sizeof(a)); } return r; @@ -5690,10 +5842,10 @@ static hipError_t capture_hipGraphAddExternalSemaphoresSignalNode(hipGraphNode_t hrr_args_hipGraphAddExternalSemaphoresSignalNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.nodeParams = reinterpret_cast(nodeParams); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDEXTERNALSEMAPHORESSIGNALNODE, &a.hdr, sizeof(a)); } return r; @@ -5786,10 +5938,10 @@ static hipError_t capture_hipGraphAddNode(hipGraphNode_t* pGraphNode, hipGraph_t hrr_args_hipGraphAddNode a{}; a.ret = static_cast(r); a.graph = reinterpret_cast(graph); + a.pDependencies = reinterpret_cast(pDependencies); a.numDependencies = static_cast(numDependencies); a.nodeParams = reinterpret_cast(nodeParams); if (pGraphNode) a.pGraphNode = reinterpret_cast(*pGraphNode); - if (pDependencies) a.pDependencies = reinterpret_cast(*pDependencies); hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDNODE, &a.hdr, sizeof(a)); } return r; @@ -5857,6 +6009,25 @@ static hipError_t capture_hipGetProcAddress(const char* symbol, void** pfn, int a.flags = static_cast(flags); a.symbolStatus = reinterpret_cast(symbolStatus); if (pfn) a.pfn = reinterpret_cast(*pfn); + if (symbol) { + size_t _n = std::strlen(symbol); + if (_n > 255u) { + static bool warned_symbol = false; + if (!warned_symbol) { + warned_symbol = true; + LogPrintfWarning( + "[HRR] hipGetProcAddress: symbol is %zu characters; " + "recording the first 255 only", _n); + hrr_cap::writer::note_unreplayable("hipGetProcAddress", + "symbol was truncated to 255 characters at capture " + "time; replay would pass a shortened string"); + } + _n = 255u; + } + std::memcpy(a.symbol_bytes, symbol, _n); + a.symbol_bytes[_n] = 0; + a.symbol_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGETPROCADDRESS, &a.hdr, sizeof(a)); } return r; @@ -5870,10 +6041,50 @@ static hipError_t capture_hipStreamBeginCaptureToGraph(hipStream_t stream, hipGr a.ret = static_cast(r); a.stream = reinterpret_cast(stream); a.graph = reinterpret_cast(graph); + a.dependencies = reinterpret_cast(dependencies); a.dependencyData = reinterpret_cast(dependencyData); a.numDependencies = static_cast(numDependencies); a.mode = static_cast(mode); - if (dependencies) a.dependencies = reinterpret_cast(*dependencies); + if (dependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_dependencies = false; + if (!warned_dependencies) { + warned_dependencies = true; + LogPrintfWarning( + "[HRR] hipStreamBeginCaptureToGraph: recording only the first 16 " + "of %u dependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipStreamBeginCaptureToGraph", + "only the first 16 dependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.dependencies_bytes, dependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.dependencies_n = _n; + a.dependencies_present = 1; + } + if (dependencyData && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_dependencyData = false; + if (!warned_dependencyData) { + warned_dependencyData = true; + LogPrintfWarning( + "[HRR] hipStreamBeginCaptureToGraph: recording only the first 16 " + "of %u dependencyData entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipStreamBeginCaptureToGraph", + "only the first 16 dependencyData entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.dependencyData_bytes, dependencyData, static_cast(_n) * sizeof(hipGraphEdgeData)); + a.dependencyData_n = _n; + a.dependencyData_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPSTREAMBEGINCAPTURETOGRAPH, &a.hdr, sizeof(a)); } return r; @@ -6010,10 +6221,30 @@ static hipError_t capture_hipDrvGraphAddMemFreeNode(hipGraphNode_t* phGraphNode, hrr_args_hipDrvGraphAddMemFreeNode a{}; a.ret = static_cast(r); a.hGraph = reinterpret_cast(hGraph); + a.dependencies = reinterpret_cast(dependencies); a.numDependencies = static_cast(numDependencies); a.dptr = static_cast(reinterpret_cast(dptr)); if (phGraphNode) a.phGraphNode = reinterpret_cast(*phGraphNode); - if (dependencies) a.dependencies = reinterpret_cast(*dependencies); + if (dependencies && numDependencies > 0) { + uint32_t _n = static_cast(numDependencies); + if (_n > 16u) { + static bool warned_dependencies = false; + if (!warned_dependencies) { + warned_dependencies = true; + LogPrintfWarning( + "[HRR] hipDrvGraphAddMemFreeNode: recording only the first 16 " + "of %u dependencies entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipDrvGraphAddMemFreeNode", + "only the first 16 dependencies entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.dependencies_bytes, dependencies, static_cast(_n) * sizeof(hipGraphNode_t)); + a.dependencies_n = _n; + a.dependencies_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPDRVGRAPHADDMEMFREENODE, &a.hdr, sizeof(a)); } return r; @@ -6029,6 +6260,10 @@ static hipError_t capture_hipDrvGraphExecMemcpyNodeSetParams(hipGraphExec_t hGra a.hNode = reinterpret_cast(hNode); a.copyParams = reinterpret_cast(copyParams); a.ctx = reinterpret_cast(ctx); + if (copyParams) { + std::memcpy(a.copyParams_bytes, copyParams, sizeof(HIP_MEMCPY3D)); + a.copyParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPDRVGRAPHEXECMEMCPYNODESETPARAMS, &a.hdr, sizeof(a)); } return r; @@ -6044,6 +6279,10 @@ static hipError_t capture_hipDrvGraphExecMemsetNodeSetParams(hipGraphExec_t hGra a.hNode = reinterpret_cast(hNode); a.memsetParams = reinterpret_cast(memsetParams); a.ctx = reinterpret_cast(ctx); + if (memsetParams) { + std::memcpy(a.memsetParams_bytes, memsetParams, sizeof(hipMemsetParams)); + a.memsetParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPDRVGRAPHEXECMEMSETNODESETPARAMS, &a.hdr, sizeof(a)); } return r; @@ -6124,6 +6363,10 @@ static hipError_t capture_hipDrvGraphMemcpyNodeSetParams(hipGraphNode_t hNode, c a.ret = static_cast(r); a.hNode = reinterpret_cast(hNode); a.nodeParams = reinterpret_cast(nodeParams); + if (nodeParams) { + std::memcpy(a.nodeParams_bytes, nodeParams, sizeof(HIP_MEMCPY3D)); + a.nodeParams_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPDRVGRAPHMEMCPYNODESETPARAMS, &a.hdr, sizeof(a)); } return r; @@ -6150,8 +6393,16 @@ static hipError_t capture_hipDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthIn hrr_args_hipDeviceGetTexture1DLinearMaxWidth a{}; a.ret = static_cast(r); a.maxWidthInElements = reinterpret_cast(maxWidthInElements); - a.fmtDesc = 0; // non-castable type skipped + a.fmtDesc = reinterpret_cast(fmtDesc); a.device = static_cast(device); + if (fmtDesc) { + std::memcpy(a.fmtDesc_bytes, fmtDesc, sizeof(hipChannelFormatDesc)); + a.fmtDesc_present = 1; + } + if (maxWidthInElements) { + std::memcpy(a.maxWidthInElements_bytes, maxWidthInElements, sizeof(size_t)); + a.maxWidthInElements_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPDEVICEGETTEXTURE1DLINEARMAXWIDTH, &a.hdr, sizeof(a)); } return r; @@ -6167,26 +6418,31 @@ static hipError_t capture_hipStreamBatchMemOp(hipStream_t stream, unsigned int c a.count = static_cast(count); a.paramArray = reinterpret_cast(paramArray); a.flags = static_cast(flags); + if (paramArray && count > 0) { + uint32_t _n = static_cast(count); + if (_n > 16u) { + static bool warned_paramArray = false; + if (!warned_paramArray) { + warned_paramArray = true; + LogPrintfWarning( + "[HRR] hipStreamBatchMemOp: recording only the first 16 " + "of %u paramArray entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipStreamBatchMemOp", + "only the first 16 paramArray entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.paramArray_bytes, paramArray, static_cast(_n) * sizeof(hipStreamBatchMemOpParams)); + a.paramArray_n = _n; + a.paramArray_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPSTREAMBATCHMEMOP, &a.hdr, sizeof(a)); } return r; } -// Generated shim -static hipError_t capture_hipGraphAddBatchMemOpNode(hipGraphNode_t* phGraphNode, hipGraph_t hGraph, const hipGraphNode_t* dependencies, size_t numDependencies, const hipBatchMemOpNodeParams* nodeParams) { - hipError_t r = g_real_table.hipGraphAddBatchMemOpNode_fn(phGraphNode, hGraph, dependencies, numDependencies, nodeParams); - if (r == hipSuccess) { - hrr_args_hipGraphAddBatchMemOpNode a{}; - a.ret = static_cast(r); - a.hGraph = reinterpret_cast(hGraph); - a.numDependencies = static_cast(numDependencies); - a.nodeParams = reinterpret_cast(nodeParams); - if (phGraphNode) a.phGraphNode = reinterpret_cast(*phGraphNode); - if (dependencies) a.dependencies = reinterpret_cast(*dependencies); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHADDBATCHMEMOPNODE, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipGraphBatchMemOpNodeGetParams(hipGraphNode_t hNode, hipBatchMemOpNodeParams* nodeParams_out) { @@ -6201,51 +6457,8 @@ static hipError_t capture_hipGraphBatchMemOpNodeGetParams(hipGraphNode_t hNode, return r; } -// Generated shim -static hipError_t capture_hipGraphBatchMemOpNodeSetParams(hipGraphNode_t hNode, hipBatchMemOpNodeParams* nodeParams) { - hipError_t r = g_real_table.hipGraphBatchMemOpNodeSetParams_fn(hNode, nodeParams); - if (r == hipSuccess) { - hrr_args_hipGraphBatchMemOpNodeSetParams a{}; - a.ret = static_cast(r); - a.hNode = reinterpret_cast(hNode); - a.nodeParams = reinterpret_cast(nodeParams); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHBATCHMEMOPNODESETPARAMS, &a.hdr, sizeof(a)); - } - return r; -} -// Generated shim -static hipError_t capture_hipGraphExecBatchMemOpNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t hNode, const hipBatchMemOpNodeParams* nodeParams) { - hipError_t r = g_real_table.hipGraphExecBatchMemOpNodeSetParams_fn(hGraphExec, hNode, nodeParams); - if (r == hipSuccess) { - hrr_args_hipGraphExecBatchMemOpNodeSetParams a{}; - a.ret = static_cast(r); - a.hGraphExec = reinterpret_cast(hGraphExec); - a.hNode = reinterpret_cast(hNode); - a.nodeParams = reinterpret_cast(nodeParams); - hrr_cap::writer::write_event_raw(HRR_API_HIPGRAPHEXECBATCHMEMOPNODESETPARAMS, &a.hdr, sizeof(a)); - } - return r; -} -// Generated shim -static hipError_t capture_hipLinkAddData(hipLinkState_t state, hipJitInputType type, void* data, size_t size, const char* name, unsigned int numOptions, hipJitOption* options, void** optionValues) { - hipError_t r = g_real_table.hipLinkAddData_fn(state, type, data, size, name, numOptions, options, optionValues); - if (r == hipSuccess) { - hrr_args_hipLinkAddData a{}; - a.ret = static_cast(r); - a.state = 0; // non-castable type skipped - a.type = static_cast(type); - a.data = reinterpret_cast(data); - a.size = static_cast(size); - a.name = reinterpret_cast(name); - a.numOptions = static_cast(numOptions); - a.options = reinterpret_cast(options); - if (optionValues) a.optionValues = reinterpret_cast(*optionValues); - hrr_cap::writer::write_event_raw(HRR_API_HIPLINKADDDATA, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipLinkAddFile(hipLinkState_t state, hipJitInputType type, const char* path, unsigned int numOptions, hipJitOption* options, void** optionValues) { @@ -6253,12 +6466,71 @@ static hipError_t capture_hipLinkAddFile(hipLinkState_t state, hipJitInputType t if (r == hipSuccess) { hrr_args_hipLinkAddFile a{}; a.ret = static_cast(r); - a.state = 0; // non-castable type skipped + a.state = reinterpret_cast(state); a.type = static_cast(type); a.path = reinterpret_cast(path); a.numOptions = static_cast(numOptions); a.options = reinterpret_cast(options); if (optionValues) a.optionValues = reinterpret_cast(*optionValues); + if (path) { + size_t _n = std::strlen(path); + if (_n > 255u) { + static bool warned_path = false; + if (!warned_path) { + warned_path = true; + LogPrintfWarning( + "[HRR] hipLinkAddFile: path is %zu characters; " + "recording the first 255 only", _n); + hrr_cap::writer::note_unreplayable("hipLinkAddFile", + "path was truncated to 255 characters at capture " + "time; replay would pass a shortened string"); + } + _n = 255u; + } + std::memcpy(a.path_bytes, path, _n); + a.path_bytes[_n] = 0; + a.path_present = 1; + } + if (options && numOptions > 0) { + uint32_t _n = static_cast(numOptions); + if (_n > 32u) { + static bool warned_options = false; + if (!warned_options) { + warned_options = true; + LogPrintfWarning( + "[HRR] hipLinkAddFile: recording only the first 32 " + "of %u options entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipLinkAddFile", + "only the first 32 options entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 32u; + } + std::memcpy(a.options_bytes, options, static_cast(_n) * sizeof(hipJitOption)); + a.options_n = _n; + a.options_present = 1; + } + if (optionValues && numOptions > 0) { + uint32_t _n = static_cast(numOptions); + if (_n > 32u) { + static bool warned_optionValues = false; + if (!warned_optionValues) { + warned_optionValues = true; + LogPrintfWarning( + "[HRR] hipLinkAddFile: recording only the first 32 " + "of %u optionValues entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipLinkAddFile", + "only the first 32 optionValues entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 32u; + } + std::memcpy(a.optionValues_bytes, optionValues, static_cast(_n) * sizeof(void*)); + a.optionValues_n = _n; + a.optionValues_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPLINKADDFILE, &a.hdr, sizeof(a)); } return r; @@ -6270,7 +6542,7 @@ static hipError_t capture_hipLinkComplete(hipLinkState_t state, void** hipBinOut if (r == hipSuccess) { hrr_args_hipLinkComplete a{}; a.ret = static_cast(r); - a.state = 0; // non-castable type skipped + a.state = reinterpret_cast(state); a.sizeOut = reinterpret_cast(sizeOut); if (hipBinOut) a.hipBinOut = reinterpret_cast(*hipBinOut); hrr_cap::writer::write_event_raw(HRR_API_HIPLINKCOMPLETE, &a.hdr, sizeof(a)); @@ -6286,8 +6558,48 @@ static hipError_t capture_hipLinkCreate(unsigned int numOptions, hipJitOption* o a.ret = static_cast(r); a.numOptions = static_cast(numOptions); a.options = reinterpret_cast(options); - a.stateOut = 0; // non-castable type skipped if (optionValues) a.optionValues = reinterpret_cast(*optionValues); + if (stateOut) a.stateOut = reinterpret_cast(*stateOut); + if (options && numOptions > 0) { + uint32_t _n = static_cast(numOptions); + if (_n > 32u) { + static bool warned_options = false; + if (!warned_options) { + warned_options = true; + LogPrintfWarning( + "[HRR] hipLinkCreate: recording only the first 32 " + "of %u options entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipLinkCreate", + "only the first 32 options entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 32u; + } + std::memcpy(a.options_bytes, options, static_cast(_n) * sizeof(hipJitOption)); + a.options_n = _n; + a.options_present = 1; + } + if (optionValues && numOptions > 0) { + uint32_t _n = static_cast(numOptions); + if (_n > 32u) { + static bool warned_optionValues = false; + if (!warned_optionValues) { + warned_optionValues = true; + LogPrintfWarning( + "[HRR] hipLinkCreate: recording only the first 32 " + "of %u optionValues entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipLinkCreate", + "only the first 32 optionValues entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 32u; + } + std::memcpy(a.optionValues_bytes, optionValues, static_cast(_n) * sizeof(void*)); + a.optionValues_n = _n; + a.optionValues_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPLINKCREATE, &a.hdr, sizeof(a)); } return r; @@ -6299,7 +6611,7 @@ static hipError_t capture_hipLinkDestroy(hipLinkState_t state) { if (r == hipSuccess) { hrr_args_hipLinkDestroy a{}; a.ret = static_cast(r); - a.state = 0; // non-castable type skipped + a.state = reinterpret_cast(state); hrr_cap::writer::write_event_raw(HRR_API_HIPLINKDESTROY, &a.hdr, sizeof(a)); } return r; @@ -6319,34 +6631,7 @@ static hipError_t capture_hipEventRecordWithFlags(hipEvent_t event, hipStream_t return r; } -// Generated shim -static hipError_t capture_hipLaunchKernelExC(const hipLaunchConfig_t* config, const void* fPtr, void** args) { - hipError_t r = g_real_table.hipLaunchKernelExC_fn(config, fPtr, args); - if (r == hipSuccess) { - hrr_args_hipLaunchKernelExC a{}; - a.ret = static_cast(r); - a.config = reinterpret_cast(config); - a.fPtr = reinterpret_cast(fPtr); - if (args) a.args = reinterpret_cast(*args); - hrr_cap::writer::write_event_raw(HRR_API_HIPLAUNCHKERNELEXC, &a.hdr, sizeof(a)); - } - return r; -} -// Generated shim -static hipError_t capture_hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipFunction_t f, void** params, void** extra) { - hipError_t r = g_real_table.hipDrvLaunchKernelEx_fn(config, f, params, extra); - if (r == hipSuccess) { - hrr_args_hipDrvLaunchKernelEx a{}; - a.ret = static_cast(r); - a.config = reinterpret_cast(config); - a.f = reinterpret_cast(f); - if (params) a.params = reinterpret_cast(*params); - if (extra) a.extra = reinterpret_cast(*extra); - hrr_cap::writer::write_event_raw(HRR_API_HIPDRVLAUNCHKERNELEX, &a.hdr, sizeof(a)); - } - return r; -} // Generated shim static hipError_t capture_hipMemGetHandleForAddressRange(void* handle, hipDeviceptr_t dptr, size_t size, hipMemRangeHandleType handleType, unsigned long long flags) { @@ -6519,6 +6804,110 @@ static hipError_t capture_hipMemcpyBatchAsync(void** dsts, void** srcs, size_t* a.stream = reinterpret_cast(stream); if (dsts) a.dsts = reinterpret_cast(*dsts); if (srcs) a.srcs = reinterpret_cast(*srcs); + if (dsts && count > 0) { + uint32_t _n = static_cast(count); + if (_n > 16u) { + static bool warned_dsts = false; + if (!warned_dsts) { + warned_dsts = true; + LogPrintfWarning( + "[HRR] hipMemcpyBatchAsync: recording only the first 16 " + "of %u dsts entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipMemcpyBatchAsync", + "only the first 16 dsts entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.dsts_bytes, dsts, static_cast(_n) * sizeof(void*)); + a.dsts_n = _n; + a.dsts_present = 1; + } + if (srcs && count > 0) { + uint32_t _n = static_cast(count); + if (_n > 16u) { + static bool warned_srcs = false; + if (!warned_srcs) { + warned_srcs = true; + LogPrintfWarning( + "[HRR] hipMemcpyBatchAsync: recording only the first 16 " + "of %u srcs entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipMemcpyBatchAsync", + "only the first 16 srcs entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.srcs_bytes, srcs, static_cast(_n) * sizeof(void*)); + a.srcs_n = _n; + a.srcs_present = 1; + } + if (sizes && count > 0) { + uint32_t _n = static_cast(count); + if (_n > 16u) { + static bool warned_sizes = false; + if (!warned_sizes) { + warned_sizes = true; + LogPrintfWarning( + "[HRR] hipMemcpyBatchAsync: recording only the first 16 " + "of %u sizes entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipMemcpyBatchAsync", + "only the first 16 sizes entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.sizes_bytes, sizes, static_cast(_n) * sizeof(size_t)); + a.sizes_n = _n; + a.sizes_present = 1; + } + if (attrs && numAttrs > 0) { + uint32_t _n = static_cast(numAttrs); + if (_n > 16u) { + static bool warned_attrs = false; + if (!warned_attrs) { + warned_attrs = true; + LogPrintfWarning( + "[HRR] hipMemcpyBatchAsync: recording only the first 16 " + "of %u attrs entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipMemcpyBatchAsync", + "only the first 16 attrs entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.attrs_bytes, attrs, static_cast(_n) * sizeof(hipMemcpyAttributes)); + a.attrs_n = _n; + a.attrs_present = 1; + } + if (attrsIdxs && numAttrs > 0) { + uint32_t _n = static_cast(numAttrs); + if (_n > 16u) { + static bool warned_attrsIdxs = false; + if (!warned_attrsIdxs) { + warned_attrsIdxs = true; + LogPrintfWarning( + "[HRR] hipMemcpyBatchAsync: recording only the first 16 " + "of %u attrsIdxs entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipMemcpyBatchAsync", + "only the first 16 attrsIdxs entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.attrsIdxs_bytes, attrsIdxs, static_cast(_n) * sizeof(size_t)); + a.attrsIdxs_n = _n; + a.attrsIdxs_present = 1; + } + if (failIdx) { + std::memcpy(a.failIdx_bytes, failIdx, sizeof(size_t)); + a.failIdx_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPMEMCPYBATCHASYNC, &a.hdr, sizeof(a)); } return r; @@ -6535,6 +6924,30 @@ static hipError_t capture_hipMemcpy3DBatchAsync(size_t numOps, struct hipMemcpy3 a.failIdx = reinterpret_cast(failIdx); a.flags = static_cast(flags); a.stream = reinterpret_cast(stream); + if (opList && numOps > 0) { + uint32_t _n = static_cast(numOps); + if (_n > 16u) { + static bool warned_opList = false; + if (!warned_opList) { + warned_opList = true; + LogPrintfWarning( + "[HRR] hipMemcpy3DBatchAsync: recording only the first 16 " + "of %u opList entries; replay of this call will be " + "incomplete", _n); + hrr_cap::writer::note_unreplayable("hipMemcpy3DBatchAsync", + "only the first 16 opList entries were recorded at " + "capture time; replay would pass a partial array"); + } + _n = 16u; + } + std::memcpy(a.opList_bytes, opList, static_cast(_n) * sizeof(hipMemcpy3DBatchOp)); + a.opList_n = _n; + a.opList_present = 1; + } + if (failIdx) { + std::memcpy(a.failIdx_bytes, failIdx, sizeof(size_t)); + a.failIdx_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPMEMCPY3DBATCHASYNC, &a.hdr, sizeof(a)); } return r; @@ -6603,7 +7016,7 @@ static hipError_t capture_hipMemPrefetchAsync_v2(const void* dev_ptr, size_t cou a.ret = static_cast(r); a.dev_ptr = reinterpret_cast(dev_ptr); a.count = static_cast(count); - a.location = 0; // non-castable type skipped + std::memcpy(a.location_bytes, &location, sizeof(location)); a.flags = static_cast(flags); a.stream = reinterpret_cast(stream); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMPREFETCHASYNC_V2, &a.hdr, sizeof(a)); @@ -6620,7 +7033,7 @@ static hipError_t capture_hipMemAdvise_v2(const void* dev_ptr, size_t count, hip a.dev_ptr = reinterpret_cast(dev_ptr); a.count = static_cast(count); a.advice = static_cast(advice); - a.device = 0; // non-castable type skipped + std::memcpy(a.device_bytes, &device, sizeof(device)); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMADVISE_V2, &a.hdr, sizeof(a)); } return r; @@ -6795,6 +7208,25 @@ static hipError_t capture_hipGetProcAddress_spt(const char* symbol, void** pfn, a.flags = static_cast(flags); a.symbolStatus = reinterpret_cast(symbolStatus); if (pfn) a.pfn = reinterpret_cast(*pfn); + if (symbol) { + size_t _n = std::strlen(symbol); + if (_n > 255u) { + static bool warned_symbol = false; + if (!warned_symbol) { + warned_symbol = true; + LogPrintfWarning( + "[HRR] hipGetProcAddress_spt: symbol is %zu characters; " + "recording the first 255 only", _n); + hrr_cap::writer::note_unreplayable("hipGetProcAddress_spt", + "symbol was truncated to 255 characters at capture " + "time; replay would pass a shortened string"); + } + _n = 255u; + } + std::memcpy(a.symbol_bytes, symbol, _n); + a.symbol_bytes[_n] = 0; + a.symbol_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPGETPROCADDRESS_SPT, &a.hdr, sizeof(a)); } return r; @@ -6857,9 +7289,13 @@ static hipError_t capture_hipMemSetMemPool(hipMemLocation* location, hipMemAlloc if (r == hipSuccess) { hrr_args_hipMemSetMemPool a{}; a.ret = static_cast(r); - a.location = 0; // non-castable type skipped + a.location = reinterpret_cast(location); a.type = static_cast(type); a.pool = reinterpret_cast(pool); + if (location) { + std::memcpy(a.location_bytes, location, sizeof(hipMemLocation)); + a.location_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPMEMSETMEMPOOL, &a.hdr, sizeof(a)); } return r; @@ -6871,9 +7307,13 @@ static hipError_t capture_hipMemGetMemPool(hipMemPool_t* pool, hipMemLocation* l if (r == hipSuccess) { hrr_args_hipMemGetMemPool a{}; a.ret = static_cast(r); - a.location = 0; // non-castable type skipped + a.location = reinterpret_cast(location); a.type = static_cast(type); if (pool) a.pool = reinterpret_cast(*pool); + if (location) { + std::memcpy(a.location_bytes, location, sizeof(hipMemLocation)); + a.location_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPMEMGETMEMPOOL, &a.hdr, sizeof(a)); } return r; @@ -6944,7 +7384,7 @@ static hipError_t capture_hipMemPrefetchBatchAsync(void** dev_ptrs, size_t* size a.ret = static_cast(r); a.sizes = reinterpret_cast(sizes); a.count = static_cast(count); - a.prefetch_locs = 0; // non-castable type skipped + a.prefetch_locs = reinterpret_cast(prefetch_locs); a.prefetch_loc_idxs = reinterpret_cast(prefetch_loc_idxs); a.num_prefetch_locs = static_cast(num_prefetch_locs); a.flags = static_cast(flags); @@ -7262,7 +7702,7 @@ static hipError_t capture_hipMemDiscardAndPrefetchBatchAsync(void** dptrs, size_ a.ret = static_cast(r); a.sizes = reinterpret_cast(sizes); a.count = static_cast(count); - a.prefetchLocs = 0; // non-castable type skipped + a.prefetchLocs = reinterpret_cast(prefetchLocs); a.prefetchLocIdxs = reinterpret_cast(prefetchLocIdxs); a.numPrefetchLocs = static_cast(numPrefetchLocs); a.flags = static_cast(flags); @@ -7282,7 +7722,7 @@ static hipError_t capture_hipDrvMemDiscardAndPrefetchBatchAsync(hipDeviceptr_t* a.dptrs = static_cast(reinterpret_cast(dptrs)); a.sizes = reinterpret_cast(sizes); a.count = static_cast(count); - a.prefetchLocs = 0; // non-castable type skipped + a.prefetchLocs = reinterpret_cast(prefetchLocs); a.prefetchLocIdxs = reinterpret_cast(prefetchLocIdxs); a.numPrefetchLocs = static_cast(numPrefetchLocs); a.flags = static_cast(flags); @@ -7298,7 +7738,7 @@ static hipError_t capture_hipMemGetDefaultMemPool(hipMemPool_t* memPool, hipMemL if (r == hipSuccess) { hrr_args_hipMemGetDefaultMemPool a{}; a.ret = static_cast(r); - a.location = 0; // non-castable type skipped + a.location = reinterpret_cast(location); a.type = static_cast(type); if (memPool) a.memPool = reinterpret_cast(*memPool); hrr_cap::writer::write_event_raw(HRR_API_HIPMEMGETDEFAULTMEMPOOL, &a.hdr, sizeof(a)); @@ -7443,6 +7883,25 @@ static void capture___hipRegisterVar(void** modules, void* var, char* hostVar, c a.constant = static_cast(constant); a.global = static_cast(global); if (modules) a.modules = reinterpret_cast(*modules); + if (deviceVar) { + size_t _n = std::strlen(deviceVar); + if (_n > 255u) { + static bool warned_deviceVar = false; + if (!warned_deviceVar) { + warned_deviceVar = true; + LogPrintfWarning( + "[HRR] __hipRegisterVar: deviceVar is %zu characters; " + "recording the first 255 only", _n); + hrr_cap::writer::note_unreplayable("__hipRegisterVar", + "deviceVar was truncated to 255 characters at capture " + "time; replay would pass a shortened string"); + } + _n = 255u; + } + std::memcpy(a.deviceVar_bytes, deviceVar, _n); + a.deviceVar_bytes[_n] = 0; + a.deviceVar_present = 1; + } hrr_cap::writer::write_event_raw(HRR_API_HIPREGISTERVAR, &a.hdr, sizeof(a)); } } @@ -7458,9 +7917,17 @@ extern hipError_t capture_hipArrayCreate(hipArray_t* pHandle, const HIP_ARRAY_DE extern hipError_t capture_hipDrvMemcpy2DUnaligned(const hip_Memcpy2D* pCopy); extern hipError_t capture_hipDrvMemcpy3D(const HIP_MEMCPY3D* pCopy); extern hipError_t capture_hipDrvMemcpy3DAsync(const HIP_MEMCPY3D* pCopy, hipStream_t stream); +extern hipError_t capture_hipGraphAddKernelNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const hipKernelNodeParams* pNodeParams); +extern hipError_t capture_hipGraphAddMemcpyNodeFromSymbol(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* symbol, size_t count, size_t offset, hipMemcpyKind kind); +extern hipError_t capture_hipGraphAddMemcpyNodeToSymbol(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind); +extern hipError_t capture_hipGraphExecKernelNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t node, const hipKernelNodeParams* pNodeParams); +extern hipError_t capture_hipGraphExecMemcpyNodeSetParamsToSymbol(hipGraphExec_t hGraphExec, hipGraphNode_t node, const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind); +extern hipError_t capture_hipGraphKernelNodeSetParams(hipGraphNode_t node, const hipKernelNodeParams* pNodeParams); +extern hipError_t capture_hipGraphMemcpyNodeSetParamsToSymbol(hipGraphNode_t node, const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind); extern hipError_t capture_hipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags); extern hipError_t capture_hipHostUnregister(void* hostPtr); extern hipError_t capture_hipLaunchByPtr(const void* func); +extern hipError_t capture_hipLaunchCooperativeKernel(const void* f, dim3 gridDim, dim3 blockDimX, void** kernelParams, unsigned int sharedMemBytes, hipStream_t stream); extern hipError_t capture_hipLaunchKernel(const void* function_address, dim3 numBlocks, dim3 dimBlocks, void** args, size_t sharedMemBytes, hipStream_t stream); extern hipError_t capture_hipMemGetAllocationGranularity(size_t* granularity, const hipMemAllocationProp* prop, hipMemAllocationGranularity_flags option); extern hipError_t capture_hipMemPoolCreate(hipMemPool_t* mem_pool, const hipMemPoolProps* pool_props); @@ -7477,14 +7944,26 @@ extern hipError_t capture_hipMemcpyDtoH(void* dst, hipDeviceptr_t src, size_t si extern hipError_t capture_hipMemcpyDtoHAsync(void* dst, hipDeviceptr_t src, size_t sizeBytes, hipStream_t stream); extern hipError_t capture_hipMemcpyHtoD(hipDeviceptr_t dst, const void* src, size_t sizeBytes); extern hipError_t capture_hipMemcpyHtoDAsync(hipDeviceptr_t dst, const void* src, size_t sizeBytes, hipStream_t stream); +extern hipError_t capture_hipMemcpyParam2D(const hip_Memcpy2D* pCopy); +extern hipError_t capture_hipMemcpyParam2DAsync(const hip_Memcpy2D* pCopy, hipStream_t stream); extern hipError_t capture_hipMemcpyWithStream(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind, hipStream_t stream); +extern hipError_t capture_hipModuleLaunchCooperativeKernel(hipFunction_t f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, hipStream_t stream, void** kernelParams); extern hipError_t capture_hipModuleLaunchKernel(hipFunction_t f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, hipStream_t stream, void** kernelParams, void** extra); extern hipError_t capture_hipModuleLoad(hipModule_t* module, const char* fname); extern hipError_t capture_hipModuleLoadData(hipModule_t* module, const void* image); extern hipError_t capture_hipModuleLoadDataEx(hipModule_t* module, const void* image, unsigned int numOptions, hipJitOption* options, void** optionValues); +extern hipError_t capture_hipModuleUnload(hipModule_t module); extern hipError_t capture_hipExtModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX, uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, uint32_t localWorkSizeX, uint32_t localWorkSizeY, uint32_t localWorkSizeZ, size_t sharedMemBytes, hipStream_t hStream, void** kernelParams, void** extra, hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags); extern hipError_t capture_hipMemcpy3D_spt(const struct hipMemcpy3DParms* p); extern hipError_t capture_hipMemcpy3DAsync_spt(const hipMemcpy3DParms* p, hipStream_t stream); +extern hipError_t capture_hipLaunchCooperativeKernel_spt(const void* f, dim3 gridDim, dim3 blockDim, void** kernelParams, uint32_t sharedMemBytes, hipStream_t hStream); +extern hipError_t capture_hipLaunchKernel_spt(const void* function_address, dim3 numBlocks, dim3 dimBlocks, void** args, size_t sharedMemBytes, hipStream_t stream); +extern hipError_t capture_hipGraphAddBatchMemOpNode(hipGraphNode_t* phGraphNode, hipGraph_t hGraph, const hipGraphNode_t* dependencies, size_t numDependencies, const hipBatchMemOpNodeParams* nodeParams); +extern hipError_t capture_hipGraphBatchMemOpNodeSetParams(hipGraphNode_t hNode, hipBatchMemOpNodeParams* nodeParams); +extern hipError_t capture_hipGraphExecBatchMemOpNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t hNode, const hipBatchMemOpNodeParams* nodeParams); +extern hipError_t capture_hipLinkAddData(hipLinkState_t state, hipJitInputType type, void* data, size_t size, const char* name, unsigned int numOptions, hipJitOption* options, void** optionValues); +extern hipError_t capture_hipLaunchKernelExC(const hipLaunchConfig_t* config, const void* fPtr, void** args); +extern hipError_t capture_hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipFunction_t f, void** params, void** extra); extern hipError_t capture_hipStreamSetAttribute(hipStream_t stream, hipStreamAttrID attr, const hipStreamAttrValue* value); extern hipError_t capture___hipPushCallConfiguration(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream); extern void** capture___hipRegisterFatBinary(const void* data); diff --git a/projects/clr/hipamd/src/hrr/hip_capture_metadata.cpp b/projects/clr/hipamd/src/hrr/hip_capture_metadata.cpp index b802df68d816..a6678e5ea4ef 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_metadata.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture_metadata.cpp @@ -15,6 +15,16 @@ #include #include +// Kept out of hip_capture.cpp: glibc's includes , whose +// Elf64_* typedefs collide with the ones CLR's ELF headers put in the global +// namespace. +#if defined(__linux__) +#include +#ifndef AT_SECURE +#define AT_SECURE 23 // Linux ABI value; same fallback as ROCr's runtime.cpp +#endif +#endif + namespace hip { class Device; extern std::vector g_devices; @@ -24,7 +34,6 @@ extern hipError_t ihipGetDeviceProperties(hipDeviceProp_t* props, hipDevice_t de namespace hrr_cap { namespace metadata { -namespace { std::string json_escape(const char* s) { std::string out; @@ -54,7 +63,9 @@ std::string json_escape(const char* s) { return out; } -std::string json_escape(const std::string& s) { return json_escape(s.c_str()); } +namespace { + +std::string json_escape(const std::string& s) { return metadata::json_escape(s.c_str()); } std::string quote(const std::string& s) { return "\"" + json_escape(s) + "\""; } @@ -190,5 +201,13 @@ std::string collect_json() { return os.str(); } +bool secure_exec() { +#if defined(__linux__) + return getauxval(AT_SECURE) != 0; +#else + return false; +#endif +} + } // namespace metadata } // namespace hrr_cap diff --git a/projects/clr/hipamd/src/hrr/hip_capture_metadata.h b/projects/clr/hipamd/src/hrr/hip_capture_metadata.h index 26b2e9663fa6..3e97fd184b67 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_metadata.h +++ b/projects/clr/hipamd/src/hrr/hip_capture_metadata.h @@ -10,11 +10,20 @@ namespace hrr_cap { namespace metadata { +// True when the kernel started this process in secure-execution mode (set-user-ID, +// set-group-ID, file capabilities or an LSM transition). Always false off Linux. +bool secure_exec(); + // Collects best-effort capture environment metadata for the HRR manifest. // The collector is safe to call from hip_capture_init(): it reads HIP runtime // constants and initialized internal device state rather than calling public HIP // APIs that would re-enter hip::init(). std::string collect_json(); +// Escapes `s` for use inside a JSON string literal (quotes not included): +// backslash, double quote, the short control escapes and any other byte below +// 0x20 as \u00XX. A null `s` yields "". +std::string json_escape(const char* s); + } // namespace metadata } // namespace hrr_cap diff --git a/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp b/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp index 821504a552cc..5bbfbea486c4 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp @@ -18,6 +18,9 @@ * its absence marks the archive as crash-truncated for the reader, which * recovers all complete records. * + * Disk space: capture stops, and the archive is marked incomplete, before the + * file system holding it falls below a reserve; see "Disk space" below. + * * Thread-safety: write_event_raw() and write_blob() acquire the file mutex. * open()/close()/flush() are called from a single thread (init/shutdown). */ @@ -35,7 +38,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -82,6 +87,7 @@ static inline uint64_t current_parent_process_id() { # include # include # include +# include # include # include # define HRR_OPEN(p) ::open((p), O_WRONLY | O_CREAT | O_TRUNC, 0644) @@ -216,6 +222,15 @@ static std::atomic g_blob_count{0}; static std::mutex g_blob_mu; static std::unordered_set g_written_blobs; +// APIs recorded in this archive that replay cannot reproduce (note_unreplayable). +// Listed in manifest.json so the gap is a property of the archive rather than +// something only visible in a replay log. +static std::mutex g_unreplayable_mu; +// api -> distinct reasons it was noted for. One API can be unreplayable for +// several reasons (e.g. a different truncated argument on different calls), +// and each is worth reporting. +static std::map> g_unreplayable_apis; + // --------------------------------------------------------------------------- // Low-level fd helpers // --------------------------------------------------------------------------- @@ -499,6 +514,26 @@ static void write_manifest_stdio(const char* output_dir, bool complete) { complete ? "true" : "false", static_cast(g_event_count.load()), static_cast(g_blob_count.load())); + { + std::lock_guard lk(g_unreplayable_mu); + if (!g_unreplayable_apis.empty()) { + fprintf(mf, ",\n \"unreplayable_apis\": {"); + bool first_api = true; + for (const auto& [api, reasons] : g_unreplayable_apis) { + fprintf(mf, "%s\n \"%s\": [", first_api ? "" : ",", + metadata::json_escape(api.c_str()).c_str()); + bool first_reason = true; + for (const auto& reason : reasons) { + fprintf(mf, "%s\"%s\"", first_reason ? "" : ", ", + metadata::json_escape(reason.c_str()).c_str()); + first_reason = false; + } + fprintf(mf, "]"); + first_api = false; + } + fprintf(mf, "\n }"); + } + } if (g_metadata_json_len > 0) { fprintf(mf, ",\n \"metadata\": %.*s\n", static_cast(g_metadata_json_len), g_metadata_json); @@ -605,12 +640,187 @@ static void update_root_manifest() { (void)atomic_write_file(g_base_dir + "/manifest.json", json.data(), json.size()); } +// --------------------------------------------------------------------------- +// Disk space +// +// Capture stops rather than fill the file system it writes to. It keeps free +// the smaller of 15% of that file system and 4 GiB, the default systemd-journald +// uses for SystemKeepFree. Free space is read when the archive opens, and again +// before any write that brings the bytes counted since the last check to +// g_space_check_bytes, so less than that is ever written unchecked. The interval +// is at most a quarter of the reserve, which bounds how far a capture can go +// into it. +// --------------------------------------------------------------------------- + +static constexpr uint64_t kKeepFreeMax = 4ull << 30; +static constexpr uint64_t kSpaceCheckMax = 64ull << 20; +static constexpr uint64_t kFileBlockDefault = 4096; +// Set by open(); g_keep_free stays 0 when free space cannot be read at all. +static uint64_t g_keep_free = 0; +static uint64_t g_space_check_bytes = kSpaceCheckMax; +// Every blob and code object is a file of its own and takes whole blocks, so they +// are counted in blocks of the archive's file system. +static std::atomic g_file_block_bytes{kFileBlockDefault}; +// Events and blobs both count here. Protected by g_file_mu. +static uint64_t g_bytes_since_space_check = 0; +// Bytes already accepted by reserve_space and not yet finished writing. Concurrent +// writers all count against the free-space check until they release. +static std::atomic g_bytes_reserved{0}; +static std::atomic g_out_of_space{false}; + +static bool fs_space(const std::string& dir, uint64_t* avail, uint64_t* total, + uint64_t* block = nullptr) { + uint64_t bs = 0; +#ifdef _WIN32 + ULARGE_INTEGER a{}, t{}, f{}; + // A UNC path is only accepted with a trailing backslash. + const std::string d = dir.empty() || dir.back() == '\\' ? dir : dir + '\\'; + if (!GetDiskFreeSpaceExA(d.c_str(), &a, &t, &f)) return false; + *avail = a.QuadPart; + *total = t.QuadPart; + // Files take whole clusters, which can be 64 KiB or more on NTFS and ReFS. + // GetDiskFreeSpaceA() reports the cluster size but wants the volume's root. + if (block != nullptr) { + char full[MAX_PATH], root[MAX_PATH]; + DWORD sectors = 0, sector_bytes = 0, free_clusters = 0, clusters = 0; + const DWORD n = GetFullPathNameA(d.c_str(), MAX_PATH, full, nullptr); + if (n > 0 && n < MAX_PATH && GetVolumePathNameA(full, root, MAX_PATH) && + GetDiskFreeSpaceA(root, §ors, §or_bytes, &free_clusters, &clusters)) + bs = static_cast(sectors) * sector_bytes; + } +#else + struct statvfs sv{}; + if (::statvfs(dir.c_str(), &sv) != 0) return false; + *avail = static_cast(sv.f_bavail) * sv.f_frsize; + *total = static_cast(sv.f_blocks) * sv.f_frsize; + bs = sv.f_frsize; +#endif + if (block != nullptr && bs != 0) *block = bs; + return true; +} + +// Flush what is buffered, close events.bin and mark the archive incomplete. +// Every write path already treats a closed events fd as "capture off". Both flags +// are raised under g_file_mu, so a writer that sees the stop flag and then takes +// the mutex finds events.bin already closed, and flush() cannot find it closed +// without also seeing the archive marked incomplete. +static void stop_for_space(uint64_t keep_free) { + char reason[160]; + snprintf(reason, sizeof(reason), + "stopped writing: less than %llu MiB would stay free on the file system " + "holding the archive", + static_cast(keep_free >> 20)); + BufWriteGuard lk; + if (g_out_of_space.exchange(true)) return; + if (g_events_fd >= 0) { + flush_buffer_locked(); + HRR_FSYNC(g_events_fd); + HRR_CLOSE(g_events_fd); + g_events_fd = -1; + } + mark_incomplete(reason); + // Not gated on AMD_LOG_LEVEL, like the refusal in open(). + fprintf(stderr, + "[HRR capture] Capture stopped: less than %llu MiB would stay free on the file " + "system holding %s. The archive is incomplete.\n", + static_cast(keep_free >> 20), g_output_dir.c_str()); +} + +// Size the reserve for the archive's file system. False when less than the +// reserve is free to begin with. +static bool init_space_reserve() { + g_keep_free = 0; + g_bytes_since_space_check = 0; + g_bytes_reserved.store(0, std::memory_order_relaxed); + uint64_t avail = 0, total = 0, block = kFileBlockDefault; + if (!fs_space(g_output_dir, &avail, &total, &block)) return true; + g_file_block_bytes.store(block, std::memory_order_relaxed); + g_keep_free = std::min(total / 100 * 15, kKeepFreeMax); + g_space_check_bytes = std::max(std::min(kSpaceCheckMax, g_keep_free / 4), 1); + return avail >= g_keep_free; +} + +// Count `len` bytes toward the next free-space check and return true when one is +// due. Caller must hold g_file_mu. +static bool space_check_due_locked(uint64_t len) { + g_bytes_since_space_check += len; + if (len < g_space_check_bytes && g_bytes_since_space_check < g_space_check_bytes) return false; + g_bytes_since_space_check = 0; + return true; +} + +// Read free space and return false, having stopped the capture, if `pending` more +// bytes, on top of the events still buffered, would eat into the reserve. Must not +// be called with g_file_mu held. +static bool check_space(uint64_t pending) { + { + // Read before the free space, so a flush in between is counted twice, not missed. + std::lock_guard lk(g_file_mu); + pending += g_buf_len; + } + uint64_t avail = 0, total = 0; + if (!fs_space(g_output_dir, &avail, &total)) return true; + if (avail >= g_keep_free && avail - g_keep_free >= pending) return true; + stop_for_space(g_keep_free); + return false; +} + +static uint64_t file_bytes(uint64_t len) { + const uint64_t block = g_file_block_bytes.load(std::memory_order_relaxed); + return (len + block - 1) / block * block; +} + +// Account for a file of `len` bytes and return false, having stopped the +// capture, if writing it would eat into the reserve. Must not be called with +// g_file_mu held. On success the bytes stay reserved until release_space(len). +static bool reserve_space(uint64_t len) { + if (g_out_of_space.load(std::memory_order_relaxed)) return false; + if (g_keep_free == 0) return true; + len = file_bytes(len); + const uint64_t reserved = + g_bytes_reserved.fetch_add(len, std::memory_order_acq_rel) + len; + bool due; + { + std::lock_guard lk(g_file_mu); + due = space_check_due_locked(len); + } + if (!due || check_space(reserved)) return true; + g_bytes_reserved.fetch_sub(len, std::memory_order_acq_rel); + return false; +} + +static void release_space(uint64_t len) { + if (g_keep_free == 0 || len == 0) return; + g_bytes_reserved.fetch_sub(file_bytes(len), std::memory_order_acq_rel); +} + +// Held for the rest of a writer after reserve_space(len) succeeds, so every way +// out of it, early returns and exceptions included, releases the bytes. +struct SpaceReservation { + uint64_t len; + ~SpaceReservation() { release_space(len); } +}; + +// For open() refusing for lack of space. A forked child inherits these paths and +// atexit(hip_capture_shutdown), so without this its flush() would finalize the +// parent's archive, or one the child never opened. An empty g_base_dir also stops +// atfork_child() trying again in later forks. +static void clear_archive_paths() { + g_base_dir.clear(); + g_output_dir.clear(); + g_manifest_path[0] = '\0'; +} + // --------------------------------------------------------------------------- // open / close / flush / checkpoint // --------------------------------------------------------------------------- bool open(const char* output_dir) { if (g_events_fd >= 0) return true; // already open — guard against double-invocation + if (g_out_of_space.load(std::memory_order_relaxed)) { // e.g. a child after fork + clear_archive_paths(); + return false; + } #ifndef _WIN32 install_atfork_handlers_once(); #endif @@ -633,6 +843,19 @@ bool open(const char* output_dir) { ensure_dir(g_output_dir + "/blobs"); ensure_dir(g_output_dir + "/code_objects"); + if (!init_space_reserve()) { + LogPrintfError("[HRR capture] Less than %llu MiB free on the file system holding %s", + static_cast(g_keep_free >> 20), g_output_dir.c_str()); + fprintf(stderr, "[HRR capture] Capture disabled: less than %llu MiB free on the file " + "system holding %s.\n", static_cast(g_keep_free >> 20), + g_output_dir.c_str()); + // Only removes directories that are still empty. + std::error_code ec; + for (const char* dir : {"/blobs", "/code_objects", ""}) fs::remove(g_output_dir + dir, ec); + clear_archive_paths(); + return false; + } + std::string events_path = g_output_dir + "/events.bin"; std::string manifest_path = g_output_dir + "/manifest.json"; snprintf(g_manifest_path, sizeof(g_manifest_path), "%s", manifest_path.c_str()); @@ -737,27 +960,49 @@ void checkpoint() { void mark_incomplete(const char* reason) { // Record once; the loud, AMD_LOG_LEVEL-routed message is emitted by the caller // (e.g. serialize_kernel_launch) which has the relevant context. Here we only - // need the durable flag and a single stderr breadcrumb so a bare run still - // surfaces it. + // need the durable flag and a single breadcrumb so a bare run still surfaces + // it. Error level: the archive is not faithful and replay must not treat it + // as one. log_printf appends its own newline, so the format string omits it. + // stop_for_space() calls this with g_file_mu held, so it must not take that + // mutex. if (!g_capture_incomplete.exchange(true, std::memory_order_relaxed)) { - fprintf(stderr, - "[HRR capture] Archive marked INCOMPLETE: %s. The clean-shutdown " - "trailer will be omitted and manifest.complete=false so replay " - "cannot treat this capture as faithful.\n", - reason ? reason : "(unspecified)"); + LogPrintfError( + "[HRR capture] Archive marked INCOMPLETE: %s. The clean-shutdown " + "trailer will be omitted and manifest.complete=false so replay " + "cannot treat this capture as faithful", + reason ? reason : "(unspecified)"); } } bool is_incomplete() { return g_capture_incomplete.load(std::memory_order_relaxed); } +void note_unreplayable(const char* api, const char* reason) { + if (!api) return; + if (!reason) reason = "(unspecified)"; + { + std::lock_guard lk(g_unreplayable_mu); + if (!g_unreplayable_apis[api].insert(reason).second) return; + } + // Warning, not Error: unlike mark_incomplete() the archive is well-formed and + // every event is present — only the ability to re-execute this one call is + // lost. That is a degradation, not a failure. + // log_printf appends its own newline, so the format string omits it. + LogPrintfWarning( + "[HRR capture] %s cannot be replayed: %s. The call is recorded, but " + "replay will report it as unreplayable rather than reproduce it", + api, reason); +} + void flush(const char* /*output_dir*/) { // Always finalize the *effective* directory this process actually wrote to // (g_output_dir), which is always a pid- sub-archive. The caller passes // the base HIP_HRR_CAPTURE_OUTPUT path. - const bool incomplete = g_capture_incomplete.load(std::memory_order_relaxed); + bool incomplete; std::string out_dir; { BufWriteGuard lk; + // Must be read under the lock, see stop_for_space(). + incomplete = g_capture_incomplete.load(std::memory_order_relaxed); out_dir = g_output_dir; // Skip the clean-shutdown trailer when the capture is known incomplete: its // absence is exactly how the reader detects a non-faithful archive. @@ -814,7 +1059,10 @@ static size_t append_lit(char* out, size_t off, const char* s) { } void emergency_finalize(bool clean_shutdown) { - if (g_events_fd < 0) return; + // A space stop has already closed events.bin, but a crash after it must still + // leave a manifest that says the archive is incomplete. + if (g_events_fd < 0 && (clean_shutdown || !g_out_of_space.load(std::memory_order_relaxed))) + return; // Flush the in-memory buffer only if no writer thread is mid-mutation. // We must NOT touch g_file_mu here: if the crash interrupts a writer mid-lock, @@ -837,7 +1085,9 @@ void emergency_finalize(bool clean_shutdown) { } g_buf_busy.clear(std::memory_order_release); } - HRR_FSYNC(g_events_fd); + // After a space stop there is no descriptor, and on Windows _commit(-1) calls + // the CRT invalid parameter handler, which ends the process by default. + if (g_events_fd >= 0) HRR_FSYNC(g_events_fd); // Best-effort manifest via raw open/write only. A clean shutdown writes // complete:true (the trailer is present); a crash writes complete:false — its @@ -883,6 +1133,29 @@ void emergency_finalize(bool clean_shutdown) { // Fills all header fields then copies the whole struct into the app buffer. // --------------------------------------------------------------------------- +// Assign sequence_id and buffer the record atomically so IDs are only consumed for +// events that are actually written. A full record is always appended under the +// lock, so the buffer never holds a torn record — which is what makes the +// crash-callback flush in emergency_finalize() safe. Caller holds BufWriteGuard and +// has seen g_events_fd open. +// +// The checkpoint flush+fsync happens inside the same lock scope. An earlier +// version released the lock and re-acquired it for the fsync, which let two +// threads that both crossed the kCheckpointEvents boundary race into back-to- +// back fsyncs (a thundering herd at every 4096-event boundary). Doing the +// fsync under the lock blocks other writers for the duration of the syscall, +// but guarantees exactly one fsync per checkpoint and removes the race. +static void append_event_locked(hrr_event_header* hdr, uint32_t payload_len) { + hdr->sequence_id = g_seq_id.fetch_add(1, std::memory_order_relaxed); + buffer_append_locked(hdr, payload_len); + g_event_count.fetch_add(1, std::memory_order_relaxed); + if (++g_events_since_ckpt >= kCheckpointEvents) { + flush_buffer_locked(); + HRR_FSYNC(g_events_fd); + g_events_since_ckpt = 0; + } +} + void write_event_raw(uint16_t api_id, hrr_event_header* hdr, uint32_t payload_len) { // Fill fields that don't require the lock (timestamp and thread_id are // cheap and per-thread; getting them outside the lock keeps contention low). @@ -892,29 +1165,27 @@ void write_event_raw(uint16_t api_id, hrr_event_header* hdr, uint32_t payload_le hdr->payload_length = payload_len; memset(hdr->reserved, 0, sizeof(hdr->reserved)); - // Acquire once: assign sequence_id and buffer the record atomically so IDs are - // only consumed for events that are actually written. A full record is always - // appended under the lock, so the buffer never holds a torn record — which is - // what makes the crash-callback flush in emergency_finalize() safe. - // - // The checkpoint flush+fsync happens inside this single lock scope. An earlier - // version released the lock and re-acquired it for the fsync, which let two - // threads that both crossed the kCheckpointEvents boundary race into back-to- - // back fsyncs (a thundering herd at every 4096-event boundary). Doing the - // fsync under the lock blocks other writers for the duration of the syscall, - // but guarantees exactly one fsync per checkpoint and removes the race. + // A record is counted toward the next free-space check before it is written. When + // it makes a check due, the check runs first, so it covers the record, as for a + // blob: the bytes written between two checks stay under one interval. { BufWriteGuard lk; if (g_events_fd < 0) return; - hdr->sequence_id = g_seq_id.fetch_add(1, std::memory_order_relaxed); - buffer_append_locked(hdr, payload_len); - g_event_count.fetch_add(1, std::memory_order_relaxed); - if (++g_events_since_ckpt >= kCheckpointEvents) { - flush_buffer_locked(); - HRR_FSYNC(g_events_fd); - g_events_since_ckpt = 0; + if (g_keep_free == 0 || !space_check_due_locked(payload_len)) { + append_event_locked(hdr, payload_len); + return; } } + // Outside g_file_mu, since check_space may take it to stop the capture. The + // record's bytes stay reserved until it is buffered, so a concurrent check counts + // them. + const uint64_t reserved = + g_bytes_reserved.fetch_add(payload_len, std::memory_order_acq_rel) + payload_len; + if (check_space(reserved)) { + BufWriteGuard lk; + if (g_events_fd >= 0) append_event_locked(hdr, payload_len); + } + g_bytes_reserved.fetch_sub(payload_len, std::memory_order_acq_rel); } // --------------------------------------------------------------------------- @@ -964,7 +1235,15 @@ Hash128 write_blob(const void* data, size_t len) { { std::lock_guard lk(g_blob_mu); - if (!g_written_blobs.insert(key).second) return h; // already written + if (g_written_blobs.count(key) != 0) return h; // already written + } + // Publish the key only after the space is reserved: a caller that finds it returns + // the hash at once, so the blob must not be abandoned after that. + if (!reserve_space(len)) return {}; + const SpaceReservation reservation{len}; + { + std::lock_guard lk(g_blob_mu); + if (!g_written_blobs.insert(key).second) return h; // written meanwhile } // blobs/<2-char-prefix>/.blob @@ -1000,7 +1279,14 @@ Hash128 write_code_object(const void* image, size_t image_size) { { std::lock_guard lk(g_blob_mu); - if (!g_written_blobs.insert(key).second) return h; // already written + if (g_written_blobs.count(key) != 0) return h; // already written + } + // As in write_blob(), the key is published only after the space is reserved. + if (!reserve_space(image_size)) return {}; + const SpaceReservation reservation{image_size}; + { + std::lock_guard lk(g_blob_mu); + if (!g_written_blobs.insert(key).second) return h; // written meanwhile } std::string path = g_output_dir + "/code_objects/" + hex + ".hsaco"; diff --git a/projects/clr/hipamd/src/hrr/hip_capture_writer.h b/projects/clr/hipamd/src/hrr/hip_capture_writer.h index 325d45c83892..f21a759f9668 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_writer.h +++ b/projects/clr/hipamd/src/hrr/hip_capture_writer.h @@ -16,10 +16,12 @@ namespace hrr_cap { namespace writer { // Open events.bin for writing in output_dir. Creates directory tree. -// Must be called before any write_* functions. +// Must be called before any write_* functions. Returns false, leaving capture +// off, when the file system holding the archive has less than the reserve free. bool open(const char* output_dir); -// Returns true if open() has been called successfully. +// True while events.bin is open: from a successful open() until close(), or +// until capture stops because the file system fell to the reserve. bool is_open(); // Cache process-level metadata JSON for manifest writers. The emergency @@ -71,11 +73,28 @@ void mark_incomplete(const char* reason); // Returns true if mark_incomplete() has been called. bool is_incomplete(); -// Write a buffer as a content-addressed blob. Returns hash. +// Record that `api` was captured but cannot be replayed faithfully — the call +// is in the archive, its effect is not reproducible (a host callback belonging +// to this process, a handle only meaningful to another rank, an argument +// truncated at capture, ...). Warns once per distinct (api, reason) at capture +// time, so the gap is visible when the recording is made rather than only when +// someone tries to replay it, and lists it in manifest.json as +// "unreplayable_apis": {"": ["", ...]}. Both strings are +// JSON-escaped on write, so any text is safe to pass. +// +// Deliberately NOT mark_incomplete(): that means an event was dropped or torn +// and suppresses the clean-shutdown trailer. Here every event is present and +// the archive is well-formed; what is missing is the ability to re-execute +// this particular call. Thread-safe; the warning fires once per (api, reason) +// per process. +void note_unreplayable(const char* api, const char* reason); + +// Write a buffer as a content-addressed blob. Returns hash, or {} when the +// writer is not open or capture has stopped for lack of space. // Thread-safe. Skips write if blob already exists on disk. Hash128 write_blob(const void* data, size_t len); -// Write a code object (.hsaco) blob. Returns hash. +// Write a code object (.hsaco) blob. Returns hash, or {} as write_blob() does. Hash128 write_code_object(const void* image, size_t image_size); // Number of events written so far. diff --git a/projects/clr/rocclr/device/rocm/rocblit.cpp b/projects/clr/rocclr/device/rocm/rocblit.cpp index 152131480e9d..59b6b54c7bb9 100644 --- a/projects/clr/rocclr/device/rocm/rocblit.cpp +++ b/projects/clr/rocclr/device/rocm/rocblit.cpp @@ -2848,14 +2848,14 @@ bool KernelBlitManager::ShaderCopyBufferBatchRaw( } const size_t max_operations_per_dispatch = (kernarg_pool_chunk_size - kernarg_reservation) / sizeof(CopyBufferBatchDescriptor); - const size_t descriptor_buffer_bytes = - max_operations_per_dispatch * sizeof(CopyBufferBatchDescriptor); bool attach_signal = false; for (size_t operation_offset = 0; operation_offset < copy_operations.size(); operation_offset += max_operations_per_dispatch) { const size_t operation_count = std::min(max_operations_per_dispatch, copy_operations.size() - operation_offset); + // Reserve only the size this dispatch requires. + const size_t descriptor_buffer_bytes = operation_count * sizeof(CopyBufferBatchDescriptor); void* descriptor_buffer = gpu().allocKernArg(descriptor_buffer_bytes, kCBAlignment); CopyBufferBatchDescriptor* descriptors = static_cast(descriptor_buffer); diff --git a/projects/clr/rocclr/device/rocm/rocdevice.cpp b/projects/clr/rocclr/device/rocm/rocdevice.cpp index 5952365f0d2c..b389f8d02d4b 100644 --- a/projects/clr/rocclr/device/rocm/rocdevice.cpp +++ b/projects/clr/rocclr/device/rocm/rocdevice.cpp @@ -125,7 +125,7 @@ bool NullDevice::create(const amd::Isa& isa) { std::stringstream ss; ss << AMD_BUILD_STRING " (HSA,LC) [Offline]"; ::strncpy(info_.driverVersion_, ss.str().c_str(), sizeof(info_.driverVersion_) - 1); - info_.version_ = "OpenCL " OPENCL_VERSION_STR " "; + info_.version_ = "OpenCL " OPENCL_VERSION_STR " " AMD_PLATFORM_INFO; return true; } @@ -1449,12 +1449,12 @@ bool Device::populateOCLDeviceConstants() { info_.version_ = "OpenCL " /*OPENCL_VERSION_STR*/ "2.0" - " "; + " " AMD_PLATFORM_INFO; } else { info_.version_ = "OpenCL " /*OPENCL_VERSION_STR*/ "1.2" - " "; + " " AMD_PLATFORM_INFO; } info_.builtInKernels_ = ""; diff --git a/projects/clr/rocclr/device/rocm/rocvirtual.cpp b/projects/clr/rocclr/device/rocm/rocvirtual.cpp index 76b95143870f..16912bc970bc 100644 --- a/projects/clr/rocclr/device/rocm/rocvirtual.cpp +++ b/projects/clr/rocclr/device/rocm/rocvirtual.cpp @@ -3250,6 +3250,9 @@ void VirtualGPU::submitSvmPrefetchAsync(amd::SvmPrefetchAsyncCommand& cmd) { profilingBegin(cmd); if (dev().info().hmmSupported_) { + // ROCr orders the prefetch only by the wait signals, so a signal must cover all earlier + // work in the queue + releaseGpuMemoryFence(kSkipCpuWait); // Initialize signal for the barrier auto wait_events = Barriers().WaitingSignal(HwQueueEngine::Unknown); hsa_signal_t active = Barriers().ActiveSignal(kInitSignalValueOne, timestamp_); @@ -3289,6 +3292,9 @@ void VirtualGPU::SubmitSvmPrefetchBatchAsync(amd::SvmPrefetchBatchAsyncCommand& std::scoped_lock lock(execution()); profilingBegin(command); + // ROCr orders the prefetch only by the wait signals, so a signal must cover all earlier + // work in the queue + releaseGpuMemoryFence(kSkipCpuWait); auto wait_events = Barriers().WaitingSignal(HwQueueEngine::Unknown); hsa_signal_t active = Barriers().ActiveSignal(command.Count(), timestamp_); @@ -3329,6 +3335,9 @@ void VirtualGPU::SubmitSvmDiscardBatchAsync(amd::SvmDiscardBatchAsyncCommand& co std::scoped_lock lock(execution()); profilingBegin(command); + // ROCr orders the discard only by the wait signals, so a signal must cover all earlier + // work in the queue + releaseGpuMemoryFence(kSkipCpuWait); auto wait_events = Barriers().WaitingSignal(HwQueueEngine::Unknown); hsa_signal_t active = Barriers().ActiveSignal(kInitSignalValueOne, timestamp_); @@ -4361,6 +4370,15 @@ void VirtualGPU::submitBatchMemoryOperation(amd::BatchMemoryOperationCommand& cm std::scoped_lock lock(execution()); profilingBegin(cmd); + // System scope is needed here to order this packet against writes made by EARLIER + // packets on the same stream, not for the write performed by this one. The blit kernel + // already stores the value with system scope, so the value itself reaches a peer agent + // without any flush -- which is precisely the hazard: data left in this agent's L2 by a + // preceding agent-scope dispatch (e.g. fillBuffer) has not drained yet, so the value + // overtakes it. The system-scope acquire makes this agent's caches coherent before the + // packet runs. Matches submitStreamOperation(), which does the same for its blit paths. + addSystemScope(); + bool result = blitMgr().batchMemOps(cmd.getParamPtr(), cmd.paramSize(), cmd.count()); if (!result) { LogError("submitBatchMemoryOperation failed!"); diff --git a/projects/clr/rocclr/utils/flags.hpp b/projects/clr/rocclr/utils/flags.hpp index c3470c42853b..6d83af4be9a5 100644 --- a/projects/clr/rocclr/utils/flags.hpp +++ b/projects/clr/rocclr/utils/flags.hpp @@ -108,8 +108,8 @@ release(bool, DISABLE_DEFERRED_ALLOC, false, \ "Disables deferred memory allocation on device") \ release(int, AMD_GPU_FORCE_SINGLE_FP_DENORM, -1, \ "Force denorm for single precision: -1 - don't force, 0 - disable, 1 - enable") \ -release(uint, OCL_SET_SVM_SIZE, 256*1024, \ - "set SVM space size for discrete GPU") \ +release(uint, OCL_SET_SVM_SIZE, 4 * 1024 * 1024, \ + "Set SVM space size for discrete GPU (MiB)") \ release(uint, GPU_WAVES_PER_SIMD, 0, \ "Force the number of waves per SIMD (1-10)") \ release(bool, OCL_STUB_PROGRAMS, false, \ diff --git a/projects/cuid/.pre-commit-config.yaml b/projects/cuid/.pre-commit-config.yaml index 6cab99eb57b0..26c1d8bd254f 100644 --- a/projects/cuid/.pre-commit-config.yaml +++ b/projects/cuid/.pre-commit-config.yaml @@ -71,19 +71,3 @@ repos: # care about (quoting, unset variables, portability). files: '^projects/cuid/.*\.(sh|sh\.in)$' types: [file] - -- repo: local - hooks: - - id: cuid-sha256-drift - name: CUID/rocprofiler-sdk SHA-256 drift - description: >- - libamdcuid carries a copy of rocprofiler-sdk's SHA-256. A silent - divergence would not fail to build, it would change every CUID - ever issued, so the two are compared function by function. - entry: python3 projects/cuid/tests/check_sha256_drift.py - language: system - pass_filenames: false - # always_run rather than a `files` pattern: the global scope above would - # hide a change made only on the rocprofiler-sdk side, which is exactly - # the drift this is meant to catch. The check takes milliseconds. - always_run: true diff --git a/projects/cuid/CMakeLists.txt b/projects/cuid/CMakeLists.txt index d47daac0e505..eea030ed2b06 100644 --- a/projects/cuid/CMakeLists.txt +++ b/projects/cuid/CMakeLists.txt @@ -82,8 +82,8 @@ if(WIN32) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) # Import libs endif() -# No third-party dependencies. SHA-256 and HMAC-SHA-256 are built from -# lib/src/sha256.cc; random key material comes from the platform CSPRNG +# No third-party dependencies. SHA-256 and HMAC-SHA-256 are built from source +# out of shared/sha256; random key material comes from the platform CSPRNG # (bcrypt on Windows, getrandom(2)//dev/urandom elsewhere). message("Build Configuration:") diff --git a/projects/cuid/README.md b/projects/cuid/README.md index d261ff2417e6..359be47522cd 100644 --- a/projects/cuid/README.md +++ b/projects/cuid/README.md @@ -183,7 +183,7 @@ sphinx-build docs docs/_build/html #### Notes -- No third-party crypto dependency: SHA-256 and HMAC-SHA-256 are built from `lib/src/sha256.cc`, so the library links no TLS stack on any platform +- No third-party crypto dependency: SHA-256 and HMAC-SHA-256 are built from source out of the shared `shared/sha256` component, whose objects are absorbed into `libamdcuid_static.a`, so the shipped archive is self-contained and the library links no TLS stack on any platform - Root/administrator privileges are required for full functionality (ACPI tables, SMBIOS UUID, PCI config space access) - systemd is only needed for `udevd` reloads and optional service management; the daemon can also be started using other mechanisms such as an `@reboot` cron job diff --git a/projects/cuid/lib/CMakeLists.txt b/projects/cuid/lib/CMakeLists.txt index 52695707168a..d323839e5a84 100644 --- a/projects/cuid/lib/CMakeLists.txt +++ b/projects/cuid/lib/CMakeLists.txt @@ -34,7 +34,6 @@ set(CUID_SOURCES ${SRC_DIR}/cuid_platform.cc ${SRC_DIR}/cuid_file.cc ${SRC_DIR}/hmac.cc - ${SRC_DIR}/sha256.cc ${SRC_DIR}/pci_util.cc ${SRC_DIR}/acpi_parser.cc ${SRC_DIR}/smbios_util.cc @@ -53,7 +52,6 @@ set(CUID_HEADERS ${SRC_DIR}/cuid_platform.h ${SRC_DIR}/cuid_file.h ${SRC_DIR}/hmac.h - ${SRC_DIR}/sha256.h ${SRC_DIR}/ipc_protocol.h ${SRC_DIR}/pci_util.h ${SRC_DIR}/acpi_parser.h @@ -73,8 +71,34 @@ target_include_directories(amdcuid_static PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PRI # the HMAC key is shared and CUIDs remain stable across version upgrades. target_compile_definitions(amdcuid_static PRIVATE AMDCUID_CONFIG_DIR="${AMDCUID_SHARED_CONFIG_DIR}") -# SHA-256/HMAC comes from src/sha256.cc, so the only external dependency left -# is the platform CSPRNG: bcrypt on Windows, getrandom(2)//dev/urandom on POSIX. +# SHA-256/HMAC comes from shared/sha256, built from source. The cache variable +# is project-prefixed so that a build which configures cuid and another +# rocm-systems project together does not have them fight over one entry. +set(CUID_ROCM_SHA256_DIR + "${CMAKE_CURRENT_SOURCE_DIR}/../../../shared/sha256" + CACHE PATH + "Path to the shared SHA-256 component" +) +if(NOT TARGET rocm-sha256) + if(NOT EXISTS "${CUID_ROCM_SHA256_DIR}/CMakeLists.txt") + message( + FATAL_ERROR + "Shared SHA-256 not found at ${CUID_ROCM_SHA256_DIR}. Set " + "CUID_ROCM_SHA256_DIR when building cuid outside the rocm-systems tree." + ) + endif() + add_subdirectory(${CUID_ROCM_SHA256_DIR} ${CMAKE_CURRENT_BINARY_DIR}/rocm-sha256) +endif() + +# PRIVATE so the OBJECT library's objects land in amdcuid_static and the +# installed archive defines SHA-256 symbols rather than importing. +# Include dir is PUBLIC + BUILD_INTERFACE so in-tree targets that link +# amdcuid_static and include rocm/sha2/sha256.h directly can find it; +# shared/sha256 headers are never installed, and sha256.h is not part of the installed public API. +target_link_libraries(amdcuid_static PRIVATE rocm-sha256-objects) +target_include_directories(amdcuid_static PUBLIC $) + +# The only external dependency left is the platform CSPRNG. if(WIN32) target_link_libraries(amdcuid_static PUBLIC bcrypt) endif() diff --git a/projects/cuid/lib/src/hmac.cc b/projects/cuid/lib/src/hmac.cc index 747f1b6c8a11..66dd4df03ba4 100644 --- a/projects/cuid/lib/src/hmac.cc +++ b/projects/cuid/lib/src/hmac.cc @@ -16,7 +16,9 @@ #include #include -#include "sha256.h" +#include "cuid_util.h" +#include "rocm/sha2/log.h" +#include "rocm/sha2/sha256.h" #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN @@ -46,6 +48,21 @@ namespace { +// rocm::sha2 has no logging dependency of its own; it reports diagnostics +// (e.g. update() after finalize(), which should never happen in this library) +// through a swappable handler that writes straight to stderr by default. Route +// it into cuid's own Logger instead, so it is subject to the same level +// filtering as the rest of the library and downstream apps aren't surprised by +// unconditional stderr output from a dependency they don't call directly. +void sha2_log_handler(const char* message) { LOG(ERROR, message); } + +// Idempotent; called from every cuid_hmac constructor so the handler is +// installed before any sha256 use regardless of construction order. +void init_sha2_logging() { + static std::once_flag once; + std::call_once(once, [] { rocm::sha2::set_log_handler(&sha2_log_handler); }); +} + // Used when no secret is provisioned. Byte-identical to CUID_DEFAULT_SEED in // the kernel's amdgpu_cuid.c, so sysfs and this library agree on an // unprovisioned machine. Not a substitute for provisioning; callers can check @@ -116,6 +133,8 @@ struct cuid_hmac::Impl { cuid_hmac::cuid_hmac() : impl_(nullptr), key(nullptr), key_len(key_length), valid(false), using_default_key(false) { + init_sha2_logging(); + // getenv races only against setenv, which this library never calls. // NOLINTNEXTLINE(concurrency-mt-unsafe) const char* env_path = std::getenv("AMDCUID_HMAC_KEY_PATH"); @@ -171,6 +190,8 @@ void cuid_hmac::use_default_key() { cuid_hmac::cuid_hmac(uint8_t key_data[key_length]) : impl_(nullptr), key(nullptr), key_len(key_length), valid(false), using_default_key(false) { + init_sha2_logging(); + impl_ = new Impl(); impl_->digest_name = "SHA256"; diff --git a/projects/cuid/lib/src/sha256.cc b/projects/cuid/lib/src/sha256.cc deleted file mode 100644 index 1836f3a3fcfd..000000000000 --- a/projects/cuid/lib/src/sha256.cc +++ /dev/null @@ -1,236 +0,0 @@ -// MIT License -// -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// The compression function below is verbatim from -// projects/rocprofiler-sdk/source/lib/common/sha256.cpp. See sha256.h for why -// there are two copies and what the plan is for collapsing them. - -#include "sha256.h" - -#include -#include -#include - -namespace rocm { -namespace sha2 { - -/// FIPS 180-4 section 5.3.3 initial hash value. -constexpr std::array sha256_initial_state = { - 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; - -void secure_zero(void* p, size_t n) { - auto* volatile q = static_cast(p); - while (n-- > 0) *q++ = 0; -} - -sha256::sha256() { reset(); } - -sha256::sha256(const std::string& data) { - reset(); - update(data); - finalize(); -} - -void sha256::update(const uint8_t* data, size_t len) { - if (m_finalized) return; - for (size_t i = 0; i < len; ++i) { - m_data[m_datalen++] = data[i]; - if (m_datalen == 64) { - transform(); - m_bitlen += 512; - m_datalen = 0; - } - } -} - -void sha256::update(const std::string& data) { - if (m_finalized) return; - update(reinterpret_cast(data.data()), data.size()); -} - -void sha256::finalize() { - if (m_finalized) return; - - uint32_t idx = m_datalen; - if (m_datalen < 56) { - m_data[idx++] = 0x80; - while (idx < 56) m_data[idx++] = 0x00; - } else { - m_data[idx++] = 0x80; - while (idx < 64) m_data[idx++] = 0x00; - transform(); - std::memset(m_data.data(), 0, 56); - } - - m_bitlen += static_cast(m_datalen) * 8; - for (int j = 0; j < 8; ++j) m_data[63 - j] = static_cast((m_bitlen >> (8 * j)) & 0xFF); - - transform(); - - m_finalized = true; -} - -std::string sha256::hexdigest() { - finalize(); - - auto oss = std::ostringstream{}; - for (int j = 0; j < 8; ++j) oss << std::hex << std::setfill('0') << std::setw(8) << m_state[j]; - return oss.str(); -} - -std::array sha256::rawdigest() { - finalize(); - - return m_state; -} - -std::array sha256::digest() { - finalize(); - - auto out = std::array{}; - for (size_t j = 0; j < m_state.size(); ++j) { - out[j * 4 + 0] = static_cast((m_state[j] >> 24) & 0xFF); - out[j * 4 + 1] = static_cast((m_state[j] >> 16) & 0xFF); - out[j * 4 + 2] = static_cast((m_state[j] >> 8) & 0xFF); - out[j * 4 + 3] = static_cast((m_state[j]) & 0xFF); - } - return out; -} - -uint32_t sha256::rotr(uint32_t x, uint32_t n) { return (x >> n) | (x << (32 - n)); } - -uint32_t sha256::ch(uint32_t x, uint32_t y, uint32_t z) { return (x & y) ^ (~x & z); } - -uint32_t sha256::maj(uint32_t x, uint32_t y, uint32_t z) { return (x & y) ^ (x & z) ^ (y & z); } - -uint32_t sha256::sig0(uint32_t x) { return rotr(x, 2) ^ rotr(x, 13) ^ rotr(x, 22); } - -uint32_t sha256::sig1(uint32_t x) { return rotr(x, 6) ^ rotr(x, 11) ^ rotr(x, 25); } - -uint32_t sha256::theta0(uint32_t x) { return rotr(x, 7) ^ rotr(x, 18) ^ (x >> 3); } - -uint32_t sha256::theta1(uint32_t x) { return rotr(x, 17) ^ rotr(x, 19) ^ (x >> 10); } - -void sha256::transform() { - uint32_t m[64]; - for (int i = 0; i < 16; ++i) { - m[i] = (static_cast(m_data[static_cast(i) * 4]) << 24) | - (static_cast(m_data[static_cast(i) * 4 + 1]) << 16) | - (static_cast(m_data[static_cast(i) * 4 + 2]) << 8) | - (static_cast(m_data[static_cast(i) * 4 + 3])); - } - for (int i = 16; i < 64; ++i) { - m[i] = theta1(m[i - 2]) + m[i - 7] + theta0(m[i - 15]) + m[i - 16]; - } - - uint32_t a = m_state[0]; - uint32_t b = m_state[1]; - uint32_t c = m_state[2]; - uint32_t d = m_state[3]; - uint32_t e = m_state[4]; - uint32_t f = m_state[5]; - uint32_t g = m_state[6]; - uint32_t h = m_state[7]; - - for (int i = 0; i < 64; ++i) { - uint32_t t1 = h + sig1(e) + ch(e, f, g) + m_k[i] + m[i]; - uint32_t t2 = sig0(a) + maj(a, b, c); - h = g; - g = f; - f = e; - e = d + t1; - d = c; - c = b; - b = a; - a = t1 + t2; - } - - m_state[0] += a; - m_state[1] += b; - m_state[2] += c; - m_state[3] += d; - m_state[4] += e; - m_state[5] += f; - m_state[6] += g; - m_state[7] += h; - - secure_zero(m, sizeof(m)); -} - -void sha256::reset() { - m_state = sha256_initial_state; - m_data = {}; - m_datalen = 0; - m_bitlen = 0; - m_finalized = false; -} - -void sha256_digest(const uint8_t* data, size_t data_len, uint8_t out[SHA256_DIGEST_SIZE]) { - if (!out) return; - if (!data && data_len > 0) { - secure_zero(out, SHA256_DIGEST_SIZE); - return; - } - sha256 h; - if (data_len > 0) h.update(data, data_len); - const auto d = h.digest(); - std::memcpy(out, d.data(), d.size()); -} - -void hmac_sha256(const uint8_t* key, size_t key_len, const uint8_t* msg, size_t msg_len, - uint8_t out[SHA256_DIGEST_SIZE]) { - uint8_t k0[SHA256_BLOCK_SIZE] = {}; - - if (key_len > SHA256_BLOCK_SIZE) { - sha256_digest(key, key_len, k0); - } else if (key_len > 0) { - std::memcpy(k0, key, key_len); - } - - uint8_t pad[SHA256_BLOCK_SIZE]; - uint8_t inner[SHA256_DIGEST_SIZE]; - - for (size_t i = 0; i < SHA256_BLOCK_SIZE; ++i) pad[i] = static_cast(k0[i] ^ 0x36); - { - sha256 h; - h.update(pad, SHA256_BLOCK_SIZE); - if (msg_len > 0) h.update(msg, msg_len); - const auto d = h.digest(); - std::memcpy(inner, d.data(), d.size()); - } - - for (size_t i = 0; i < SHA256_BLOCK_SIZE; ++i) pad[i] = static_cast(k0[i] ^ 0x5c); - { - sha256 h; - h.update(pad, SHA256_BLOCK_SIZE); - h.update(inner, SHA256_DIGEST_SIZE); - const auto d = h.digest(); - std::memcpy(out, d.data(), d.size()); - } - - secure_zero(k0, sizeof(k0)); - secure_zero(pad, sizeof(pad)); - secure_zero(inner, sizeof(inner)); -} - -} // namespace sha2 -} // namespace rocm diff --git a/projects/cuid/lib/src/sha256.h b/projects/cuid/lib/src/sha256.h deleted file mode 100644 index b633b6f85bef..000000000000 --- a/projects/cuid/lib/src/sha256.h +++ /dev/null @@ -1,112 +0,0 @@ -// MIT License -// -// Copyright (c) 2026 Advanced Micro Devices, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// SHA-256 (FIPS 180-4) and HMAC-SHA-256 (RFC 2104). -// -// Copy of projects/rocprofiler-sdk/source/lib/common/sha256.{hpp,cpp}, so CUID -// needs nothing beyond the standard library. Kept API-compatible with it and in -// a neutral namespace so the two can be merged later by moving these files. -// tests/check_sha256_drift.py enforces that they stay identical; only update() -// differs, by rocprofiler's logging call. -// -// Not constant-time. For identifier derivation, not bulk secret processing. - -#ifndef ROCM_SHA2_SHA256_H -#define ROCM_SHA2_SHA256_H - -#include -#include -#include -#include - -namespace rocm { -namespace sha2 { - -/// Size of a SHA-256 digest, in bytes. -constexpr size_t SHA256_DIGEST_SIZE = 32; -/// SHA-256 compression block size, in bytes. Also the HMAC key block size. -constexpr size_t SHA256_BLOCK_SIZE = 64; - -/// Overwrite a buffer in a way the optimiser is not permitted to elide. -void secure_zero(void* p, size_t n); - -// --- SHA-256 Implementation --- -class sha256 { - public: - sha256(); - explicit sha256(const std::string& data); - - void update(const uint8_t* data, size_t len); - void update(const std::string& data); - - void finalize(); - std::string hexdigest(); - std::array rawdigest(); - - /// Serialise the digest big-endian: the FIPS 180-4 byte order, and the one - /// every other SHA-256 implementation emits. Finalises if not already done. - std::array digest(); - - private: - bool m_finalized = false; - std::array m_data = {}; - std::array m_state = {}; - uint32_t m_datalen = 0; - uint64_t m_bitlen = 0; - - static constexpr std::array m_k = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, - 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, - 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, - 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, - 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, - 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, - 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, - 0xc67178f2}; - - static uint32_t rotr(uint32_t x, uint32_t n); - static uint32_t ch(uint32_t x, uint32_t y, uint32_t z); - static uint32_t maj(uint32_t x, uint32_t y, uint32_t z); - static uint32_t sig0(uint32_t x); - static uint32_t sig1(uint32_t x); - static uint32_t theta0(uint32_t x); - static uint32_t theta1(uint32_t x); - - void transform(); - void reset(); -}; - -/// One-shot SHA-256 over a single buffer. -void sha256_digest(const uint8_t* data, size_t data_len, uint8_t out[SHA256_DIGEST_SIZE]); - -/// HMAC-SHA-256 (FIPS 198-1 / RFC 2104). Any key length is accepted: keys -/// longer than the 64-byte block are hashed down first, shorter keys are -/// zero-padded, per the standard. -void hmac_sha256(const uint8_t* key, size_t key_len, const uint8_t* msg, size_t msg_len, - uint8_t out[SHA256_DIGEST_SIZE]); - -} // namespace sha2 -} // namespace rocm - -#endif // ROCM_SHA2_SHA256_H diff --git a/projects/cuid/tests/check_sha256_drift.py b/projects/cuid/tests/check_sha256_drift.py deleted file mode 100755 index 2da61abeb9ca..000000000000 --- a/projects/cuid/tests/check_sha256_drift.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -"""Fail if the two copies of SHA-256 in this repo have drifted apart. - -libamdcuid carries a copy of rocprofiler-sdk's SHA-256. A silent divergence -would not fail to build -- it would change every CUID ever issued. - - python3 projects/cuid/tests/check_sha256_drift.py -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path - -# Both paths are relative to the repository root. -UPSTREAM = Path("projects/rocprofiler-sdk/source/lib/common/sha256.cpp") -COPY = Path("projects/cuid/lib/src/sha256.cc") - -# update() may differ only by rocprofiler's logging call. Anything else is drift. -ALLOWED_UPSTREAM_ONLY = ("ROCP_CI_LOG_IF",) - - -def strip_comments(src: str) -> str: - src = re.sub(r"//[^\n]*", "", src) - return re.sub(r"/\*.*?\*/", "", src, flags=re.S) - - -def member_bodies(path: Path) -> dict[tuple[str, str], str]: - """Map (name, signature) -> body text, for every member defined in the file. - - The signature includes the parameter list and any trailing cv/ref - qualifiers, so overloads (e.g. two sha256::update()) get distinct keys - instead of colliding on the member name alone. - """ - src = strip_comments(path.read_text(encoding="utf-8")) - bodies: dict[tuple[str, str], str] = {} - for match in re.finditer(r"\bsha256::(\w+)\s*\(", src): - name = match.group(1) - - depth, idx = 1, match.end() - while idx < len(src) and depth > 0: - if src[idx] == "(": - depth += 1 - elif src[idx] == ")": - depth -= 1 - idx += 1 - params_end = idx - - open_brace = src.find("{", params_end) - if open_brace < 0: - continue - signature = re.sub(r"\s+", "", src[match.end() - 1 : open_brace]) - - depth, idx = 0, open_brace - while idx < len(src): - if src[idx] == "{": - depth += 1 - elif src[idx] == "}": - depth -= 1 - if depth == 0: - break - idx += 1 - bodies.setdefault((name, signature), src[open_brace + 1 : idx]) - return bodies - - -def normalise(body: str, drop_allowed: bool) -> str: - lines = [] - for line in body.split("\n"): - if drop_allowed and any(tok in line for tok in ALLOWED_UPSTREAM_ONLY): - continue - lines.append(line) - return re.sub(r"\s+", "", "\n".join(lines)) - - -def main() -> int: - root = Path(__file__).resolve().parents[3] - upstream_path, copy_path = root / UPSTREAM, root / COPY - - for path in (upstream_path, copy_path): - if not path.is_file(): - print(f"error: {path} not found", file=sys.stderr) - return 2 - - upstream = member_bodies(upstream_path) - copy = member_bodies(copy_path) - - if not upstream or not copy: - print( - "error: parsed no sha256:: members; the extractor needs updating", - file=sys.stderr, - ) - return 2 - - problems: list[str] = [] - - missing = sorted(set(upstream) - set(copy)) - if missing: - names = ", ".join(f"{name}{sig}" for name, sig in missing) - problems.append(f"{COPY} is missing member(s) present upstream: {names}") - - for key in sorted(set(upstream) & set(copy)): - if normalise(upstream[key], drop_allowed=True) != normalise(copy[key], False): - name, sig = key - problems.append(f"sha256::{name}{sig} differs between the two copies") - - if problems: - print("SHA-256 implementations have drifted apart:\n", file=sys.stderr) - for problem in problems: - print(f" - {problem}", file=sys.stderr) - print( - f"\n upstream: {UPSTREAM}\n copy: {COPY}\n\n" - "Reconcile them, or record an intended difference in ALLOWED_UPSTREAM_ONLY.", - file=sys.stderr, - ) - return 1 - - print( - f"sha256 copies agree: {len(set(upstream) & set(copy))} member functions checked" - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/projects/cuid/tests/unit/sha256_test.cc b/projects/cuid/tests/unit/sha256_test.cc index 695219f7f821..3e44b98c9e75 100644 --- a/projects/cuid/tests/unit/sha256_test.cc +++ b/projects/cuid/tests/unit/sha256_test.cc @@ -37,8 +37,8 @@ #include #include +#include "rocm/sha2/sha256.h" #include "src/hmac.h" -#include "src/sha256.h" namespace { diff --git a/projects/hip-tests/CONTRACT_COVERAGE.md b/projects/hip-tests/CONTRACT_COVERAGE.md index 8f9157d7e5a7..f62499997696 100644 --- a/projects/hip-tests/CONTRACT_COVERAGE.md +++ b/projects/hip-tests/CONTRACT_COVERAGE.md @@ -13,22 +13,22 @@ contract tier intentionally pins only small, portable semantic guarantees. ## Snapshot - Snapshot date: 2026-08-26 - Snapshot commit: `920418c484` -- Contract tests: 612 +- Contract tests: 616 - Contract domains: 118 -- Declared HIP runtime APIs parsed from `hip_runtime_api.h`: 498 -- Declared HIP runtime APIs directly exercised by contract tests: 487 -- Intentionally uncovered, allowlisted APIs: 11 -- Approximate declared API-name coverage: 97.8% +- Declared HIP runtime APIs parsed from `hip_runtime_api.h`: 501 +- Declared HIP runtime APIs directly exercised by contract tests: 488 +- Intentionally uncovered, allowlisted APIs: 13 +- Approximate declared API-name coverage: 97.4% - Additional public macro exercised: `hipLaunchKernelGGL` - Additional non-runtime-header APIs exercised: HIPRTC (`hiprtcCreateProgram`, `hiprtcCompileProgram`, `hiprtcGetCodeSize`, `hiprtcGetCode`, @@ -104,7 +104,7 @@ binaries instead of exposing portable state to assert. | `context_config` | 6 | | `context_mutation` | 5 | | `copy3d` | 4 | -| `device` | 7 | +| `device` | 11 | | `device_config` | 6 | | `device_identity` | 9 | | `device_lifecycle` | 7 | diff --git a/projects/hip-tests/README.md b/projects/hip-tests/README.md index 5edd9fe2654a..f7db6bb42f9f 100644 --- a/projects/hip-tests/README.md +++ b/projects/hip-tests/README.md @@ -209,26 +209,32 @@ When `ENABLE_YAML_TAGS` is not defined (the default for standalone builds), test Each test case in the YAML config is assigned a level (`level_0` through `level_4`) that controls test parameters such as memory sizes, block sizes, iteration counts, and warmups. Level definitions live in `catch/config/configs/definitions.yaml`. For example, `level_0` is a quick smoke test with small sizes, while `level_2` is a comprehensive run with sizes up to 2 GB. -When building through cmake, levels work automatically. The build system generates `hip_test_parameters.hh` from `definitions.yaml`, and the event listener in `hip_test_listener.cc` detects the active level from ctest tags (e.g., `ctest -L level_0`) and loads the corresponding parameters into `TestParameterStore`. +When building through cmake, levels work automatically. The build system generates `hip_test_parameters.hh` from `definitions.yaml`. The level is resolved from a strict priority order: an explicit `[level_N]` command-line filter (e.g., `./test "[level_0]"`), then the `HIP_TEST_LEVEL` environment variable, and finally the hardcoded default `level_2`. The resolved level's parameters are loaded into `TestParameterStore` by the event listener in `hip_test_level.cc`'s companion `hip_test_listener.cc`. -For standalone builds, levels are an opt-in feature that requires compiling additional source files from `catch/hipTestMain/`: +Only an *explicitly requested* level also narrows which test cases run, and that narrowing happens in `main()` rather than the listener, because Catch2 has already parsed the test spec by the time any listener event fires. The hardcoded default never narrows: a bare `./test` and plain `ctest` run every test case in the binary and simply use `level_2` parameters. + +For standalone builds levels remain opt-in, and require compiling additional source files from `catch/hipTestMain/`. Add `hip_test_features.cc` as well if the test under build needs feature detection: ```bash amdclang++ \ -I $ROCM_SYSTEMS_DIR/projects/hip-tests/catch/external/picojson \ -I $ROCM_SYSTEMS_DIR/projects/hip-tests/catch/include \ + -I /catch_tests/include \ -x hip $ROCM_SYSTEMS_DIR/projects/hip-tests/catch/hipTestMain/main.cc \ -x hip $ROCM_SYSTEMS_DIR/projects/hip-tests/catch/hipTestMain/hip_test_context.cc \ -x hip $ROCM_SYSTEMS_DIR/projects/hip-tests/catch/hipTestMain/hip_test_features.cc \ -x hip $ROCM_SYSTEMS_DIR/projects/hip-tests/catch/hipTestMain/hip_test_params.cc \ -x hip $ROCM_SYSTEMS_DIR/projects/hip-tests/catch/hipTestMain/hip_test_listener.cc \ + -x hip $ROCM_SYSTEMS_DIR/projects/hip-tests/catch/hipTestMain/hip_test_level.cc \ -L /usr/local/lib \ --offload-arch=native \ -lCatch2 \ -x hip ``` -This requires `hip_test_parameters.hh` to have been generated by a prior cmake build (it lives in the build directory). Without these additional files, standalone builds still work but all tests use hardcoded default parameters regardless of level. +This requires `hip_test_parameters.hh` to have been generated by a prior cmake build (it lives in the build directory, under `catch_tests/include`). Without these additional files, standalone builds still work but all tests use hardcoded default parameters regardless of level. + +Note that level-based *selection* additionally needs `-DENABLE_YAML_TAGS`, since it is compiled out of `main()` otherwise and the test cases carry no tags to match against. Parameters work without it. ### Building with address sanitizer diff --git a/projects/hip-tests/TESTING.md b/projects/hip-tests/TESTING.md index a00f9c508a9e..50eebf131bc7 100644 --- a/projects/hip-tests/TESTING.md +++ b/projects/hip-tests/TESTING.md @@ -17,7 +17,7 @@ This document does not define testing strategy for ROCm math libraries, communic ## Development workflow -All tests are C++ built on Catch2 and discovered and run through CTest. Test executables are grouped by CMake `BUILD_*` options and by CTest labels generated from Catch2/YAML tags. Workload size is parameterized by level, selected at runtime with `HIP_TEST_LEVEL=level_N` or a Catch2 filter such as `[level_0]`. +All tests are C++ built on Catch2 and discovered and run through CTest. Test executables are grouped by CMake `BUILD_*` options and by CTest labels generated from Catch2/YAML tags. Workload size is parameterized by level. Explicitly setting `HIP_TEST_LEVEL=level_N` (or a Catch2 filter such as `[level_0]`) both selects which tests run and their parameters; left unset, the default level (`level_2`) supplies parameters only and leaves test selection untouched. | You changed | Run this before pushing | Needs GPU | |---|---|---| @@ -46,7 +46,7 @@ Contract tests live in `catch/contract/` and are built ON by default through `BU ### Unit and functional testing -Unit and functional tests live under `catch/unit/` feature-area directories and are built ON by default through `BUILD_UNIT_TESTS`. They validate functional correctness of individual HIP APIs and features across many feature areas, including memory, stream, graph, module, texture, cooperative groups, peer-to-peer, RTC, and virtual memory: arguments accepted or rejected, values returned, and observable side effects such as data movement and synchronization. Tests use the in-tree `HIP_CHECK` and `HIP_ASSERT` helpers, are discovered with `catch_discover_tests`, and run under CTest as one process per case. Workload size is parameterized by level and selected at runtime with `HIP_TEST_LEVEL=level_N` or a Catch2 filter such as `[level_0]`; `catch/config/configs/definitions.yaml` holds the generated per-level parameters and `catch/README.md` is the canonical reference for level semantics. +Unit and functional tests live under `catch/unit/` feature-area directories and are built ON by default through `BUILD_UNIT_TESTS`. They validate functional correctness of individual HIP APIs and features across many feature areas, including memory, stream, graph, module, texture, cooperative groups, peer-to-peer, RTC, and virtual memory: arguments accepted or rejected, values returned, and observable side effects such as data movement and synchronization. Tests use the in-tree `HIP_CHECK` and `HIP_ASSERT` helpers, are discovered with `catch_discover_tests`, and run under CTest as one process per case. Workload size is parameterized by level. Explicitly setting `HIP_TEST_LEVEL=level_N` (or a Catch2 filter such as `[level_0]`) both selects which tests run and their parameters, while the unset default only sets parameters; `catch/config/configs/definitions.yaml` holds the generated per-level parameters and `catch/README.md` is the canonical reference for level semantics. ### Integration testing @@ -81,7 +81,7 @@ When you change public HIP API behavior, run the contract tests for the touched A flaky or quarantined case is treated as unstable signal: it does not block merge, but it must have an owner, a tracking issue, and a removal or fix plan before it can be promoted back to trusted signal. Use `HIP_SKIP_TEST` for a genuine capability gap such as an unsupported device or runtime path, too few GPUs, or no image support, so the case skips cleanly instead of failing. Never leave a test that crashes the process. Any test that intentionally triggers a HIP error must consume the sticky thread-local error with `(void)hipGetLastError()` before returning so it does not leak into later cases in the same process. -Disabling a test case should be rare and explicit. It is appropriate when a test cannot execute on a platform or architecture because a required capability, OS facility, graphics environment, or runtime mode is absent; when a known product or infrastructure defect has a tracking issue; or when keeping the test enabled would crash the process and hide later signal. Prefer runtime capability checks plus `HIP_SKIP_TEST` for environment-dependent behavior, and use YAML `disabled:` entries only when a case should be hidden for a specific platform, architecture, sanitizer mode, or tracked known issue. Do not disable a test only to make CI green. Every disabled or quarantined case should record why it is disabled, where it is tracked, and what condition allows it to be re-enabled. +Disabling a test case should be rare and explicit. It is appropriate when a test cannot execute on a platform or architecture because a required capability, OS facility, graphics environment, or runtime mode is absent; when a known product or infrastructure defect has a tracking issue; or when keeping the test enabled would crash the process and hide later signal. Prefer runtime capability checks plus `HIP_SKIP_TEST` for environment-dependent behavior, and use the YAML skip fields only when a case should be hidden for a specific platform, architecture, sanitizer mode, or tracked known issue. The `disabled:` and `unsupported:` fields share the same syntax and produce the same skip; they differ only in intent. Do not disable a test only to make CI green. Every skipped case should record why it is skipped, where it is tracked when a `disabled:` entry points at a bug. ## Coverage @@ -91,7 +91,7 @@ Code coverage and test coverage are different signals. Code coverage measures wh ## Nightly validation -Nightly and multi-arch runs extend the per-PR build-and-test flow across a broader architecture set and longer-running suites. Today, if `HIP_TEST_LEVEL` is unset, the listener infers the active level from the first generated `[level_N]` tag it sees, unless a scheduled or on-demand workflow explicitly overrides the level. The CI handling of levels is expected to change when the pending level-selection workflow update in ROCm/rocm-systems#8932 lands, so this section should be reviewed in the same PR or immediately after that merge. On-demand runs such as multi-arch sweeps, WSL runtime checks, and performance or stress runs are invoked explicitly rather than on every PR. CI level cadence is not yet fully wired to `HIP_TEST_LEVEL`. +Nightly and multi-arch runs extend the per-PR build-and-test flow across a broader architecture set and longer-running suites. If `HIP_TEST_LEVEL` is unset, `main()` falls back to the default level (`level_2`) for workload parameters and leaves test selection alone. Setting `HIP_TEST_LEVEL=level_N` selects the tests as well as their parameters — under CTest the tests outside that level are reported as skipped — and is the single dial for level cadence. On-demand runs such as multi-arch sweeps, WSL runtime checks, and performance or stress runs are invoked explicitly rather than on every PR. CI level cadence is not yet fully wired to `HIP_TEST_LEVEL`. ## Supported configurations diff --git a/projects/hip-tests/catch/README.md b/projects/hip-tests/catch/README.md index 0b04351721d5..d0e5800e68f9 100644 --- a/projects/hip-tests/catch/README.md +++ b/projects/hip-tests/catch/README.md @@ -69,7 +69,9 @@ All configuration for the test cases is done through the `hip_tests_config.yaml` Every test case has its own entry. Currently supported options are: - level : Specify to which level the case belongs to (e.g. Level_2 is a standard test) - tags : List all Catch2 tags that the case is associated with -- disabled : List all platforms where the case should be disabled +- disabled : Temporarily skip the case on the listed targets (a regression or tracked bug that is expected to be re-enabled) +- unsupported : Permanently skip the case on the listed targets, where the platform, architecture, or backend cannot support it +`disabled` and `unsupported` share the same syntax and produce the same skip; they differ only in intent (temporary regression versus permanent capability gap). The group name is automatically added as a tag for every case. Changing the configuration file will retrigger the build, so we have an up to date configuration every time. @@ -80,8 +82,12 @@ unit: Unit_atomicExch_system_Positive_Peer_GPUs: <<: *level_2 tags: [multigpu] - # SWDEV-435667: Below tests failing randomly in stress test on 01/12/23 - disabled: [amd_wsl] + disabled: + targets: [amd_wsl] + reason: "SWDEV-435667: fails randomly in stress test" + unsupported: + targets: [gfx900] + reason: requires a hardware feature not present on this architecture ``` will be generated (on an AMD linux machine) as: ```cpp @@ -112,17 +118,45 @@ Tests are assigned a level in the YAML configuration (`level: 0` through `level: ### Running at a specific level -Use the `HIP_TEST_LEVEL` environment variable: - -```bash -HIP_TEST_LEVEL=level_0 ./MemoryTest1 "Unit_hipMemsetSync" -``` - -Or use Catch2 tag filters: - -```bash -./MemoryTest1 "[level_0]" -``` +The active level always supplies the workload parameters. It selects which +tests run **only when it was requested explicitly**. The level is resolved once +per process, in strict priority order: + +1. A Catch2 tag filter on the command line: `./MemoryTest1 "[level_0]"` +2. The `HIP_TEST_LEVEL` environment variable: `HIP_TEST_LEVEL=level_0 ./MemoryTest1` +3. The default, `level_2` + +| Invocation | Parameters | Tests that run | +|---|---|---| +| `./MemoryTest1` | `level_2` (default) | every test in the binary | +| `./MemoryTest1 Unit_hipMemsetSync` | `level_2` (default) | `Unit_hipMemsetSync` | +| `./MemoryTest1 "[level_0]"` | `level_0` | only `[level_0]`-tagged | +| `HIP_TEST_LEVEL=level_0 ./MemoryTest1` | `level_0` | only `[level_0]`-tagged | +| `HIP_TEST_LEVEL=level_0 ./MemoryTest1 Unit_hipMemsetSync` | `level_0` | `Unit_hipMemsetSync` if it is tagged `level_0`, otherwise nothing | +| `HIP_TEST_LEVEL=level_0,level_2 ./MemoryTest1` | `level_2` (highest requested) | `[level_0]` **or** `[level_2]` | +| `HIP_TEST_LEVEL=~level_4 ./MemoryTest1` | `level_3` (highest remaining) | `level_0` through `level_3` | +| `ctest` | `level_2` (default) | every test | +| `HIP_TEST_LEVEL=level_0 ctest` | `level_0` | `level_0` tests run, the rest reported **Skipped** | +| `ctest -L level_0` | `level_2` (default) | the `level_0`-labelled entries, on default parameters | +| `ctest -N`, `--list-tests`, `--help` | — | no level is resolved at all | + +Two rules explain every row. First, naming a test is itself a statement about +what to run, so a merely-default level never overrides it — which is also why +plain `ctest` behaves exactly as it always has, since every ctest entry names a +test. Second, when several levels are requested at once, **all** of them run +while the **highest** supplies the parameters: there is one parameter store per +process, so one level has to win, but nothing that was asked for is dropped +from the run. + +An explicit request that matches nothing is not a failure: the process exits +with code 4 ("all tests skipped"), and CTest reports such entries as **Skipped** +rather than **Failed**. Given that levels partition rather than nest, this is +how `HIP_TEST_LEVEL=level_0 ctest` leaves every non-`level_0` entry. + +> `ctest -L level_0` selects the level_0-labelled entries but still runs them +> with the default `level_2` parameters — `-L` is consumed by ctest and never +> reaches the test binary. Use `HIP_TEST_LEVEL=level_0 ctest`, which selects and +> parameterises in one go and needs no `-L`. ### Writing level-aware tests with `isQuickLevel()` @@ -140,7 +174,7 @@ HIP_TEST_CASE(Unit_hipMemsetSync) { #### Guidelines for `isQuickLevel()` trimming - **Prefer reducing iteration counts and buffer sizes** over skipping code paths, so the same API surface is exercised with less data. -- **Call `isQuickLevel()` inside the test function**, not in global/static variable initializers. The test level is detected at runtime by the Catch2 listener, which runs after static initialization. +- **Call `isQuickLevel()` inside the test function**, not in global/static variable initializers. The test level is resolved in `main()` before any test runs, which is after static initialization. - **Use reproducible seeds** when tests involve randomization. Log the seed and allow overriding (e.g., via `HIP_TEST_SEED`) so failures can be replayed. #### Example: reducing a memset size diff --git a/projects/hip-tests/catch/TEST_PLAN.md b/projects/hip-tests/catch/TEST_PLAN.md index f61d50ad1589..b5fdf71e1195 100644 --- a/projects/hip-tests/catch/TEST_PLAN.md +++ b/projects/hip-tests/catch/TEST_PLAN.md @@ -13,8 +13,8 @@ Each row is one `HIP_TEST_CASE`. The API and invariant come from the `// @assert | Tier | Cases | Tagged | Missing `@asserts` | |---|---:|---:|---:| -| `contract` | 612 | 612 | 0 | -| **total** | **612** | **612** | **0** | +| `contract` | 616 | 616 | 0 | +| **total** | **616** | **616** | **0** | ## Tier: `contract` @@ -135,10 +135,14 @@ Each row is one `HIP_TEST_CASE`. The API and invariant come from the `// @assert | `Contract_Copy3D_HipMemcpy3D_Default_HostDeviceRoundTripsExtent` | hipMemcpy3D | a full-extent H2D then D2H 3D copy round-trips all bytes unchanged | | `Contract_Copy3D_HipMemcpy3D_Default_SingleSliceRoundTripsBytes` | hipMemcpy3D | a single-slice (depth=1) H2D then D2H 3D copy round-trips that slice's bytes | -### `device` (7 cases) +### `device` (11 cases) | Case | API | Asserts | |---|---|---| +| `Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_InvalidScope_IsRejected` | hipDeviceFlushGPUDirectRDMAWrites | an out-of-range scope is rejected as an invalid argument | +| `Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_InvalidTarget_IsRejected` | hipDeviceFlushGPUDirectRDMAWrites | an out-of-range target is rejected as an invalid argument | +| `Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_ToAllDevices_AcceptedOrUnsupported` | hipDeviceFlushGPUDirectRDMAWrites | a flush to all-devices scope is accepted or reports unsupported | +| `Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_ToOwner_AcceptedOrUnsupported` | hipDeviceFlushGPUDirectRDMAWrites | a flush to owner scope is accepted or reports unsupported | | `Contract_Device_HipDeviceGetAttribute_WarpSize_MatchesProperties` | hipDeviceGetAttribute | hipDeviceAttributeWarpSize matches the warp size from hipGetDeviceProperties | | `Contract_Device_HipGetDeviceProperties_GetProperties_SucceedsForCurrentDevice` | hipGetDeviceProperties | succeeds in populating properties for the current device | | `Contract_Device_HipGetDeviceProperties_MultiProcessorCount_IsPositive` | hipGetDeviceProperties | reported multiprocessor count is positive | diff --git a/projects/hip-tests/catch/cmake/hip-tests.cmake b/projects/hip-tests/catch/cmake/hip-tests.cmake index 4b0fb46f179b..bf12bb42e4e2 100644 --- a/projects/hip-tests/catch/cmake/hip-tests.cmake +++ b/projects/hip-tests/catch/cmake/hip-tests.cmake @@ -124,7 +124,12 @@ function(hip_gen_exe_target) endforeach() # add binary to global list of binaries to install set_property(GLOBAL APPEND PROPERTY G_INSTALL_EXE_TARGETS ${_EXE_NAME}) - set(_DISCOVER_PROPERTIES "") + # Catch2 already defaults discovered tests to SKIP_RETURN_CODE 4 + # (extras/Catch.cmake) unless SKIP_IS_FAILURE is passed. Restated + # here because main() deliberately returns 4 when the active level + # matches none of the requested tests - this is the contract that + # makes those entries report as skipped rather than failed. + set(_DISCOVER_PROPERTIES SKIP_RETURN_CODE 4) if (DEFINED HIP_TEST_LABELS) list(APPEND _DISCOVER_PROPERTIES LABELS "${HIP_TEST_LABELS}") endif() diff --git a/projects/hip-tests/catch/config/check_config.py b/projects/hip-tests/catch/config/check_config.py index a3a5520baf34..838e859f650d 100644 --- a/projects/hip-tests/catch/config/check_config.py +++ b/projects/hip-tests/catch/config/check_config.py @@ -26,11 +26,29 @@ def main(): configs_path = args.configs_path missing = [] + invalid_skip_fields = [] for group, cases in iter_group_configs(configs_path): for case_name, case_config in cases.items(): if "level" not in case_config: missing.append(f" {group}/{case_name}") + for field in ("disabled", "unsupported"): + if field not in case_config: + continue + value = case_config[field] + # A skip field is either a flat list of targets, or a mapping + # with a 'targets' list plus an optional 'reason' scalar. The + # reason is metadata and is intentionally NOT enforced yet — + # deferred to a later phase (AIRUNTIME-2744): + # https://amd-hub.atlassian.net/browse/AIRUNTIME-2744 + # We only guard that the targets are a list, since a scalar + # breaks tag generation (a string is iterated char-by-char) and + # a malformed mapping fails the parser's list concatenation. + if isinstance(value, list): + continue + if isinstance(value, dict) and isinstance(value.get("targets"), list): + continue + invalid_skip_fields.append(f" {group}/{case_name}: '{field}'") if missing: print( @@ -41,6 +59,15 @@ def main(): print(f"[check_config] {ERROR}{entry}{RESET}", file=sys.stderr) sys.exit(1) + if invalid_skip_fields: + print( + f"[check_config] {ERROR}ERROR: The following test cases have a 'disabled'/'unsupported' field that is neither a YAML list nor a mapping with a 'targets' list:{RESET}", + file=sys.stderr, + ) + for entry in invalid_skip_fields: + print(f"[check_config] {ERROR}{entry}{RESET}", file=sys.stderr) + sys.exit(1) + if __name__ == "__main__": main() diff --git a/projects/hip-tests/catch/config/configs/contract.yaml b/projects/hip-tests/catch/config/configs/contract.yaml index 0b536f0cdd1c..8b2c610b6921 100644 --- a/projects/hip-tests/catch/config/configs/contract.yaml +++ b/projects/hip-tests/catch/config/configs/contract.yaml @@ -108,6 +108,18 @@ contract: Contract_Device_HipDeviceGetAttribute_WarpSize_MatchesProperties: <<: *level_0 tags: [device, P0] + Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_ToOwner_AcceptedOrUnsupported: + <<: *level_0 + tags: [device, P0] + Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_ToAllDevices_AcceptedOrUnsupported: + <<: *level_0 + tags: [device, P0] + Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_InvalidScope_IsRejected: + <<: *level_0 + tags: [device, P0] + Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_InvalidTarget_IsRejected: + <<: *level_0 + tags: [device, P0] Contract_Device_HipGetDevice_CurrentOrdinal_IsWithinDeviceCount: <<: *level_0 tags: [device, P0] @@ -311,19 +323,19 @@ contract: tags: [stream_cu_mask, P0] Contract_StreamAttach_HipStreamAttachMemAsync_ManagedOnCreatedStream_Succeeds: <<: *level_0 - tags: [stream_attach, P0] + tags: [stream_attach, P0, managed_memory] Contract_StreamAttach_HipStreamAttachMemAsync_NullStreamAttachGlobal_Succeeds: <<: *level_0 - tags: [stream_attach, P0] + tags: [stream_attach, P0, managed_memory] Contract_StreamAttach_HipStreamAttachMemAsync_NonZeroLengthAttachSingle_Succeeds: <<: *level_0 - tags: [stream_attach, P0] + tags: [stream_attach, P0, managed_memory] Contract_StreamAttach_HipStreamAttachMemAsync_NullDevPtr_IsRejected: <<: *level_0 tags: [stream_attach, P0] Contract_StreamAttach_HipStreamAttachMemAsync_NullStreamAttachSingle_IsRejected: <<: *level_0 - tags: [stream_attach, P0] + tags: [stream_attach, P0, managed_memory] Contract_AsyncTransfer_HipMemcpyAsync_HostToDeviceToHost_RoundTripsAfterStreamSynchronize: <<: *level_0 tags: [async_transfer, P0] @@ -359,16 +371,16 @@ contract: tags: [mem_batch_copy_3d, P0] Contract_MemBatchDiscard_HipMemDiscardBatchAsync_DiscardBatch_IsAcceptedOrUnsupported: <<: *level_0 - tags: [mem_batch_discard, P0] + tags: [mem_batch_discard, P0, managed_memory] Contract_MemBatchDiscard_HipMemDiscardAndPrefetchBatchAsync_DiscardAndPrefetchBatch_IsAcceptedOrUnsupported: <<: *level_0 - tags: [mem_batch_discard, P0] + tags: [mem_batch_discard, P0, managed_memory] Contract_MemBatchDiscard_HipDrvMemDiscardBatchAsync_DrvDiscardBatch_IsAcceptedOrUnsupported: <<: *level_0 - tags: [mem_batch_discard, P0] + tags: [mem_batch_discard, P0, managed_memory] Contract_MemBatchDiscard_HipDrvMemDiscardAndPrefetchBatchAsync_DrvDiscardAndPrefetchBatch_IsAcceptedOrUnsupported: <<: *level_0 - tags: [mem_batch_discard, P0] + tags: [mem_batch_discard, P0, managed_memory] Contract_MemBatchDiscard_HipMemDiscardBatchAsync_NullPointer_IsRejectedOrUnsupported: <<: *level_0 tags: [mem_batch_discard, P0] @@ -377,7 +389,7 @@ contract: tags: [mem_batch_discard, P0] Contract_MemBatchDiscard_HipMemPrefetchBatchAsync_PrefetchBatch_IsAcceptedOrUnsupported: <<: *level_0 - tags: [mem_batch_discard, P0] + tags: [mem_batch_discard, P0, managed_memory] Contract_Memset_HipMemset_DeviceBuffer_IsFilledWithBytePattern: <<: *level_0 tags: [memset, P0] @@ -1163,31 +1175,31 @@ contract: tags: [managed_memory, P0] Contract_MemAdvise_HipMemAdvise_SetReadMostly_RangeAttributeReflectsAdvice: <<: *level_0 - tags: [mem_advise, P0] + tags: [mem_advise, P0, managed_memory] Contract_MemAdvise_HipMemAdvise_SetPreferredLocation_RangeAttributeReturnsDevice: <<: *level_0 - tags: [mem_advise, P0] + tags: [mem_advise, P0, managed_memory] Contract_MemAdvise_HipMemAdvise_SetAccessedBy_RangeAttributeReflectsDevice: <<: *level_0 - tags: [mem_advise, P0] + tags: [mem_advise, P0, managed_memory] Contract_MemAdvise_HipMemRangeGetAttributes_MultipleAttributes_Succeed: <<: *level_0 - tags: [mem_advise, P0] + tags: [mem_advise, P0, managed_memory] Contract_MemAdvise_HipMemAdvise_NullPointer_IsRejected: <<: *level_0 tags: [mem_advise, P0] Contract_MemAdvise_HipMemRangeGetAttribute_NullData_IsRejected: <<: *level_0 - tags: [mem_advise, P0] + tags: [mem_advise, P0, managed_memory] Contract_MemAdviseV2_HipMemAdviseV2_SetReadMostly_IsAcceptedOrUnsupported: <<: *level_0 - tags: [mem_advise_v2, P0] + tags: [mem_advise_v2, P0, managed_memory] Contract_MemAdviseV2_HipMemAdviseV2_SetAndUnsetPreferredLocation_IsAcceptedOrUnsupported: <<: *level_0 - tags: [mem_advise_v2, P0] + tags: [mem_advise_v2, P0, managed_memory] Contract_MemAdviseV2_HipMemPrefetchAsyncV2_PrefetchAsync_IsAcceptedOrUnsupported: <<: *level_0 - tags: [mem_advise_v2, P0] + tags: [mem_advise_v2, P0, managed_memory] Contract_MemAdviseV2_HipMemAdviseV2_NullPointer_IsRejected: <<: *level_0 tags: [mem_advise_v2, P0] @@ -1199,7 +1211,7 @@ contract: tags: [pointer_info, P0] Contract_PointerInfo_HipPointerGetAttributes_ManagedAllocation_ReportsManagedOrUnified: <<: *level_0 - tags: [pointer_info, P0] + tags: [pointer_info, P0, managed_memory] Contract_PointerInfo_HipPointerGetAttribute_MemoryType_MatchesGetAttributes: <<: *level_0 tags: [pointer_info, P0] diff --git a/projects/hip-tests/catch/config/configs/unit/atomics.yaml b/projects/hip-tests/catch/config/configs/unit/atomics.yaml index f5059469ed4e..e95a343de212 100644 --- a/projects/hip-tests/catch/config/configs/unit/atomics.yaml +++ b/projects/hip-tests/catch/config/configs/unit/atomics.yaml @@ -222,105 +222,6 @@ atomics: Unit_unsafeAtomicMax_Positive_Multi_Kernel_Scattered_Addresses: <<: *level_2 tags: [deviceLib, ops] - Unit___hip_atomic_fetch_min_Positive_Wavefront_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_min_Positive_Wavefront_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_min_Positive_Wavefront_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_min_Positive_Workgroup_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_min_Positive_Workgroup_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_min_Positive_Workgroup_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_max_Positive_Wavefront_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_max_Positive_Wavefront_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_max_Positive_Wavefront_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_max_Positive_Workgroup_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_max_Positive_Workgroup_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_max_Positive_Workgroup_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit_AtomicBuiltins_Negative_Parameters_RTC: - <<: *level_2 - tags: [builtin, deviceLib] - Unit___hip_atomic_load_store_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_exchange_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_compare_exchange_strong_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_compare_exchange_weak_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_add_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_and_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_or_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_xor_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_min_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_max_Positive_Acquire_Release: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_load_store_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_exchange_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_compare_exchange_strong_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_compare_exchange_weak_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_add_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_and_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_or_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_xor_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_min_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] - Unit___hip_atomic_fetch_max_Positive_Sequential_Consistency: - <<: *level_2 - tags: [deviceLib, memorder] Unit_atomicAdd_Positive: <<: *level_2 tags: [deviceLib, ops] @@ -392,18 +293,6 @@ atomics: Unit_atomicCAS_system_Positive_Host_And_Peer_GPUs: <<: *level_2 tags: [deviceLib, system, multigpu] - Unit___hip_atomic_fetch_add_Positive_Wavefront: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_add_Positive_Workgroup: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_compare_exchange_strong_Positive_Wavefront: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_compare_exchange_strong_Positive_Workgroup: - <<: *level_2 - tags: [deviceLib, builtin] Unit_atomicExch_Positive_Same_Address_Compile_Time: <<: *level_2 # SWDEV-435667: Below tests failing randomly in stress test on 01/12/23 @@ -438,66 +327,6 @@ atomics: Unit_atomicExch_system_Negative_Parameters_RTC: <<: *level_2 tags: [deviceLib, system] - Unit___hip_atomic_fetch_and_Positive_Wavefront_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_and_Positive_Wavefront_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_and_Positive_Wavefront_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_and_Positive_Workgroup_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_and_Positive_Workgroup_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_and_Positive_Workgroup_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_or_Positive_Wavefront_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_or_Positive_Wavefront_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_or_Positive_Wavefront_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_or_Positive_Workgroup_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_or_Positive_Workgroup_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_or_Positive_Workgroup_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_xor_Positive_Wavefront_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_xor_Positive_Wavefront_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_xor_Positive_Wavefront_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_xor_Positive_Workgroup_SameAddress: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_xor_Positive_Workgroup_Adjacent_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_fetch_xor_Positive_Workgroup_Scattered_Addresses: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_exchange_Positive_Wavefront: - <<: *level_2 - tags: [deviceLib, builtin] - Unit___hip_atomic_exchange_Positive_Workgroup: - <<: *level_2 - tags: [deviceLib, builtin] Unit_atomicInc_Positive: <<: *level_2 tags: [deviceLib, ops] diff --git a/projects/hip-tests/catch/config/configs/unit/cluster.yaml b/projects/hip-tests/catch/config/configs/unit/cluster.yaml index 13d154c38c55..b5bc2676e578 100644 --- a/projects/hip-tests/catch/config/configs/unit/cluster.yaml +++ b/projects/hip-tests/catch/config/configs/unit/cluster.yaml @@ -8,5 +8,13 @@ cluster: disabled: [amd_linux, amd_windows, amd_wsl] Unit_hipClusterLaunch_CompilerDirective_Basic: <<: *level_2 + Unit_hipClusterLaunch_CompilerDirective_Negative_OversizedCluster: + <<: *level_2 + Unit_hipClusterLaunch_CompilerDirective_Negative_ZeroClusterDim: + <<: *level_2 + Unit_hipClusterLaunch_CompilerDirective_Negative_ZeroClusterDim_Graph: + <<: *level_2 + Unit_hipClusterLaunch_CompilerDirective_NoCluster: + <<: *level_2 Unit_hipClusterLaunch_LaunchApi_Basic: <<: *level_2 diff --git a/projects/hip-tests/catch/config/configs/unit/cooperativeGrps.yaml b/projects/hip-tests/catch/config/configs/unit/cooperativeGrps.yaml index 36101bc43a9d..52100497eaa0 100644 --- a/projects/hip-tests/catch/config/configs/unit/cooperativeGrps.yaml +++ b/projects/hip-tests/catch/config/configs/unit/cooperativeGrps.yaml @@ -303,6 +303,10 @@ cooperativeGrps: <<: *level_2 # Rock_Window_Failures_on_gfx1151 disabled: [amd_windows, amd_wsl] + Unit_Thread_Block_Tile_Scan_Random_Bool_Less_Greater: + <<: *level_2 + # Rock_Window_Failures_on_gfx1151 + disabled: [amd_windows, amd_wsl] Unit_Thread_Block_Tile_Scan_Random_boolean: <<: *level_2 # Rock_Window_Failures_on_gfx1151 diff --git a/projects/hip-tests/catch/config/configs/unit/device.yaml b/projects/hip-tests/catch/config/configs/unit/device.yaml index 762c3a1ea757..6b8b4b0bc96b 100644 --- a/projects/hip-tests/catch/config/configs/unit/device.yaml +++ b/projects/hip-tests/catch/config/configs/unit/device.yaml @@ -6,6 +6,48 @@ device: Unit_hipChooseDevice_NegTst: <<: *level_2 tags: [query, P0] + Unit_hipDeviceFlushGPUDirectRDMAWrites_AttributeDomains: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_AttributesMatchProperties: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_CapabilityConsistency: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_Basic: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_Repeated: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_MultiDevice: + <<: *level_2 + tags: [multigpu, query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_MultiThreaded: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_DoesNotSynchronize: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_PayloadVisibleToKernel: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_AfterDeviceReset: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_NotCapturedByGraph: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Negative_Parameters: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Negative_FailureSetsLastErrorOnly: + <<: *level_2 + tags: [query] + Unit_hipDeviceFlushGPUDirectRDMAWrites_Negative_AttributeQuery: + <<: *level_2 + tags: [query] Unit_hipDeviceComputeCapability_Negative: <<: *level_2 tags: [query, P0] diff --git a/projects/hip-tests/catch/config/configs/unit/deviceLib.yaml b/projects/hip-tests/catch/config/configs/unit/deviceLib.yaml index 8d1369a75c8b..7ec2e8b51934 100644 --- a/projects/hip-tests/catch/config/configs/unit/deviceLib.yaml +++ b/projects/hip-tests/catch/config/configs/unit/deviceLib.yaml @@ -304,6 +304,12 @@ deviceLib: Unit_half_min_max_host: <<: *level_2 tags: [types] + Unit_hipTestHalfIntegralAssign_Host: + <<: *level_2 + tags: [types] + Unit_hipTestHalfIntegralAssign_Device: + <<: *level_2 + tags: [types] Unit_bf16_atomic_add: <<: *level_2 tags: [atomics, types] diff --git a/projects/hip-tests/catch/config/configs/unit/graph.yaml b/projects/hip-tests/catch/config/configs/unit/graph.yaml index 0adee952ce21..16c2951506e1 100644 --- a/projects/hip-tests/catch/config/configs/unit/graph.yaml +++ b/projects/hip-tests/catch/config/configs/unit/graph.yaml @@ -724,6 +724,48 @@ graph: Unit_hipStreamBeginCapture_Positive_SelfWaitOnForkedStream: <<: *level_2 tags: [capture] + Unit_hipStreamBeginCapture_Positive_SelfWaitWithInterveningWork: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Positive_DestroyForkedStreamDuringCapture: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Positive_DestroyOriginStreamDuringCapture: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Negative_SelfWaitOnInvalidatedForkedStream: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Positive_CycleAmongForkedStreams: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Negative_CrossCaptureWaitIsRejected: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Negative_BeginCaptureOnInvalidatedStream: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Positive_DestroyNestedForkDuringCapture: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Positive_ConcurrentForkIntoOneCapture: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Positive_ConcurrentWaitsAccumulateDependencies: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Negative_WaitDoesNotJoinInvalidatedCapture: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Negative_StreamOffenceInvalidatesWholeCapture: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Negative_GlobalOffenceInvalidatesWholeCapture: + <<: *level_2 + tags: [capture] + Unit_hipStreamBeginCapture_Negative_InvalidationReachesSiblingForks: + <<: *level_2 + tags: [capture] Unit_hipGraphAddMemcpyNode1D_Functional: <<: *level_2 tags: [node] @@ -936,6 +978,9 @@ graph: Unit_hipStreamBeginCaptureToGraph_EndingWhileCaptureInProgress: <<: *level_2 tags: [capture] + Unit_hipStreamBeginCaptureToGraph_Positive_CaptureTeardownDestroysGraph: + <<: *level_2 + tags: [capture] Unit_hipStreamBeginCaptureToGraph_MultipleFlags: <<: *level_2 tags: [capture] @@ -1098,7 +1143,7 @@ graph: tags: [node] Unit_hipGraphAddMemsetNode_hipMallocManaged: <<: *level_2 - tags: [node] + tags: [node, managed_memory] Unit_hipGraphAddKernelNode_Negative: <<: *level_2 tags: [node, P0] @@ -1309,7 +1354,7 @@ graph: tags: [capture] Unit_hipLaunchHostFunc_KernelWritesManagedMem_HostNodeReads: <<: *level_2 - tags: [capture] + tags: [capture, managed_memory] Unit_hipUserObjectCreate_Functional_1: <<: *level_2 tags: [userobj] @@ -1573,6 +1618,26 @@ graph: Unit_hipGraphKernelNodeSetAttribute_Negative_Parameters: <<: *level_2 tags: [node] + Unit_hipGraphKernelNodeSetAttribute_Positive_Priority: + <<: *level_2 + tags: [node] + Unit_hipGraphKernelNodeSetAttribute_Negative_Priority_OutOfRange: + <<: *level_2 + tags: [node] + Unit_hipGraphKernelNodePriority_CaptureStreamPriorityCopied: + <<: *level_2 + tags: [node] + Unit_hipGraphKernelNodeCopyAttributes_MultiAttr: + <<: *level_2 + tags: [node] + Unit_hipGraphKernelNodePriority_ForkJoinWithFlag: + <<: *level_2 + tags: [node] + disabled: [amd_windows] + Unit_hipGraphKernelNodePriority_ChildGraphWithFlag: + <<: *level_2 + tags: [node] + disabled: [amd_windows] Unit_hipGraphMem_Alloc_Free_NodeGetParams_Functional: <<: *level_2 tags: [query] @@ -1780,7 +1845,7 @@ graph: <<: *level_2 # Below tests are failing PSDB disabled: [amd_windows, amd_wsl] - tags: [node] + tags: [node, managed_memory] Unit_hipDrvGraphAddMemcpyNode_Negative: <<: *level_2 tags: [node] diff --git a/projects/hip-tests/catch/config/configs/unit/library.yaml b/projects/hip-tests/catch/config/configs/unit/library.yaml index 5877c1d90db6..074edfc6f87d 100644 --- a/projects/hip-tests/catch/config/configs/unit/library.yaml +++ b/projects/hip-tests/catch/config/configs/unit/library.yaml @@ -43,13 +43,13 @@ library: tags: [module] Unit_hipLibraryGetManaged_Negative: <<: *level_2 - tags: [module] + tags: [module, managed_memory] Unit_hipLibraryGetManaged_Positive: <<: *level_2 - tags: [module] + tags: [module, managed_memory] Unit_hipLibraryGetManaged_HipMemcpyInterop: <<: *level_2 - tags: [module] + tags: [module, managed_memory] Unit_hipLibraryGetKernel_CountAndEnumerate: <<: *level_2 tags: [module] @@ -67,10 +67,10 @@ library: tags: [module] Unit_hipLibraryGetManaged_CO_Negative: <<: *level_2 - tags: [module] + tags: [module, managed_memory] Unit_hipLibraryGetManaged_CO_Values: <<: *level_2 - tags: [module] + tags: [module, managed_memory] Unit_hipLibraryEnumerateKernels_ZeroMax: <<: *level_2 tags: [module] diff --git a/projects/hip-tests/catch/config/configs/unit/memory.yaml b/projects/hip-tests/catch/config/configs/unit/memory.yaml index 791f48d3d897..8fb2de9e241a 100644 --- a/projects/hip-tests/catch/config/configs/unit/memory.yaml +++ b/projects/hip-tests/catch/config/configs/unit/memory.yaml @@ -187,7 +187,7 @@ memory: disabled: [amd_windows, amd_wsl] Unit_hipMemcpy2D_H2D_D2D_D2H_Managed_WithOffset: <<: *level_2 - tags: [P1] + tags: [P1, managed_memory] # Below tests are failing PSDB disabled: [amd_windows, amd_wsl] Unit_hipMemcpy2D_multiDevice_Basic_Size_Test: @@ -495,35 +495,35 @@ memory: tags: [alloc] Unit_hipMallocManaged_HostDeviceConcurrent: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_MultiChunkSingleDevice: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_MultiChunkMultiDevice: <<: *level_2 - tags: [alloc, multigpu, P1] + tags: [alloc, multigpu, P1, managed_memory] Unit_hipMallocManaged_OverSubscription: <<: *level_2 # Huge memory usage, no point enabling it with ASAN disabled: [asan] - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_Negative: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_TwoPointers: <<: *level_2 - tags: [P1] + tags: [P1, managed_memory] Unit_hipMallocManaged_DeviceContextChange: <<: *level_2 - tags: [multigpu, P1] + tags: [multigpu, P1, managed_memory] # Note: Windows disabled disabled: [amd_windows, amd_wsl] Unit_hipManagedMultiGpuVaConsistency_MultiDevice: <<: *level_2 - tags: [alloc, multigpu] + tags: [alloc, multigpu, managed_memory] Unit_hipManagedMultiGpuVaConsistency_HostAllocNoDivergence: <<: *level_2 - tags: [alloc, multigpu] + tags: [alloc, multigpu, managed_memory] Unit_hipMemset_Negative_InvalidPtr: <<: *level_2 tags: [memset, P1] @@ -664,104 +664,104 @@ memory: tags: [memset, P1] Unit_hipMallocManaged_FlgParam: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_AccessMultiStream: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMemPrefetchAsyncAdviseFlgTst: <<: *level_2 - tags: [multigpu, query, P1] + tags: [multigpu, query, P1, managed_memory] Unit_hipMemPrefetchAsyncAccsdByTst: <<: *level_2 - tags: [multigpu, query, P1] + tags: [multigpu, query, P1, managed_memory] Unit_hipMemPrefetchAsyncNegativeTst: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemPrefetchAsync_NonPageSz: <<: *level_2 # Disabling below tests temporarily due to change in API behavior disabled: [amd_windows, amd_wsl, amd_linux] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_MmapMem: <<: *level_2 tags: [query, P1] Unit_hipMallocManaged_Basic: <<: *level_2 - tags: [alloc, smoke, P0] + tags: [alloc, smoke, P0, managed_memory] Unit_hipMallocManaged_Advanced: <<: *level_2 # Note: Windows disabled disabled: [amd_windows, amd_wsl] - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_Large: <<: *level_2 # Huge memory usage, no point enabling it with ASAN disabled: [asan] - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_ReadMostly_Basic: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_ReadMostly_Partial_Range: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_PreferredLocation_Basic: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_PreferredLocation_CPU: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_PreferredLocation_Partial_Range: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_LastPrefetchLocation_Basic: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_LastPrefetchLocation_CPU: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_LastPrefetchLocation_Partial_Range: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_AccessedBy_Basic: <<: *level_2 # NOTE: The following 2 tests are disabled due to defect - EXSWHTEC-238 disabled: [amd_linux, amd_wsl] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_AccessedBy_Partial_Range: <<: *level_2 # NOTE: The following 2 tests are disabled due to defect - EXSWHTEC-238 disabled: [amd_linux, amd_wsl] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Positive_AccessedBy_MultiDevice: <<: *level_2 - tags: [multigpu, query, P1] + tags: [multigpu, query, P1, managed_memory] Unit_hipMemRangeGetAttribute_Negative_Parameters: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_TstCountParam: <<: *level_2 # Huge memory usage, no point enabling it with ASAN disabled: [asan] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_AccessedBy1: <<: *level_2 disabled: [amd_windows, amd_wsl] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttribute_4: <<: *level_2 disabled: [amd_windows, amd_wsl] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipHostMalloc_CoherentTst: <<: *level_2 tags: [query, P0] Unit_hipMallocManaged_CoherentTst: <<: *level_2 - tags: [query, P0] + tags: [query, P0, managed_memory] Unit_hipMallocManaged_CoherentTstWthAdvise: <<: *level_2 # Note: Windows disabled disabled: [amd_windows, amd_wsl] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMalloc_CoherentTst: <<: *level_2 # Note: Windows disabled @@ -1052,7 +1052,7 @@ memory: tags: [query, P1] Unit_hipPointerGetAttributes_GpuIter_Managed__Memory: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipPointerGetAttributes_GpuIter_Unregistered_Memory: <<: *level_2 tags: [query, P1] @@ -1072,22 +1072,22 @@ memory: <<: *level_2 # Does not play well with ASAN disabled: [asan] - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_MGpuMThread: <<: *level_2 - tags: [alloc, multigpu, P1] + tags: [alloc, multigpu, P1, managed_memory] Unit_hipMallocManaged_MultiKrnlComnHmm: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_MultiKrnlComnMalloc: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_MultiThrdMultiStrm: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMallocManaged_TwoKrnlsComnHmmMem: <<: *level_2 - tags: [alloc, P1] + tags: [alloc, P1, managed_memory] Unit_hipMemVmm_Basic: <<: *level_2 # SWDEV-396616 hipMemMap returns invalid error @@ -1175,7 +1175,7 @@ memory: tags: [P1] Unit_hipGetProcAddress_MemoryApisManagedMemory: <<: *level_2 - tags: [P1] + tags: [P1, managed_memory] Unit_hipGetProcAddress_MemoryApisStreamOrderedMemory: <<: *level_2 tags: [P1] @@ -1189,22 +1189,22 @@ memory: Unit_hipGetSymbolSize_Negative_Parameters: *level_2 Unit_hipMemPrefetchAsync_v2_Device_Host: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemPrefetchAsync_v2_HostNuma_HostNumaCurrent: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemPrefetchAsync_v2_Negative: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemAdvise_v2_Device_Host: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemAdvise_v2_HostNuma_HostNumaCurrent: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemAdvise_v2_Negative: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipPointerSetAttribute_Positive_SyncMemops: <<: *level_2 # Below test is disabled due to defect EXSWHTEC-347 @@ -2010,7 +2010,7 @@ memory: tags: [query, image, P0] Unit_hipPointerGetAttribute_DeferredManagedVar: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipPointerGetAttribute_KernelUpdation: <<: *level_2 tags: [query, P1] @@ -2044,60 +2044,88 @@ memory: tags: [query, P1] Unit_hipMemPrefetchAsync_Basic_AllDevices: <<: *level_2 - tags: [] + tags: [managed_memory] Unit_hipMemPrefetchAsync_Sync_Behavior: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemPrefetchAsync_Rounding_Behavior: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemPrefetchAsync_Negative_Parameters: <<: *level_2 - tags: [query, P1] - Unit_hipMemPrefetchBatchAsync_SingleOperationSingleLocation: + tags: [query, P1, managed_memory] + Unit_hipMemPrefetchBatchAsync_Sync_Behavior: <<: *level_2 tags: [query] + Unit_hipMemPrefetchBatchAsync_SingleOperationSingleLocation: + <<: *level_2 + tags: [query, managed_memory] Unit_hipMemPrefetchBatchAsync_LocationDistribution: <<: *level_2 disabled: [amd_windows, amd_wsl] - tags: [query] + tags: [query, managed_memory] Unit_hipMemPrefetchBatchAsync_RoundTripDataIntegrity: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemPrefetchBatchAsync_Negative_NullAndInvalidPointers: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemPrefetchBatchAsync_Negative_IndexArrayConstraints: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemPrefetchBatchAsync_Negative_ParameterValidation: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemPrefetchBatchAsync_Negative_DeviceCapabilities: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemPrefetchBatchAsync_MultiDevice: <<: *level_2 - tags: [query, multigpu] + tags: [query, multigpu, managed_memory] Unit_hipMemPrefetchBatchAsync_ManagedGlobalVariable: <<: *level_2 - tags: [query] + tags: [query, managed_memory] Unit_hipMemDiscardBatchAsync_NegativeTests: <<: *level_2 - tags: [query] - Unit_hipMemDiscardBatchAsync_SingleDiscard: *level_2 - Unit_hipMemDiscardBatchAsync_MultipleDiscards: *level_2 - Unit_hipMemDiscardBatchAsync_LargeCount: *level_2 - Unit_hipMemDiscardBatchAsync_DiscardThenPrefetch: *level_2 - Unit_hipMemDiscardBatchAsync_StreamOrdering: *level_2 - Unit_hipMemDiscardBatchAsync_SubrangeDiscard: *level_2 - Unit_hipMemDiscardBatchAsync_SamePointerTwice: *level_2 - Unit_hipMemDiscardBatchAsync_MinimalSize: *level_2 - Unit_hipMemDiscardBatchAsync_DrvApiParity: *level_2 - Unit_hipMemDiscardBatchAsync_ContentAfterDiscard: *level_2 + tags: [query, managed_memory] + Unit_hipMemDiscardBatchAsync_SingleDiscard: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_MultipleDiscards: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_LargeCount: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_DiscardThenPrefetch: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_StreamOrdering: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_Sync_Behavior: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_SubrangeDiscard: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_SamePointerTwice: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_MinimalSize: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_DrvApiParity: + <<: *level_2 + tags: [managed_memory] + Unit_hipMemDiscardBatchAsync_ContentAfterDiscard: + <<: *level_2 + tags: [managed_memory] Unit_hipMemDiscardBatchAsync_PageableDeviceMemory: *level_2 Unit_hipMemDiscardBatchAsync_PageableSystemMemory: *level_2 - Unit_hipMemDiscardBatchAsync_PageableMixedBatch: *level_2 + Unit_hipMemDiscardBatchAsync_PageableMixedBatch: + <<: *level_2 + tags: [managed_memory] Unit_hipMemDiscardAndPrefetchBatchAsync_NegativeTests: <<: *level_2 tags: [query] @@ -2227,19 +2255,19 @@ memory: tags: [memset, P1] Unit_hipMemAdvise_TstAccessedByPeer: <<: *level_2 - tags: [multigpu, query, P1] + tags: [multigpu, query, P1, managed_memory] Unit_hipMemAdvise_TstAccessedByFlg2: <<: *level_2 - tags: [query, multigpu, P1] + tags: [query, multigpu, P1, managed_memory] Unit_hipMemAdvise_TstAccessedByFlg3: <<: *level_2 - tags: [query, multigpu, P1] + tags: [query, multigpu, P1, managed_memory] Unit_hipMemAdvise_TstAccessedByFlg4: <<: *level_2 # Note: Windows disabled # AIRUNTIME-2291: temporarily disabled (hangs in CI 942) disabled: [amd_windows, amd_wsl, amd_linux] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_TstAlignedAllocMem: <<: *level_2 tags: [query, P1] @@ -2250,45 +2278,45 @@ memory: tags: [query, P1] Unit_hipMemAdvise_ReadMosltyMgpuTst: <<: *level_2 - tags: [multigpu, query, P1] + tags: [multigpu, query, P1, managed_memory] # Note: Windows disabled disabled: [amd_windows, amd_wsl] Unit_hipMemAdvise_Set_Unset_Basic: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_No_Flag_Interference: <<: *level_2 # (amd_linux) NOTE: The following test is disabled due to defect - EXSWHTEC-244 # (amd_windows) Note: intermittent Seg fault failure # (amd_wsl) Note: intermittent Seg fault failure disabled: [amd_linux, amd_windows, amd_wsl] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_Rounding: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_Flags_Do_Not_Cause_Prefetch: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_Read_Write_After_Advise: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_Prefetch_After_Advise: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_AccessedBy_All_Devices: <<: *level_2 # Note: intermittent Seg fault failure disabled: [amd_wsl] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemAdvise_Negative_Parameters: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttributes_Positive_Basic: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemRangeGetAttributes_Negative_Parameters: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipFreeAsync_Negative_Parameters: <<: *level_2 # Below tests are failing PSDB @@ -2365,25 +2393,25 @@ memory: tags: [alloc, P1] Unit_hipStreamAttachMemAsync_Positive_Basic: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipStreamAttachMemAsync_Positive_Pageable: <<: *level_2 tags: [query, P1] Unit_hipStreamAttachMemAsync_Positive_AttachGlobal: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipStreamAttachMemAsync_Positive_AttachHost: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipStreamAttachMemAsync_Positive_AttachSingle: <<: *level_2 - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipStreamAttachMemAsync_Negative_Parameters: <<: *level_2 # Below tests soft hang in stress test on 13/09/23 # Asan test disabled temporarily until PR #5287 is merged disabled: [amd_wsl, asan] - tags: [query, P1] + tags: [query, P1, managed_memory] Unit_hipMemGetAddressRange_Positive: <<: *level_2 # Note: intermittent Seg fault failure diff --git a/projects/hip-tests/catch/config/parse_config.py b/projects/hip-tests/catch/config/parse_config.py index 0651ba0940c7..1f43316b0af5 100644 --- a/projects/hip-tests/catch/config/parse_config.py +++ b/projects/hip-tests/catch/config/parse_config.py @@ -42,17 +42,39 @@ def parse_args(): "--asan", action="store_true", help="Target is an Address Sanitizer (ASAN) build. Test cases that " - "list 'asan' in their 'disabled' field are skipped.", + "list 'asan' in their 'disabled' or 'unsupported' field are skipped.", ) return parser.parse_args() +def _skip_targets(field): + """Return the platform/arch/config targets from a skip field. + + A skip field is either a flat list of targets ([amd_windows]) or a mapping + with a 'targets' list plus an optional 'reason' scalar + ({targets: [...], reason: ...}). Only the targets drive tag generation; the + reason is metadata for other tooling and is ignored here. + """ + if isinstance(field, dict): + return field.get("targets", []) + return field + + def create_test_definition( group, case_name, case_config, platform, os_name, arch, asan=False ): level = case_config.get("level", 2) tags = case_config.get("tags", []) disabled = case_config.get("disabled", []) + unsupported = case_config.get("unsupported", []) + # Extract the targets from each skip field (list form or {targets, reason} + # mapping form) and merge into one ordered local (disabled first) so the + # match and promotion logic below is identical for either field. Do not + # mutate the source lists. "disabled" is temporary/regressions, "unsupported" + # is permanent; both produce the same [disabled] skip tag and + # [exclude_] promotions that CI and the compute-utils/WSL runners + # depend on. + skip_targets = _skip_targets(disabled) + _skip_targets(unsupported) tags_str = "" @@ -62,9 +84,9 @@ def create_test_definition( tags_str += f"[{group}]" if ( - f"{platform}_{os_name}" in disabled - or arch in disabled - or (asan and "asan" in disabled) + f"{platform}_{os_name}" in skip_targets + or arch in skip_targets + or (asan and "asan" in skip_targets) ): # Disabled on this platform (e.g. amd_linux) or arch (e.g. gfx1260). # Use the [disabled] tag (no leading dot) so it is visible in --list-tests @@ -78,7 +100,7 @@ def create_test_definition( # specific labels a test is disabled for (incl. OS labels dropped above). # Prefix is "exclude_" (not "disabled_") so it does not substring-match a # ctest -LE disabled filter; consumers should match anchored ^exclude_$. - for entry in disabled: + for entry in skip_targets: tags_str += f"[exclude_{entry}]" return f'#define {case_name} "{case_name}", "{tags_str}"' @@ -96,6 +118,7 @@ def generate_parameter_header(cmd_options, output_path): f.write("// DO NOT EDIT - This file is generated at build time\n") f.write("// Contains compile-time test parameters for each level\n\n") f.write("#pragma once\n\n") + f.write("#include \n") f.write("#include \n") f.write("#include \n") f.write("#include \n") @@ -114,33 +137,33 @@ def generate_parameter_header(cmd_options, output_path): # Memory sizes if "memory_sizes" in options: sizes = [parse_size_string(s) for s in options["memory_sizes"]] - f.write(f"inline const std::vector {level_name}_memory_sizes = {{\n") + f.write(f"inline constexpr std::array {level_name}_memory_sizes = {{\n") f.write(" " + ",\n ".join(str(s) for s in sizes) + "\n") f.write("};\n\n") # Block sizes if "block_sizes" in options: sizes = options["block_sizes"] - f.write(f"inline const std::vector {level_name}_block_sizes = {{\n") + f.write(f"inline constexpr std::array {level_name}_block_sizes = {{\n") f.write(" " + ", ".join(str(s) for s in sizes) + "\n") f.write("};\n\n") # Iterations if "iterations" in options: - f.write(f"inline const int {level_name}_iterations = {options['iterations']};\n\n") + f.write(f"inline constexpr int {level_name}_iterations = {options['iterations']};\n\n") # Warmups if "warmups" in options: - f.write(f"inline const int {level_name}_warmups = {options['warmups']};\n\n") + f.write(f"inline constexpr int {level_name}_warmups = {options['warmups']};\n\n") # Max memory if "max_memory" in options: max_mem = parse_size_string(str(options["max_memory"])) - f.write(f"inline const size_t {level_name}_max_memory = {max_mem};\n\n") + f.write(f"inline constexpr size_t {level_name}_max_memory = {max_mem};\n\n") # Reduction factor if "reduction_factor" in options: - f.write(f"inline const double {level_name}_reduction_factor = {options['reduction_factor']};\n\n") + f.write(f"inline constexpr double {level_name}_reduction_factor = {options['reduction_factor']};\n\n") # Generate LevelParameters struct and initialization function f.write(f"// {'=' * 76}\n") @@ -162,8 +185,10 @@ def generate_parameter_header(cmd_options, output_path): for level_name in levels_found: f.write(f" // {level_name}\n") f.write(f" params[\"{level_name}\"] = {{\n") - f.write(f" {level_name}_memory_sizes,\n") - f.write(f" {level_name}_block_sizes,\n") + f.write(f" std::vector({level_name}_memory_sizes.begin(), " + f"{level_name}_memory_sizes.end()),\n") + f.write(f" std::vector({level_name}_block_sizes.begin(), " + f"{level_name}_block_sizes.end()),\n") f.write(f" {level_name}_iterations,\n") f.write(f" {level_name}_warmups,\n") f.write(f" {level_name}_max_memory,\n") diff --git a/projects/hip-tests/catch/contract/README.md b/projects/hip-tests/catch/contract/README.md index a5c20a8c5fb3..51ebd3992581 100644 --- a/projects/hip-tests/catch/contract/README.md +++ b/projects/hip-tests/catch/contract/README.md @@ -11,7 +11,7 @@ The first domains are: - `driver_memcpy`: driver-style directed 1D synchronous and stream-ordered copy contracts - `driver_memcpy_2d`: driver-style struct-based 2D synchronous and stream-ordered copy contracts - `runtime`: runtime initialization, device visibility, version, and error-state contracts -- `device`: portable current-device property contracts +- `device`: portable current-device property contracts, plus GPUDirect RDMA write-flush accepted-or-unsupported and invalid-enumerator contracts - `device_identity`: device PCI identity, device selection, and single-device peer-query contracts - `device_texture_query`: image-gated device texture-width query contracts - `peer_query`: peer-to-peer attribute and AMD link-type query invalid-input contracts diff --git a/projects/hip-tests/catch/contract/device/test_hip_device_contract.cc b/projects/hip-tests/catch/contract/device/test_hip_device_contract.cc index 04fc94ec84fd..d59ee2252822 100644 --- a/projects/hip-tests/catch/contract/device/test_hip_device_contract.cc +++ b/projects/hip-tests/catch/contract/device/test_hip_device_contract.cc @@ -79,3 +79,58 @@ HIP_TEST_CASE(Contract_Device_HipGetDevice_CurrentOrdinal_IsWithinDeviceCount) { REQUIRE(current_device >= 0); REQUIRE(current_device < device_count); } + +// hipDeviceFlushGPUDirectRDMAWrites is a host-ordered visibility barrier on inbound +// GPUDirect RDMA writes. Proving that remote writes actually became visible needs an +// RDMA-capable NIC writing into device memory, which a device-only harness cannot +// arrange, so that behavioral coverage lives in +// unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc. What is portable here is that a +// well-formed call is accepted or cleanly reports no flush path, and that a malformed +// enumerator is rejected as an invalid argument rather than as a missing capability. +namespace { +void RequireFlushAcceptedOrUnsupported(hipFlushGPUDirectRDMAWritesScope scope) { + const hipError_t status = + hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, scope); + + if (status == hipErrorNotSupported) { + (void)hipGetLastError(); + HIP_SKIP_TEST("device does not advertise a host GPUDirect RDMA flush path."); + return; + } + + REQUIRE(status == hipSuccess); +} +} // namespace + +// @asserts: hipDeviceFlushGPUDirectRDMAWrites - a flush to owner scope is accepted or reports unsupported +HIP_TEST_CASE(Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_ToOwner_AcceptedOrUnsupported) { + RequireFlushAcceptedOrUnsupported(hipFlushGPUDirectRDMAWritesToOwner); +} + +// @asserts: hipDeviceFlushGPUDirectRDMAWrites - a flush to all-devices scope is accepted or reports unsupported +HIP_TEST_CASE( + Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_ToAllDevices_AcceptedOrUnsupported) { + RequireFlushAcceptedOrUnsupported(hipFlushGPUDirectRDMAWritesToAllDevices); +} + +// @asserts: hipDeviceFlushGPUDirectRDMAWrites - an out-of-range scope is rejected as an invalid argument +HIP_TEST_CASE(Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_InvalidScope_IsRejected) { + // Argument validation must precede the capability check, so a device with no flush path + // still reports the bad enumerator rather than hipErrorNotSupported. + const hipError_t status = + hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + static_cast(0x7fff)); + + REQUIRE(status == hipErrorInvalidValue); + (void)hipGetLastError(); +} + +// @asserts: hipDeviceFlushGPUDirectRDMAWrites - an out-of-range target is rejected as an invalid argument +HIP_TEST_CASE(Contract_Device_HipDeviceFlushGPUDirectRDMAWrites_InvalidTarget_IsRejected) { + const hipError_t status = + hipDeviceFlushGPUDirectRDMAWrites(static_cast(0x7fff), + hipFlushGPUDirectRDMAWritesToOwner); + + REQUIRE(status == hipErrorInvalidValue); + (void)hipGetLastError(); +} diff --git a/projects/hip-tests/catch/contract/graph_debug/test_hip_graph_debug_contract.cc b/projects/hip-tests/catch/contract/graph_debug/test_hip_graph_debug_contract.cc index b0552f36ccac..ac3d44c5f641 100644 --- a/projects/hip-tests/catch/contract/graph_debug/test_hip_graph_debug_contract.cc +++ b/projects/hip-tests/catch/contract/graph_debug/test_hip_graph_debug_contract.cc @@ -12,18 +12,17 @@ #include #include +#include #include namespace { constexpr size_t kByteCount = 64; -// Builds a unique temporary file path for the exported dot graph so concurrent -// test binaries do not collide on a shared filename. -std::string UniqueDotPath() { +// Temp dir: an installed test tree may be read-only. +std::string DotPath() { int device = 0; HIP_CHECK(hipGetDevice(&device)); - return std::string("hip_contract_graph_debug_") + std::to_string(device) + "_" + - std::to_string(static_cast(reinterpret_cast(&device))) + ".dot"; + return hip_test::TempPath("hip_contract_graph_debug_" + std::to_string(device), ".dot"); } hipMemsetParams MakeByteMemsetParams(void* device_ptr, unsigned int value) { @@ -61,8 +60,7 @@ HIP_TEST_CASE(Contract_GraphDebug_HipGraphDebugDotPrint_Default_WritesNonEmptyFi hipMemsetParams memset_params = MakeByteMemsetParams(device_ptr, 0x5A); HIP_CHECK(hipGraphAddMemsetNode(&node, graph, nullptr, 0, &memset_params)); - const std::string path = UniqueDotPath(); - std::remove(path.c_str()); + const std::string path = DotPath(); // Exporting a non-empty graph to a dot file must succeed (or report the // feature unsupported) and, on success, produce a non-empty file on disk. @@ -87,8 +85,7 @@ HIP_TEST_CASE(Contract_GraphDebug_HipGraphDebugDotPrint_Default_VerboseFlagIsAcc cleanup.Add([graph] { (void)hipGraphDestroy(graph); }); HIP_CHECK(hipGraphAddEmptyNode(&node, graph, nullptr, 0)); - const std::string path = UniqueDotPath(); - std::remove(path.c_str()); + const std::string path = DotPath(); // The verbose flag augments the output but must not change the success // contract: a valid graph still exports to a non-empty file. diff --git a/projects/hip-tests/catch/contract/library_file/test_hip_library_file_contract.cc b/projects/hip-tests/catch/contract/library_file/test_hip_library_file_contract.cc index bf627ffd0471..2858f37d72a8 100644 --- a/projects/hip-tests/catch/contract/library_file/test_hip_library_file_contract.cc +++ b/projects/hip-tests/catch/contract/library_file/test_hip_library_file_contract.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -68,15 +69,15 @@ bool CompileLibrarySource(std::vector& code) { return true; } -// Compiles the library source and writes it to a unique per-test file in the -// current working directory, returning the path. Skips when HIPRTC is -// unavailable. The caller must remove the file when done. +// Compile to a temp file: an installed test tree may be read-only. +// Caller removes the returned path. std::string WriteCodeObjectFile(const char* suffix) { std::vector code; if (!CompileLibrarySource(code)) { HIP_SKIP_TEST("HIPRTC compilation is not supported by this device/runtime path."); } - const std::string path = std::string("hip-contract-library-file-") + suffix + ".code"; + const std::string path = + hip_test::TempPath("hip-contract-library-file-" + std::string(suffix), ".code"); std::ofstream out(path, std::ios::binary); REQUIRE(out.is_open()); out.write(code.data(), static_cast(code.size())); diff --git a/projects/hip-tests/catch/contract/mem_advise_v2/test_hip_mem_advise_v2_contract.cc b/projects/hip-tests/catch/contract/mem_advise_v2/test_hip_mem_advise_v2_contract.cc index dce295f3c6ba..1c23b3de41c2 100644 --- a/projects/hip-tests/catch/contract/mem_advise_v2/test_hip_mem_advise_v2_contract.cc +++ b/projects/hip-tests/catch/contract/mem_advise_v2/test_hip_mem_advise_v2_contract.cc @@ -131,8 +131,6 @@ HIP_TEST_CASE(Contract_MemAdviseV2_HipMemPrefetchAsyncV2_PrefetchAsync_IsAccepte // @asserts: hipMemAdvise_v2 - advising a null range does not silently succeed and returns a non-success status HIP_TEST_CASE(Contract_MemAdviseV2_HipMemAdviseV2_NullPointer_IsRejected) { - SkipIfManagedMemoryUnsupported(); - // Advising a null range must not silently succeed. The exact error code is // backend-specific, so only a non-success status is required. const hipMemLocation location = CurrentDeviceLocation(); diff --git a/projects/hip-tests/catch/contract/mem_batch_discard/test_hip_mem_batch_discard_contract.cc b/projects/hip-tests/catch/contract/mem_batch_discard/test_hip_mem_batch_discard_contract.cc index c494253ada53..5ef6eec8428e 100644 --- a/projects/hip-tests/catch/contract/mem_batch_discard/test_hip_mem_batch_discard_contract.cc +++ b/projects/hip-tests/catch/contract/mem_batch_discard/test_hip_mem_batch_discard_contract.cc @@ -204,7 +204,6 @@ HIP_TEST_CASE(Contract_MemBatchDiscard_HipDrvMemDiscardAndPrefetchBatchAsync_Drv // @asserts: hipMemDiscardBatchAsync - a batch with a null range never returns hipSuccess (rejected, or reported unsupported) HIP_TEST_CASE(Contract_MemBatchDiscard_HipMemDiscardBatchAsync_NullPointer_IsRejectedOrUnsupported) { - SkipIfManagedMemoryUnsupported(); hip::contract::ContractCleanup cleanup; hipStream_t stream = nullptr; diff --git a/projects/hip-tests/catch/hipTestMain/CMakeLists.txt b/projects/hip-tests/catch/hipTestMain/CMakeLists.txt index 35485f40978d..f52cee599cd1 100644 --- a/projects/hip-tests/catch/hipTestMain/CMakeLists.txt +++ b/projects/hip-tests/catch/hipTestMain/CMakeLists.txt @@ -11,6 +11,7 @@ set(MAIN_SRC hip_test_context.cc hip_test_features.cc hip_test_params.cc + hip_test_level.cc hip_test_listener.cc) add_library(Main_Object EXCLUDE_FROM_ALL OBJECT ${MAIN_SRC}) set_source_files_properties(${MAIN_SRC} PROPERTIES LANGUAGE ${GPGPU_LANGUAGE}) @@ -23,3 +24,5 @@ target_link_libraries(Main_Object PRIVATE Catch2::Catch2 hip::host hip::device) target_include_directories(Main_Object PRIVATE ${CATCH_INCLUDE_BINARY_DIR}) add_dependencies(Main_Object hip_tests_config) + +add_subdirectory(tests) diff --git a/projects/hip-tests/catch/hipTestMain/hip_test_level.cc b/projects/hip-tests/catch/hipTestMain/hip_test_level.cc new file mode 100644 index 000000000000..dc061c1f3bdc --- /dev/null +++ b/projects/hip-tests/catch/hipTestMain/hip_test_level.cc @@ -0,0 +1,275 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +#include + +#include +#include + +namespace HipTestLevel { +namespace { + +/** + * @brief Split a Catch2 test spec into its OR'd filters. + * + * Only unescaped commas separate filters. CatchAddTests.cmake escapes '\', ',', + * '[', ']' and ';' in generated test names, so a name may legitimately contain + * "\," - splitting on it would corrupt the filter. + */ +std::vector splitFilters(const std::string& spec) { + std::vector filters; + std::string filter; + for (std::size_t i = 0; i < spec.size(); ++i) { + if (spec[i] == '\\' && i + 1 < spec.size()) { + // Escaped character: copy the pair verbatim, never split on it. + filter += spec[i]; + filter += spec[i + 1]; + ++i; + } else if (spec[i] == ',') { + filters.push_back(filter); + filter.clear(); + } else { + filter += spec[i]; + } + } + filters.push_back(filter); + return filters; +} + +/// @brief Join @p parts into one Catch2 spec string, i.e. OR them together. +std::string joinWithCommas(const std::vector& parts) { + std::string joined; + for (const auto& part : parts) { + if (!joined.empty()) { + joined += ','; + } + joined += part; + } + return joined; +} + +} // namespace + +bool isSupportedLevel(const std::string& level) { + for (const char* supported : kSupportedLevels) { + if (level == supported) { + return true; + } + } + return false; +} + +std::set collectLevels(const std::string& text, bool allowBareTags) { + static const std::regex bracketedPattern("(~?)\\[([^\\[\\]]*)\\]"); + static const std::regex barePattern("\\s*(~?)\\s*([^\\s]+)\\s*"); + + // Level named by a pattern, or -1 when it does not name one. Anything but + // an exact "level_N" is rejected; the digit count is bounded so that the + // conversion cannot overflow. + const auto parseLevel = [](const std::string& tag) { + static const std::regex levelRegex("level_([0-9]{1,9})"); + std::smatch match; + return std::regex_match(tag, match, levelRegex) ? std::stoi(match[1].str()) : -1; + }; + + std::set levels; + std::stringstream stream(text); + std::string filter; + while (std::getline(stream, filter, ',')) { + std::set included, excluded; + + // Sort this filter's level patterns into included and excluded. + const auto sortPattern = [&](const std::string& negation, const std::string& tag) { + const int level = parseLevel(tag); + if (level >= 0) { + (negation.empty() ? included : excluded).insert(level); + } + }; + + bool bracketed = false; + for (auto it = std::sregex_iterator(filter.begin(), filter.end(), bracketedPattern); + it != std::sregex_iterator(); ++it) { + bracketed = true; + sortPattern((*it)[1].str(), (*it)[2].str()); + } + if (!bracketed && allowBareTags) { + std::smatch match; + if (std::regex_match(filter, match, barePattern)) { + sortPattern(match[1].str(), match[2].str()); + } + } + + if (included.empty()) { + if (excluded.empty()) { + continue; // filter says nothing about the level + } + // Exclusion only: everything the suite supports is still in play. + for (const char* supported : kSupportedLevels) { + const int level = parseLevel(supported); + if (level >= 0) { + included.insert(level); + } + } + } + for (const int level : included) { + if (excluded.count(level) == 0) { + levels.insert(level); + } + } + } + return levels; +} + +std::string highestLevel(const std::set& levels) { + if (levels.empty()) { + return ""; + } + return "level_" + std::to_string(*levels.rbegin()); +} + +LevelResolution resolveLevel(const std::vector& testsOrTags, const char* envLevel) { + LevelResolution resolution; + + // Priority 1: Catch2 command-line tag filter (e.g. ./test "[level_2]"). + std::set cliLevels; + for (const auto& arg : testsOrTags) { + const auto found = collectLevels(arg, false); + cliLevels.insert(found.begin(), found.end()); + } + if (!cliLevels.empty()) { + resolution.levels = cliLevels; + resolution.level = highestLevel(cliLevels); + resolution.source = LevelSource::kCommandLine; + return resolution; + } + + // Priority 2: HIP_TEST_LEVEL environment variable. + if (envLevel != nullptr) { + resolution.envValue = envLevel; + const auto envLevels = collectLevels(envLevel, true); + if (!envLevels.empty()) { + resolution.levels = envLevels; + resolution.level = highestLevel(envLevels); + resolution.source = LevelSource::kEnvironment; + return resolution; + } + resolution.envRejected = true; + } + + // Priority 3: Hardcoded default. Parsed rather than hardcoded as a number so + // kDefaultLevel stays the single source of truth for the default. + resolution.levels = collectLevels(kDefaultLevel, true); + resolution.level = kDefaultLevel; + resolution.source = LevelSource::kDefault; + return resolution; +} + +bool LevelResolution::multipleSelected() const { return levels.size() > 1; } + +std::string LevelResolution::firstUnsupportedLevel() const { + for (const int selected : levels) { + const std::string name = "level_" + std::to_string(selected); + if (!isSupportedLevel(name)) { + return name; + } + } + return ""; +} + +std::vector LevelResolution::describe() const { + const char* sourceName = "default"; + switch (source) { + case LevelSource::kCommandLine: + sourceName = "command line"; + break; + case LevelSource::kEnvironment: + sourceName = "HIP_TEST_LEVEL"; + break; + case LevelSource::kDefault: + break; + } + + std::vector lines; + if (envRejected) { + lines.push_back("[Level Filter] HIP_TEST_LEVEL='" + envValue + + "' has no valid level, ignoring"); + } + if (multipleSelected()) { + std::string selected; + for (const int level : levels) { + if (!selected.empty()) { + selected += ", "; + } + selected += "level_" + std::to_string(level); + } + lines.push_back("[Level Filter] Multiple levels selected via " + std::string(sourceName) + + " (" + selected + "); all of them run, parameters use the highest (" + level + + ")"); + } + lines.push_back("[Level Filter] Detected from " + std::string(sourceName) + ": " + level); + return lines; +} + +bool applyLevelFilter(std::vector& testsOrTags, const LevelResolution& resolution) { + // The spec already names the level; re-applying would AND in a second one. + if (resolution.source == LevelSource::kCommandLine) { + return false; + } + + // The default level never narrows the run - it only supplies parameters. A + // bare "./AtomicsTest" therefore runs every test in the binary, as it always + // has, and so does plain ctest. Only an explicit request selects. + if (resolution.source != LevelSource::kEnvironment) { + return false; + } + + // One tag per selected level. Catch2 has no OR inside a filter, so these are + // combined by repeating filters, not by concatenating tags. + std::vector tags; + tags.reserve(resolution.levels.size()); + for (const int level : resolution.levels) { + tags.push_back("[level_" + std::to_string(level) + "]"); + } + if (tags.empty()) { + return false; + } + + // Nothing else was asked for, so the levels are the only selector available. + if (testsOrTags.empty()) { + testsOrTags.push_back(joinWithCommas(tags)); + return true; + } + + // Cross product: every filter of the spec, once per selected level. + for (auto& spec : testsOrTags) { + std::vector rewritten; + for (const auto& filter : splitFilters(spec)) { + // An empty filter constrains nothing; Catch2 discards it too. + if (filter.empty()) { + continue; + } + for (const auto& tag : tags) { + rewritten.push_back(filter + tag); + } + } + spec = joinWithCommas(rewritten); + } + return true; +} + +int mapLevelFilterExitCode(int exitCode, bool levelFilterApplied) { + // Catch2 exit codes are file-local constants in catch_session.cpp:37-42 and + // are not exported, so they are restated here. + constexpr int kCatchNoTestsRunExitCode = 2; + constexpr int kCatchAllTestsSkippedExitCode = 4; + + if (levelFilterApplied && exitCode == kCatchNoTestsRunExitCode) { + return kCatchAllTestsSkippedExitCode; + } + return exitCode; +} + +} // namespace HipTestLevel diff --git a/projects/hip-tests/catch/hipTestMain/hip_test_listener.cc b/projects/hip-tests/catch/hipTestMain/hip_test_listener.cc index aefd889180b4..342d8ef3444f 100644 --- a/projects/hip-tests/catch/hipTestMain/hip_test_listener.cc +++ b/projects/hip-tests/catch/hipTestMain/hip_test_listener.cc @@ -1,186 +1,58 @@ /* -Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ #include #include -#include +#include #include #include -#include +#include +#include +#include #include -#include /** - * @brief Event listener for HIP test parameter initialization - * - * This listener hooks into Catch2 v3 events to: - * - Initialize test parameters before test execution - * - Detect level filter from command-line args - * - Load level-specific configs based on filter - * - Clean up resources after testing - * - * Usage: - * ./test "[level_0]" -> Quick smoke tests (3 memory sizes) - * ./test "[level_1]" -> Standard regression (7 memory sizes) - * ./test "[level_2]" -> Comprehensive (14 memory sizes) - * - * Or override via environment: - * HIP_TEST_LEVEL=level_1 ./test "[device]" + * @brief Loads the active level's test parameters into TestParameterStore. + * + * Only the parameters live here. Test *selection* cannot: Catch2 has already + * parsed the test spec by the time any listener event fires, so the spec is + * rewritten in main() instead (see HipTestLevel::applyLevelFilter). + * + * The level is resolved the same way main() resolves it, from the same two + * inputs, so the two agree. Where main() rewrote the spec, that rewrite names + * the levels it applied, so resolving again from the rewritten spec yields the + * same answer. */ class HipTestParameterListener : public Catch::EventListenerBase { -public: - using Catch::EventListenerBase::EventListenerBase; - -private: - std::string filterLevel; - - /** - * @brief Extract level from a filter string like "[level_1]" - */ - std::string extractLevelFromFilter(const std::string& filter) { - std::regex levelRegex("\\[level_(\\d+)\\]"); - std::smatch match; - if (std::regex_search(filter, match, levelRegex)) { - return "level_" + match[1].str(); - } - return ""; - } - - /** - * @brief Detect level filter from environment or command line - * - * Priority: - * 1. HIP_TEST_LEVEL environment variable - * 2. Command line argument (parsed from /proc/self/cmdline on Linux) - */ - std::string detectLevelFilter() { - // Priority 1: Check environment variable - if (const char* envLevel = std::getenv("HIP_TEST_LEVEL")) { - std::string level = envLevel; - LogPrintf("[Level Filter] Detected from HIP_TEST_LEVEL: %s\n", level.c_str()); - return level; - } - - // Priority 2: Read from /proc/self/cmdline on Linux - #ifdef __linux__ - std::ifstream cmdline("/proc/self/cmdline"); - if (cmdline) { - std::string arg; - while (std::getline(cmdline, arg, '\0')) { - std::string level = extractLevelFromFilter(arg); - if (!level.empty()) { - LogPrintf("[Level Filter] Detected from command line: %s\n", level.c_str()); - return level; - } - } - } - #endif - - return ""; + public: + using Catch::EventListenerBase::EventListenerBase; + + /** + * @brief Called once when the test run begins. + * Initializes TestParameterStore and loads the resolved level's parameters. + */ + void testRunStarting(Catch::TestRunInfo const&) override { + const std::vector testsOrTags = + (m_config != nullptr) ? m_config->getTestsOrTags() : std::vector{}; + const auto resolution = HipTestLevel::resolveLevel(testsOrTags, std::getenv("HIP_TEST_LEVEL")); + + for (const auto& line : resolution.describe()) { + LogPrintf("%s\n", line.c_str()); } -public: + // An unsupported level is rejected in main() before the run starts; by the + // time we get here the level is known good, so load it and move on. + auto& params = TestParameterStore::instance(); + params.initialize(); + LogPrintf("[Level Filter] Applying test parameters level: %s\n", resolution.level.c_str()); + params.loadLevelConfig(resolution.level); + } - /** - * @brief Called once when the test run begins - * Initializes TestParameterStore and detects level filter - */ - void testRunStarting(Catch::TestRunInfo const&) override { - TestParameterStore::instance().initialize(); - - // Detect level filter from environment or command-line - filterLevel = detectLevelFilter(); - - // If level was specified, load it immediately - if (!filterLevel.empty()) { - LogPrintf("[Level Filter] Applying global level: %s\n", filterLevel.c_str()); - TestParameterStore::instance().loadLevelConfig(filterLevel); - } - } - - /** - * @brief Called before each test case starts - * Uses filter level if specified, otherwise detects from test tags - */ - void testCaseStarting(Catch::TestCaseInfo const& testInfo) override { - auto& params = TestParameterStore::instance(); - - // Priority 1: Use filter level if explicitly set (from env or detected) - if (!filterLevel.empty()) { - // Filter level takes precedence - all tests use same parameters - if (params.currentTestLevel != filterLevel) { - LogPrintf("[Level Filter] Test: %s -> Using filter level: %s\n", - testInfo.name.c_str(), filterLevel.c_str()); - params.loadLevelConfig(filterLevel); - } - return; - } - - // Priority 2: Auto-detect from first test's level tag (filter inference) - std::string detectedLevel = ""; - for (const auto& tag : testInfo.tags) { - std::string tagStr = std::string(tag.original); - - // Remove brackets: "[level_0]" -> "level_0" - if (tagStr.size() > 2 && tagStr.front() == '[' && tagStr.back() == ']') { - tagStr = tagStr.substr(1, tagStr.size() - 2); - } - - // Check if it's a level tag - if (tagStr.find("level_") == 0) { - detectedLevel = tagStr; - break; - } - } - - // If this is the first test with a level tag, set it as filter level - if (!detectedLevel.empty() && filterLevel.empty()) { - filterLevel = detectedLevel; - LogPrintf("[Level Auto-Detection] Inferred filter level: %s from test: %s\n", - filterLevel.c_str(), testInfo.name.c_str()); - } - - // Load level-specific config if detected - if (!detectedLevel.empty()) { - if (params.currentTestLevel != detectedLevel) { - LogPrintf("[Level Detection] Test: %s -> Level: %s\n", - testInfo.name.c_str(), detectedLevel.c_str()); - params.loadLevelConfig(detectedLevel); - } - } else { - // Reset to defaults if no level tag - if (!params.currentTestLevel.empty()) { - params.currentTestLevel = ""; - } - } - } - - /** - * @brief Called when test run ends - * Cleanup resources - */ - void testRunEnded(Catch::TestRunStats const&) override { - TestParameterStore::instance().clear(); - } + /// @brief Called when the test run ends. Releases the loaded parameters. + void testRunEnded(Catch::TestRunStats const&) override { TestParameterStore::instance().clear(); } }; // Register the listener - it will be automatically activated diff --git a/projects/hip-tests/catch/hipTestMain/hip_test_params.cc b/projects/hip-tests/catch/hipTestMain/hip_test_params.cc index f15338a60d5d..925cf9ca5f14 100644 --- a/projects/hip-tests/catch/hipTestMain/hip_test_params.cc +++ b/projects/hip-tests/catch/hipTestMain/hip_test_params.cc @@ -40,19 +40,6 @@ void TestParameterStore::initialize() { params.block_sizes.size(), params.iterations); } - // Set defaults (use level_0 as fallback if available, otherwise hardcoded) - if (levelMemorySizes.count("level_0")) { - defaultMemorySizes = levelMemorySizes["level_0"]; - defaultBlockSizes = levelBlockSizes["level_0"]; - defaultIterations = levelIterations["level_0"]; - defaultWarmups = levelWarmups["level_0"]; - } else { - // Hardcoded fallback if no levels defined - defaultMemorySizes = {1024, 1048576, 10485760}; // 1K, 1M, 10M - defaultBlockSizes = {64, 256}; - LogPrintf("[TestParameterStore] Warning: No level_0 defined, using hardcoded defaults\n%s", ""); - } - LogPrintf("[TestParameterStore] Initialization complete - %zu levels loaded\n", allParams.size()); } @@ -93,14 +80,16 @@ const std::vector& TestParameterStore::getMemorySizesForCurrentLevel() c if (!currentTestLevel.empty() && levelMemorySizes.count(currentTestLevel)) { return levelMemorySizes.at(currentTestLevel); } - return defaultMemorySizes; + static const std::vector fallback(defaultMemorySizes.begin(), defaultMemorySizes.end()); + return fallback; } const std::vector& TestParameterStore::getBlockSizesForCurrentLevel() const { if (!currentTestLevel.empty() && levelBlockSizes.count(currentTestLevel)) { return levelBlockSizes.at(currentTestLevel); } - return defaultBlockSizes; + static const std::vector fallback(defaultBlockSizes.begin(), defaultBlockSizes.end()); + return fallback; } int TestParameterStore::getIterationsForCurrentLevel() const { diff --git a/projects/hip-tests/catch/hipTestMain/main.cc b/projects/hip-tests/catch/hipTestMain/main.cc index b720d0ceee58..1720b949ff9c 100755 --- a/projects/hip-tests/catch/hipTestMain/main.cc +++ b/projects/hip-tests/catch/hipTestMain/main.cc @@ -7,9 +7,24 @@ #define CATCH_CONFIG_RUNNER #include #include +#include +#include +#include +#include +#include CmdOptions cmd_options; +namespace { + +/// @brief Whether Catch2 will only print information instead of running tests. +bool isListingOnly(const Catch::ConfigData& configData) { + return configData.showHelp || configData.libIdentify || configData.listTests || + configData.listTags || configData.listReporters || configData.listListeners; +} + +} // namespace + int main(int argc, char** argv) { auto& context = TestContext::get(); @@ -62,7 +77,50 @@ int main(int argc, char** argv) { session.cli(cli); - int out = session.run(argc, argv); + int out = session.applyCommandLine(argc, argv); + if (out == 0) { + auto& configData = session.configData(); + if (isListingOnly(configData)) { + // Nothing runs, so the level is irrelevant + out = session.run(); + } else { +#ifdef ENABLE_YAML_TAGS + // Test selection by level has to happen here: Catch2 parses the test spec + // before any listener event fires, so a listener cannot narrow the run. + // Loading the level's parameters can wait, and does - see + // hip_test_listener.cc, which is not part of the standalone build. + // + // Without YAML tags every TEST_CASE has an empty tag string, so a level + // filter would match nothing; the whole block is compiled out, which is + // also what keeps the standalone build free of hip_test_level.cc. + const auto resolution = + HipTestLevel::resolveLevel(configData.testsOrTags, std::getenv("HIP_TEST_LEVEL")); + + const std::string unsupported = resolution.firstUnsupportedLevel(); + if (!unsupported.empty()) { + std::fprintf(stderr, "[Level Filter] ERROR: '%s' is not a supported level. Aborting.\n", + unsupported.c_str()); + context.cleanContext(); + return EXIT_FAILURE; + } + + const bool levelFilterApplied = + HipTestLevel::applyLevelFilter(configData.testsOrTags, resolution); + + out = session.run(); + + const int reported = HipTestLevel::mapLevelFilterExitCode(out, levelFilterApplied); + if (reported != out) { + LogPrintf("[Level Filter] No test matched the selected level(s); reporting as skipped%s", + ""); + out = reported; + } +#else + out = session.run(); +#endif + } + } + context.cleanContext(); return out; } diff --git a/projects/hip-tests/catch/hipTestMain/tests/CMakeLists.txt b/projects/hip-tests/catch/hipTestMain/tests/CMakeLists.txt new file mode 100644 index 000000000000..f3374eb1405d --- /dev/null +++ b/projects/hip-tests/catch/hipTestMain/tests/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +set(LEVEL_TEST_SRC hip_test_level_test.cc ../hip_test_level.cc) +set_source_files_properties(${LEVEL_TEST_SRC} PROPERTIES LANGUAGE ${GPGPU_LANGUAGE}) + +add_executable(HipTestLevelUnitTest EXCLUDE_FROM_ALL ${LEVEL_TEST_SRC}) +target_link_libraries(HipTestLevelUnitTest PRIVATE Catch2::Catch2WithMain) +add_dependencies(build_tests HipTestLevelUnitTest) +add_test(NAME HipTestLevelUnitTest COMMAND HipTestLevelUnitTest) +set_tests_properties(HipTestLevelUnitTest PROPERTIES LABELS "hosttest") diff --git a/projects/hip-tests/catch/hipTestMain/tests/hip_test_level_test.cc b/projects/hip-tests/catch/hipTestMain/tests/hip_test_level_test.cc new file mode 100644 index 000000000000..c1dcb36a71b3 --- /dev/null +++ b/projects/hip-tests/catch/hipTestMain/tests/hip_test_level_test.cc @@ -0,0 +1,325 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +#include +#include + +#include +#include +#include + +using HipTestLevel::LevelSource; + +namespace { +/// Build a single-level resolution without going through the priority chain. +HipTestLevel::LevelResolution resolutionFrom(LevelSource source, const std::string& level) { + HipTestLevel::LevelResolution resolution; + resolution.levels = HipTestLevel::collectLevels(level, true); + resolution.level = level; + resolution.source = source; + return resolution; +} + +/// Build a multi-level resolution; @p levels supplies selection, the highest +/// supplies parameters. +HipTestLevel::LevelResolution resolutionFrom(LevelSource source, const std::set& levels) { + HipTestLevel::LevelResolution resolution; + resolution.levels = levels; + resolution.level = HipTestLevel::highestLevel(levels); + resolution.source = source; + return resolution; +} +} // namespace + +TEST_CASE("LevelParse_OrFiltersAreUnioned") { + REQUIRE(HipTestLevel::collectLevels("[level_1],[level_2]", false) == std::set{1, 2}); +} + +TEST_CASE("LevelParse_ExclusionOnlyLeavesTheRest") { + REQUIRE(HipTestLevel::collectLevels("~[level_3]~[level_4]", false) == std::set{0, 1, 2}); +} + +TEST_CASE("LevelParse_SelfCancellingFilterSelectsNothing") { + REQUIRE(HipTestLevel::collectLevels("[level_0]~[level_0]", false).empty()); +} + +TEST_CASE("LevelParse_MalformedTagIsIgnored") { + REQUIRE(HipTestLevel::collectLevels("[level_2x]", false).empty()); +} + +TEST_CASE("LevelParse_TestNameIsIgnored") { + REQUIRE(HipTestLevel::collectLevels("Unit_atomicAnd_Positive_SameAddress", false).empty()); +} + +TEST_CASE("LevelParse_BareTagRejectedWhenNotAllowed") { + REQUIRE(HipTestLevel::collectLevels("level_2", false).empty()); +} + +TEST_CASE("LevelParse_BareTagAcceptedWhenAllowed") { + REQUIRE(HipTestLevel::collectLevels("level_2", true) == std::set{2}); +} + +TEST_CASE("LevelParse_BareExclusionAcceptedWhenAllowed") { + REQUIRE(HipTestLevel::collectLevels("~level_4", true) == std::set{0, 1, 2, 3}); +} + +TEST_CASE("LevelResolve_DefaultsWhenNothingProvided") { + const auto resolution = HipTestLevel::resolveLevel({}, nullptr); + REQUIRE(resolution.level == "level_2"); + REQUIRE(resolution.source == LevelSource::kDefault); + REQUIRE_FALSE(resolution.envRejected); +} + +TEST_CASE("LevelResolve_EnvironmentUsedWhenCommandLineSilent") { + const auto resolution = HipTestLevel::resolveLevel({"Unit_foo"}, "level_1"); + REQUIRE(resolution.level == "level_1"); + REQUIRE(resolution.source == LevelSource::kEnvironment); +} + +TEST_CASE("LevelResolve_CommandLineBeatsEnvironment") { + const auto resolution = HipTestLevel::resolveLevel({"[level_0]"}, "level_1"); + REQUIRE(resolution.level == "level_0"); + REQUIRE(resolution.source == LevelSource::kCommandLine); +} + +TEST_CASE("LevelResolve_HighestWinsForParametersButAllLevelsAreKept") { + // Parameters need a single level, so the highest wins. Selection must not + // silently drop the others - they are kept for applyLevelFilter to use. + const auto resolution = HipTestLevel::resolveLevel({"[level_1]", "[level_3]"}, nullptr); + REQUIRE(resolution.level == "level_3"); + REQUIRE(resolution.levels == std::set{1, 3}); + REQUIRE(resolution.multipleSelected()); +} + +TEST_CASE("LevelResolve_SingleLevelIsNotFlaggedAsMultiple") { + const auto resolution = HipTestLevel::resolveLevel({}, "level_1"); + REQUIRE(resolution.levels == std::set{1}); + REQUIRE_FALSE(resolution.multipleSelected()); +} + +TEST_CASE("LevelResolve_EnvironmentExclusionKeepsEveryRemainingLevel") { + // "~level_4" asks for everything below level_4: all four must run, while the + // parameters come from the highest of them. + const auto resolution = HipTestLevel::resolveLevel({}, "~level_4"); + REQUIRE(resolution.levels == std::set{0, 1, 2, 3}); + REQUIRE(resolution.level == "level_3"); + REQUIRE(resolution.source == LevelSource::kEnvironment); +} + +TEST_CASE("LevelResolve_DefaultSelectsExactlyTheDefaultLevel") { + const auto resolution = HipTestLevel::resolveLevel({"Unit_foo"}, nullptr); + REQUIRE(resolution.levels == std::set{2}); + REQUIRE(resolution.level == HipTestLevel::kDefaultLevel); +} + +TEST_CASE("LevelResolve_GarbageEnvironmentFallsBackToDefault") { + const auto resolution = HipTestLevel::resolveLevel({}, "not_a_level"); + REQUIRE(resolution.level == "level_2"); + REQUIRE(resolution.source == LevelSource::kDefault); + REQUIRE(resolution.envRejected); +} + +TEST_CASE("LevelResolve_OutOfRangeEnvironmentIsReportedUnsupported") { + const auto resolution = HipTestLevel::resolveLevel({}, "level_9"); + REQUIRE(resolution.level == "level_9"); + REQUIRE(resolution.source == LevelSource::kEnvironment); + REQUIRE(resolution.firstUnsupportedLevel() == "level_9"); +} + +TEST_CASE("LevelResolve_UnsupportedLevelIsCaughtEvenAlongsideSupportedOnes") { + // Selection now uses every level, so validation must cover every level - not + // just the highest that supplies the parameters. + const auto resolution = HipTestLevel::resolveLevel({}, "level_1,level_7"); + REQUIRE(resolution.levels == std::set{1, 7}); + REQUIRE(resolution.firstUnsupportedLevel() == "level_7"); +} + +TEST_CASE("LevelResolve_AllSupportedLevelsPassValidation") { + REQUIRE(HipTestLevel::resolveLevel({}, "~level_4").firstUnsupportedLevel().empty()); + REQUIRE(HipTestLevel::resolveLevel({}, nullptr).firstUnsupportedLevel().empty()); +} + +TEST_CASE("LevelDescribe_ReportsSourceAndLevel") { + const auto lines = HipTestLevel::resolveLevel({}, "level_1").describe(); + REQUIRE(lines.size() == 1); + REQUIRE(lines.back() == "[Level Filter] Detected from HIP_TEST_LEVEL: level_1"); +} + +TEST_CASE("LevelDescribe_ReportsEveryLevelAndWhichSuppliesParameters") { + const auto lines = HipTestLevel::resolveLevel({}, "level_1,level_3").describe(); + REQUIRE(lines.size() == 2); + REQUIRE(lines.front() == + "[Level Filter] Multiple levels selected via HIP_TEST_LEVEL (level_1, level_3); " + "all of them run, parameters use the highest (level_3)"); +} + +TEST_CASE("LevelDescribe_ReportsARejectedEnvironmentValue") { + const auto lines = HipTestLevel::resolveLevel({}, "not_a_level").describe(); + REQUIRE(lines.size() == 2); + REQUIRE(lines.front() == + "[Level Filter] HIP_TEST_LEVEL='not_a_level' has no valid level, " + "ignoring"); + REQUIRE(lines.back() == "[Level Filter] Detected from default: level_2"); +} + +TEST_CASE("LevelSupport_KnownAndUnknown") { + REQUIRE(HipTestLevel::isSupportedLevel("level_0")); + REQUIRE(HipTestLevel::isSupportedLevel("level_4")); + REQUIRE_FALSE(HipTestLevel::isSupportedLevel("level_5")); + REQUIRE_FALSE(HipTestLevel::isSupportedLevel("")); +} + +TEST_CASE("LevelResolve_SourceIsReportedForEveryPath") { + // main() branches on the source: a command-line level is already expressed + // in the spec, so it must be distinguishable from the other two. + REQUIRE(HipTestLevel::resolveLevel({"[level_1]"}, nullptr).source == LevelSource::kCommandLine); + REQUIRE(HipTestLevel::resolveLevel({"Unit_foo"}, "level_1").source == LevelSource::kEnvironment); + REQUIRE(HipTestLevel::resolveLevel({"Unit_foo"}, nullptr).source == LevelSource::kDefault); +} + +TEST_CASE("LevelFilter_DefaultLevelNeverNarrowsABareRun") { + // A bare "./AtomicsTest" must run every test in the binary and take its + // parameters from level_2. Narrowing here would empty any binary whose tests + // all sit at another level - the contract tier is entirely level_0. + std::vector spec; + REQUIRE_FALSE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kDefault, "level_2"))); + REQUIRE(spec.empty()); +} + +TEST_CASE("LevelFilter_EmptySpecGetsTheEnvironmentLevelTag") { + std::vector spec; + REQUIRE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kEnvironment, "level_0"))); + REQUIRE(spec == std::vector{"[level_0]"}); +} + +TEST_CASE("LevelFilter_DefaultLevelLeavesANamedTestAlone") { + // ./AtomicsTest Unit_foo and plain ctest must keep running the named test + // whatever its tags; the default level only supplies the parameters. + std::vector spec{"Unit_foo"}; + REQUIRE_FALSE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kDefault, "level_2"))); + REQUIRE(spec == std::vector{"Unit_foo"}); +} + +TEST_CASE("LevelFilter_CommandLineLevelLeavesTheSpecAlone") { + std::vector spec{"[level_0]"}; + REQUIRE_FALSE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kCommandLine, "level_0"))); + REQUIRE(spec == std::vector{"[level_0]"}); +} + +TEST_CASE("LevelFilter_EnvironmentLevelIsAndedWithANamedTest") { + // Patterns inside one filter are ANDed by Catch2, so this narrows the run + // to the named test rather than widening it to every level_0 test. + std::vector spec{"Unit_atomicAnd_Positive_SameAddress"}; + REQUIRE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kEnvironment, "level_0"))); + REQUIRE(spec == std::vector{"Unit_atomicAnd_Positive_SameAddress[level_0]"}); +} + +TEST_CASE("LevelFilter_EachOrFilterGetsTheTag") { + // Appending once to the whole string would bind the tag to the last filter only. + std::vector spec{"Unit_a,Unit_b"}; + REQUIRE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kEnvironment, "level_1"))); + REQUIRE(spec == std::vector{"Unit_a[level_1],Unit_b[level_1]"}); +} + +TEST_CASE("LevelFilter_EscapedCommaIsNotASeparator") { + // CatchAddTests.cmake escapes \ , [ ] ; in generated test names. + std::vector spec{"Unit_a\\,b"}; + REQUIRE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kEnvironment, "level_1"))); + REQUIRE(spec == std::vector{"Unit_a\\,b[level_1]"}); +} + +TEST_CASE("LevelFilter_TrailingUnpairedBackslashIsKeptAndTagged") { + // Not reachable from CatchAddTests.cmake-generated specs, which always emit + // matched backslash pairs; pinned here only so a change in this corner does + // not go unnoticed. A trailing backslash has no following character to form + // a pair with, so the loop's escape branch never triggers for it: it is + // copied through like an ordinary character, and the tag is appended after + // it rather than being swallowed into an escape of the tag itself. + std::vector spec{"Unit_a\\"}; + REQUIRE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kEnvironment, "level_1"))); + REQUIRE(spec == std::vector{"Unit_a\\[level_1]"}); +} + +TEST_CASE("LevelFilter_ExclusionKeepsItsScope") { + // Catch2 resets exclusion per pattern, so the appended tag is required, + // not forbidden. + std::vector spec{"~[disabled]"}; + REQUIRE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kEnvironment, "level_2"))); + REQUIRE(spec == std::vector{"~[disabled][level_2]"}); +} + +TEST_CASE("LevelFilter_MultipleSpecEntriesAreAllRewritten") { + std::vector spec{"Unit_a", "Unit_b"}; + REQUIRE( + HipTestLevel::applyLevelFilter(spec, resolutionFrom(LevelSource::kEnvironment, "level_4"))); + REQUIRE(spec == std::vector{"Unit_a[level_4]", "Unit_b[level_4]"}); +} + +TEST_CASE("LevelFilter_EmptySpecGetsEverySelectedLevel") { + // Catch2 ORs comma-separated filters, so this selects level_1 or level_3. + std::vector spec; + REQUIRE(HipTestLevel::applyLevelFilter( + spec, resolutionFrom(LevelSource::kEnvironment, std::set{1, 3}))); + REQUIRE(spec == std::vector{"[level_1],[level_3]"}); +} + +TEST_CASE("LevelFilter_NamedTestIsRepeatedOncePerSelectedLevel") { + // Catch2 has no OR inside a filter, so "Unit_foo AND (level_1 OR level_3)" + // has to be written as two filters. + std::vector spec{"Unit_foo"}; + REQUIRE(HipTestLevel::applyLevelFilter( + spec, resolutionFrom(LevelSource::kEnvironment, std::set{1, 3}))); + REQUIRE(spec == std::vector{"Unit_foo[level_1],Unit_foo[level_3]"}); +} + +TEST_CASE("LevelFilter_CrossProductCoversEveryFilterAndLevel") { + std::vector spec{"Unit_a,Unit_b"}; + REQUIRE(HipTestLevel::applyLevelFilter( + spec, resolutionFrom(LevelSource::kEnvironment, std::set{0, 2}))); + REQUIRE(spec == std::vector{"Unit_a[level_0],Unit_a[level_2]," + "Unit_b[level_0],Unit_b[level_2]"}); +} + +TEST_CASE("LevelFilter_EverySelectedLevelSurvivesARoundTrip") { + std::vector spec; + const auto resolution = HipTestLevel::resolveLevel({}, "~level_4"); + REQUIRE(HipTestLevel::applyLevelFilter(spec, resolution)); + REQUIRE(HipTestLevel::collectLevels(spec.at(0), false) == std::set{0, 1, 2, 3}); +} + +TEST_CASE("LevelFilter_TagRoundTrips") { + std::vector spec; + const auto resolution = HipTestLevel::resolveLevel({}, "level_3"); + REQUIRE(HipTestLevel::applyLevelFilter(spec, resolution)); + REQUIRE(HipTestLevel::collectLevels(spec.at(0), false) == std::set{3}); +} + +TEST_CASE("LevelExitCode_NoTestsBecomesSkippedWhenFiltered") { + // Catch2 returns NoTestsRunExitCode when the level filter matched nothing; + // ctest must see a skip, not a failure. + REQUIRE(HipTestLevel::mapLevelFilterExitCode(2, true) == 4); +} + +TEST_CASE("LevelExitCode_NoTestsIsLeftAloneWhenNotFiltered") { + // No filter was applied, so an empty run means the user's own spec matched + // nothing - a genuine failure worth reporting. + REQUIRE(HipTestLevel::mapLevelFilterExitCode(2, false) == 2); +} + +TEST_CASE("LevelExitCode_OtherCodesPassThrough") { + REQUIRE(HipTestLevel::mapLevelFilterExitCode(0, true) == 0); + REQUIRE(HipTestLevel::mapLevelFilterExitCode(1, true) == 1); + REQUIRE(HipTestLevel::mapLevelFilterExitCode(42, true) == 42); +} diff --git a/projects/hip-tests/catch/include/hip_test_common.hh b/projects/hip-tests/catch/include/hip_test_common.hh index 37b30ac23948..f98229108f56 100644 --- a/projects/hip-tests/catch/include/hip_test_common.hh +++ b/projects/hip-tests/catch/include/hip_test_common.hh @@ -315,7 +315,7 @@ static void initHipCtx(hipCtx_t* pcontext) { #define HIP_TEST_DRIVER_INIT() #endif -#if defined(__gfx1250__) || defined(__gfx1250_strict__) || defined(__gfx1251__) +#if defined(__gfx1250__) || defined(__gfx1250_strict__) // Wrap __cluster_dims__ so a test's host code is NOT compiled away when the // offload-arch string mixes archs that support clusters with ones that don't // (e.g. gfx950). The attribute is only emitted for targets with cluster support. diff --git a/projects/hip-tests/catch/include/hip_test_level.hh b/projects/hip-tests/catch/include/hip_test_level.hh new file mode 100644 index 000000000000..5da77865a130 --- /dev/null +++ b/projects/hip-tests/catch/include/hip_test_level.hh @@ -0,0 +1,177 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include +#include +#include + +/** + * @brief Resolution of the single test level used for a whole run. + * + * The active level is resolved once, before the run starts, using a strict + * priority order: + * 1. Catch2 command-line tag filter, e.g. ./test "[level_2]" + * 2. HIP_TEST_LEVEL environment variable, e.g. HIP_TEST_LEVEL=level_2 + * 3. Hardcoded default (kDefaultLevel) + * + * A higher-priority source is used whenever it yields a level; the lower ones + * are then ignored (strict precedence, no merging across sources). + */ +namespace HipTestLevel { + +/// Levels understood by the test suite (must match definitions.yaml). +inline constexpr const char* kSupportedLevels[] = {"level_0", "level_1", "level_2", "level_3", + "level_4"}; + +/// Level used when neither the command line nor HIP_TEST_LEVEL specify one. +inline constexpr const char* kDefaultLevel = "level_2"; + +/// Which source the active level came from. +enum class LevelSource { kCommandLine, kEnvironment, kDefault }; + +/** + * @brief Outcome of resolving the active level, including what to report. + * + * A request may select more than one level ("HIP_TEST_LEVEL=level_1,level_3", + * "~level_4"). The two consumers treat that differently: + * + * - Test parameters take the single highest level (@ref level). There is one + * TestParameterStore per run, so one set of sizes and iteration counts has + * to win, and the highest is the widest. + * - Test selection uses every selected level (@ref levels), OR'd together, so + * nothing the request asked for is silently dropped from the run. + */ +struct LevelResolution { + /// Every selected level, ascending. Never empty after resolveLevel(). + std::set levels; + /// Highest of @ref levels as "level_N" - the level whose parameters are used. + /// Not necessarily supported; callers must still validate (see + /// @ref firstUnsupportedLevel). + std::string level; + /// Where @ref levels came from. + LevelSource source = LevelSource::kDefault; + /// Value of HIP_TEST_LEVEL as seen by resolveLevel(), empty when unset. + /// Retained so @ref describe can report a rejected value without re-reading + /// the environment. + std::string envValue; + /// HIP_TEST_LEVEL was set but named no valid level, so it was skipped. + bool envRejected = false; + + /// @brief Whether more than one level was selected. + bool multipleSelected() const; + + /// @brief First selected level the suite does not support, as "level_N". + /// @return "" when every selected level is supported. + std::string firstUnsupportedLevel() const; + + /// @brief Human-readable lines describing how the level was resolved. + /// @return One to three lines, most specific last. Never empty. + std::vector describe() const; +}; + +/// @brief Whether @p level is one of kSupportedLevels. +bool isSupportedLevel(const std::string& level); + +/** + * @brief Collect every level a tag expression can select. + * + * The expression is read the way Catch2 reads a test spec: a comma separated + * list of filters OR'd together, each filter a sequence of patterns AND'd + * together, each pattern optionally negated with '~'. Patterns that do not read + * exactly "level_N" (test names, other tags, and malformed tags such as + * "[level_2x]") place no constraint on the level and are ignored. + * + * A filter naming levels contributes those levels, minus the ones it excludes; + * a filter that only excludes levels contributes every supported level except + * the excluded ones; a filter mentioning no level at all contributes nothing. + * Note that filters are OR'd, so "[level_1],~[level_2]" also selects level_3 and + * level_4, exactly as Catch2 would. + * + * @param allowBareTags when true, a pattern may also be written without the + * surrounding brackets ("level_2", "~level_2"), which HIP_TEST_LEVEL + * accepts. Brackets, when present, must still be balanced. + * @return Distinct level numbers selectable, sorted ascending (empty if the + * expression places no constraint on the level). + */ +std::set collectLevels(const std::string& text, bool allowBareTags); + +/** + * @brief Reduce a set of levels to a single level string ("level_N"). + * @return "level_N" for the highest level, or "" if the set is empty. + */ +std::string highestLevel(const std::set& levels); + +/** + * @brief Resolve the active level using the strict priority order. + * @param testsOrTags the Catch2 test spec entries, as parsed from the command line. + * @param envLevel value of HIP_TEST_LEVEL, or nullptr when unset. + */ +LevelResolution resolveLevel(const std::vector& testsOrTags, const char* envLevel); + +/** + * @brief Let the active level select tests, when it was explicitly requested. + * + * An explicitly requested level narrows the run; the hardcoded default only + * supplies parameters. Concretely: + * + * - kCommandLine: the spec already names the levels. Nothing to do. + * - kDefault: nothing to do either. The default level supplies parameters + * and never narrows, so a bare "./test" and plain ctest run + * every test in the binary, as they always have. + * - kEnvironment, empty spec: becomes "[level_A],[level_B],..." over every + * selected level. + * - kEnvironment, non-empty spec: every filter of every entry is expanded once + * per selected level, so "Unit_foo" with levels {1,3} becomes + * "Unit_foo[level_1],Unit_foo[level_3]". + * + * Selection uses every level in LevelResolution::levels, not just the highest. + * Catch2 has no OR inside a single filter, so a multi-level request is encoded + * by repeating each filter once per level - the cross product of the spec's + * filters and the selected levels. + * + * Only an explicit request selects. A test binary cannot tell a hand-typed + * "./test Unit_foo" from the identical command line ctest generates for each + * registered test, so "was a level explicitly asked for?" is the only signal + * available. It also matters that level tags partition the suite rather than + * nesting: a default that narrowed would empty any binary whose tests all sit + * at some level other than the default. + * + * Patterns inside one Catch2 filter are AND'd, and filters (separated by + * unescaped commas) are OR'd, so the tag must be appended per filter. Appending + * it once as a new spec entry would instead OR in every test carrying the level. + * + * Backslash escapes are respected when splitting: CatchAddTests.cmake escapes + * '\', ',', '[', ']' and ';' in generated test names, so a name may legitimately + * contain "\,", which is not a filter separator. + * + * @param testsOrTags spec entries, rewritten in place. + * @param resolution the active level and where it came from. + * @return true if a filter was applied, so that a zero-test run means "nothing + * matched the level" rather than "nothing matched the user's spec". + */ +bool applyLevelFilter(std::vector& testsOrTags, const LevelResolution& resolution); + +/** + * @brief Translate "no tests ran" into "skipped" when a level filter caused it. + * + * When applyLevelFilter() narrowed the spec and nothing matched, Catch2 exits + * with NoTestsRunExitCode, which CTest reports as a failure. The run is really + * a skip: the test exists, it just does not belong to the active level. Mapping + * it onto Catch2's own AllTestsSkippedExitCode lets a single CTest + * SKIP_RETURN_CODE cover both cases. + * + * Both codes are file-local constants in Catch2's catch_session.cpp:37-42 and + * are not exported, so they are restated in the implementation. + * + * @param exitCode value returned by Catch::Session::run(). + * @param levelFilterApplied whether applyLevelFilter() rewrote the spec. + * @return the exit code to return from main(). + */ +int mapLevelFilterExitCode(int exitCode, bool levelFilterApplied); + +} // namespace HipTestLevel diff --git a/projects/hip-tests/catch/include/hip_test_params.hh b/projects/hip-tests/catch/include/hip_test_params.hh index 47a3d5cd8dc3..3a9d9935fb15 100644 --- a/projects/hip-tests/catch/include/hip_test_params.hh +++ b/projects/hip-tests/catch/include/hip_test_params.hh @@ -22,6 +22,8 @@ THE SOFTWARE. #pragma once +#include +#include #include #include #include @@ -30,9 +32,10 @@ THE SOFTWARE. /** * @brief Global parameter store for test configuration. * - * Parameters are loaded from compile-time constants generated from - * definitions.yaml at build time. The event listener detects the test - * level from command-line filters and loads appropriate parameters. + * Parameters are loaded from compile-time constants generated from + * definitions.yaml at build time. main() (see hip_test_level.cc) resolves + * the active test level before any test runs and loads the corresponding + * parameters. * * Thread Safety: * This class is designed for single-threaded test execution (Catch2 default). @@ -66,13 +69,13 @@ public: /** * @brief Initialize parameter store from generated compile-time constants - * Called once at test startup by event listener + * Called once at test startup by main() */ void initialize(); /** * @brief Load parameters for a specific level - * Called by event listener when [level_X] tag is detected + * Called by main() once the active level has been resolved * @param level Level name (e.g., "level_0", "level_1") */ void loadLevelConfig(const std::string& level); @@ -113,7 +116,7 @@ public: void clear(); /** - * @brief Current test level (set by event listener) + * @brief Current test level (set by main()) */ std::string currentTestLevel; @@ -134,11 +137,15 @@ private: TestParameterStore& operator=(const TestParameterStore&) = delete; /** - * @brief Fallback parameters (if no level specified) + * @brief Fallback parameters (if no level specified) - mirror level_2. */ - std::vector defaultMemorySizes; - std::vector defaultBlockSizes; - int defaultIterations = 1000; - int defaultWarmups = 100; - size_t defaultMaxMemory = 2147483648; // 2GB + // 64, 256, 1K, 4K, 16K, 64K, 256K, 1M, 10M, 50M, 100M, 500M, 1G, 2G + static constexpr std::array defaultMemorySizes = {64, 256, 1024, 4096, + 16384, 65536, 262144, 1048576, + 10485760, 52428800, 104857600, 524288000, + 1073741824, 2147483648}; + static constexpr std::array defaultBlockSizes = {32, 64, 96, 128, 192, 256, 384, 512, 768, 1024}; + static constexpr int defaultIterations = 5; + static constexpr int defaultWarmups = 5; + static constexpr size_t defaultMaxMemory = 8589934592; // 8G }; diff --git a/projects/hip-tests/catch/include/hip_test_temp_path.hh b/projects/hip-tests/catch/include/hip_test_temp_path.hh new file mode 100644 index 000000000000..87385c64c98a --- /dev/null +++ b/projects/hip-tests/catch/include/hip_test_temp_path.hh @@ -0,0 +1,45 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include "hip_test_filesystem.hh" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#define HIP_TEST_GETPID _getpid +#else +#include +#define HIP_TEST_GETPID getpid +#endif + +// Naming for temporary test artifacts. The temp directory is shared, so names +// must be unguessable: a predictable path can be pre-seeded with a symlink. +namespace hip_test { + +// 64 bits from std::random_device as 16 hex chars. The pid keeps the name +// unique if random_device degenerates to a fixed sequence, which the standard +// permits; live pids in a namespace are distinct, clock readings are not. +inline std::string RandomTag() { + std::random_device dev; + uint64_t value = (static_cast(dev()) << 32) ^ dev(); + value ^= static_cast(HIP_TEST_GETPID()); + char buf[17]; + snprintf(buf, sizeof(buf), "%016llx", static_cast(value)); + return std::string(buf); +} + +// /-. +inline std::string TempPath(const std::string& stem, const std::string& extension) { + return (fs::temp_directory_path() / (stem + "-" + RandomTag() + extension)).string(); +} + +} // namespace hip_test diff --git a/projects/hip-tests/catch/include/utils.hh b/projects/hip-tests/catch/include/utils.hh index 1b4debc778ac..e8026f9fbcb5 100644 --- a/projects/hip-tests/catch/include/utils.hh +++ b/projects/hip-tests/catch/include/utils.hh @@ -140,6 +140,16 @@ static __global__ void Delay(uint32_t interval, const uint32_t ticks_per_ms) { } } +// Spins until the host stores a nonzero value to `release`, which keeps the stream busy for as +// long as the host decides, independent of any timing. `release` must point to host memory that +// is coherent with the device, such as hipHostMalloc memory allocated with default flags. The +// host must set the flag before any check that can throw, otherwise the cleanup that follows +// waits forever for the kernel. +[[maybe_unused]] static __global__ void WaitForHostRelease(volatile int* release) { + while (*release == 0) { + } +} + template __global__ void Iota(T* const out, size_t pitch, size_t w, size_t h, size_t d) { const auto x = blockIdx.x * blockDim.x + threadIdx.x; diff --git a/projects/hip-tests/catch/include/warp_common.hh b/projects/hip-tests/catch/include/warp_common.hh index 1d8a72c3452a..cef4cb2decf9 100644 --- a/projects/hip-tests/catch/include/warp_common.hh +++ b/projects/hip-tests/catch/include/warp_common.hh @@ -182,6 +182,13 @@ struct DistributionType { using type = std::uniform_int_distribution; }; +template <> +struct DistributionType { + // there is not std::uniform_int_distribution on MSVC; we need to define an + // specialization + using type = std::uniform_int_distribution; +}; + // there is no std::uniform_real_distribution for 'half' type, so we cast from // unsigned short, avoiding Nan and Infinity template <> @@ -199,6 +206,10 @@ struct DistributionType { using type = std::uniform_real_distribution; }; +template <> +struct DistributionType { + using type = std::uniform_int_distribution; +}; template struct MinOp { @@ -394,6 +405,11 @@ void genRandomBuffers(LinearAllocGuard& d_buf, for (int i = 0; i < numItems; i++) { buf.ptr()[i] = genRandomHalf(dist, gen, exponent); } + } else if constexpr (std::is_same::value) { + for (int i = 0; i < numItems; i++) { + auto val = dist(gen); + buf.ptr()[i] = val == 1; + } } else { for (int i = 0; i < numItems; i++) { buf.ptr()[i] = dist(gen); @@ -483,7 +499,11 @@ void scanIdentity(T& id) result = std::numeric_limits::lowest(); } } else if constexpr (std::is_same>::value) { - result = ~result; + if constexpr(std::is_same::value) { + result = true; + } else { + result = ~result; + } } else if constexpr (std::is_same>::value) { } else { std::memset(&result, 0, sizeof(T)); @@ -693,6 +713,43 @@ void compareFloatingPoint(const T& result, const T& expected, unsigned long long } } +// To test scans/reduces we use a very reduce range of numbers to avoid overflows +template +struct TestRange { +}; + +template +struct TestRange>> { + // for float16, we generate any random unsigned short, but cap the exponent later on + // On the rest of the types, just use a bigger reduced range of numbers to avoid overflows too + static inline unsigned short minimum = std::numeric_limits::lowest(); + static inline unsigned short maximum = std::numeric_limits::max(); +}; + +template +struct TestRange>> { + static inline int minimum = 0; + static inline int maximum = 1; +}; + +template +struct TestRange>> { + static inline unsigned char minimum = 0; + static inline unsigned char maximum = 255; +}; + +template +struct TestRange && sizeof(T) >= 4>> { + static inline T minimum = -1023; + static inline T maximum = 1023; +}; + +template +struct TestRange && sizeof(T) >= 4>> { + static inline T minimum = 0; + static inline T maximum = 1023; +}; + // @tparam Reduce a functor; abstracts away kernel dispatching // (via hiprtc or normal execution) template class Op> @@ -705,12 +762,8 @@ void runTestReduce(int iteration, Reduce reduce) LinearAllocGuard d_output(LinearAllocs::hipMalloc, kNumReduces * wavefrontSize * sizeof(T)); LinearAllocGuard output(LinearAllocs::malloc, kNumReduces * wavefrontSize * sizeof(T)); std::mt19937_64 gen(iteration); - // for float16, we generate any random unsigned short, but cap the exponent later on - // On the rest of the types, just use a bigger reduced range of numbers to avoid overflows too - typename distribution::result_type a = std::is_same::value? std::numeric_limits::lowest() : - (std::is_signed::value? -1023 : 0); - typename distribution::result_type b = std::is_same::value? std::numeric_limits::max() : - 1023; + typename distribution::result_type a = TestRange::minimum; + typename distribution::result_type b = TestRange::maximum; distribution dist(a, b); LinearAllocGuard input, d_input; LinearAllocGuard masks, d_masks; diff --git a/projects/hip-tests/catch/performance/scenarios/memory/hipPerfMemFill.cc b/projects/hip-tests/catch/performance/scenarios/memory/hipPerfMemFill.cc index 886c79fd0d84..ae7b9114e231 100644 --- a/projects/hip-tests/catch/performance/scenarios/memory/hipPerfMemFill.cc +++ b/projects/hip-tests/catch/performance/scenarios/memory/hipPerfMemFill.cc @@ -118,7 +118,7 @@ template class hipPerfMemFill { << " " << props_.name << " with " << props_.multiProcessorCount << " CUs, large bar: " << supportLargeBar() << ", managed memory: " << supportManagedMemory() - << ", DeviceMallocFinegrained: " << supportDeviceMallocFinegrained() << std::endl; + << ", Device has FineGrained Attribute: " << deviceAttributeSupportFineGrained() << std::endl; return true; } @@ -369,19 +369,12 @@ template class hipPerfMemFill { return true; } - /* This function should be via device attribute query*/ - bool supportDeviceMallocFinegrained() { + bool deviceAttributeSupportFineGrained() { #ifdef __HIP_PLATFORM_AMD__ - T* A = nullptr; - hipError_t err; - - err = - hipExtMallocWithFlags(reinterpret_cast(&A), sizeof(T), hipDeviceMallocFinegrained); - if (err || !A) { - return false; - } - HIP_CHECK(hipFree(A)); - return true; + int deviceId, deviceFineGrain; + HIP_CHECK(hipGetDevice(&deviceId)); + HIP_CHECK(hipDeviceGetAttribute(&deviceFineGrain, hipDeviceAttributeFineGrainSupport, deviceId)); + return deviceFineGrain > 0; #else return false; #endif @@ -477,7 +470,7 @@ template class hipPerfMemFill { } #ifdef __HIP_PLATFORM_AMD__ - if (supportDeviceMallocFinegrained()) { + if (deviceAttributeSupportFineGrained()) { if (!testExtDeviceMemory()) { return false; } diff --git a/projects/hip-tests/catch/unit/atomics/CMakeLists.txt b/projects/hip-tests/catch/unit/atomics/CMakeLists.txt index 235bcdc2e2be..2efecc4a41fe 100644 --- a/projects/hip-tests/catch/unit/atomics/CMakeLists.txt +++ b/projects/hip-tests/catch/unit/atomics/CMakeLists.txt @@ -16,11 +16,6 @@ if(HIP_PLATFORM MATCHES "amd") atomicMax_system.cc unsafeAtomicMin.cc unsafeAtomicMax.cc - __hip_atomic_fetch_min.cc - __hip_atomic_fetch_max.cc - atomic_builtins.cc - acquire_release.cc - sequential_consistency.cc atomicAdd.cc atomicAdd_system.cc safeAtomicAdd.cc @@ -28,14 +23,8 @@ if(HIP_PLATFORM MATCHES "amd") atomicSub_system.cc atomicCAS.cc atomicCAS_system.cc - __hip_atomic_fetch_add.cc - __hip_atomic_compare_exchange_strong.cc atomicExch.cc atomicExch_system.cc - __hip_atomic_fetch_and.cc - __hip_atomic_fetch_or.cc - __hip_atomic_fetch_xor.cc - __hip_atomic_exchange.cc atomicInc.cc atomicDec.cc ) @@ -63,7 +52,6 @@ if(HIP_PLATFORM MATCHES "amd") "atomicXor_negative_kernels.cc" "atomicMin_negative_kernels.cc" "atomicMax_negative_kernels.cc" - "atomic_builtin_kernels.cc" "atomicAdd_negative_kernels.cc" "atomicSub_negative_kernels.cc" "atomicInc_negative_kernels.cc" diff --git a/projects/hip-tests/catch/unit/atomics/__hip_atomic_compare_exchange_strong.cc b/projects/hip-tests/catch/unit/atomics/__hip_atomic_compare_exchange_strong.cc deleted file mode 100644 index 5b461cdd0247..000000000000 --- a/projects/hip-tests/catch/unit/atomics/__hip_atomic_compare_exchange_strong.cc +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include "arithmetic_common.hh" - -#include - -/** - * @addtogroup __hip_atomic_compare_exchange_strong __hip_atomic_compare_exchange_strong - * @{ - * @ingroup AtomicsTest - */ - -// Helper function to run __hip_atomic_compare_exchange_strong tests with WAVEFRONT scope -template static void runHipAtomicCompareExchangeStrongWavefrontTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - - DYNAMIC_SECTION("Adjacent addresses " << current) { - SingleDeviceSingleKernelTest(warp_size, sizeof(TestType)); - } - - DYNAMIC_SECTION("Scattered addresses " << current) { - SingleDeviceSingleKernelTest(warp_size, cache_line_size); - } - } -} - -// Helper function to run __hip_atomic_compare_exchange_strong tests with WORKGROUP scope -template static void runHipAtomicCompareExchangeStrongWorkgroupTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - - DYNAMIC_SECTION("Adjacent addresses " << current) { - SingleDeviceSingleKernelTest(warp_size, sizeof(TestType)); - } - - DYNAMIC_SECTION("Scattered addresses " << current) { - SingleDeviceSingleKernelTest(warp_size, cache_line_size); - } - } -} - -/** - * Test Description - * ------------------------ - * - Executes a single kernel on a single device wherein all threads will perform an atomic - * addition on a target memory location. Each thread will add the same value to the memory location, - * storing the return value into a separate output array slot corresponding to it. Once complete, - * the output array and target memory is validated to contain all the expected values. Several - * memory access patterns are tested: - * -# All threads add to a single, compile time deducible, memory location - * -# Each thread targets an array containing warp_size elements, using tid % warp_size - * for indexing - * -# Same as the above, but the elements are spread out by L1 cache line size bytes. - * - * - The test is run for: - * - All overloads of __hip_atomic_compare_exchange_strong - * - hipMalloc, hipMallocManaged, hipHostMalloc and hipHostRegister allocated memory - * - Shared memory - * - WAVEFRONT memory scope. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_compare_exchange_strong.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_compare_exchange_strong_Positive_Wavefront) { - SECTION("int") { runHipAtomicCompareExchangeStrongWavefrontTest(); } - SECTION("unsigned int") { runHipAtomicCompareExchangeStrongWavefrontTest(); } - SECTION("unsigned long") { runHipAtomicCompareExchangeStrongWavefrontTest(); } - SECTION("unsigned long long") { - runHipAtomicCompareExchangeStrongWavefrontTest(); - } - SECTION("float") { runHipAtomicCompareExchangeStrongWavefrontTest(); } - SECTION("double") { runHipAtomicCompareExchangeStrongWavefrontTest(); } -} - -/** - * Test Description - * ------------------------ - * - Executes a single kernel on a single device wherein all threads will perform an atomic - * addition on a target memory location. Each thread will add the same value to the memory location, - * storing the return value into a separate output array slot corresponding to it. Once complete, - * the output array and target memory is validated to contain all the expected values. Several - * memory access patterns are tested: - * -# All threads add to a single, compile time deducible, memory location - * -# Each thread targets an array containing warp_size elements, using tid % warp_size - * for indexing - * -# Same as the above, but the elements are spread out by L1 cache line size bytes. - * - * - The test is run for: - * - All overloads of __hip_atomic_compare_exchange_strong - * - hipMalloc, hipMallocManaged, hipHostMalloc and hipHostRegister allocated memory - * - Shared memory - * - WORKGROUP memory scope. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_compare_exchange_strong.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_compare_exchange_strong_Positive_Workgroup) { - SECTION("int") { runHipAtomicCompareExchangeStrongWorkgroupTest(); } - SECTION("unsigned int") { runHipAtomicCompareExchangeStrongWorkgroupTest(); } - SECTION("unsigned long") { runHipAtomicCompareExchangeStrongWorkgroupTest(); } - SECTION("unsigned long long") { - runHipAtomicCompareExchangeStrongWorkgroupTest(); - } - SECTION("float") { runHipAtomicCompareExchangeStrongWorkgroupTest(); } - SECTION("double") { runHipAtomicCompareExchangeStrongWorkgroupTest(); } -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/__hip_atomic_exchange.cc b/projects/hip-tests/catch/unit/atomics/__hip_atomic_exchange.cc deleted file mode 100644 index 22232cb49ff9..000000000000 --- a/projects/hip-tests/catch/unit/atomics/__hip_atomic_exchange.cc +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include "atomicExch_common.hh" - -/** - * @addtogroup __hip_atomic_exchange __hip_atomic_exchange - * @{ - * @ingroup AtomicsTest - * ________________________ - * Test cases from other modules: - * - @ref Unit_AtomicBuiltins_Negative_Parameters_RTC - */ - -// Helper function to run __hip_atomic_exchange tests with WAVEFRONT scope -template static void runHipAtomicExchangeWavefrontTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - AtomicExchSingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - - DYNAMIC_SECTION("Adjacent addresses " << current) { - AtomicExchSingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - - DYNAMIC_SECTION("Scattered addresses " << current) { - AtomicExchSingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -// Helper function to run __hip_atomic_exchange tests with WORKGROUP scope -template static void runHipAtomicExchangeWorkgroupTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - AtomicExchSingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - - DYNAMIC_SECTION("Adjacent addresses " << current) { - AtomicExchSingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - - DYNAMIC_SECTION("Scattered addresses " << current) { - AtomicExchSingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -/** - * Test Description - * ------------------------ - * - Executes a single kernel on a single device wherein all threads will perform an atomic - * exchange into a runtime determined memory location. Each thread will exchange its own grid wide - * linear index + offset into the memory location, storing the return value into a separate output - * array slot corresponding to it. Once complete, the union of output array and exchange memory is - * validated to contain all values in the range [0, number_of_threads + - * number_of_exchange_memory_slots). Several memory access patterns are tested: - * -# All threads exchange to a single memory location - * -# Each thread exchanges into an array containing warp_size elements, using tid % warp_size - * for indexing - * -# Same as the above, but the exchange elements are spread out by L1 cache line size bytes. - * - * - The test is run for: - * - All overloads of atomicExch - * - hipMalloc, hipMallocManaged, hipHostMalloc and hipHostRegister allocated exchange memory - * - Exchange memory located in shared memory - * - WAVEFRONT memory scope - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_exchange.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_exchange_Positive_Wavefront) { - SECTION("int") { runHipAtomicExchangeWavefrontTest(); } - SECTION("unsigned int") { runHipAtomicExchangeWavefrontTest(); } - SECTION("unsigned long") { runHipAtomicExchangeWavefrontTest(); } - SECTION("unsigned long long") { runHipAtomicExchangeWavefrontTest(); } - SECTION("float") { runHipAtomicExchangeWavefrontTest(); } - SECTION("double") { runHipAtomicExchangeWavefrontTest(); } -} - -/** - * Test Description - * ------------------------ - * - Executes a single kernel on a single device wherein all threads will perform an atomic - * exchange into a runtime determined memory location. Each thread will exchange its own grid wide - * linear index + offset into the memory location, storing the return value into a separate output - * array slot corresponding to it. Once complete, the union of output array and exchange memory is - * validated to contain all values in the range [0, number_of_threads + - * number_of_exchange_memory_slots). Several memory access patterns are tested: - * -# All threads exchange to a single memory location - * -# Each thread exchanges into an array containing warp_size elements, using tid % warp_size - * for indexing - * -# Same as the above, but the exchange elements are spread out by L1 cache line size bytes. - * - * - The test is run for: - * - All overloads of atomicExch - * - hipMalloc, hipMallocManaged, hipHostMalloc and hipHostRegister allocated exchange memory - * - Exchange memory located in shared memory - * - WORKGROUP memory scope - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_exchange.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_exchange_Positive_Workgroup) { - SECTION("int") { runHipAtomicExchangeWorkgroupTest(); } - SECTION("unsigned int") { runHipAtomicExchangeWorkgroupTest(); } - SECTION("unsigned long") { runHipAtomicExchangeWorkgroupTest(); } - SECTION("unsigned long long") { runHipAtomicExchangeWorkgroupTest(); } - SECTION("float") { runHipAtomicExchangeWorkgroupTest(); } - SECTION("double") { runHipAtomicExchangeWorkgroupTest(); } -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_add.cc b/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_add.cc deleted file mode 100644 index f893b5988dad..000000000000 --- a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_add.cc +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include "arithmetic_common.hh" - -#include - -/** - * @addtogroup __hip_atomic_fetch_add __hip_atomic_fetch_add - * @{ - * @ingroup AtomicsTest - * ________________________ - * Test cases from other modules: - * - @ref Unit_AtomicBuiltins_Negative_Parameters_RTC - */ - -// Helper function to run __hip_atomic_fetch_add tests with WAVEFRONT scope -template -static void runHipAtomicFetchAddWavefrontTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - - DYNAMIC_SECTION("Adjacent addresses " << current) { - SingleDeviceSingleKernelTest(warp_size, sizeof(TestType)); - } - - DYNAMIC_SECTION("Scattered addresses " << current) { - SingleDeviceSingleKernelTest(warp_size, cache_line_size); - } - } -} - -// Helper function to run __hip_atomic_fetch_add tests with WORKGROUP scope -template -static void runHipAtomicFetchAddWorkgroupTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - - DYNAMIC_SECTION("Adjacent addresses " << current) { - SingleDeviceSingleKernelTest(warp_size, sizeof(TestType)); - } - - DYNAMIC_SECTION("Scattered addresses " << current) { - SingleDeviceSingleKernelTest(warp_size, cache_line_size); - } - } -} - -/** - * Test Description - * ------------------------ - * - Executes a single kernel on a single device wherein all threads will perform an atomic - * addition on a target memory location. Each thread will add the same value to the memory location, - * storing the return value into a separate output array slot corresponding to it. Once complete, - * the output array and target memory is validated to contain all the expected values. Several - * memory access patterns are tested: - * -# All threads add to a single, compile time deducible, memory location - * -# Each thread targets an array containing warp_size elements, using tid % warp_size - * for indexing - * -# Same as the above, but the elements are spread out by L1 cache line size bytes. - * - * - The test is run for: - * - All overloads of __hip_atomic_fetch_add - * - hipMalloc, hipMallocManaged, hipHostMalloc and hipHostRegister allocated memory - * - Shared memory - * - WAVEFRONT memory scope. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_add.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_add_Positive_Wavefront) { - SECTION("int") { runHipAtomicFetchAddWavefrontTest(); } - SECTION("unsigned int") { runHipAtomicFetchAddWavefrontTest(); } - SECTION("unsigned long") { runHipAtomicFetchAddWavefrontTest(); } - SECTION("unsigned long long") { runHipAtomicFetchAddWavefrontTest(); } - SECTION("float") { runHipAtomicFetchAddWavefrontTest(); } - SECTION("double") { runHipAtomicFetchAddWavefrontTest(); } -} - -/** - * Test Description - * ------------------------ - * - Executes a single kernel on a single device wherein all threads will perform an atomic - * addition on a target memory location. Each thread will add the same value to the memory location, - * storing the return value into a separate output array slot corresponding to it. Once complete, - * the output array and target memory is validated to contain all the expected values. Several - * memory access patterns are tested: - * -# All threads add to a single, compile time deducible, memory location - * -# Each thread targets an array containing warp_size elements, using tid % warp_size - * for indexing - * -# Same as the above, but the elements are spread out by L1 cache line size bytes. - * - * - The test is run for: - * - All overloads of __hip_atomic_fetch_add - * - hipMalloc, hipMallocManaged, hipHostMalloc and hipHostRegister allocated memory - * - Shared memory - * - WORKGROUP memory scope. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_add.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_add_Positive_Workgroup) { - SECTION("int") { runHipAtomicFetchAddWorkgroupTest(); } - SECTION("unsigned int") { runHipAtomicFetchAddWorkgroupTest(); } - SECTION("unsigned long") { runHipAtomicFetchAddWorkgroupTest(); } - SECTION("unsigned long long") { runHipAtomicFetchAddWorkgroupTest(); } - SECTION("float") { runHipAtomicFetchAddWorkgroupTest(); } - SECTION("double") { runHipAtomicFetchAddWorkgroupTest(); } -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_and.cc b/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_and.cc deleted file mode 100644 index 463aa4c9e5dc..000000000000 --- a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_and.cc +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include "bitwise_common.hh" - -#include - -/** - * @addtogroup __hip_atomic_fetch_and __hip_atomic_fetch_and - * @{ - * @ingroup AtomicsTest - */ - -// Helper function to run __hip_atomic_fetch_and tests for WAVEFRONT scope, same address -template -static void runHipAtomicFetchAndWavefrontSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - Bitwise::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_and tests for WAVEFRONT scope, adjacent addresses -template -static void runHipAtomicFetchAndWavefrontAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_and tests for WAVEFRONT scope, scattered addresses -template -static void runHipAtomicFetchAndWavefrontScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -// Helper function to run __hip_atomic_fetch_and tests for WORKGROUP scope, same address -template -static void runHipAtomicFetchAndWorkgroupSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - Bitwise::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_and tests for WORKGROUP scope, adjacent addresses -template -static void runHipAtomicFetchAndWorkgroupAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_and tests for WORKGROUP scope, scattered addresses -template -static void runHipAtomicFetchAndWorkgroupScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic AND with memory scope WAVEFRONT from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_and.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_and_Positive_Wavefront_SameAddress) { - SECTION("int") { runHipAtomicFetchAndWavefrontSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchAndWavefrontSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchAndWavefrontSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchAndWavefrontSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic AND with memory scope WAVEFRONT from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_and.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_and_Positive_Wavefront_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchAndWavefrontAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchAndWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchAndWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchAndWavefrontAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic AND with memory scope WAVEFRONT from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_and.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_and_Positive_Wavefront_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchAndWavefrontScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchAndWavefrontScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchAndWavefrontScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchAndWavefrontScatteredAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic AND with memory scope WORKGROUP from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_and.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_and_Positive_Workgroup_SameAddress) { - SECTION("int") { runHipAtomicFetchAndWorkgroupSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchAndWorkgroupSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchAndWorkgroupSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchAndWorkgroupSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic AND with memory scope WORKGROUP from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_and.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_and_Positive_Workgroup_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchAndWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchAndWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchAndWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchAndWorkgroupAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic AND with memory scope WORKGROUP from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_and.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_and_Positive_Workgroup_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchAndWorkgroupScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchAndWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchAndWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchAndWorkgroupScatteredAddressesTest(); } -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_max.cc b/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_max.cc deleted file mode 100644 index 0618f7a23aa5..000000000000 --- a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_max.cc +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include "min_max_common.hh" - -#include - -/** - * @addtogroup __hip_atomic_fetch_max __hip_atomic_fetch_max - * @{ - * @ingroup AtomicsTest - */ - -// Helper function to run __hip_atomic_fetch_max tests for WAVEFRONT scope, same address -template -static void runHipAtomicFetchMaxWavefrontSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - MinMax::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_max tests for WAVEFRONT scope, adjacent addresses -template -static void runHipAtomicFetchMaxWavefrontAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - MinMax::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_max tests for WAVEFRONT scope, scattered addresses -template -static void runHipAtomicFetchMaxWavefrontScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - MinMax::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -// Helper function to run __hip_atomic_fetch_max tests for WORKGROUP scope, same address -template -static void runHipAtomicFetchMaxWorkgroupSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - MinMax::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_max tests for WORKGROUP scope, adjacent addresses -template -static void runHipAtomicFetchMaxWorkgroupAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - MinMax::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_max tests for WORKGROUP scope, scattered addresses -template -static void runHipAtomicFetchMaxWorkgroupScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - MinMax::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MAX with memory scope WAVEFRONT from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_max.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_max_Positive_Wavefront_SameAddress) { - SECTION("int") { runHipAtomicFetchMaxWavefrontSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchMaxWavefrontSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchMaxWavefrontSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMaxWavefrontSameAddressTest(); } - SECTION("float") { runHipAtomicFetchMaxWavefrontSameAddressTest(); } - SECTION("double") { runHipAtomicFetchMaxWavefrontSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MAX with memory scope WAVEFRONT from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_max.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_max_Positive_Wavefront_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchMaxWavefrontAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchMaxWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchMaxWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMaxWavefrontAdjacentAddressesTest(); } - SECTION("float") { runHipAtomicFetchMaxWavefrontAdjacentAddressesTest(); } - SECTION("double") { runHipAtomicFetchMaxWavefrontAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MAX with memory scope WAVEFRONT from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_max.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_max_Positive_Wavefront_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchMaxWavefrontScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchMaxWavefrontScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchMaxWavefrontScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMaxWavefrontScatteredAddressesTest(); } - SECTION("float") { runHipAtomicFetchMaxWavefrontScatteredAddressesTest(); } - SECTION("double") { runHipAtomicFetchMaxWavefrontScatteredAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MAX with memory scope WORKGROUP from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_max.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_max_Positive_Workgroup_SameAddress) { - SECTION("int") { runHipAtomicFetchMaxWorkgroupSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchMaxWorkgroupSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchMaxWorkgroupSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMaxWorkgroupSameAddressTest(); } - SECTION("float") { runHipAtomicFetchMaxWorkgroupSameAddressTest(); } - SECTION("double") { runHipAtomicFetchMaxWorkgroupSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MAX with memory scope WORKGROUP from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_max.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_max_Positive_Workgroup_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchMaxWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchMaxWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchMaxWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMaxWorkgroupAdjacentAddressesTest(); } - SECTION("float") { runHipAtomicFetchMaxWorkgroupAdjacentAddressesTest(); } - SECTION("double") { runHipAtomicFetchMaxWorkgroupAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MAX with memory scope WORKGROUP from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_max.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_max_Positive_Workgroup_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchMaxWorkgroupScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchMaxWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchMaxWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMaxWorkgroupScatteredAddressesTest(); } - SECTION("float") { runHipAtomicFetchMaxWorkgroupScatteredAddressesTest(); } - SECTION("double") { runHipAtomicFetchMaxWorkgroupScatteredAddressesTest(); } -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_min.cc b/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_min.cc deleted file mode 100644 index a8287fd03021..000000000000 --- a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_min.cc +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include "min_max_common.hh" - -#include - -/** - * @addtogroup __hip_atomic_fetch_min __hip_atomic_fetch_min - * @{ - * @ingroup AtomicsTest - */ - -// Helper function to run __hip_atomic_fetch_min tests for WAVEFRONT scope, same address -template -static void runHipAtomicFetchMinWavefrontSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - MinMax::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_min tests for WAVEFRONT scope, adjacent addresses -template -static void runHipAtomicFetchMinWavefrontAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - MinMax::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_min tests for WAVEFRONT scope, scattered addresses -template -static void runHipAtomicFetchMinWavefrontScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - MinMax::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -// Helper function to run __hip_atomic_fetch_min tests for WORKGROUP scope, same address -template -static void runHipAtomicFetchMinWorkgroupSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - MinMax::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_min tests for WORKGROUP scope, adjacent addresses -template -static void runHipAtomicFetchMinWorkgroupAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - MinMax::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_min tests for WORKGROUP scope, scattered addresses -template -static void runHipAtomicFetchMinWorkgroupScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - MinMax::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MIN with memory scope WAVEFRONT from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_min.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_min_Positive_Wavefront_SameAddress) { - SECTION("int") { runHipAtomicFetchMinWavefrontSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchMinWavefrontSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchMinWavefrontSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMinWavefrontSameAddressTest(); } - SECTION("float") { runHipAtomicFetchMinWavefrontSameAddressTest(); } - SECTION("double") { runHipAtomicFetchMinWavefrontSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MIN with memory scope WAVEFRONT from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_min.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_min_Positive_Wavefront_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchMinWavefrontAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchMinWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchMinWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMinWavefrontAdjacentAddressesTest(); } - SECTION("float") { runHipAtomicFetchMinWavefrontAdjacentAddressesTest(); } - SECTION("double") { runHipAtomicFetchMinWavefrontAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MIN with memory scope WAVEFRONT from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_min.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_min_Positive_Wavefront_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchMinWavefrontScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchMinWavefrontScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchMinWavefrontScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMinWavefrontScatteredAddressesTest(); } - SECTION("float") { runHipAtomicFetchMinWavefrontScatteredAddressesTest(); } - SECTION("double") { runHipAtomicFetchMinWavefrontScatteredAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MIN with memory scope WORKGROUP from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_min.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_min_Positive_Workgroup_SameAddress) { - SECTION("int") { runHipAtomicFetchMinWorkgroupSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchMinWorkgroupSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchMinWorkgroupSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMinWorkgroupSameAddressTest(); } - SECTION("float") { runHipAtomicFetchMinWorkgroupSameAddressTest(); } - SECTION("double") { runHipAtomicFetchMinWorkgroupSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MIN with memory scope WORKGROUP from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_min.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_min_Positive_Workgroup_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchMinWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchMinWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchMinWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMinWorkgroupAdjacentAddressesTest(); } - SECTION("float") { runHipAtomicFetchMinWorkgroupAdjacentAddressesTest(); } - SECTION("double") { runHipAtomicFetchMinWorkgroupAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic MIN with memory scope WORKGROUP from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_min.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_min_Positive_Workgroup_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchMinWorkgroupScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchMinWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchMinWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchMinWorkgroupScatteredAddressesTest(); } - SECTION("float") { runHipAtomicFetchMinWorkgroupScatteredAddressesTest(); } - SECTION("double") { runHipAtomicFetchMinWorkgroupScatteredAddressesTest(); } -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_or.cc b/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_or.cc deleted file mode 100644 index 8c2d010d9b55..000000000000 --- a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_or.cc +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include "bitwise_common.hh" - -#include - -/** - * @addtogroup __hip_atomic_fetch_or __hip_atomic_fetch_or - * @{ - * @ingroup AtomicsTest - */ - -// Helper function to run __hip_atomic_fetch_or tests for WAVEFRONT scope, same address -template -static void runHipAtomicFetchOrWavefrontSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - Bitwise::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_or tests for WAVEFRONT scope, adjacent addresses -template -static void runHipAtomicFetchOrWavefrontAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_or tests for WAVEFRONT scope, scattered addresses -template -static void runHipAtomicFetchOrWavefrontScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -// Helper function to run __hip_atomic_fetch_or tests for WORKGROUP scope, same address -template -static void runHipAtomicFetchOrWorkgroupSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - Bitwise::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_or tests for WORKGROUP scope, adjacent addresses -template -static void runHipAtomicFetchOrWorkgroupAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_or tests for WORKGROUP scope, scattered addresses -template -static void runHipAtomicFetchOrWorkgroupScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic OR with memory scope WAVEFRONT from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_or.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_or_Positive_Wavefront_SameAddress) { - SECTION("int") { runHipAtomicFetchOrWavefrontSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchOrWavefrontSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchOrWavefrontSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchOrWavefrontSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic OR with memory scope WAVEFRONT from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_or.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_or_Positive_Wavefront_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchOrWavefrontAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchOrWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchOrWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchOrWavefrontAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic OR with memory scope WAVEFRONT from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_or.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_or_Positive_Wavefront_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchOrWavefrontScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchOrWavefrontScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchOrWavefrontScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchOrWavefrontScatteredAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic OR with memory scope WORKGROUP from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_or.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_or_Positive_Workgroup_SameAddress) { - SECTION("int") { runHipAtomicFetchOrWorkgroupSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchOrWorkgroupSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchOrWorkgroupSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchOrWorkgroupSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic OR with memory scope WORKGROUP from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_or.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_or_Positive_Workgroup_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchOrWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchOrWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchOrWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchOrWorkgroupAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic OR with memory scope WORKGROUP from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_or.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_or_Positive_Workgroup_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchOrWorkgroupScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchOrWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchOrWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchOrWorkgroupScatteredAddressesTest(); } -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_xor.cc b/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_xor.cc deleted file mode 100644 index 8aa932e27903..000000000000 --- a/projects/hip-tests/catch/unit/atomics/__hip_atomic_fetch_xor.cc +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include "bitwise_common.hh" - -#include - -/** - * @addtogroup __hip_atomic_fetch_xor __hip_atomic_fetch_xor - * @{ - * @ingroup AtomicsTest - */ - -// Helper function to run __hip_atomic_fetch_xor tests for WAVEFRONT scope, same address -template -static void runHipAtomicFetchXorWavefrontSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - Bitwise::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_xor tests for WAVEFRONT scope, adjacent addresses -template -static void runHipAtomicFetchXorWavefrontAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_xor tests for WAVEFRONT scope, scattered addresses -template -static void runHipAtomicFetchXorWavefrontScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -// Helper function to run __hip_atomic_fetch_xor tests for WORKGROUP scope, same address -template -static void runHipAtomicFetchXorWorkgroupSameAddressTest() { - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Same address " << current) { - Bitwise::SingleDeviceSingleKernelTest(1, sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_xor tests for WORKGROUP scope, adjacent addresses -template -static void runHipAtomicFetchXorWorkgroupAdjacentAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Adjacent address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - sizeof(TestType)); - } - } -} - -// Helper function to run __hip_atomic_fetch_xor tests for WORKGROUP scope, scattered addresses -template -static void runHipAtomicFetchXorWorkgroupScatteredAddressesTest() { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - const auto cache_line_size = 128u; - - for (auto current = 0; current < cmd_options.iterations; ++current) { - DYNAMIC_SECTION("Scattered address " << current) { - Bitwise::SingleDeviceSingleKernelTest(warp_size, - cache_line_size); - } - } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic XOR with memory scope WAVEFRONT from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_xor.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_xor_Positive_Wavefront_SameAddress) { - SECTION("int") { runHipAtomicFetchXorWavefrontSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchXorWavefrontSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchXorWavefrontSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchXorWavefrontSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic XOR with memory scope WAVEFRONT from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_xor.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_xor_Positive_Wavefront_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchXorWavefrontAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchXorWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchXorWavefrontAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchXorWavefrontAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic XOR with memory scope WAVEFRONT from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_xor.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_xor_Positive_Wavefront_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchXorWavefrontScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchXorWavefrontScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchXorWavefrontScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchXorWavefrontScatteredAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic XOR with memory scope WORKGROUP from multiple threads on the same - * address. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_xor.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_xor_Positive_Workgroup_SameAddress) { - SECTION("int") { runHipAtomicFetchXorWorkgroupSameAddressTest(); } - SECTION("unsigned int") { runHipAtomicFetchXorWorkgroupSameAddressTest(); } - SECTION("unsigned long") { runHipAtomicFetchXorWorkgroupSameAddressTest(); } - SECTION("unsigned long long") { runHipAtomicFetchXorWorkgroupSameAddressTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic XOR with memory scope WORKGROUP from multiple threads on adjacent - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_xor.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_xor_Positive_Workgroup_Adjacent_Addresses) { - SECTION("int") { runHipAtomicFetchXorWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchXorWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchXorWorkgroupAdjacentAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchXorWorkgroupAdjacentAddressesTest(); } -} - -/** - * Test Description - * ------------------------ - * - Performs a builtin atomic XOR with memory scope WORKGROUP from multiple threads on scattered - * addresses. - * - Uses only one device and launches one kernel. - * Test source - * ------------------------ - * - unit/atomics/__hip_atomic_fetch_xor.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit___hip_atomic_fetch_xor_Positive_Workgroup_Scattered_Addresses) { - SECTION("int") { runHipAtomicFetchXorWorkgroupScatteredAddressesTest(); } - SECTION("unsigned int") { runHipAtomicFetchXorWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long") { runHipAtomicFetchXorWorkgroupScatteredAddressesTest(); } - SECTION("unsigned long long") { runHipAtomicFetchXorWorkgroupScatteredAddressesTest(); } -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/acquire_release.cc b/projects/hip-tests/catch/unit/atomics/acquire_release.cc deleted file mode 100644 index 24dd63f9b5e9..000000000000 --- a/projects/hip-tests/catch/unit/atomics/acquire_release.cc +++ /dev/null @@ -1,535 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include - -#include "memory_order_common.hh" - -HIP_TEST_CASE(Unit___hip_atomic_load_store_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_exchange_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_compare_exchange_strong_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_compare_exchange_weak_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_add_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_and_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_or_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_xor_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_min_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_max_Positive_Acquire_Release) { - SECTION("ACQUIRE/RELEASE") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("ACQ_REL") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } - SECTION("SEQ_CST") { - SECTION("WAVEFRONT") { - AcquireRelease::Test(); - } - SECTION("WORKGROUP") { - AcquireRelease::Test(); - } - SECTION("AGENT") { - AcquireRelease::Test(); - } - SECTION("SYSTEM") { - AcquireRelease::SystemTest(); - } - } -} diff --git a/projects/hip-tests/catch/unit/atomics/arithmetic_common.hh b/projects/hip-tests/catch/unit/atomics/arithmetic_common.hh index 668d11108fa9..6305a80eb0d0 100644 --- a/projects/hip-tests/catch/unit/atomics/arithmetic_common.hh +++ b/projects/hip-tests/catch/unit/atomics/arithmetic_common.hh @@ -24,9 +24,7 @@ enum class AtomicOperation { kUnsafeAdd, kSafeAdd, kCASAdd, - kCASAddSystem, - kBuiltinAdd, - kBuiltinCAS + kCASAddSystem }; // Constants that are passed as operands to the atomic operations @@ -72,29 +70,9 @@ __device__ TestType CASAtomicAddSystem(TestType* address, TestType val) { return old; } -// Implements an atomic addition via __hip_atomic_compare_exchange_strong -template -__device__ TestType BuiltinCASAtomicAdd(TestType* address, TestType val) { - TestType old = *address, assumed; - - const auto builtin_cas = [](TestType* address, TestType assumed, TestType val) { - __hip_atomic_compare_exchange_strong(address, &assumed, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - memory_scope); - return assumed; - }; - - do { - assumed = old; - old = builtin_cas(address, assumed, val + assumed); - } while (assumed != old); - - return old; -} - // Performs an atomic operation on parameter `mem` based on the `operation` enumerator. -// `memory_scope` is forwarded to the builtin operations and is by default device-wide. template -__device__ TestType PerformAtomicOperation(TestType* const mem, const LinearAllocs allocType) { +__device__ TestType PerformAtomicOperation(TestType* const mem) { const auto val = GetTestValue(); if constexpr (operation == AtomicOperation::kAdd) { @@ -117,16 +95,6 @@ __device__ TestType PerformAtomicOperation(TestType* const mem, const LinearAllo return CASAtomicAdd(mem, val); } else if constexpr (operation == AtomicOperation::kCASAddSystem) { return CASAtomicAddSystem(mem, val); - } else if constexpr (operation == AtomicOperation::kBuiltinAdd) { - if (std::is_floating_point_v && allocType == LinearAllocs::hipHostMalloc) { - HIP_TEST_ATOMIC_BACKWARD_COMPAT_MEMORY { - return __hip_atomic_fetch_add(mem, val, __ATOMIC_RELAXED, memory_scope); - } - } else { - return __hip_atomic_fetch_add(mem, val, __ATOMIC_RELAXED, memory_scope); - } - } else if constexpr (operation == AtomicOperation::kBuiltinCAS) { - return BuiltinCASAtomicAdd(mem, val); } } @@ -137,8 +105,7 @@ __device__ TestType PerformAtomicOperation(TestType* const mem, const LinearAllo // operations are executed on shared memory, and the result is copied back to `global_mem`. template -__global__ void TestKernel(TestType* const global_mem, TestType* const old_vals, - const LinearAllocs allocType) { +__global__ void TestKernel(TestType* const global_mem, TestType* const old_vals) { __shared__ TestType shared_mem; const auto tid = cg::this_grid().thread_rank(); @@ -150,7 +117,7 @@ __global__ void TestKernel(TestType* const global_mem, TestType* const old_vals, __syncthreads(); } - old_vals[tid] = PerformAtomicOperation(mem, allocType); + old_vals[tid] = PerformAtomicOperation(mem); if constexpr (use_shared_mem) { @@ -201,8 +168,7 @@ __device__ void GenerateMemoryTraffic(uint8_t* const begin_addr, uint8_t* const template __global__ void TestKernel(TestType* const global_mem, TestType* const old_vals, - const unsigned int width, const unsigned pitch, - const LinearAllocs allocType) { + const unsigned int width, const unsigned pitch) { extern __shared__ uint8_t shared_mem[]; const auto tid = cg::this_grid().thread_rank(); @@ -223,7 +189,7 @@ __global__ void TestKernel(TestType* const global_mem, TestType* const old_vals, if (tid < n) { old_vals[tid] = PerformAtomicOperation( - PitchedOffset(mem, pitch, tid % width), allocType); + PitchedOffset(mem, pitch, tid % width)); } else { uint8_t* const begin_addr = reinterpret_cast(atomic_addr + 1); uint8_t* const end_addr = reinterpret_cast(atomic_addr) + pitch; @@ -278,9 +244,7 @@ std::tuple, std::vector> TestKernelHostRef(const if constexpr (operation == AtomicOperation::kAdd || operation == AtomicOperation::kAddSystem || operation == AtomicOperation::kUnsafeAdd || operation == AtomicOperation::kSafeAdd || operation == AtomicOperation::kCASAdd || - operation == AtomicOperation::kCASAddSystem || - operation == AtomicOperation::kBuiltinAdd || - operation == AtomicOperation::kBuiltinCAS) { + operation == AtomicOperation::kCASAddSystem) { res = res + val; } else if constexpr (operation == AtomicOperation::kSub || operation == AtomicOperation::kSubSystem) { @@ -336,11 +300,10 @@ void LaunchKernel(const TestParams& p, hipStream_t stream, TestType* const mem_p const auto shared_mem_size = use_shared_mem ? p.width * p.pitch : 0u; if (p.width == 1 && p.pitch == sizeof(TestType)) TestKernel - <<>>(mem_ptr, old_vals, p.alloc_type); + <<>>(mem_ptr, old_vals); else TestKernel - <<>>(mem_ptr, old_vals, p.width, p.pitch, - p.alloc_type); + <<>>(mem_ptr, old_vals, p.width, p.pitch); } // Performs a host atomic operation on parameter `mem` based on the `operation` enumerator. @@ -351,9 +314,7 @@ void HostAtomicOperation(const unsigned int iterations, TestType* mem, TestType* for (auto i = 0u; i < iterations; ++i) { if constexpr (operation == AtomicOperation::kAddSystem || - operation == AtomicOperation::kCASAddSystem || - operation == AtomicOperation::kBuiltinAdd || - operation == AtomicOperation::kBuiltinCAS) { + operation == AtomicOperation::kCASAddSystem) { old_vals[i] = __atomic_fetch_add(PitchedOffset(mem, pitch, i % width), val, __ATOMIC_RELAXED); } else if constexpr (operation == AtomicOperation::kSubSystem) { old_vals[i] = __atomic_fetch_sub(PitchedOffset(mem, pitch, i % width), val, __ATOMIC_RELAXED); @@ -467,32 +428,12 @@ void SingleDeviceSingleKernelTest(const unsigned int width, const unsigned int p TestParams params; params.num_devices = 1; params.kernel_count = 1; - if constexpr ((operation == AtomicOperation::kBuiltinAdd || - operation == AtomicOperation::kBuiltinCAS) && - memory_scope == __HIP_MEMORY_SCOPE_SINGLETHREAD) { - params.threads = 1; - } else if constexpr ((operation == AtomicOperation::kBuiltinAdd || - operation == AtomicOperation::kBuiltinCAS) && - memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT) { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - params.threads = dim3(warp_size); - } else { - params.threads = GenerateThreadDimensions(); - } + params.threads = GenerateThreadDimensions(); params.width = width; params.pitch = pitch; SECTION("Global memory") { - if constexpr ((operation == AtomicOperation::kBuiltinAdd || - operation == AtomicOperation::kBuiltinCAS) && - (memory_scope == __HIP_MEMORY_SCOPE_SINGLETHREAD || - memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT || - memory_scope == __HIP_MEMORY_SCOPE_WORKGROUP)) { - params.blocks = dim3(1); - } else { - params.blocks = GenerateBlockDimensions(); - } + params.blocks = GenerateBlockDimensions(); using LA = LinearAllocs; for (const auto alloc_type : {LA::hipMalloc}) { diff --git a/projects/hip-tests/catch/unit/atomics/atomicExch_common.hh b/projects/hip-tests/catch/unit/atomics/atomicExch_common.hh index 70cafc41a16c..d8114c8b342d 100644 --- a/projects/hip-tests/catch/unit/atomics/atomicExch_common.hh +++ b/projects/hip-tests/catch/unit/atomics/atomicExch_common.hh @@ -13,7 +13,7 @@ #include #include -enum class AtomicScopes { device, system, builtin }; +enum class AtomicScopes { device, system }; template __device__ T perform_atomic_exch(T* address, T val) { @@ -21,8 +21,6 @@ __device__ T perform_atomic_exch(T* address, T val) { return atomicExch(address, val); } else if (scope == AtomicScopes::system) { return atomicExch_system(address, val); - } else if (scope == AtomicScopes::builtin) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, memory_scope); } } @@ -297,28 +295,12 @@ void AtomicExchSingleDeviceSingleKernelTest(const unsigned int width, const unsi AtomicExchParams params; params.num_devices = 1; params.kernel_count = 1; - if constexpr (scope == AtomicScopes::builtin && memory_scope == __HIP_MEMORY_SCOPE_SINGLETHREAD) { - params.threads = 1; - } else if constexpr (scope == AtomicScopes::builtin && - memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT) { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - params.threads = dim3(warp_size); - } else { - params.threads = GenerateAtomicExchThreadDimensions(); - } + params.threads = GenerateAtomicExchThreadDimensions(); params.width = width; params.pitch = pitch; SECTION("Global memory") { - if constexpr (scope == AtomicScopes::builtin && - (memory_scope == __HIP_MEMORY_SCOPE_SINGLETHREAD || - memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT || - memory_scope == __HIP_MEMORY_SCOPE_WORKGROUP)) { - params.blocks = dim3(1); - } else { - params.blocks = GenerateAtomicExchBlockDimensions(); - } + params.blocks = GenerateAtomicExchBlockDimensions(); using LA = LinearAllocs; for (const auto alloc_type : {LA::hipMalloc, LA::hipHostMalloc, LA::hipMallocManaged}) { diff --git a/projects/hip-tests/catch/unit/atomics/atomic_builtin_kernels.cc b/projects/hip-tests/catch/unit/atomics/atomic_builtin_kernels.cc deleted file mode 100644 index 81fe1e65cc28..000000000000 --- a/projects/hip-tests/catch/unit/atomics/atomic_builtin_kernels.cc +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include - -constexpr int kMemOrder = __ATOMIC_RELAXED; -constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - -// Trivially-copyable class. -class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; -}; - -class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} -}; - -__global__ void StoreCompileKernel(int* x) { - // Valid combinations - __hip_atomic_store(x, 1, __ATOMIC_RELAXED, kMemScope); - __hip_atomic_store(x, 1, __ATOMIC_RELEASE, kMemScope); - __hip_atomic_store(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - // Pointer to a non-const type - __hip_atomic_store(reinterpret_cast(x), 1, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - __hip_atomic_store(*x, 1, kMemOrder, kMemScope); - // Consume not allowed by C++1 for store - __hip_atomic_store(x, 1, __ATOMIC_CONSUME, kMemScope); - // Acquire not allowed by C++11 for store - __hip_atomic_store(x, 1, __ATOMIC_ACQUIRE, kMemScope); - // Acquire-Release not allowed by C++11 for store - __hip_atomic_store(x, 1, __ATOMIC_ACQ_REL, kMemScope); - // Memory order is out of bounds - __hip_atomic_store(x, 1, -1, kMemScope); - __hip_atomic_store(x, 1, 10, kMemScope); - // Memory scope is out of bounds - __hip_atomic_store(x, 1, kMemOrder, -1); - __hip_atomic_store(x, 1, kMemOrder, 10); - - // Storing an object that is not trivially-copyable - Dummy dummy_a{}; - Dummy dummy_b{}; - __hip_atomic_store(&dummy_a, dummy_b, kMemOrder, kMemScope); - - // Storing an object that is trivially-copyable - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - __hip_atomic_store(&dummytc_a, dummytc_b, kMemOrder, kMemScope); -} - -__global__ void LoadCompileKernel(int* x, int* y) { - // Valid combinations - *y = __hip_atomic_load(x, __ATOMIC_RELAXED, kMemScope); - *y = __hip_atomic_load(x, __ATOMIC_CONSUME, kMemScope); - *y = __hip_atomic_load(x, __ATOMIC_ACQUIRE, kMemScope); - *y = __hip_atomic_load(x, __ATOMIC_SEQ_CST, kMemScope); - - // Value instead of pointer to the atomic builtin for 1st parameter - *y = __hip_atomic_load(*x, kMemOrder, kMemScope); - // Release not allowed by C++11 for load - *y = __hip_atomic_load(x, __ATOMIC_RELEASE, kMemScope); - // Acquire-Release not allowed by C++11 for load - *y = __hip_atomic_load(x, __ATOMIC_ACQ_REL, kMemScope); - // Memory order is out of bounds - *y = __hip_atomic_load(x, -1, kMemScope); - *y = __hip_atomic_load(x, 10, kMemScope); - // Memory scope is out of bounds - *y = __hip_atomic_load(x, kMemOrder, -1); - *y = __hip_atomic_load(x, kMemOrder, 10); - - // Loading an object that is not trivially-copyable - Dummy dummy_a{}; - Dummy dummy_b{}; - dummy_a = __hip_atomic_load(&dummy_b, kMemOrder, kMemScope); - - // Loading an object that is trivially-copyable - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - dummytc_a = __hip_atomic_load(&dummytc_b, kMemOrder, kMemScope); -} - -__global__ void CompareWeakCompileKernel(int* x, int* expected) { - bool res{false}; - // Valid combinations - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_CONSUME, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_CONSUME, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_ACQ_REL, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST, - kMemScope); - - // Release not allowed on fail by C++11 - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, __ATOMIC_RELEASE, kMemScope); - // Acquire-Release not allowed on fail by C++11 - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, __ATOMIC_ACQ_REL, kMemScope); - // Fail stronger than success - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELAXED, __ATOMIC_SEQ_CST, - kMemScope); - // Pointer to a non-const type - res = __hip_atomic_compare_exchange_weak(reinterpret_cast(x), expected, 1, kMemOrder, - kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - res = __hip_atomic_compare_exchange_weak(*x, expected, 1, kMemOrder, kMemOrder, kMemScope); - // Memory order on success is out of bounds - res = __hip_atomic_compare_exchange_weak(x, expected, 1, -1, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, 10, kMemOrder, kMemScope); - // Memory order on failure is out of bounds - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, -1, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, 10, kMemScope); - // Memory scope is out of bounds - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, kMemOrder, -1); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, kMemOrder, 10); - - // User-defined class is not trivially-copyable and therefore cannot be atomically copied - Dummy dummy_a{}; - Dummy dummy_b{}; - Dummy dummy_c{}; - res = __hip_atomic_compare_exchange_weak(&dummy_a, &dummy_b, dummy_c, kMemOrder, kMemOrder, - kMemScope); - // User-defined class is trivially-copyable and can be atomically copied - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - DummyTC dummytc_c{}; - res = __hip_atomic_compare_exchange_weak(&dummytc_a, &dummytc_b, dummytc_c, kMemOrder, kMemOrder, - kMemScope); -} - -__global__ void CompareStrongCompileKernel(int* x, int* expected) { - bool res{false}; - // Valid combinations - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_CONSUME, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_CONSUME, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_ACQ_REL, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST, - kMemScope); - - // Release not allowed on fail by C++11 - res = - __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, __ATOMIC_RELEASE, kMemScope); - // Acquire-Release not allowed on fail by C++11 - res = - __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, __ATOMIC_ACQ_REL, kMemScope); - // Fail stronger than success - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELAXED, __ATOMIC_SEQ_CST, - kMemScope); - // Pointer to a non-const type - res = __hip_atomic_compare_exchange_strong(reinterpret_cast(x), expected, 1, - kMemOrder, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin for 1st parameter - res = __hip_atomic_compare_exchange_strong(*x, expected, 1, kMemOrder, kMemOrder, kMemScope); - // Memory order on success is out of bounds - res = __hip_atomic_compare_exchange_strong(x, expected, 1, -1, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, 10, kMemOrder, kMemScope); - // Memory order on failure is out of bounds - res = __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, -1, kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, 10, kMemScope); - // Memory scope is out of bounds - res = __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, kMemOrder, -1); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, kMemOrder, 10); - - // User-defined class is not trivially-copyable and therefore cannot be atomically copied - Dummy dummy_a{}; - Dummy dummy_b{}; - Dummy dummy_c{}; - res = __hip_atomic_compare_exchange_strong(&dummy_a, &dummy_b, dummy_c, kMemOrder, kMemOrder, - kMemScope); - // User-defined class is trivially-copyable and can be atomically copied - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - DummyTC dummytc_c{}; - res = __hip_atomic_compare_exchange_strong(&dummytc_a, &dummytc_b, dummytc_c, kMemOrder, - kMemOrder, kMemScope); -} - -__global__ void ExchangeCompileKernel(int* x) { - int old{}; - // Valid combinations - old = __hip_atomic_exchange(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - // Pointer to a non-const type - old = __hip_atomic_exchange(reinterpret_cast(x), 1, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - old = __hip_atomic_exchange(*x, 1, kMemOrder, kMemScope); - // Memory order out of bounds - old = __hip_atomic_exchange(x, 1, -1, kMemScope); - old = __hip_atomic_exchange(x, 1, 10, kMemScope); - // Memory scope out of bounds - old = __hip_atomic_exchange(x, 1, kMemOrder, -1); - old = __hip_atomic_exchange(x, 1, kMemOrder, 10); - - // User-defined class is not trivially-copyable and therefore cannot be atomically copied - Dummy dummy_a{}; - Dummy dummy_b{}; - dummy_b = __hip_atomic_exchange(&dummy_a, dummy_b, kMemOrder, kMemScope); - - // User-defined class is trivially-copyable and can be atomically copied - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - dummytc_b = __hip_atomic_exchange(&dummytc_a, dummytc_b, kMemOrder, kMemScope); -} - -__global__ void FetchAddCompileKernel(int* x) { - int old{}; - // Valid combinations - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - // Pointer to a non-const type - old = __hip_atomic_fetch_add(reinterpret_cast(x), 1, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - old = __hip_atomic_fetch_add(*x, 1, kMemOrder, kMemScope); - // Memory order out of bounds - old = __hip_atomic_fetch_add(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_add(x, 1, 10, kMemScope); - // Memory scope out of bounds - old = __hip_atomic_fetch_add(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_add(x, 1, kMemOrder, 10); - - Dummy dummy{}; - old = __hip_atomic_fetch_add(&dummy, 1, kMemOrder, kMemScope); -} - -__global__ void FetchAndCompileKernel(int* x) { - int old{}; - // Valid combinations - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - // Pointer to a non-const type - old = __hip_atomic_fetch_and(reinterpret_cast(x), 1, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - old = __hip_atomic_fetch_and(*x, 1, kMemOrder, kMemScope); - // Memory order out of bounds - old = __hip_atomic_fetch_and(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_and(x, 1, 10, kMemScope); - // Memory scope out of bounds - old = __hip_atomic_fetch_and(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_and(x, 1, kMemOrder, 10); - - // Value must be an integer - Dummy dummy{}; - old = __hip_atomic_fetch_and(&dummy, 1, kMemOrder, kMemScope); - float float_var{1.5f}; - old = __hip_atomic_fetch_and(&float_var, 1, kMemOrder, kMemScope); - double double_var{1.5}; - old = __hip_atomic_fetch_and(&double_var, 1, kMemOrder, kMemScope); -} - -__global__ void FetchOrCompileKernel(int* x) { - int old{}; - // Valid combinations - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - // Pointer to a non-const type - old = __hip_atomic_fetch_or(reinterpret_cast(x), 1, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - old = __hip_atomic_fetch_or(*x, 1, kMemOrder, kMemScope); - // Memory order out of bounds - old = __hip_atomic_fetch_or(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_or(x, 1, 10, kMemScope); - // Memory scope out of bounds - old = __hip_atomic_fetch_or(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_or(x, 1, kMemOrder, 10); - - // Value must be an integer - Dummy dummy{}; - old = __hip_atomic_fetch_or(&dummy, 1, kMemOrder, kMemScope); - float float_var{1.5f}; - old = __hip_atomic_fetch_or(&float_var, 1, kMemOrder, kMemScope); - double double_var{1.5}; - old = __hip_atomic_fetch_or(&double_var, 1, kMemOrder, kMemScope); -} - -__global__ void FetchXorCompileKernel(int* x) { - int old{}; - // Valid combinations - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - // Pointer to a non-const type - old = __hip_atomic_fetch_xor(reinterpret_cast(x), 1, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - old = __hip_atomic_fetch_xor(*x, 1, kMemOrder, kMemScope); - // Memory order out of bounds - old = __hip_atomic_fetch_xor(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, 10, kMemScope); - // Memory scope out of bounds - old = __hip_atomic_fetch_xor(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_xor(x, 1, kMemOrder, 10); - - // Value must be an integer - Dummy dummy{}; - old = __hip_atomic_fetch_xor(&dummy, 1, kMemOrder, kMemScope); - float float_var{1.5f}; - old = __hip_atomic_fetch_xor(&float_var, 1, kMemOrder, kMemScope); - double double_var{1.5}; - old = __hip_atomic_fetch_xor(&double_var, 1, kMemOrder, kMemScope); -} - -__global__ void FetchMaxCompileKernel(int* x) { - int old{}; - // Valid combinations - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - // Pointer to a non-const type - old = __hip_atomic_fetch_max(reinterpret_cast(x), 1, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - old = __hip_atomic_fetch_max(*x, 1, kMemOrder, kMemScope); - // Memory order out of bounds - old = __hip_atomic_fetch_max(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_max(x, 1, 10, kMemScope); - // Memory scope out of bounds - old = __hip_atomic_fetch_max(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_max(x, 1, kMemOrder, 10); - - // Value must be integer or floating point type - Dummy dummy{}; - old = __hip_atomic_fetch_max(&dummy, 1, kMemOrder, kMemScope); -} - -__global__ void FetchMinCompileKernel(int* x) { - int old{}; - // Valid combinations - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - // Pointer to a non-const type - old = __hip_atomic_fetch_min(reinterpret_cast(x), 1, kMemOrder, kMemScope); - // Value instead of pointer to the atomic builtin - old = __hip_atomic_fetch_min(*x, 1, kMemOrder, kMemScope); - // Memory order out of bounds - old = __hip_atomic_fetch_min(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_min(x, 1, 10, kMemScope); - // Memory scope out of bounds - old = __hip_atomic_fetch_min(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_min(x, 1, kMemOrder, 10); - - // Value must be integer or floating point type - Dummy dummy{}; - old = __hip_atomic_fetch_min(&dummy, 1, kMemOrder, kMemScope); -} diff --git a/projects/hip-tests/catch/unit/atomics/atomic_builtins.cc b/projects/hip-tests/catch/unit/atomics/atomic_builtins.cc deleted file mode 100644 index 7244ed7c7103..000000000000 --- a/projects/hip-tests/catch/unit/atomics/atomic_builtins.cc +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include -#include - -#include "atomic_builtins_kernels_rtc.hh" - -/** - * @addtogroup __hip_atomic_fetch_add __hip_atomic_fetch_add - * @{ - * @ingroup AtomicsTest - */ - -void AtomicBuiltinsRTCWrapper(const char* program_source, int expected_errors_num, - int expected_warnings_num) { - hiprtcProgram program{}; - HIPRTC_CHECK(hiprtcCreateProgram(&program, program_source, "atomics_builtins_kernels.cc", 0, - nullptr, nullptr)); - - hiprtcResult result{hiprtcCompileProgram(program, 0, nullptr)}; - - size_t log_size{}; - HIPRTC_CHECK(hiprtcGetProgramLogSize(program, &log_size)); - std::string log(log_size, ' '); - HIPRTC_CHECK(hiprtcGetProgramLog(program, log.data())); - int error_count{0}; - int warning_count{0}; - - std::string error_message{"error:"}; - std::string warning_message{"warning:"}; - - size_t npos_e = log.find(error_message, 0); - while (npos_e != std::string::npos) { - ++error_count; - npos_e = log.find(error_message, npos_e + 1); - } - - size_t npos_w = log.find(warning_message, 0); - while (npos_w != std::string::npos) { - ++warning_count; - npos_w = log.find(warning_message, npos_w + 1); - } - - HIPRTC_CHECK(hiprtcDestroyProgram(&program)); - HIPRTC_CHECK_ERROR(result, HIPRTC_ERROR_COMPILATION); - REQUIRE(error_count == expected_errors_num); - REQUIRE(warning_count == expected_warnings_num); -} - -/** - * Test Description - * ------------------------ - * - Compiles atomic builtins while passing parameters that shall cause: - * -# Compiler warnings - * -# Compiler errors - * Test source - * ------------------------ - * - unit/atomics/atomic_builtins.cc - * Test requirements - * ------------------------ - * - HIP_VERSION >= 5.2 - */ -HIP_TEST_CASE(Unit_AtomicBuiltins_Negative_Parameters_RTC) { - AtomicBuiltinsRTCWrapper(kBuiltinStore, 5, 5); - AtomicBuiltinsRTCWrapper(kBuiltinLoad, 4, 4); - AtomicBuiltinsRTCWrapper(kBuiltinCompExWeak, 5, 7); - AtomicBuiltinsRTCWrapper(kBuiltinCompExStrong, 5, 7); - AtomicBuiltinsRTCWrapper(kBuiltinExchange, 5, 2); - AtomicBuiltinsRTCWrapper(kBuiltinFetchAdd, 5, 2); - AtomicBuiltinsRTCWrapper(kBuiltinFetchAnd, 7, 2); - AtomicBuiltinsRTCWrapper(kBuiltinFetchOr, 7, 2); - AtomicBuiltinsRTCWrapper(kBuiltinFetchXor, 7, 2); - AtomicBuiltinsRTCWrapper(kBuiltinFetchMax, 5, 2); - AtomicBuiltinsRTCWrapper(kBuiltinFetchMin, 5, 2); -} - -/** - * End doxygen group AtomicsTest. - * @} - */ diff --git a/projects/hip-tests/catch/unit/atomics/atomic_builtins_kernels_rtc.hh b/projects/hip-tests/catch/unit/atomics/atomic_builtins_kernels_rtc.hh deleted file mode 100644 index 6fe759f5c375..000000000000 --- a/projects/hip-tests/catch/unit/atomics/atomic_builtins_kernels_rtc.hh +++ /dev/null @@ -1,574 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#pragma once - -/* -Positive and negative kernels used for the builtin atomic Test Cases that are using RTC. -*/ - -static constexpr auto kBuiltinStore{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void StoreCompileKernel(int* x) { - __hip_atomic_store(x, 1, __ATOMIC_RELAXED, kMemScope); - __hip_atomic_store(x, 1, __ATOMIC_RELEASE, kMemScope); - __hip_atomic_store(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - __hip_atomic_store(reinterpret_cast(x), 1, kMemOrder, kMemScope); - __hip_atomic_store(*x, 1, kMemOrder, kMemScope); - __hip_atomic_store(x, 1, __ATOMIC_CONSUME, kMemScope); - __hip_atomic_store(x, 1, __ATOMIC_ACQUIRE, kMemScope); - __hip_atomic_store(x, 1, __ATOMIC_ACQ_REL, kMemScope); - __hip_atomic_store(x, 1, -1, kMemScope); - __hip_atomic_store(x, 1, 10, kMemScope); - __hip_atomic_store(x, 1, kMemOrder, -1); - __hip_atomic_store(x, 1, kMemOrder, 10); - - Dummy dummy_a{}; - Dummy dummy_b{}; - __hip_atomic_store(&dummy_a, dummy_b, kMemOrder, kMemScope); - - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - __hip_atomic_store(&dummytc_a, dummytc_b, kMemOrder, kMemScope); - } -)"}; - -static constexpr auto kBuiltinLoad{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void LoadCompileKernel(int* x, int* y) { - *y = __hip_atomic_load(x, __ATOMIC_RELAXED, kMemScope); - *y = __hip_atomic_load(x, __ATOMIC_CONSUME, kMemScope); - *y = __hip_atomic_load(x, __ATOMIC_ACQUIRE, kMemScope); - *y = __hip_atomic_load(x, __ATOMIC_SEQ_CST, kMemScope); - - *y = __hip_atomic_load(*x, kMemOrder, kMemScope); - *y = __hip_atomic_load(x, __ATOMIC_RELEASE, kMemScope); - *y = __hip_atomic_load(x, __ATOMIC_ACQ_REL, kMemScope); - *y = __hip_atomic_load(x, -1, kMemScope); - *y = __hip_atomic_load(x, 10, kMemScope); - *y = __hip_atomic_load(x, kMemOrder, -1); - *y = __hip_atomic_load(x, kMemOrder, 10); - - Dummy dummy_a{}; - Dummy dummy_b{}; - dummy_a = __hip_atomic_load(&dummy_b, kMemOrder, kMemScope); - - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - dummytc_a = __hip_atomic_load(&dummytc_b, kMemOrder, kMemScope); - } -)"}; - -static constexpr auto kBuiltinCompExWeak{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void CompareWeakCompileKernel(int* x, int* expected) { - bool res{false}; - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_CONSUME, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_CONSUME, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_ACQ_REL, - kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST, - kMemScope); - - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, __ATOMIC_RELEASE, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, __ATOMIC_ACQ_REL, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, __ATOMIC_RELAXED, __ATOMIC_SEQ_CST, - kMemScope); - res = __hip_atomic_compare_exchange_weak(reinterpret_cast(x), expected, 1, kMemOrder, - kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_weak(*x, expected, 1, kMemOrder, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, -1, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, 10, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, -1, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, 10, kMemScope); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, kMemOrder, -1); - res = __hip_atomic_compare_exchange_weak(x, expected, 1, kMemOrder, kMemOrder, 10); - - Dummy dummy_a{}; - Dummy dummy_b{}; - Dummy dummy_c{}; - res = __hip_atomic_compare_exchange_weak(&dummy_a, &dummy_b, dummy_c, kMemOrder, kMemOrder, - kMemScope); - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - DummyTC dummytc_c{}; - res = __hip_atomic_compare_exchange_weak(&dummytc_a, &dummytc_b, dummytc_c, kMemOrder, kMemOrder, - kMemScope); - } -)"}; - -static constexpr auto kBuiltinCompExStrong{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void CompareStrongCompileKernel(int* x, int* expected) { - bool res{false}; - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_CONSUME, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_CONSUME, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_CONSUME, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_ACQ_REL, - kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST, - kMemScope); - - res = - __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, __ATOMIC_RELEASE, kMemScope); - res = - __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, __ATOMIC_ACQ_REL, kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, __ATOMIC_RELAXED, __ATOMIC_SEQ_CST, - kMemScope); - res = __hip_atomic_compare_exchange_strong(reinterpret_cast(x), expected, 1, - kMemOrder, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_strong(*x, expected, 1, kMemOrder, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, -1, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, 10, kMemOrder, kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, -1, kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, 10, kMemScope); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, kMemOrder, -1); - res = __hip_atomic_compare_exchange_strong(x, expected, 1, kMemOrder, kMemOrder, 10); - - Dummy dummy_a{}; - Dummy dummy_b{}; - Dummy dummy_c{}; - res = __hip_atomic_compare_exchange_strong(&dummy_a, &dummy_b, dummy_c, kMemOrder, kMemOrder, - kMemScope); - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - DummyTC dummytc_c{}; - res = __hip_atomic_compare_exchange_strong(&dummytc_a, &dummytc_b, dummytc_c, kMemOrder, - kMemOrder, kMemScope); - } -)"}; - -static constexpr auto kBuiltinExchange{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void ExchangeCompileKernel(int* x) { - int old{}; - old = __hip_atomic_exchange(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_exchange(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - old = __hip_atomic_exchange(reinterpret_cast(x), 1, kMemOrder, kMemScope); - old = __hip_atomic_exchange(*x, 1, kMemOrder, kMemScope); - old = __hip_atomic_exchange(x, 1, -1, kMemScope); - old = __hip_atomic_exchange(x, 1, 10, kMemScope); - old = __hip_atomic_exchange(x, 1, kMemOrder, -1); - old = __hip_atomic_exchange(x, 1, kMemOrder, 10); - - Dummy dummy_a{}; - Dummy dummy_b{}; - dummy_b = __hip_atomic_exchange(&dummy_a, dummy_b, kMemOrder, kMemScope); - - DummyTC dummytc_a{}; - DummyTC dummytc_b{}; - dummytc_b = __hip_atomic_exchange(&dummytc_a, dummytc_b, kMemOrder, kMemScope); - } -)"}; - -static constexpr auto kBuiltinFetchAdd{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void FetchAddCompileKernel(int* x) { - int old{}; - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_add(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - old = __hip_atomic_fetch_add(reinterpret_cast(x), 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_add(*x, 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_add(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_add(x, 1, 10, kMemScope); - old = __hip_atomic_fetch_add(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_add(x, 1, kMemOrder, 10); - - Dummy dummy{}; - old = __hip_atomic_fetch_add(&dummy, 1, kMemOrder, kMemScope); - } -)"}; - -static constexpr auto kBuiltinFetchAnd{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void FetchAndCompileKernel(int* x) { - int old{}; - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_and(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - old = __hip_atomic_fetch_and(reinterpret_cast(x), 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_and(*x, 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_and(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_and(x, 1, 10, kMemScope); - old = __hip_atomic_fetch_and(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_and(x, 1, kMemOrder, 10); - - Dummy dummy{}; - old = __hip_atomic_fetch_and(&dummy, 1, kMemOrder, kMemScope); - float float_var{1.5f}; - old = __hip_atomic_fetch_and(&float_var, 1, kMemOrder, kMemScope); - double double_var{1.5}; - old = __hip_atomic_fetch_and(&double_var, 1, kMemOrder, kMemScope); - } -)"}; - -static constexpr auto kBuiltinFetchOr{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void FetchOrCompileKernel(int* x) { - int old{}; - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_or(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - old = __hip_atomic_fetch_or(reinterpret_cast(x), 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_or(*x, 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_or(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_or(x, 1, 10, kMemScope); - old = __hip_atomic_fetch_or(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_or(x, 1, kMemOrder, 10); - - Dummy dummy{}; - old = __hip_atomic_fetch_or(&dummy, 1, kMemOrder, kMemScope); - float float_var{1.5f}; - old = __hip_atomic_fetch_or(&float_var, 1, kMemOrder, kMemScope); - double double_var{1.5}; - old = __hip_atomic_fetch_or(&double_var, 1, kMemOrder, kMemScope); - } -)"}; - -static auto constexpr kBuiltinFetchXor{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void FetchXorCompileKernel(int* x) { - int old{}; - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - old = __hip_atomic_fetch_xor(reinterpret_cast(x), 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_xor(*x, 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, 10, kMemScope); - old = __hip_atomic_fetch_xor(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_xor(x, 1, kMemOrder, 10); - - Dummy dummy{}; - old = __hip_atomic_fetch_xor(&dummy, 1, kMemOrder, kMemScope); - float float_var{1.5f}; - old = __hip_atomic_fetch_xor(&float_var, 1, kMemOrder, kMemScope); - double double_var{1.5}; - old = __hip_atomic_fetch_xor(&double_var, 1, kMemOrder, kMemScope); - } -)"}; - -static constexpr auto kBuiltinFetchMax{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void FetchMaxCompileKernel(int* x) { - int old{}; - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_max(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - old = __hip_atomic_fetch_max(reinterpret_cast(x), 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_max(*x, 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_max(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_max(x, 1, 10, kMemScope); - old = __hip_atomic_fetch_max(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_max(x, 1, kMemOrder, 10); - - Dummy dummy{}; - old = __hip_atomic_fetch_max(&dummy, 1, kMemOrder, kMemScope); - } -)"}; - -static constexpr auto kBuiltinFetchMin{R"( - constexpr int kMemOrder = __ATOMIC_RELAXED; - constexpr int kMemScope = __HIP_MEMORY_SCOPE_SYSTEM; - - class DummyTC { - public: - __device__ DummyTC() {} - __device__ ~DummyTC() = default; - __device__ DummyTC(const DummyTC&) = default; - __device__ DummyTC& operator=(const DummyTC&) = default; - __device__ DummyTC(DummyTC&&) = default; - __device__ DummyTC& operator=(DummyTC&&) = default; - }; - - class Dummy { - public: - __device__ Dummy() {} - __device__ ~Dummy() {} - }; - - __global__ void FetchMinCompileKernel(int* x) { - int old{}; - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_RELAXED, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_CONSUME, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_ACQUIRE, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_RELEASE, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_ACQ_REL, kMemScope); - old = __hip_atomic_fetch_min(x, 1, __ATOMIC_SEQ_CST, kMemScope); - - old = __hip_atomic_fetch_min(reinterpret_cast(x), 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_min(*x, 1, kMemOrder, kMemScope); - old = __hip_atomic_fetch_min(x, 1, -1, kMemScope); - old = __hip_atomic_fetch_min(x, 1, 10, kMemScope); - old = __hip_atomic_fetch_min(x, 1, kMemOrder, -1); - old = __hip_atomic_fetch_min(x, 1, kMemOrder, 10); - - Dummy dummy{}; - old = __hip_atomic_fetch_min(&dummy, 1, kMemOrder, kMemScope); - } -)"}; diff --git a/projects/hip-tests/catch/unit/atomics/bitwise_common.hh b/projects/hip-tests/catch/unit/atomics/bitwise_common.hh index f1eb0b2d9815..1210f8a1c49e 100644 --- a/projects/hip-tests/catch/unit/atomics/bitwise_common.hh +++ b/projects/hip-tests/catch/unit/atomics/bitwise_common.hh @@ -20,10 +20,7 @@ enum class AtomicOperation { kOr, kOrSystem, kXor, - kXorSystem, - kBuiltinAnd, - kBuiltinOr, - kBuiltinXor + kXorSystem }; constexpr auto kMask = 0xAAAA; @@ -55,12 +52,6 @@ __device__ TestType PerformAtomicOperation(TestType* const mem) { return atomicXor(mem, mask); } else if constexpr (operation == AtomicOperation::kXorSystem) { return atomicXor_system(mem, mask); - } else if constexpr (operation == AtomicOperation::kBuiltinAnd) { - return __hip_atomic_fetch_and(mem, mask, __ATOMIC_RELAXED, memory_scope); - } else if constexpr (operation == AtomicOperation::kBuiltinOr) { - return __hip_atomic_fetch_or(mem, mask, __ATOMIC_RELAXED, memory_scope); - } else if constexpr (operation == AtomicOperation::kBuiltinXor) { - return __hip_atomic_fetch_xor(mem, mask, __ATOMIC_RELAXED, memory_scope); } } @@ -173,16 +164,14 @@ std::tuple, std::vector> TestKernelHostRef(const auto& res = res_vals[tid % p.width + (i * p.width)]; old_vals.push_back(res); - if constexpr (operation == AtomicOperation::kAnd || operation == AtomicOperation::kAndSystem || - operation == AtomicOperation::kBuiltinAnd) { + if constexpr (operation == AtomicOperation::kAnd || + operation == AtomicOperation::kAndSystem) { res = res & mask; } else if constexpr (operation == AtomicOperation::kOr || - operation == AtomicOperation::kOrSystem || - operation == AtomicOperation::kBuiltinOr) { + operation == AtomicOperation::kOrSystem) { res = res | mask; } else if constexpr (operation == AtomicOperation::kXor || - operation == AtomicOperation::kXorSystem || - operation == AtomicOperation::kBuiltinXor) { + operation == AtomicOperation::kXorSystem) { res = res ^ mask; } } @@ -292,35 +281,12 @@ void SingleDeviceSingleKernelTest(const unsigned int width, const unsigned int p TestParams params; params.num_devices = 1; params.kernel_count = 1; - if constexpr ((operation == AtomicOperation::kBuiltinAnd || - operation == AtomicOperation::kBuiltinOr || - operation == AtomicOperation::kBuiltinXor) && - memory_scope == __HIP_MEMORY_SCOPE_SINGLETHREAD) { - params.threads = 1; - } else if constexpr ((operation == AtomicOperation::kBuiltinAnd || - operation == AtomicOperation::kBuiltinOr || - operation == AtomicOperation::kBuiltinXor) && - memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT) { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - params.threads = dim3(warp_size); - } else { - params.threads = GenerateThreadDimensions(); - } + params.threads = GenerateThreadDimensions(); params.width = width; params.pitch = pitch; SECTION("Global memory") { - if constexpr ((operation == AtomicOperation::kBuiltinAnd || - operation == AtomicOperation::kBuiltinOr || - operation == AtomicOperation::kBuiltinXor) && - (memory_scope == __HIP_MEMORY_SCOPE_SINGLETHREAD || - memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT || - memory_scope == __HIP_MEMORY_SCOPE_WORKGROUP)) { - params.blocks = dim3(1); - } else { - params.blocks = GenerateBlockDimensions(); - } + params.blocks = GenerateBlockDimensions(); using LA = LinearAllocs; for (const auto alloc_type : {LA::hipMalloc}) { diff --git a/projects/hip-tests/catch/unit/atomics/memory_order_common.hh b/projects/hip-tests/catch/unit/atomics/memory_order_common.hh deleted file mode 100644 index 2d0794c81033..000000000000 --- a/projects/hip-tests/catch/unit/atomics/memory_order_common.hh +++ /dev/null @@ -1,480 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#pragma once - -#include -#include - -enum class BuiltinAtomicOperation { - kLoadStore = 0, - kExchange, - kCompareExchangeStrong, - kCompareExchangeWeak, - kAdd, - kAnd, - kOr, - kXor, - kMin, - kMax -}; - -template -__host__ __device__ void SetFlag(int* const flag) { -#ifdef __HIP_DEVICE_COMPILE__ - if constexpr (operation == BuiltinAtomicOperation::kLoadStore) { - static_assert(memory_order != __ATOMIC_ACQ_REL); - __hip_atomic_store(flag, 1, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kExchange) { - __hip_atomic_exchange(flag, 1, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kCompareExchangeStrong) { - int compare = 0; - __hip_atomic_compare_exchange_strong(flag, &compare, 1, memory_order, __ATOMIC_RELAXED, - memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kCompareExchangeWeak) { - int compare = 0; - while (!__hip_atomic_compare_exchange_weak(flag, &compare, 1, memory_order, __ATOMIC_RELAXED, - memory_scope)) - compare = 0; - } else if constexpr (operation == BuiltinAtomicOperation::kAdd) { - __hip_atomic_fetch_add(flag, 1, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kAnd) { - __hip_atomic_fetch_and(flag, 0x0, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kOr) { - __hip_atomic_fetch_or(flag, 0x1, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kXor) { - __hip_atomic_fetch_xor(flag, 0x1, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kMin) { - __hip_atomic_fetch_min(flag, -1, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kMax) { - __hip_atomic_fetch_max(flag, 1, memory_order, memory_scope); - } -#else - if constexpr (operation == BuiltinAtomicOperation::kAnd) { - __atomic_store_n(flag, 0, __ATOMIC_RELEASE); - } else { - __atomic_store_n(flag, 1, __ATOMIC_RELEASE); - } -#endif -} - -template -__host__ __device__ int FetchFlag(int* const flag) { -#ifdef __HIP_DEVICE_COMPILE__ - if constexpr (operation == BuiltinAtomicOperation::kLoadStore) { - static_assert(memory_order != __ATOMIC_ACQ_REL); - return __hip_atomic_load(flag, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kExchange) { - return __hip_atomic_exchange(flag, 0, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kCompareExchangeStrong) { - int compare = 1; - __hip_atomic_compare_exchange_strong( - flag, &compare, 1, memory_order, - memory_order == __ATOMIC_ACQ_REL ? __ATOMIC_ACQUIRE : memory_order, memory_scope); - return compare; - } else if constexpr (operation == BuiltinAtomicOperation::kCompareExchangeWeak) { - int compare = 1; - __hip_atomic_compare_exchange_weak( - flag, &compare, 1, memory_order, - memory_order == __ATOMIC_ACQ_REL ? __ATOMIC_ACQUIRE : memory_order, memory_scope); - return compare; - } else if constexpr (operation == BuiltinAtomicOperation::kAdd) { - return __hip_atomic_fetch_add(flag, 0, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kAnd) { - return !__hip_atomic_fetch_and(flag, 0x1, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kOr) { - return __hip_atomic_fetch_or(flag, 0x0, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kXor) { - return __hip_atomic_fetch_xor(flag, 0x0, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kMin) { - return __hip_atomic_fetch_min(flag, 0, memory_order, memory_scope); - } else if constexpr (operation == BuiltinAtomicOperation::kMax) { - return __hip_atomic_fetch_max(flag, 0, memory_order, memory_scope); - } -#else - if constexpr (operation == BuiltinAtomicOperation::kAnd) { - return !__atomic_load_n(flag, __ATOMIC_ACQUIRE); - } else { - return __atomic_load_n(flag, __ATOMIC_ACQUIRE); - } -#endif -} - -namespace AcquireRelease { - -constexpr auto kTestValue = 42; - -template -__host__ __device__ void Producer(int* const flag, int* const data) { - constexpr int actual_memory_order = - memory_order == __ATOMIC_ACQUIRE ? __ATOMIC_RELEASE : memory_order; - - data[0] = kTestValue; - SetFlag(flag); -} - -template -__host__ __device__ void Consumer(int* const flag, int* const data, int* const ret) { - while (!FetchFlag(flag)) - ; - - ret[0] = data[0]; -} - -template -__global__ void TestKernel(int* const flag, int* data, int* const ret) { - __shared__ int shared_mem; - - if (data == nullptr) { - data = &shared_mem; - } - __syncthreads(); - - bool producer = false, consumer = false; - - if constexpr (memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT) { - producer = blockIdx.x == 0 && threadIdx.x == 0; - consumer = blockIdx.x == 0 && threadIdx.x == 1; - } else if constexpr (memory_scope == __HIP_MEMORY_SCOPE_WORKGROUP) { - producer = blockIdx.x == 0 && threadIdx.x == 0; - consumer = blockIdx.x == 0 && threadIdx.x == warpSize; - } else if constexpr (memory_scope == __HIP_MEMORY_SCOPE_AGENT) { - producer = blockIdx.x == 0 && threadIdx.x == 0; - consumer = blockIdx.x == 1 && threadIdx.x == 0; - } - - if (producer) { - Producer(flag, data); - } - - if (consumer) { - Consumer(flag, data, ret); - } -} - -template -__global__ void ProducerKernel(int* const flag, int* const data) { - if (!(blockIdx.x == 0 && threadIdx.x == 0)) { - return; - } - - Producer(flag, data); -} - -template -__global__ void ConsumerKernel(int* const flag, int* const data, int* const ret) { - if (!(blockIdx.x == 0 && threadIdx.x == 0)) { - return; - } - - Consumer(flag, data, ret); -} - -template void Test() { - int blocks = 1, threads = 1; - if (memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT) { - blocks = 1; - threads = 2; - } else if (memory_scope == __HIP_MEMORY_SCOPE_WORKGROUP) { - blocks = 1; - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - threads = warp_size * 2; - } else if (memory_scope == __HIP_MEMORY_SCOPE_AGENT) { - blocks = 2; - threads = 1; - } - - LinearAllocGuard flag(LinearAllocs::hipMalloc, sizeof(int)); - int flagvalue = 0; - - if constexpr (operation == BuiltinAtomicOperation::kAnd) { - flagvalue = 1; - } - HIP_CHECK(hipMemcpy(flag.ptr(), &flagvalue, sizeof(int), hipMemcpyHostToDevice)); - - LinearAllocGuard ret(LinearAllocs::hipMallocManaged, sizeof(int)); - - SECTION("Global memory") { - const auto alloc_type = LinearAllocs::hipMalloc; - LinearAllocGuard data(alloc_type, sizeof(int)); - - HIP_CHECK(hipMemset(data.ptr(), 0, sizeof(int))); - TestKernel - <<>>(flag.ptr(), data.ptr(), ret.ptr()); - } - - if constexpr (memory_scope != __HIP_MEMORY_SCOPE_AGENT && memory_scope != __HIP_MEMORY_SCOPE_SYSTEM) { - SECTION("Shared memory") { - TestKernel - <<>>(flag.ptr(), nullptr, ret.ptr()); - } - } - - HIP_CHECK(hipDeviceSynchronize()); - - REQUIRE(ret.ptr()[0] == kTestValue); -} - -template void SystemTest() { - // The test need xnack on to make managed memory be host coherent. - // If xnack is off, managed memory has coarse grained access. - if (!HipTest::isXnackOn()) { - std::cerr << "GPU is not xnack enabled hence skipping system test\n"; - return; - } - std::thread host_thread; - - LinearAllocGuard flag(LinearAllocs::hipMallocManaged, sizeof(int)); - LinearAllocGuard ret(LinearAllocs::hipMallocManaged, sizeof(int)); - - HIP_CHECK(hipMemset(flag.ptr(), 0, sizeof(int))); - - SECTION("Global memory") { - const auto alloc_type = GENERATE(LinearAllocs::hipHostMalloc , LinearAllocs::hipMallocManaged); - LinearAllocGuard data(alloc_type, sizeof(int)); - - HIP_CHECK(hipMemset(data.ptr(), 0, sizeof(int))); - - if constexpr(operation == BuiltinAtomicOperation::kAnd) { - flag.ptr()[0] = 1; - } - - SECTION("Host producer - Device consumer") { - host_thread = std::thread([&] { - Producer(flag.host_ptr(), data.host_ptr()); - }); - ConsumerKernel - <<<1, 1>>>(flag.ptr(), data.ptr(), ret.ptr()); - - HIP_CHECK(hipDeviceSynchronize()); - host_thread.join(); - } - - SECTION("Device producer - Host consumer") { - host_thread = std::thread([&] { - Consumer(flag.host_ptr(), data.host_ptr(), - ret.host_ptr()); - }); - ProducerKernel - <<<1, 1>>>(flag.ptr(), data.ptr()); - - HIP_CHECK(hipDeviceSynchronize()); - host_thread.join(); - } - } - - REQUIRE(ret.ptr()[0] == kTestValue); -} - -} /* namespace AcquireRelease */ - -namespace SequentialConsistency { - -template -__host__ __device__ void Producer(int* const flag) { - #ifdef __HIP_DEVICE_COMPILE__ - if constexpr (operation == BuiltinAtomicOperation::kAnd) { - __hip_atomic_store(flag, 0, __ATOMIC_SEQ_CST, memory_scope); - } - else { - __hip_atomic_store(flag, 1, __ATOMIC_SEQ_CST, memory_scope); - } - #else - if constexpr (operation == BuiltinAtomicOperation::kAnd) { - __atomic_store_n(flag, 0, __ATOMIC_SEQ_CST); - } - else { - __atomic_store_n(flag, 1, __ATOMIC_SEQ_CST); - } - #endif -} - -template -__host__ __device__ void Consumer(int* const flag1, int* const counter) { - while (!FetchFlag(flag1)) {}; - -#ifdef __HIP_DEVICE_COMPILE__ - __hip_atomic_fetch_add(counter, 1, __ATOMIC_SEQ_CST, memory_scope); -#else - __atomic_fetch_add(counter, 1, __ATOMIC_SEQ_CST); -#endif -} - -template -__global__ void TestKernel(int* flag1, int* flag2, int* const counter1, int* const counter2) { - __shared__ int shared_mem[2]; - - if (flag1 == nullptr && flag2 == nullptr) { - flag1 = &shared_mem[0]; - flag2 = &shared_mem[1]; - if (threadIdx.x == 0 ) { - if constexpr (operation == BuiltinAtomicOperation::kAnd) { - *flag1 = 1; - *flag2 = 1; - } else { - *flag1 = 0; - *flag2 = 0; - } - } - } - __syncthreads(); - - bool producer1 = false, producer2 = false, consumer1 = false, consumer2 = false; - - if constexpr (memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT) { - producer1 = blockIdx.x == 0 && threadIdx.x == 0; - consumer1 = blockIdx.x == 0 && threadIdx.x == 1; - producer2 = blockIdx.x == 0 && threadIdx.x == 2; - consumer2 = blockIdx.x == 0 && threadIdx.x == 3; - } else if constexpr (memory_scope == __HIP_MEMORY_SCOPE_WORKGROUP) { - producer1 = blockIdx.x == 0 && threadIdx.x == 0; - consumer1 = blockIdx.x == 0 && threadIdx.x == warpSize; - producer2 = blockIdx.x == 0 && threadIdx.x == warpSize * 2; - consumer2 = blockIdx.x == 0 && threadIdx.x == warpSize * 3; - } else if constexpr (memory_scope == __HIP_MEMORY_SCOPE_AGENT) { - producer1 = blockIdx.x == 0 && threadIdx.x == 0; - consumer1 = blockIdx.x == 1 && threadIdx.x == 0; - producer2 = blockIdx.x == 2 && threadIdx.x == 0; - consumer2 = blockIdx.x == 3 && threadIdx.x == 0; - } - - if (producer1) { - Producer(flag1); - } - - if (consumer1) { - Consumer(flag1, counter1); - } - - if (producer2) { - Producer(flag2); - } - - if (consumer2) { - Consumer(flag2, counter2); - } -} - -template -__global__ void ProducerKernel(int* const flag) { - if (!(blockIdx.x == 0 && threadIdx.x == 0)) { - return; - } - - Producer(flag); -} - -template -__global__ void ConsumerKernel(int* const flag1, int* const counter) { - if (!(blockIdx.x == 0 && threadIdx.x == 0)) { - return; - } - - Consumer(flag1, counter); -} - -template void Test() { - int blocks = 1, threads = 1; - if (memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT) { - blocks = 1; - threads = 4; - } else if (memory_scope == __HIP_MEMORY_SCOPE_WORKGROUP) { - blocks = 1; - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - threads = warp_size * 4; - } else if (memory_scope == __HIP_MEMORY_SCOPE_AGENT) { - blocks = 4; - threads = 1; - } - - LinearAllocGuard counter1(LinearAllocs::hipMallocManaged, sizeof(int)); - LinearAllocGuard counter2(LinearAllocs::hipMallocManaged, sizeof(int)); - - HIP_CHECK(hipMemset(counter1.ptr(), 0, sizeof(int))); - HIP_CHECK(hipMemset(counter2.ptr(), 0, sizeof(int))); - - SECTION("Global memory") { - const auto alloc_type = LinearAllocs::hipMalloc; - LinearAllocGuard flag1(alloc_type, sizeof(int)); - LinearAllocGuard flag2(alloc_type, sizeof(int)); - int flagvalue = 0; - if constexpr (operation == BuiltinAtomicOperation::kAnd) { - flagvalue = 1; - } - HIP_CHECK(hipMemcpy(flag1.ptr(), &flagvalue, sizeof(int), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(flag2.ptr(), &flagvalue, sizeof(int), hipMemcpyHostToDevice)); - - TestKernel - <<>>(flag1.ptr(), flag2.ptr(), counter1.ptr(), counter2.ptr()); - } - - if constexpr (memory_scope != __HIP_MEMORY_SCOPE_AGENT && memory_scope != __HIP_MEMORY_SCOPE_SYSTEM) { - SECTION("Shared memory") { - TestKernel<<>>(nullptr, nullptr, counter1.ptr(), counter2.ptr()); - } - } - - HIP_CHECK(hipDeviceSynchronize()); - - REQUIRE(counter1.ptr()[0] != 0); - REQUIRE(counter2.ptr()[0] != 0); -} - -template void SystemTest() { - // The test need xnack on to make managed memory be host coherent. - // If xnack is off, managed memory has coarse grained access. - if (!HipTest::isXnackOn()) { - std::cerr << "GPU is not xnack enabled hence skipping system test\n"; - return; - } - std::thread host_producer, host_consumer; - - LinearAllocGuard counter1(LinearAllocs::hipMallocManaged, sizeof(int)); - LinearAllocGuard counter2(LinearAllocs::hipMallocManaged, sizeof(int)); - std::vector streams; - - HIP_CHECK(hipMemset(counter1.ptr(), 0, sizeof(int))); - HIP_CHECK(hipMemset(counter2.ptr(), 0, sizeof(int))); - - for (auto j = 0; j < 2; ++j) { - streams.emplace_back(Streams::created); - } - - SECTION("Global memory") { - const auto alloc_type = LinearAllocs::hipMallocManaged; - LinearAllocGuard flag1(alloc_type, sizeof(int)); - LinearAllocGuard flag2(alloc_type, sizeof(int)); - int flagvalue = 0; - if constexpr (operation == BuiltinAtomicOperation::kAnd) { - flagvalue = 1; - } - HIP_CHECK(hipMemcpy(flag1.ptr(), &flagvalue, sizeof(int), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(flag2.ptr(), &flagvalue, sizeof(int), hipMemcpyHostToDevice)); - - const auto &stream1 = streams[0].stream(); - ConsumerKernel - <<<1, 1, 0, stream1>>>(flag1.ptr(), counter1.ptr()); - host_consumer = std::thread([&] { - Consumer(flag2.ptr(), counter2.ptr()); - }); - - const auto &stream2 = streams[1].stream(); - ProducerKernel<<<1, 1, 0 , stream2>>>(flag2.ptr()); - host_producer = - std::thread([&] { Producer(flag1.ptr()); }); - } - - HIP_CHECK(hipDeviceSynchronize()); - host_producer.join(); - host_consumer.join(); - - REQUIRE(counter1.ptr()[0] != 0); - REQUIRE(counter2.ptr()[0] != 0); -} - -} // namespace SequentialConsistency diff --git a/projects/hip-tests/catch/unit/atomics/min_max_common.hh b/projects/hip-tests/catch/unit/atomics/min_max_common.hh index 552378bb3d0f..9d59e109231d 100644 --- a/projects/hip-tests/catch/unit/atomics/min_max_common.hh +++ b/projects/hip-tests/catch/unit/atomics/min_max_common.hh @@ -22,9 +22,7 @@ enum class AtomicOperation { kSafeMin, kUnsafeMin, kSafeMax, - kUnsafeMax, - kBuiltinMin, - kBuiltinMax + kUnsafeMax }; constexpr auto kIntegerTestValue = 5; @@ -45,7 +43,8 @@ __host__ __device__ TestType GetTestValue() { } template -__device__ TestType PerformAtomicOperation(TestType* const mem, const LinearAllocs allocType) { +__device__ TestType PerformAtomicOperation(TestType* const mem, + const LinearAllocs /*allocType*/) { const auto val = GetTestValue(); if constexpr (operation == AtomicOperation::kMin) { @@ -64,20 +63,6 @@ __device__ TestType PerformAtomicOperation(TestType* const mem, const LinearAllo return unsafeAtomicMax(mem, val); } else if constexpr (operation == AtomicOperation::kSafeMax) { return safeAtomicMax(mem, val); - } else if constexpr (operation == AtomicOperation::kBuiltinMin) { - if (std::is_floating_point_v && allocType == LinearAllocs::hipHostMalloc) - HIP_TEST_ATOMIC_BACKWARD_COMPAT_MEMORY { - return __hip_atomic_fetch_min(mem, val, __ATOMIC_RELAXED, memory_scope); - } else { - return __hip_atomic_fetch_min(mem, val, __ATOMIC_RELAXED, memory_scope); - } - } else if constexpr (operation == AtomicOperation::kBuiltinMax) { - if (std::is_floating_point_v && allocType == LinearAllocs::hipHostMalloc) - HIP_TEST_ATOMIC_BACKWARD_COMPAT_MEMORY { - return __hip_atomic_fetch_max(mem, val, __ATOMIC_RELAXED, memory_scope); - } else { - return __hip_atomic_fetch_max(mem, val, __ATOMIC_RELAXED, memory_scope); - } } } @@ -195,14 +180,12 @@ std::tuple, std::vector> TestKernelHostRef(const if constexpr (operation == AtomicOperation::kMin || operation == AtomicOperation::kMinSystem || operation == AtomicOperation::kUnsafeMin || - operation == AtomicOperation::kSafeMin || - operation == AtomicOperation::kBuiltinMin) { + operation == AtomicOperation::kSafeMin) { res = std::min(res, val); } else if constexpr (operation == AtomicOperation::kMax || operation == AtomicOperation::kMaxSystem || operation == AtomicOperation::kUnsafeMax || - operation == AtomicOperation::kSafeMax || - operation == AtomicOperation::kBuiltinMax) { + operation == AtomicOperation::kSafeMax) { res = std::max(res, val); } } @@ -316,32 +299,12 @@ void SingleDeviceSingleKernelTest(const unsigned int width, const unsigned int p TestParams params; params.num_devices = 1; params.kernel_count = 1; - if constexpr ((operation == AtomicOperation::kBuiltinMin || - operation == AtomicOperation::kBuiltinMax) && - memory_scope == __HIP_MEMORY_SCOPE_SINGLETHREAD) { - params.threads = 1; - } else if constexpr ((operation == AtomicOperation::kBuiltinMin || - operation == AtomicOperation::kBuiltinMax) && - memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT) { - int warp_size = 0; - HIP_CHECK(hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, 0)); - params.threads = dim3(warp_size); - } else { - params.threads = GenerateThreadDimensions(); - } + params.threads = GenerateThreadDimensions(); params.width = width; params.pitch = pitch; SECTION("Global memory") { - if constexpr ((operation == AtomicOperation::kBuiltinMin || - operation == AtomicOperation::kBuiltinMax) && - (memory_scope == __HIP_MEMORY_SCOPE_SINGLETHREAD || - memory_scope == __HIP_MEMORY_SCOPE_WAVEFRONT || - memory_scope == __HIP_MEMORY_SCOPE_WORKGROUP)) { - params.blocks = dim3(1); - } else { - params.blocks = GenerateBlockDimensions(); - } + params.blocks = GenerateBlockDimensions(); using LA = LinearAllocs; for (const auto alloc_type : {LA::hipMalloc}) { diff --git a/projects/hip-tests/catch/unit/atomics/sequential_consistency.cc b/projects/hip-tests/catch/unit/atomics/sequential_consistency.cc deleted file mode 100644 index eae32817f6a6..000000000000 --- a/projects/hip-tests/catch/unit/atomics/sequential_consistency.cc +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. - * - * SPDX-License-Identifier: MIT - */ - -#include - -#include "memory_order_common.hh" - -HIP_TEST_CASE(Unit___hip_atomic_load_store_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { SequentialConsistency::SystemTest(); } -} - -HIP_TEST_CASE(Unit___hip_atomic_exchange_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { SequentialConsistency::SystemTest(); } -} - -HIP_TEST_CASE(Unit___hip_atomic_compare_exchange_strong_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { - SequentialConsistency::SystemTest(); - } -} - -HIP_TEST_CASE(Unit___hip_atomic_compare_exchange_weak_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { - SequentialConsistency::SystemTest(); - } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_add_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { SequentialConsistency::SystemTest(); } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_and_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { SequentialConsistency::SystemTest(); } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_or_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { SequentialConsistency::SystemTest(); } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_xor_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { SequentialConsistency::SystemTest(); } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_min_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { SequentialConsistency::SystemTest(); } -} - -HIP_TEST_CASE(Unit___hip_atomic_fetch_max_Positive_Sequential_Consistency) { - SECTION("WAVEFRONT") { - SequentialConsistency::Test(); - } - SECTION("WORKGROUP") { - SequentialConsistency::Test(); - } - SECTION("AGENT") { - SequentialConsistency::Test(); - } - SECTION("SYSTEM") { SequentialConsistency::SystemTest(); } -} diff --git a/projects/hip-tests/catch/unit/cluster/CMakeLists.txt b/projects/hip-tests/catch/unit/cluster/CMakeLists.txt index bf9cfe3cf8f1..111e61fc2612 100644 --- a/projects/hip-tests/catch/unit/cluster/CMakeLists.txt +++ b/projects/hip-tests/catch/unit/cluster/CMakeLists.txt @@ -30,7 +30,9 @@ set(TEST_SRC ) # Append it to the list -set(CLUSTER_SUPPORTING_GPUS "gfx1101" "gfx1250") +# Only list targets covered by the CLUSTER_DIMS gate in catch/include/hip_test_common.hh; +# others build the kernels without cluster metadata. +set(CLUSTER_SUPPORTING_GPUS "gfx1250") set(CLUSTER_SUPPORTED OFF) foreach(CLUSTER_SUPPORTING_GPU IN LISTS CLUSTER_SUPPORTING_GPUS) diff --git a/projects/hip-tests/catch/unit/cluster/hipClusterCompilerDirective.cc b/projects/hip-tests/catch/unit/cluster/hipClusterCompilerDirective.cc index d95622bfcd16..4a22d04eb28f 100644 --- a/projects/hip-tests/catch/unit/cluster/hipClusterCompilerDirective.cc +++ b/projects/hip-tests/catch/unit/cluster/hipClusterCompilerDirective.cc @@ -21,6 +21,8 @@ THE SOFTWARE. #include #include +#include + #include "ClusterHelper.hpp" /** @@ -34,6 +36,12 @@ THE SOFTWARE. #define CLY 1 #define CLZ 1 +// Fit the compiler's per-axis encoding and 16-workgroup cluster limit. +#define OVERSIZED_CLX 4 +#define OVERSIZED_CLY 4 +#define OVERSIZED_CLZ 1 +constexpr int oversizedClusterSize = OVERSIZED_CLX * OVERSIZED_CLY * OVERSIZED_CLZ; + __global__ void CLUSTER_DIMS(CLX, CLY, CLZ) ClusterLaunchKernelBasicCD(int* in, int* out, int n) { int idx = blockDim.x * blockIdx.x + threadIdx.x; if (idx < n) { @@ -41,6 +49,30 @@ __global__ void CLUSTER_DIMS(CLX, CLY, CLZ) ClusterLaunchKernelBasicCD(int* in, } } +__global__ void CLUSTER_DIMS(OVERSIZED_CLX, OVERSIZED_CLY, OVERSIZED_CLZ) + ClusterLaunchKernelOversizedCD(int* out, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = idx; + } +} + +// The compiler accepts a zero in some but not all axes and emits it into the code object. +__global__ void CLUSTER_DIMS(0, 1, 1) ClusterLaunchKernelZeroDimCD(int* out, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = idx; + } +} + +// All-zero dims mean "no cluster" and emit no cluster metadata. +__global__ void CLUSTER_DIMS(0, 0, 0) ClusterLaunchKernelNoClusterCD(int* out, int n) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx < n) { + out[idx] = idx; + } +} + /** * Test Description * ------------------------ @@ -117,6 +149,208 @@ HIP_TEST_CASE(Unit_hipClusterLaunch_CompilerDirective_Basic) { bma.DestroyDeviceMemory(dptr_out); } +/** + * Test Description + * ------------------------ + * - Launches a kernel whose __cluster_dims__ exceeds the device maximum and checks that the + * runtime rejects it with hipErrorInvalidClusterSize. + * - Without the runtime check this test hangs rather than fails. + * + * Test source + * ------------------------ + * - catch/unit/cluster/hipClusterCompilerDirective.cc + * + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.0 + */ + +HIP_TEST_CASE(Unit_hipClusterLaunch_CompilerDirective_Negative_OversizedCluster) { + if (!CheckTargetSupport()) { + INFO("Target Not Supported!"); + return; + } + + hipLaunchAttribute attribute[1]; + attribute[0].id = hipLaunchAttributeClusterDimension; + attribute[0].val.clusterDim.x = 1; + attribute[0].val.clusterDim.y = 1; + attribute[0].val.clusterDim.z = 1; + + hipLaunchConfig_t config = {}; + config.numAttrs = 1; + config.attrs = attribute; + config.blockDim = {ntib, 1, 1}; + config.gridDim = {OVERSIZED_CLX, OVERSIZED_CLY, OVERSIZED_CLZ}; + + int maxClusterSize = 0; + HIP_CHECK(hipOccupancyMaxPotentialClusterSize( + &maxClusterSize, reinterpret_cast(ClusterLaunchKernelOversizedCD), &config)); + INFO("Maximum cluster size reported by the device: " << maxClusterSize); + if (maxClusterSize >= oversizedClusterSize) { + HIP_SKIP_TEST("Device places clusters this large, so there is nothing to reject"); + return; + } + + int* dptr = nullptr; + HIP_CHECK(hipMalloc(&dptr, num_size)); + + // Grid is a whole number of clusters, so only the size can cause a rejection. + ClusterLaunchKernelOversizedCD<<>>(dptr, num_elems); + const hipError_t launch_status = hipGetLastError(); + + // Free before REQUIRE; drain first if the launch was wrongly accepted. + if (launch_status == hipSuccess) { + HIP_CHECK(hipDeviceSynchronize()); + } + HIP_CHECK(hipFree(dptr)); + + INFO("Launch returned: " << hipGetErrorString(launch_status)); + REQUIRE(launch_status == hipErrorInvalidClusterSize); +} + +/** + * Test Description + * ------------------------ + * - Launches a kernel whose __cluster_dims__ has a zero in one axis and checks that the runtime + * rejects it with hipErrorInvalidConfiguration, as it does for a zero cluster dimension passed + * through hipLaunchAttributeClusterDimension. + * + * Test source + * ------------------------ + * - catch/unit/cluster/hipClusterCompilerDirective.cc + * + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.0 + */ + +HIP_TEST_CASE(Unit_hipClusterLaunch_CompilerDirective_Negative_ZeroClusterDim) { + if (!CheckTargetSupport()) { + INFO("Target Not Supported!"); + return; + } + + int* dptr = nullptr; + HIP_CHECK(hipMalloc(&dptr, num_size)); + + ClusterLaunchKernelZeroDimCD<<>>(dptr, num_elems); + const hipError_t launch_status = hipGetLastError(); + + // Free before REQUIRE; drain first if the launch was wrongly accepted. + if (launch_status == hipSuccess) { + HIP_CHECK(hipDeviceSynchronize()); + } + HIP_CHECK(hipFree(dptr)); + + INFO("Launch returned: " << hipGetErrorString(launch_status)); + REQUIRE(launch_status == hipErrorInvalidConfiguration); +} + +/** + * Test Description + * ------------------------ + * - Adds a kernel whose __cluster_dims__ has a zero in one axis to a graph and checks that the + * rejection reaches the caller through hipGraphInstantiate or hipGraphLaunch. + * + * Test source + * ------------------------ + * - catch/unit/cluster/hipClusterCompilerDirective.cc + * + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.0 + */ + +HIP_TEST_CASE(Unit_hipClusterLaunch_CompilerDirective_Negative_ZeroClusterDim_Graph) { + if (!CheckTargetSupport()) { + INFO("Target Not Supported!"); + return; + } + + int* dptr = nullptr; + HIP_CHECK(hipMalloc(&dptr, num_size)); + int n = num_elems; + void* kernel_args[] = {&dptr, &n}; + + hipKernelNodeParams node_params = {}; + node_params.func = reinterpret_cast(ClusterLaunchKernelZeroDimCD); + node_params.gridDim = dim3(nbig); + node_params.blockDim = dim3(ntib); + node_params.kernelParams = kernel_args; + + hipGraph_t graph = nullptr; + hipGraphNode_t node = nullptr; + HIP_CHECK(hipGraphCreate(&graph, 0)); + HIP_CHECK(hipGraphAddKernelNode(&node, graph, nullptr, 0, &node_params)); + + // The kernel's command is built at instantiation or at first launch, depending on the + // graph execution path, so accept the rejection at either point. + hipGraphExec_t graph_exec = nullptr; + const hipError_t instantiate_status = + hipGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0); + hipError_t launch_status = hipSuccess; + if (instantiate_status == hipSuccess) { + launch_status = hipGraphLaunch(graph_exec, nullptr); + // Drain before freeing if the launch was wrongly accepted. + if (launch_status == hipSuccess) { + HIP_CHECK(hipDeviceSynchronize()); + } + HIP_CHECK(hipGraphExecDestroy(graph_exec)); + } + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK(hipFree(dptr)); + // Clear the last error so the rejection doesn't leak into later tests. + (void)hipGetLastError(); + + INFO("hipGraphInstantiate returned: " << hipGetErrorString(instantiate_status)); + INFO("hipGraphLaunch returned: " << hipGetErrorString(launch_status)); + const hipError_t status = + instantiate_status != hipSuccess ? instantiate_status : launch_status; + REQUIRE(status == hipErrorInvalidConfiguration); +} + +/** + * Test Description + * ------------------------ + * - Launches a kernel with __cluster_dims__(0, 0, 0), which means no cluster, and checks that it + * runs as an ordinary launch with every block computing its own index. + * + * Test source + * ------------------------ + * - catch/unit/cluster/hipClusterCompilerDirective.cc + * + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.0 + */ + +HIP_TEST_CASE(Unit_hipClusterLaunch_CompilerDirective_NoCluster) { + if (!CheckTargetSupport()) { + INFO("Target Not Supported!"); + return; + } + + // One element per thread, several blocks, so a wrong block index shows in the output. + constexpr int threads_per_block = num_elems / nbig; + + std::vector hout(num_elems, -1); + int* dptr = nullptr; + HIP_CHECK(hipMalloc(&dptr, num_size)); + HIP_CHECK(hipMemset(dptr, 0xff, num_size)); + + ClusterLaunchKernelNoClusterCD<<>>(dptr, num_elems); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + HIP_CHECK(hipMemcpy(hout.data(), dptr, num_size, hipMemcpyDeviceToHost)); + HIP_CHECK(hipFree(dptr)); + + for (size_t idx = 0; idx < num_elems; ++idx) { + INFO("Index " << idx); + REQUIRE(hout[idx] == static_cast(idx)); + } +} + /** * End doxygen group ClusterTest. * @} diff --git a/projects/hip-tests/catch/unit/compiler/CMakeLists.txt b/projects/hip-tests/catch/unit/compiler/CMakeLists.txt index 5ce208c26282..cae030bdd966 100644 --- a/projects/hip-tests/catch/unit/compiler/CMakeLists.txt +++ b/projects/hip-tests/catch/unit/compiler/CMakeLists.txt @@ -26,7 +26,8 @@ if(HIP_PLATFORM MATCHES "amd") --offload-arch=gfx9-generic --offload-arch=gfx9-4-generic:sramecc+:xnack- --offload-arch=gfx9-4-generic:sramecc-:xnack- - --offload-arch=gfx9-4-generic:xnack+ + --offload-arch=gfx9-4-generic:sramecc+:xnack+ + --offload-arch=gfx9-4-generic:sramecc-:xnack+ --offload-arch=gfx10-1-generic --offload-arch=gfx10-3-generic --offload-arch=gfx11-generic diff --git a/projects/hip-tests/catch/unit/cooperativeGrps/grid_group.cc b/projects/hip-tests/catch/unit/cooperativeGrps/grid_group.cc index 8b019c19d6dc..2b06549c1570 100644 --- a/projects/hip-tests/catch/unit/cooperativeGrps/grid_group.cc +++ b/projects/hip-tests/catch/unit/cooperativeGrps/grid_group.cc @@ -61,8 +61,8 @@ static __global__ void sync_kernel(unsigned int* atomic_val, unsigned int* per_l // until all of the other wavefronts have incremented the // per-loop atomic and hit the grid.sync() #if HT_AMD - while (__hip_atomic_load(&per_loop_atomic[iter], __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) < - (grid_blocks - 1)) { + while (__scoped_atomic_load_n(&per_loop_atomic[iter], __ATOMIC_RELAXED, + __MEMORY_SCOPE_DEVICE) < (grid_blocks - 1)) { __builtin_amdgcn_s_sleep(127); } diff --git a/projects/hip-tests/catch/unit/cooperativeGrps/hipCGMultiGridGroupType_old.cc b/projects/hip-tests/catch/unit/cooperativeGrps/hipCGMultiGridGroupType_old.cc index 6d43c8ca06cd..196654a5877f 100644 --- a/projects/hip-tests/catch/unit/cooperativeGrps/hipCGMultiGridGroupType_old.cc +++ b/projects/hip-tests/catch/unit/cooperativeGrps/hipCGMultiGridGroupType_old.cc @@ -164,8 +164,8 @@ __global__ void test_kernel(unsigned int* atomic_val, unsigned int* array, uint3 // until all of the other wavefronts have incremented the // per-loop atomic and hit the grid.sync() #if HT_AMD - while (__hip_atomic_load(&per_loop_atomic[i], __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) < - (grid_blocks - 1)) { + while (__scoped_atomic_load_n(&per_loop_atomic[i], __ATOMIC_RELAXED, + __MEMORY_SCOPE_DEVICE) < (grid_blocks - 1)) { __builtin_amdgcn_s_sleep(127); } diff --git a/projects/hip-tests/catch/unit/cooperativeGrps/thread_block_tile.cc b/projects/hip-tests/catch/unit/cooperativeGrps/thread_block_tile.cc index e52a7a5cb3ca..44aa6a7b873d 100644 --- a/projects/hip-tests/catch/unit/cooperativeGrps/thread_block_tile.cc +++ b/projects/hip-tests/catch/unit/cooperativeGrps/thread_block_tile.cc @@ -709,10 +709,8 @@ void aggregateForTypeAndOp(AggregationType aggType, std::mt19937_64 gen(Catch::rngSeed()); dim3 gridDim = { 1 }; hipError_t status; - typename distribution::result_type a = std::is_same::value? std::numeric_limits::lowest() : - (std::is_signed::value? -1023 : 0); - typename distribution::result_type b = std::is_same::value? std::numeric_limits::max() : - 1023; + typename distribution::result_type a = TestRange::minimum; + typename distribution::result_type b = TestRange::maximum; distribution distInput {a, b}; int numAggregation = 0; void* kernelPtr; @@ -879,7 +877,7 @@ HIP_TEMPLATE_TEST_CASE(Unit_Thread_Block_Tile_Reduce_Random_arithmetic, int, uns } HIP_TEMPLATE_TEST_CASE(Unit_Thread_Block_Tile_Reduce_Random_boolean, int, unsigned int, long long, - unsigned long long) + unsigned long long) { CHECK_COOPERATIVE_LAUNCH_SUPPORT @@ -1344,7 +1342,7 @@ HIP_TEST_CASE(Unit_Thread_Block_Tile_Reduce_Standard_Op_Custom_Type) } HIP_TEMPLATE_TEST_CASE(Unit_Thread_Block_Coalesced_Reduce_arithmetic, int, unsigned int, long long, - unsigned long long, float, half, double) + unsigned long long, float, half, double, unsigned char) { CHECK_COOPERATIVE_LAUNCH_SUPPORT @@ -1359,7 +1357,8 @@ HIP_TEMPLATE_TEST_CASE(Unit_Thread_Block_Coalesced_Reduce_arithmetic, int, unsig } } -HIP_TEMPLATE_TEST_CASE(Unit_Thread_Block_Coalesced_Reduce_boolean, int, unsigned int, long long, unsigned long long) +HIP_TEMPLATE_TEST_CASE(Unit_Thread_Block_Coalesced_Reduce_boolean, int, unsigned int, + long long, unsigned long long, bool) { CHECK_COOPERATIVE_LAUNCH_SUPPORT @@ -1495,7 +1494,7 @@ TEST_CASE(Unit_Thread_Block_Tile_Inclusive_Scan_Basic) } } -TEMPLATE_TEST_CASE(Unit_Thread_Block_Tile_Exclusive_Scan_Basic, int, half) +TEMPLATE_TEST_CASE(Unit_Thread_Block_Tile_Exclusive_Scan_Basic, int, half, bool, unsigned char) { CHECK_COOPERATIVE_LAUNCH_SUPPORT @@ -1534,7 +1533,7 @@ TEMPLATE_TEST_CASE(Unit_Thread_Block_Tile_Exclusive_Scan_Basic, int, half) // for all the tile sizes and all input types, using random input values, calculates the scan // values. Additionally, randomly make some threads not participate for the coalesced_threads case TEMPLATE_TEST_CASE(Unit_Thread_Block_Tile_Scan_Random_arithmetic, int, unsigned int, long long, - unsigned long long, float, half, double) + unsigned long long, float, half, double, unsigned char) { CHECK_COOPERATIVE_LAUNCH_SUPPORT @@ -1559,8 +1558,34 @@ TEMPLATE_TEST_CASE(Unit_Thread_Block_Tile_Scan_Random_arithmetic, int, unsigned } } +// checks scans that use bool values and cg::less and greater as operators which are completely +// defined e.g. false < true +TEST_CASE(Unit_Thread_Block_Tile_Scan_Random_Bool_Less_Greater) +{ + CHECK_COOPERATIVE_LAUNCH_SUPPORT + + std::tuple, + cooperative_groups::greater> types; + + SECTION("inclusive") { + if (getWarpSize() == 32) { + runAggregationRandomForOps(AggregationType::InclusiveScan, types); + } else { + runAggregationRandomForOps(AggregationType::InclusiveScan, types); + } + } + + SECTION("exclusive") { + if (getWarpSize() == 32) { + runAggregationRandomForOps(AggregationType::ExclusiveScan, types); + } else { + runAggregationRandomForOps(AggregationType::ExclusiveScan, types); + } + } +} + TEMPLATE_TEST_CASE(Unit_Thread_Block_Tile_Scan_Random_boolean, int, unsigned int, long long, - unsigned long long) + unsigned long long, bool) { CHECK_COOPERATIVE_LAUNCH_SUPPORT @@ -1622,7 +1647,7 @@ TEST_CASE(Unit_Thread_Block_Tile_2D_3D_Blocks) } TEMPLATE_TEST_CASE(Unit_Thread_Block_Coalesced_Scan_arithmetic, int, unsigned int, long long, - unsigned long long, float, half, double) + unsigned long long, float, half, double, unsigned char) { CHECK_COOPERATIVE_LAUNCH_SUPPORT @@ -1648,7 +1673,7 @@ TEMPLATE_TEST_CASE(Unit_Thread_Block_Coalesced_Scan_arithmetic, int, unsigned in } TEMPLATE_TEST_CASE(Unit_Thread_Block_Coalesced_Scan_boolean, int, unsigned int, long long, - unsigned long long) + unsigned long long, bool) { CHECK_COOPERATIVE_LAUNCH_SUPPORT diff --git a/projects/hip-tests/catch/unit/device/CMakeLists.txt b/projects/hip-tests/catch/unit/device/CMakeLists.txt index d3ebfc11feb1..e76419492bc1 100644 --- a/projects/hip-tests/catch/unit/device/CMakeLists.txt +++ b/projects/hip-tests/catch/unit/device/CMakeLists.txt @@ -22,6 +22,7 @@ set(TEST_SRC hipDeviceGetUuid.cc hipDeviceGetLuid.cc hipDeviceAPUCheck.cc + hipDeviceFlushGPUDirectRDMAWrites.cc hipDeviceGetDefaultMemPool.cc hipDeviceCanAccessPeer.cc hipDeviceEnableDisablePeerAccess.cc diff --git a/projects/hip-tests/catch/unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc b/projects/hip-tests/catch/unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc new file mode 100644 index 000000000000..31425549419e --- /dev/null +++ b/projects/hip-tests/catch/unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc @@ -0,0 +1,704 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +#include +#include // NOLINT +#include + +#include + +/** + * @addtogroup hipDeviceFlushGPUDirectRDMAWrites hipDeviceFlushGPUDirectRDMAWrites + * @{ + * @ingroup DeviceTest + * `hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTarget target, + * hipFlushGPUDirectRDMAWritesScope scope)` - + * Blocks until remote writes that were made visible to the target by a third-party + * device (typically an RDMA-capable NIC writing directly into device memory) are + * visible at the requested scope. + * + * The call is a host-ordered, blocking visibility barrier on inbound remote writes. + * It is not a synchronization primitive: it must not wait on any stream or kernel. + * + * Capability is reported through three device attributes, mirrored in + * hipDeviceProp_t: + * - hipDeviceAttributeGPUDirectRDMASupported + * - hipDeviceAttributeGPUDirectRDMAFlushWritesOptions (hipFlushGPUDirectRDMAWritesOptions mask) + * - hipDeviceAttributeGPUDirectRDMAWritesOrdering (hipGPUDirectRDMAWritesOrdering value) + */ + +namespace { + +constexpr int kFlushOptionsMask = + hipFlushGPUDirectRDMAWritesOptionHost | hipFlushGPUDirectRDMAWritesOptionMemOps; + +struct RdmaCaps { + int supported; + int flushOptions; + int writesOrdering; +}; + +RdmaCaps GetRdmaCaps(int device) { + RdmaCaps caps{}; + HIP_CHECK(hipDeviceGetAttribute(&caps.supported, hipDeviceAttributeGPUDirectRDMASupported, + device)); + HIP_CHECK(hipDeviceGetAttribute(&caps.flushOptions, + hipDeviceAttributeGPUDirectRDMAFlushWritesOptions, device)); + HIP_CHECK(hipDeviceGetAttribute(&caps.writesOrdering, + hipDeviceAttributeGPUDirectRDMAWritesOrdering, device)); + return caps; +} + +bool HostFlushSupported(int device) { + return (GetRdmaCaps(device).flushOptions & hipFlushGPUDirectRDMAWritesOptionHost) != 0; +} + +/** + * Expected return value of hipDeviceFlushGPUDirectRDMAWrites on a given device. + * + * Contract being pinned: the host-issued flush is available if and only if the + * device advertises hipFlushGPUDirectRDMAWritesOptionHost. When the bit is clear + * the call must fail loudly rather than silently succeed, otherwise a caller has + * no way to distinguish "flushed" from "cannot flush". + * + * The hipErrorNotSupported branch could not be verified against CUDA. Both parts + * measured (an RTX PRO 4000 Blackwell and a GeForce RTX 5090) advertise the Host + * option, including the GeForce which reports GPUDirectRDMASupported = 0. The + * flush-writes options and writes-ordering attributes describe the hardware write + * path and are apparently independent of whether GPUDirect RDMA itself is enabled. + */ +hipError_t ExpectedFlushStatus(int device) { + return HostFlushSupported(device) ? hipSuccess : hipErrorNotSupported; +} + +__global__ void SpinUntilFlag(volatile int* flag) { + while (*flag == 0) { + } +} + +__global__ void CheckPattern(const int* buf, size_t n, int* mismatches) { + size_t stride = static_cast(blockDim.x) * gridDim.x; + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += stride) { + if (buf[i] != static_cast(i)) { + atomicAdd(mismatches, 1); + } + } +} + +} // namespace + +/** + * Test Description + * ------------------------ + * - Validates that the three GPUDirect RDMA capability attributes are queryable on + * every device and that each reported value is inside its defined domain: + * -# hipDeviceAttributeGPUDirectRDMASupported is a boolean + * -# hipDeviceAttributeGPUDirectRDMAFlushWritesOptions is a subset of the + * hipFlushGPUDirectRDMAWritesOptions mask + * -# hipDeviceAttributeGPUDirectRDMAWritesOrdering is one of None/Owner/AllDevices + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_AttributeDomains) { + int numDevices = 0; + HIP_CHECK(hipGetDeviceCount(&numDevices)); + if (numDevices == 0) { + HIP_SKIP_TEST(HipTest::SkipReason::kNoGpuDevice); + return; + } + + for (int device = 0; device < numDevices; ++device) { + const RdmaCaps caps = GetRdmaCaps(device); + INFO("device " << device << " supported=" << caps.supported << " flushOptions=" + << caps.flushOptions << " writesOrdering=" << caps.writesOrdering); + + REQUIRE((caps.supported == 0 || caps.supported == 1)); + REQUIRE((caps.flushOptions & ~kFlushOptionsMask) == 0); + REQUIRE((caps.writesOrdering == hipGPUDirectRDMAWritesOrderingNone || + caps.writesOrdering == hipGPUDirectRDMAWritesOrderingOwner || + caps.writesOrdering == hipGPUDirectRDMAWritesOrderingAllDevices)); + } +} + +/** + * Test Description + * ------------------------ + * - Validates that hipDeviceGetAttribute and hipGetDeviceProperties report identical + * values for the three GPUDirect RDMA capabilities. Callers use them + * interchangeably, so a divergence is a defect regardless of which value is right. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_AttributesMatchProperties) { + int numDevices = 0; + HIP_CHECK(hipGetDeviceCount(&numDevices)); + if (numDevices == 0) { + HIP_SKIP_TEST(HipTest::SkipReason::kNoGpuDevice); + return; + } + + for (int device = 0; device < numDevices; ++device) { + const RdmaCaps caps = GetRdmaCaps(device); + hipDeviceProp_t props; + HIP_CHECK(hipGetDeviceProperties(&props, device)); + + INFO("device " << device); + REQUIRE(caps.supported == props.gpuDirectRDMASupported); + REQUIRE(static_cast(caps.flushOptions) == props.gpuDirectRDMAFlushWritesOptions); + REQUIRE(caps.writesOrdering == props.gpuDirectRDMAWritesOrdering); + } +} + +/** + * Test Description + * ------------------------ + * - Validates the internal consistency of the advertised capabilities. A device that + * claims GPUDirect RDMA support must offer some way to make an inbound remote write + * visible to the owning device, which is either: + * -# native ordering at owner scope or stronger, or + * -# an explicit flush path (host and/or memory-operation based) + * - A device reporting supported=1, ordering=None and flushOptions=0 advertises RDMA + * that no caller can ever safely consume. + * - The converse is legal and observed: a GeForce RTX 5090 reports supported=0 with + * flushOptions=Host and ordering=Owner. The flush attributes describe the write path, + * not the RDMA feature, so this test only constrains the supported=1 case. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_CapabilityConsistency) { + int numDevices = 0; + HIP_CHECK(hipGetDeviceCount(&numDevices)); + if (numDevices == 0) { + HIP_SKIP_TEST(HipTest::SkipReason::kNoGpuDevice); + return; + } + + for (int device = 0; device < numDevices; ++device) { + const RdmaCaps caps = GetRdmaCaps(device); + if (caps.supported == 0) { + continue; + } + INFO("device " << device << " claims GPUDirect RDMA support with flushOptions=" + << caps.flushOptions << " writesOrdering=" << caps.writesOrdering); + REQUIRE((caps.writesOrdering != hipGPUDirectRDMAWritesOrderingNone || caps.flushOptions != 0)); + } +} + +/** + * Test Description + * ------------------------ + * - Validates the basic flush call against the current device for both scopes: + * -# hipFlushGPUDirectRDMAWritesToOwner + * -# hipFlushGPUDirectRDMAWritesToAllDevices + * - Expected output: `hipSuccess` when the device advertises the host flush option, + * `hipErrorNotSupported` otherwise. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_Basic) { + int device = 0; + HIP_CHECK(hipGetDevice(&device)); + const hipError_t expected = ExpectedFlushStatus(device); + + SECTION("scope is ToOwner") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToOwner), + expected); + } + + SECTION("scope is ToAllDevices") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToAllDevices), + expected); + } +} + +/** + * Test Description + * ------------------------ + * - Validates that the flush is repeatable and leaves no residual state. The call is + * issued in a tight loop and normal device work is then exercised to confirm the + * context is still healthy. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_Repeated) { + int device = 0; + HIP_CHECK(hipGetDevice(&device)); + if (!HostFlushSupported(device)) { + HIP_SKIP_TEST("host GPUDirect RDMA flush is not supported on this device."); + return; + } + + constexpr int kIterations = 1024; + for (int i = 0; i < kIterations; ++i) { + HIP_CHECK(hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToOwner)); + } + + constexpr size_t kElements = 4096; + int* deviceBuffer = nullptr; + HIP_CHECK(hipMalloc(&deviceBuffer, kElements * sizeof(int))); + std::vector hostBuffer(kElements); + for (size_t i = 0; i < kElements; ++i) { + hostBuffer[i] = static_cast(i); + } + HIP_CHECK(hipMemcpy(deviceBuffer, hostBuffer.data(), kElements * sizeof(int), + hipMemcpyHostToDevice)); + std::vector readBack(kElements, -1); + HIP_CHECK( + hipMemcpy(readBack.data(), deviceBuffer, kElements * sizeof(int), hipMemcpyDeviceToHost)); + HIP_CHECK(hipFree(deviceBuffer)); + + REQUIRE(readBack == hostBuffer); +} + +/** + * Test Description + * ------------------------ + * - Validates that the flush applies to the current device and succeeds after every + * hipSetDevice on a multi-GPU system. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - Multi-GPU system + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_MultiDevice) { + int numDevices = 0; + HIP_CHECK(hipGetDeviceCount(&numDevices)); + if (numDevices < 2) { + HIP_SKIP_TEST(HipTest::SkipReason::kFewerThanTwoGpus); + return; + } + + int originalDevice = 0; + HIP_CHECK(hipGetDevice(&originalDevice)); + + for (int device = 0; device < numDevices; ++device) { + HIP_CHECK(hipSetDevice(device)); + INFO("current device " << device); + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToOwner), + ExpectedFlushStatus(device)); + } + + HIP_CHECK(hipSetDevice(originalDevice)); +} + +/** + * Test Description + * ------------------------ + * - Validates that the flush is safe to call concurrently from multiple host threads + * sharing a device, as an RDMA receive path routinely would. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_MultiThreaded) { + int device = 0; + HIP_CHECK(hipGetDevice(&device)); + if (!HostFlushSupported(device)) { + HIP_SKIP_TEST("host GPUDirect RDMA flush is not supported on this device."); + return; + } + + constexpr int kThreads = 8; + constexpr int kIterationsPerThread = 256; + std::vector results(kThreads, hipSuccess); + std::vector threads; + threads.reserve(kThreads); + + for (int t = 0; t < kThreads; ++t) { + threads.emplace_back([device, t, &results]() { + hipError_t status = hipSetDevice(device); + for (int i = 0; i < kIterationsPerThread && status == hipSuccess; ++i) { + status = hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToOwner); + } + results[t] = status; + }); + } + for (auto& thread : threads) { + thread.join(); + } + + for (int t = 0; t < kThreads; ++t) { + INFO("thread " << t << " returned " << hipGetErrorString(results[t])); + REQUIRE(results[t] == hipSuccess); + } +} + +/** + * Test Description + * ------------------------ + * - Validates that the flush is a memory-visibility barrier and not a synchronization + * point. A kernel is kept resident by spinning on a host-written flag; the flush must + * return while that kernel is still executing and must leave the stream incomplete. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - Host memory mapping support + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_DoesNotSynchronize) { + int device = 0; + HIP_CHECK(hipGetDevice(&device)); + if (!HostFlushSupported(device)) { + HIP_SKIP_TEST("host GPUDirect RDMA flush is not supported on this device."); + return; + } + + int* flag = nullptr; + HIP_CHECK(hipHostMalloc(&flag, sizeof(int), hipHostMallocMapped)); + *flag = 0; + + int* deviceFlag = nullptr; + HIP_CHECK(hipHostGetDevicePointer(reinterpret_cast(&deviceFlag), flag, 0)); + + hipStream_t stream = nullptr; + HIP_CHECK(hipStreamCreate(&stream)); + SpinUntilFlag<<<1, 1, 0, stream>>>(deviceFlag); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToOwner)); + + // The kernel cannot have retired: nothing has released it yet. + HIP_CHECK_ERROR(hipStreamQuery(stream), hipErrorNotReady); + + std::atomic_thread_fence(std::memory_order_release); + *static_cast(flag) = 1; + HIP_CHECK(hipStreamSynchronize(stream)); + HIP_CHECK(hipStreamDestroy(stream)); + HIP_CHECK(hipHostFree(flag)); +} + +/** + * Test Description + * ------------------------ + * - Validates that data already resident in device memory stays intact and readable by + * a kernel across a flush. This is a portable proxy for the RDMA receive sequence + * (write payload, flush, consume) that does not require an RDMA-capable NIC; it does + * not prove inbound remote-write visibility, only that the flush is non-destructive + * and correctly ordered before subsequent kernel launches. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_PayloadVisibleToKernel) { + int device = 0; + HIP_CHECK(hipGetDevice(&device)); + if (!HostFlushSupported(device)) { + HIP_SKIP_TEST("host GPUDirect RDMA flush is not supported on this device."); + return; + } + + constexpr size_t kElements = 1 << 16; + std::vector hostBuffer(kElements); + for (size_t i = 0; i < kElements; ++i) { + hostBuffer[i] = static_cast(i); + } + + int* payload = nullptr; + int* mismatches = nullptr; + HIP_CHECK(hipMalloc(&payload, kElements * sizeof(int))); + HIP_CHECK(hipMalloc(&mismatches, sizeof(int))); + HIP_CHECK(hipMemset(mismatches, 0, sizeof(int))); + HIP_CHECK( + hipMemcpy(payload, hostBuffer.data(), kElements * sizeof(int), hipMemcpyHostToDevice)); + + HIP_CHECK(hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToOwner)); + + CheckPattern<<<64, 256>>>(payload, kElements, mismatches); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + + int mismatchCount = -1; + HIP_CHECK(hipMemcpy(&mismatchCount, mismatches, sizeof(int), hipMemcpyDeviceToHost)); + HIP_CHECK(hipFree(mismatches)); + HIP_CHECK(hipFree(payload)); + + REQUIRE(mismatchCount == 0); +} + +/** + * Test Description + * ------------------------ + * - Validates handling of invalid arguments. Every enumerator outside the defined set + * must be rejected with `hipErrorInvalidValue`, including the small values adjacent + * to the valid ones. The scope enumerators are 100 and 200, so a range check written + * as `scope < 0 || scope > 200` would wrongly accept 0, 1 and 99. + * - Argument validation is required to precede the capability check, so an out-of-range + * enumerator never reports `hipErrorNotSupported`. + * - Matches observed CUDA 13.1 behaviour on an RDMA-capable device: every one of these + * combinations returns `CUDA_ERROR_INVALID_VALUE`. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Negative_Parameters) { + using Target = hipFlushGPUDirectRDMAWritesTarget; + using Scope = hipFlushGPUDirectRDMAWritesScope; + + const Target goodTarget = hipFlushGPUDirectRDMAWritesTargetCurrentDevice; + const Scope goodScope = hipFlushGPUDirectRDMAWritesToOwner; + + SECTION("scope is zero") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(goodTarget, static_cast(0)), + hipErrorInvalidValue); + (void)hipGetLastError(); + } + + SECTION("scope is one") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(goodTarget, static_cast(1)), + hipErrorInvalidValue); + (void)hipGetLastError(); + } + + SECTION("scope is just below ToOwner") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(goodTarget, static_cast(99)), + hipErrorInvalidValue); + (void)hipGetLastError(); + } + + SECTION("scope is negative") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(goodTarget, static_cast(-1)), + hipErrorInvalidValue); + (void)hipGetLastError(); + } + + SECTION("scope is far out of range") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(goodTarget, static_cast(0x7fff)), + hipErrorInvalidValue); + (void)hipGetLastError(); + } + + SECTION("target is one past the only valid enumerator") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(static_cast(1), goodScope), + hipErrorInvalidValue); + (void)hipGetLastError(); + } + + SECTION("target is negative") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(static_cast(-1), goodScope), + hipErrorInvalidValue); + (void)hipGetLastError(); + } + + SECTION("target is far out of range") { + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites(static_cast(0x7fff), goodScope), + hipErrorInvalidValue); + (void)hipGetLastError(); + } + + SECTION("target and scope are out of range") { + HIP_CHECK_ERROR( + hipDeviceFlushGPUDirectRDMAWrites(static_cast(0x7fff), static_cast(0x7fff)), + hipErrorInvalidValue); + (void)hipGetLastError(); + } +} + +/** + * Test Description + * ------------------------ + * - Validates the error accounting of a rejected flush: + * -# the error is returned to the caller + * -# it is recorded in the per-thread last-error state, as every runtime entry point does + * -# reading the last error clears it + * -# the context is otherwise undamaged and subsequent device work succeeds + * - Matches observed CUDA 13.1 behaviour: `cudaGetLastError` reports + * `cudaErrorInvalidValue` after a rejected flush, while the following `cudaMalloc` + * still returns `cudaSuccess`. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Negative_FailureSetsLastErrorOnly) { + // Discard, not HIP_CHECK: a sibling test may have left a sticky error and this case must + // start from a known-clean state rather than assert on what it inherited. + (void)hipGetLastError(); + + HIP_CHECK_ERROR( + hipDeviceFlushGPUDirectRDMAWrites(static_cast(0x7fff), + hipFlushGPUDirectRDMAWritesToOwner), + hipErrorInvalidValue); + + // The failure is recorded, and reading it clears it. + HIP_CHECK_ERROR(hipGetLastError(), hipErrorInvalidValue); + HIP_CHECK(hipGetLastError()); + + int* buffer = nullptr; + HIP_CHECK(hipMalloc(&buffer, 256)); + HIP_CHECK(hipMemset(buffer, 0, 256)); + HIP_CHECK(hipDeviceSynchronize()); + HIP_CHECK(hipFree(buffer)); +} + +/** + * Test Description + * ------------------------ + * - Validates that the flush works against a freshly recreated primary context. After + * `hipDeviceReset` destroys the context, the next flush must transparently reinitialise + * rather than fail. + * - Matches observed CUDA 13.1 behaviour: `cudaSuccess` immediately after + * `cudaDeviceReset`. Note that the driver-level entry point instead reports + * `CUDA_ERROR_CONTEXT_IS_DESTROYED` in the same situation; the runtime-level contract + * is the one HIP mirrors. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_AfterDeviceReset) { + int device = 0; + HIP_CHECK(hipGetDevice(&device)); + const hipError_t expected = ExpectedFlushStatus(device); + + HIP_CHECK(hipDeviceReset()); + + HIP_CHECK_ERROR(hipDeviceFlushGPUDirectRDMAWrites( + hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToOwner), + expected); + // Discard rather than assert: on a device with no flush path `expected` is an error and + // would legitimately be sticky here. + (void)hipGetLastError(); + + // The recreated context must be usable. + int* buffer = nullptr; + HIP_CHECK(hipMalloc(&buffer, 256)); + HIP_CHECK(hipFree(buffer)); +} + +/** + * Test Description + * ------------------------ + * - Validates behaviour inside a stream capture region. The flush is a host-side + * visibility barrier with no stream ordering, so it must: + * -# succeed while a capture is active + * -# not invalidate the capture + * -# contribute no node to the resulting graph + * - Matches observed CUDA 13.1 behaviour: success during capture, successful + * `cudaStreamEndCapture`, and a graph with zero nodes. + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Positive_NotCapturedByGraph) { + int device = 0; + HIP_CHECK(hipGetDevice(&device)); + if (!HostFlushSupported(device)) { + HIP_SKIP_TEST("host GPUDirect RDMA flush is not supported on this device."); + return; + } + + hipStream_t stream = nullptr; + HIP_CHECK(hipStreamCreate(&stream)); + HIP_CHECK(hipStreamBeginCapture(stream, hipStreamCaptureModeThreadLocal)); + + HIP_CHECK(hipDeviceFlushGPUDirectRDMAWrites(hipFlushGPUDirectRDMAWritesTargetCurrentDevice, + hipFlushGPUDirectRDMAWritesToOwner)); + + hipGraph_t graph = nullptr; + HIP_CHECK(hipStreamEndCapture(stream, &graph)); + REQUIRE(graph != nullptr); + + size_t numNodes = 0; + HIP_CHECK(hipGraphGetNodes(graph, nullptr, &numNodes)); + INFO("captured graph node count " << numNodes); + REQUIRE(numNodes == 0); + + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK(hipStreamDestroy(stream)); +} + +/** + * Test Description + * ------------------------ + * - Validates handling of invalid device ordinals when querying the capability + * attributes: + * -# When the device ordinal is negative + * -# When the device ordinal is out of bounds + * -# When the output pointer is `nullptr` + * - Expected output: return `hipErrorInvalidValue` / `hipErrorInvalidDevice` + * Test source + * ------------------------ + * - unit/device/hipDeviceFlushGPUDirectRDMAWrites.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + */ +HIP_TEST_CASE(Unit_hipDeviceFlushGPUDirectRDMAWrites_Negative_AttributeQuery) { + int numDevices = 0; + HIP_CHECK(hipGetDeviceCount(&numDevices)); + if (numDevices == 0) { + HIP_SKIP_TEST(HipTest::SkipReason::kNoGpuDevice); + return; + } + + int value = 0; + const hipDeviceAttribute_t attributes[] = {hipDeviceAttributeGPUDirectRDMASupported, + hipDeviceAttributeGPUDirectRDMAFlushWritesOptions, + hipDeviceAttributeGPUDirectRDMAWritesOrdering}; + + for (const auto attribute : attributes) { + INFO("attribute " << static_cast(attribute)); + + HIP_CHECK_ERROR(hipDeviceGetAttribute(&value, attribute, -1), hipErrorInvalidDevice); + HIP_CHECK_ERROR(hipDeviceGetAttribute(&value, attribute, numDevices), hipErrorInvalidDevice); + HIP_CHECK_ERROR(hipDeviceGetAttribute(nullptr, attribute, 0), hipErrorInvalidValue); + (void)hipGetLastError(); + } +} + +/** + * End doxygen group DeviceTest. + * @} + */ diff --git a/projects/hip-tests/catch/unit/deviceLib/CMakeLists.txt b/projects/hip-tests/catch/unit/deviceLib/CMakeLists.txt index 55c3f4cd1d9e..462f984bc05a 100644 --- a/projects/hip-tests/catch/unit/deviceLib/CMakeLists.txt +++ b/projects/hip-tests/catch/unit/deviceLib/CMakeLists.txt @@ -61,6 +61,7 @@ set(AMD_TEST_SRC hipVectorTypesDevice.cc hipTestHalf.cc hipTestHalfConstexpr.cc + hipTestHalfIntegralAssign_test.cc hipComplex.cc hipTestFMA.cc hipTestNativeHalf.cc diff --git a/projects/hip-tests/catch/unit/deviceLib/bfloat16.cc b/projects/hip-tests/catch/unit/deviceLib/bfloat16.cc index b8db25462395..66ec421ee9aa 100644 --- a/projects/hip-tests/catch/unit/deviceLib/bfloat16.cc +++ b/projects/hip-tests/catch/unit/deviceLib/bfloat16.cc @@ -662,6 +662,23 @@ __global__ void bf162_neq(float* in, char* out, size_t size) { } } +// Device counterpart of the mismatched-lane checks in Unit_bf162_operators_host. +__global__ void bf162_lane_compare(char* out) { + const auto lt_l = __float22bfloat162_rn(float2{1.0f, 4.0f}); + const auto lt_r = __float22bfloat162_rn(float2{2.0f, 3.0f}); + const auto gt_l = __float22bfloat162_rn(float2{4.0f, 1.0f}); + const auto gt_r = __float22bfloat162_rn(float2{3.0f, 2.0f}); + const auto eq_x = __float22bfloat162_rn(float2{1.0f, 5.0f}); + const auto nan_x = __float22bfloat162_rn(float2{NAN, 1.0f}); + out[0] = lt_l < lt_r; + out[1] = lt_l <= lt_r; + out[2] = gt_l > gt_r; + out[3] = gt_l >= gt_r; + out[4] = __hbneu2(lt_l, eq_x); + out[5] = __high2float(__hgt2(gt_l, gt_r)) != 0.0f; + out[6] = __high2float(__hisnan2(nan_x)) != 0.0f; +} + HIP_TEST_CASE(Unit_bf162_basic) { auto f_in = getAllBF16(); auto max_bf16_num = f_in.size(); @@ -705,6 +722,23 @@ HIP_TEST_CASE(Unit_bf162_basic) { HIP_CHECK(hipFree(in)); HIP_CHECK(hipFree(out)); } + + SECTION("Mismatched lanes on device") { + // Each relation holds in exactly one lane, so every result must be false; the + // pre-fix header returned true for all of them. + constexpr size_t kNumResults = 7; + char* out; + HIP_CHECK(hipMalloc(&out, kNumResults)); + bf162_lane_compare<<<1, 1>>>(out); + HIP_CHECK(hipGetLastError()); + std::vector result(kNumResults, 1); + HIP_CHECK(hipMemcpy(result.data(), out, kNumResults, hipMemcpyDeviceToHost)); + for (size_t i = 0; i < kNumResults; i++) { + INFO("Result index: " << i); + REQUIRE(result[i] == 0); + } + HIP_CHECK(hipFree(out)); + } } @@ -820,6 +854,61 @@ HIP_TEST_CASE(Unit_bf162_operators_host) { REQUIRE((l + -l) == __hip_bfloat162{HIPRT_ZERO_BF16, HIPRT_ZERO_BF16}); REQUIRE((l / -l) == -__hip_bfloat162{HIPRT_ONE_BF16, HIPRT_ONE_BF16}); } + + SECTION("Compare mismatched lanes") { + // Regression: the __hip_bfloat162 relational operators must compare each lane + // against its counterpart. A prior bug compared l.x against both r.x and r.y + // (ignoring l.y), so vectors whose two lanes differ were mis-compared. + __hip_bfloat162 la = {__float2bfloat16(1.0f), __float2bfloat16(4.0f)}; + __hip_bfloat162 ra = {__float2bfloat16(2.0f), __float2bfloat16(3.0f)}; + // lane0: 1<2 (true), lane1: 4<3 (false) -> overall false + REQUIRE_FALSE(la < ra); + REQUIRE_FALSE(la <= ra); + + __hip_bfloat162 lc = {__float2bfloat16(4.0f), __float2bfloat16(1.0f)}; + __hip_bfloat162 rc = {__float2bfloat16(3.0f), __float2bfloat16(2.0f)}; + // lane0: 4>3 (true), lane1: 1>2 (false) -> overall false + REQUIRE_FALSE(lc > rc); + REQUIRE_FALSE(lc >= rc); + + // sanity: both lanes satisfy the relation -> true + __hip_bfloat162 lo = {__float2bfloat16(1.0f), __float2bfloat16(2.0f)}; + __hip_bfloat162 hi = {__float2bfloat16(3.0f), __float2bfloat16(4.0f)}; + REQUIRE(lo < hi); + REQUIRE(lo <= hi); + REQUIRE(hi > lo); + REQUIRE(hi >= lo); + } + + SECTION("__hbneu2 both-lane reduction") { + // Regression: __hbneu2 (unordered not-equal) must be true only when BOTH lanes + // are not-equal (&&), not when either lane differs (||). + __hip_bfloat162 a = {__float2bfloat16(1.0f), __float2bfloat16(2.0f)}; + __hip_bfloat162 eq_x = {__float2bfloat16(1.0f), __float2bfloat16(3.0f)}; // lane0 equal + __hip_bfloat162 eq_y = {__float2bfloat16(9.0f), __float2bfloat16(2.0f)}; // lane1 equal + __hip_bfloat162 both = {__float2bfloat16(5.0f), __float2bfloat16(6.0f)}; // both differ + __hip_bfloat162 same = a; // both equal + + REQUIRE(__hbneu2(a, both)); // both lanes not-equal -> true + REQUIRE_FALSE(__hbneu2(a, eq_x)); // one lane equal -> false (was true with ||) + REQUIRE_FALSE(__hbneu2(a, eq_y)); // one lane equal -> false (was true with ||) + REQUIRE_FALSE(__hbneu2(a, same)); // both equal -> false + } + + SECTION("__hgt2 / __hisnan2 high-lane mask") { + // Regression: __hgt2 and __hisnan2 returned 1.0 for the high (.y) lane + // unconditionally (both ternary branches were HIPRT_ONE_BF16). + __hip_bfloat162 a = {__float2bfloat16(5.0f), __float2bfloat16(1.0f)}; + __hip_bfloat162 b = {__float2bfloat16(2.0f), __float2bfloat16(3.0f)}; + float2 gt = __hgt2(a, b); // lane0: 5>2 -> 1.0, lane1: 1>3 -> 0.0 + REQUIRE(gt.x == 1.0f); + REQUIRE(gt.y == 0.0f); // was 1.0f with the bug + + __hip_bfloat162 n = {__float2bfloat16(NAN), __float2bfloat16(1.0f)}; + float2 isn = __hisnan2(n); // lane0: NaN -> 1.0, lane1: not NaN -> 0.0 + REQUIRE(isn.x == 1.0f); + REQUIRE(isn.y == 0.0f); // was 1.0f with the bug + } } // Bunch of tests which make sure we are packaging stuff correctly. diff --git a/projects/hip-tests/catch/unit/deviceLib/hipTestHalfIntegralAssign_test.cc b/projects/hip-tests/catch/unit/deviceLib/hipTestHalfIntegralAssign_test.cc new file mode 100644 index 000000000000..5cb7e4acc96a --- /dev/null +++ b/projects/hip-tests/catch/unit/deviceLib/hipTestHalfIntegralAssign_test.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +/** + * Regression test for __half::operator=(T) integral assignment. + * + * The integral assignment operator template was previously __device__-only, + * while the matching constructor was __HOST_DEVICE__. This caused compilation + * failures in any __host__ __device__ function using `half_var = int_val;`. + * + * These tests verify that integral assignment works in host, device, and + * __host__ __device__ contexts. + */ + +#include +#include + +/** + * __host__ __device__ helper that exercises integral operator= for multiple + * integer types. Without the fix, the host pass of hipcc rejects this + * function because operator=(T) was __device__-only. + */ +__host__ __device__ void halfIntegralAssignHelper(float* results) { + __half h; + int i = 3; + h = i; + results[0] = __half2float(h); + + short s = 7; + h = s; + results[1] = __half2float(h); + + unsigned u = 10; + h = u; + results[2] = __half2float(h); + + long long ll = 15; + h = ll; + results[3] = __half2float(h); +} + +__global__ void halfIntegralAssignKernel(float* results) { + halfIntegralAssignHelper(results); +} + +/** + * Test Description + * ------------------------ + * - Calls the __host__ __device__ helper from host code. + * - Compilation of this test is the primary regression check. + */ +HIP_TEST_CASE(Unit_hipTestHalfIntegralAssign_Host) { + float results[4] = {0}; + halfIntegralAssignHelper(results); + + REQUIRE(results[0] == 3.0f); + REQUIRE(results[1] == 7.0f); + REQUIRE(results[2] == 10.0f); + REQUIRE(results[3] == 15.0f); +} + +/** + * Test Description + * ------------------------ + * - Launches a kernel that calls the same __host__ __device__ helper. + * - Verifies integral assignment also works on device. + */ +HIP_TEST_CASE(Unit_hipTestHalfIntegralAssign_Device) { + constexpr size_t numResults = 4; + float* results_d = nullptr; + float results_h[numResults] = {0}; + + HIP_CHECK(hipMalloc(&results_d, numResults * sizeof(float))); + + halfIntegralAssignKernel<<<1, 1>>>(results_d); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipMemcpy(results_h, results_d, numResults * sizeof(float), + hipMemcpyDeviceToHost)); + + REQUIRE(results_h[0] == 3.0f); + REQUIRE(results_h[1] == 7.0f); + REQUIRE(results_h[2] == 10.0f); + REQUIRE(results_h[3] == 15.0f); + + HIP_CHECK(hipFree(results_d)); +} diff --git a/projects/hip-tests/catch/unit/event/hipEventCreateWithFlags.cc b/projects/hip-tests/catch/unit/event/hipEventCreateWithFlags.cc index 8788900bba2e..0b2116bb25f1 100644 --- a/projects/hip-tests/catch/unit/event/hipEventCreateWithFlags.cc +++ b/projects/hip-tests/catch/unit/event/hipEventCreateWithFlags.cc @@ -20,7 +20,7 @@ template __global__ void vector_square_system_scope_atomic(const T* if (i < N_ELMTS) { #if HT_AMD T result = A_d[i] * A_d[i]; - __hip_atomic_store(&C_d[i], result, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_store_n(&C_d[i], result, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); #else C_d[i] = A_d[i] * A_d[i]; #endif diff --git a/projects/hip-tests/catch/unit/executionContext/hipExecutionCtxStreamDetached.cc b/projects/hip-tests/catch/unit/executionContext/hipExecutionCtxStreamDetached.cc index 0ed569d9da4c..e26d54a4b6ee 100644 --- a/projects/hip-tests/catch/unit/executionContext/hipExecutionCtxStreamDetached.cc +++ b/projects/hip-tests/catch/unit/executionContext/hipExecutionCtxStreamDetached.cc @@ -587,7 +587,7 @@ HIP_TEST_CASE(Unit_hipExecutionCtxStreamDetached_ActiveCapture_Invalidated) { * stream. Both streams are owned by the same ctx. After * hipExecutionCtxDestroy, both captureStatus_ slots must flip to * hipStreamCaptureStatusInvalidated. Covers requirement (d) for - * parallelCaptureStreams_. + * captureStreams_. * Test requirements * ------------------------ * - HIP_VERSION >= 7.2 diff --git a/projects/hip-tests/catch/unit/graph/CMakeLists.txt b/projects/hip-tests/catch/unit/graph/CMakeLists.txt index 034da67962e8..1523aeacc67e 100644 --- a/projects/hip-tests/catch/unit/graph/CMakeLists.txt +++ b/projects/hip-tests/catch/unit/graph/CMakeLists.txt @@ -187,6 +187,8 @@ set(TEST_SRC if(HIP_PLATFORM MATCHES "amd") set(AMD_SRC + # hipLaunchAttributePriority not available on older CUDA versions + hipGraphKernelNodePriority.cc # hipGraphAddNode, hipGraphNodeParams, hipMemcpyNodeParams are not mapped to Nvidia hipGraphAddNode.cc # hipDrvGraphAddMemsetNode, HIP_MEMSET_NODE_PARAMS are not mapped to Nvidia diff --git a/projects/hip-tests/catch/unit/graph/hipGraphDestroy.cc b/projects/hip-tests/catch/unit/graph/hipGraphDestroy.cc index 05be92a017f4..51fdd8bd16b7 100644 --- a/projects/hip-tests/catch/unit/graph/hipGraphDestroy.cc +++ b/projects/hip-tests/catch/unit/graph/hipGraphDestroy.cc @@ -38,8 +38,8 @@ HIP_TEST_CASE(Unit_hipGraphDestroy_Positive_Basic) { /** * Test Description * ------------------------ - * - Basic negative parameter test for hipGraphDestroy - * -# Expected hipErrorInvalidValue when graph is invalid + * - Verifies that hipGraphDestroy returns hipErrorInvalidValue for null and previously + * destroyed graph handles. * Test source * ------------------------ * - unit/graph/hipGraphDestroy.cc @@ -48,7 +48,18 @@ HIP_TEST_CASE(Unit_hipGraphDestroy_Positive_Basic) { * - HIP_VERSION >= 5.2 */ HIP_TEST_CASE(Unit_hipGraphDestroy_Negative_Parameters) { - HIP_CHECK_ERROR(hipGraphDestroy(static_cast(nullptr)), hipErrorInvalidValue); + SECTION("Null graph") { + HIP_CHECK_ERROR(hipGraphDestroy(static_cast(nullptr)), hipErrorInvalidValue); + } + + SECTION("Previously destroyed graph") { + hipGraph_t graph = nullptr; + HIP_CHECK(hipGraphCreate(&graph, 0)); + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK_ERROR(hipGraphDestroy(graph), hipErrorInvalidValue); + } + + (void)hipGetLastError(); } /** diff --git a/projects/hip-tests/catch/unit/graph/hipGraphKernelNodePriority.cc b/projects/hip-tests/catch/unit/graph/hipGraphKernelNodePriority.cc new file mode 100644 index 000000000000..b87cdabdf590 --- /dev/null +++ b/projects/hip-tests/catch/unit/graph/hipGraphKernelNodePriority.cc @@ -0,0 +1,455 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + * + * Tests for hipGraph kernel node priority support: + * 1. hipGraphKernelNodeSetAttribute/GetAttribute with hipLaunchAttributePriority + * accept values in [greatest, least] and reject anything outside it. + * 2. Kernels captured on a prioritized stream carry that stream's priority. + * 3. hipGraphKernelNodeCopyAttributes copies every attribute slot. + * 4. Graphs instantiated with hipGraphInstantiateFlagUseNodePriority (flat + * fork/join and graphs containing child graphs) execute correctly. Queue + * slot ordering itself is a scheduling hint and is not observable via the + * public API, so these tests validate correctness, not placement. + */ + +#include +#include + +#include + +namespace { + +constexpr int kN = 1 << 20; + +__global__ void saxpy_kernel(const float* __restrict__ x, float* __restrict__ y, + float a, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) y[i] = a * x[i] + y[i]; +} + +bool AllEqual(const float* d_ptr, int n, float expected) { + std::vector h(n); + HIP_CHECK(hipMemcpy(h.data(), d_ptr, n * sizeof(float), hipMemcpyDeviceToHost)); + for (int i = 0; i < n; ++i) { + if (h[i] != expected) return false; + } + return true; +} + +void FillOnes(float* d_ptr, int n) { + std::vector h(n, 1.f); + HIP_CHECK(hipMemcpy(d_ptr, h.data(), n * sizeof(float), hipMemcpyHostToDevice)); +} + +} // namespace + +/* -------------------------------------------------------------------------- + * Positive: SetAttribute with hipLaunchAttributePriority stores the value + * and GetAttribute reads it back correctly. + * -------------------------------------------------------------------------- */ +HIP_TEST_CASE(Unit_hipGraphKernelNodeSetAttribute_Positive_Priority) { + int lo, hi; + HIP_CHECK(hipDeviceGetStreamPriorityRange(&lo, &hi)); + if (lo == hi) { + HIP_SKIP_TEST("Device does not support stream priority"); + } + + hipGraph_t graph; + HIP_CHECK(hipGraphCreate(&graph, 0)); + + float* d = nullptr; + HIP_CHECK(hipMalloc(&d, kN * sizeof(float))); + HIP_CHECK(hipMemset(d, 0, kN * sizeof(float))); + + hipKernelNodeParams p{}; + p.func = reinterpret_cast(saxpy_kernel); + p.gridDim = dim3((kN + 255) / 256); + p.blockDim = dim3(256); + const float* dx = d; + float* dy = d; + float a = 1.f; + int n = kN; + void* args[] = {&dx, &dy, &a, &n}; + p.kernelParams = args; + + hipGraphNode_t node; + HIP_CHECK(hipGraphAddKernelNode(&node, graph, nullptr, 0, &p)); + + // Set high priority + hipKernelNodeAttrValue val_set{}; + val_set.priority = hi; + HIP_CHECK(hipGraphKernelNodeSetAttribute(node, hipLaunchAttributePriority, &val_set)); + + // Read back and verify + hipKernelNodeAttrValue val_get{}; + HIP_CHECK(hipGraphKernelNodeGetAttribute(node, hipLaunchAttributePriority, &val_get)); + REQUIRE(val_get.priority == hi); + + // Set low priority and verify + val_set.priority = lo; + HIP_CHECK(hipGraphKernelNodeSetAttribute(node, hipLaunchAttributePriority, &val_set)); + HIP_CHECK(hipGraphKernelNodeGetAttribute(node, hipLaunchAttributePriority, &val_get)); + REQUIRE(val_get.priority == lo); + + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK(hipFree(d)); +} + +/* -------------------------------------------------------------------------- + * Negative: out-of-range priority values must return hipErrorInvalidValue. + * -------------------------------------------------------------------------- */ +HIP_TEST_CASE(Unit_hipGraphKernelNodeSetAttribute_Negative_Priority_OutOfRange) { + int lo, hi; + HIP_CHECK(hipDeviceGetStreamPriorityRange(&lo, &hi)); + + hipGraph_t graph; + HIP_CHECK(hipGraphCreate(&graph, 0)); + + float* d = nullptr; + HIP_CHECK(hipMalloc(&d, sizeof(float))); + hipKernelNodeParams p{}; + p.func = reinterpret_cast(saxpy_kernel); + p.gridDim = dim3(1); p.blockDim = dim3(1); + const float* dx = d; float* dy = d; float a = 1.f; int n = 1; + void* args[] = {&dx, &dy, &a, &n}; + p.kernelParams = args; + hipGraphNode_t node; + HIP_CHECK(hipGraphAddKernelNode(&node, graph, nullptr, 0, &p)); + + hipKernelNodeAttrValue val{}; + + // Values outside [hi, lo] must be rejected. + val.priority = hi - 1; // more urgent than hi — invalid + REQUIRE(hipGraphKernelNodeSetAttribute(node, hipLaunchAttributePriority, &val) + == hipErrorInvalidValue); + (void)hipGetLastError(); // clear sticky per-thread error + + val.priority = lo + 1; // less urgent than lo — invalid + REQUIRE(hipGraphKernelNodeSetAttribute(node, hipLaunchAttributePriority, &val) + == hipErrorInvalidValue); + (void)hipGetLastError(); // clear sticky per-thread error + + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK(hipFree(d)); +} + +/* -------------------------------------------------------------------------- + * Positive: stream priority is copied to kernel nodes during stream capture. + * + * Captures a kernel on a high-priority stream, then reads the priority back + * from the graph node via hipGraphKernelNodeGetAttribute. Verifies that the + * node carries the stream's priority, not the default Normal priority. + * -------------------------------------------------------------------------- */ +HIP_TEST_CASE(Unit_hipGraphKernelNodePriority_CaptureStreamPriorityCopied) { + int lo, hi; + HIP_CHECK(hipDeviceGetStreamPriorityRange(&lo, &hi)); + if (lo == hi) { + HIP_SKIP_TEST("Device does not support stream priority"); + } + + float* d = nullptr; + HIP_CHECK(hipMalloc(&d, kN * sizeof(float))); + HIP_CHECK(hipMemset(d, 0, kN * sizeof(float))); + + // Capture stream with high priority + hipStream_t s_hi; + HIP_CHECK(hipStreamCreateWithPriority(&s_hi, hipStreamDefault, hi)); + + hipGraph_t graph; + HIP_CHECK(hipStreamBeginCapture(s_hi, hipStreamCaptureModeGlobal)); + { + dim3 block(256), grid((kN + 255) / 256); + hipLaunchKernelGGL(saxpy_kernel, grid, block, 0, s_hi, d, d, 1.f, kN); + } + HIP_CHECK(hipStreamEndCapture(s_hi, &graph)); + + // Walk graph nodes, find the kernel node, check its priority + size_t num_nodes = 0; + HIP_CHECK(hipGraphGetNodes(graph, nullptr, &num_nodes)); + REQUIRE(num_nodes > 0); + + std::vector nodes(num_nodes); + HIP_CHECK(hipGraphGetNodes(graph, nodes.data(), &num_nodes)); + + bool found_kernel = false; + for (auto node : nodes) { + hipGraphNodeType type; + HIP_CHECK(hipGraphNodeGetType(node, &type)); + if (type != hipGraphNodeTypeKernel) continue; + + hipKernelNodeAttrValue val{}; + HIP_CHECK(hipGraphKernelNodeGetAttribute(node, hipLaunchAttributePriority, &val)); + + // The node must carry the stream's priority, not the default Normal (0). + INFO("Captured node priority: " << val.priority << ", stream priority: " << hi); + CHECK(val.priority == hi); + found_kernel = true; + } + + REQUIRE(found_kernel); + + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK(hipStreamDestroy(s_hi)); + HIP_CHECK(hipFree(d)); +} + +/* -------------------------------------------------------------------------- + * Positive: hipGraphKernelNodeCopyAttributes copies all attribute slots + * (accessPolicyWindow, cooperative, priority) independently. + * + * Before the union->per-slot refactor, CopyAttr copied only the last-written + * slot, so a node with two attributes set would lose all but one. + * -------------------------------------------------------------------------- */ +HIP_TEST_CASE(Unit_hipGraphKernelNodeCopyAttributes_MultiAttr) { + int lo, hi; + HIP_CHECK(hipDeviceGetStreamPriorityRange(&lo, &hi)); + if (lo == hi) { + HIP_SKIP_TEST("Device does not support stream priority"); + } + + hipGraph_t graph; + HIP_CHECK(hipGraphCreate(&graph, 0)); + + float* d = nullptr; + HIP_CHECK(hipMalloc(&d, sizeof(float))); + + hipKernelNodeParams p{}; + p.func = reinterpret_cast(saxpy_kernel); + p.gridDim = dim3(1); p.blockDim = dim3(1); + const float* dx = d; float* dy = d; float a = 1.f; int n = 1; + void* args[] = {&dx, &dy, &a, &n}; + p.kernelParams = args; + + hipGraphNode_t src, dst; + HIP_CHECK(hipGraphAddKernelNode(&src, graph, nullptr, 0, &p)); + HIP_CHECK(hipGraphAddKernelNode(&dst, graph, nullptr, 0, &p)); + + // Set cooperative on src. + hipKernelNodeAttrValue coop{}; + coop.cooperative = 1; + HIP_CHECK(hipGraphKernelNodeSetAttribute(src, hipKernelNodeAttributeCooperative, &coop)); + + // Set priority on src. + hipKernelNodeAttrValue prio{}; + prio.priority = hi; + HIP_CHECK(hipGraphKernelNodeSetAttribute(src, hipLaunchAttributePriority, &prio)); + + // Copy all attributes from src to dst. + HIP_CHECK(hipGraphKernelNodeCopyAttributes(src, dst)); + + // Both cooperative and priority must survive the copy. + hipKernelNodeAttrValue got{}; + HIP_CHECK(hipGraphKernelNodeGetAttribute(dst, hipKernelNodeAttributeCooperative, &got)); + REQUIRE(got.cooperative == 1); + + HIP_CHECK(hipGraphKernelNodeGetAttribute(dst, hipLaunchAttributePriority, &got)); + REQUIRE(got.priority == hi); + + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK(hipFree(d)); +} + +/* -------------------------------------------------------------------------- + * Positive: a fork/join graph instantiated with hipGraphInstantiateFlagUseNodePriority + * runs to completion and produces correct output. + * + * Two branches capture saxpy on a high-priority and a low-priority stream, each + * writing its own output buffer. Verifies the flag is accepted on the segmented + * path and that both branches produce the expected values. + * -------------------------------------------------------------------------- */ +HIP_TEST_CASE(Unit_hipGraphKernelNodePriority_ForkJoinWithFlag) { + int lo, hi; + HIP_CHECK(hipDeviceGetStreamPriorityRange(&lo, &hi)); + if (lo == hi) { + HIP_SKIP_TEST("Device does not support stream priority"); + } + + constexpr int kElems = 1 << 16; + float* d_x = nullptr; + float* d_y_hi = nullptr; + float* d_y_lo = nullptr; + HIP_CHECK(hipMalloc(&d_x, kElems * sizeof(float))); + HIP_CHECK(hipMalloc(&d_y_hi, kElems * sizeof(float))); + HIP_CHECK(hipMalloc(&d_y_lo, kElems * sizeof(float))); + FillOnes(d_x, kElems); + HIP_CHECK(hipMemset(d_y_hi, 0, kElems * sizeof(float))); + HIP_CHECK(hipMemset(d_y_lo, 0, kElems * sizeof(float))); + + hipStream_t s_main, s_hi, s_lo; + HIP_CHECK(hipStreamCreate(&s_main)); + HIP_CHECK(hipStreamCreateWithPriority(&s_hi, hipStreamDefault, hi)); + HIP_CHECK(hipStreamCreateWithPriority(&s_lo, hipStreamDefault, lo)); + + // Fork: s_main -> s_hi and s_main -> s_lo; both join back to s_main. + hipEvent_t fork_hi, fork_lo, join_hi, join_lo; + HIP_CHECK(hipEventCreate(&fork_hi)); + HIP_CHECK(hipEventCreate(&fork_lo)); + HIP_CHECK(hipEventCreate(&join_hi)); + HIP_CHECK(hipEventCreate(&join_lo)); + + hipGraph_t graph; + HIP_CHECK(hipStreamBeginCapture(s_main, hipStreamCaptureModeGlobal)); + + HIP_CHECK(hipEventRecord(fork_hi, s_main)); + HIP_CHECK(hipEventRecord(fork_lo, s_main)); + HIP_CHECK(hipStreamWaitEvent(s_hi, fork_hi, 0)); + HIP_CHECK(hipStreamWaitEvent(s_lo, fork_lo, 0)); + + // Record the low-priority branch first so capture order disagrees with priority. + dim3 block(256), grid((kElems + 255) / 256); + hipLaunchKernelGGL(saxpy_kernel, grid, block, 0, s_lo, d_x, d_y_lo, 3.f, kElems); + hipLaunchKernelGGL(saxpy_kernel, grid, block, 0, s_hi, d_x, d_y_hi, 2.f, kElems); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipEventRecord(join_hi, s_hi)); + HIP_CHECK(hipEventRecord(join_lo, s_lo)); + HIP_CHECK(hipStreamWaitEvent(s_main, join_hi, 0)); + HIP_CHECK(hipStreamWaitEvent(s_main, join_lo, 0)); + + HIP_CHECK(hipStreamEndCapture(s_main, &graph)); + + hipGraphExec_t exec; + hipError_t inst_status = hipGraphInstantiateWithFlags( + &exec, graph, hipGraphInstantiateFlagUseNodePriority); + + auto cleanup = [&]() { + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK(hipEventDestroy(join_lo)); + HIP_CHECK(hipEventDestroy(join_hi)); + HIP_CHECK(hipEventDestroy(fork_lo)); + HIP_CHECK(hipEventDestroy(fork_hi)); + HIP_CHECK(hipStreamDestroy(s_lo)); + HIP_CHECK(hipStreamDestroy(s_hi)); + HIP_CHECK(hipStreamDestroy(s_main)); + HIP_CHECK(hipFree(d_y_lo)); + HIP_CHECK(hipFree(d_y_hi)); + HIP_CHECK(hipFree(d_x)); + }; + + if (inst_status == hipErrorNotSupported) { + // Clear the sticky per-thread error before skipping. + (void)hipGetLastError(); + cleanup(); + HIP_SKIP_TEST("hipGraphInstantiateFlagUseNodePriority not supported on this backend"); + } + HIP_CHECK(inst_status); + + HIP_CHECK(hipGraphLaunch(exec, s_main)); + HIP_CHECK(hipStreamSynchronize(s_main)); + + // y = a * 1 + 0 + CHECK(AllEqual(d_y_hi, kElems, 2.f)); + CHECK(AllEqual(d_y_lo, kElems, 3.f)); + + HIP_CHECK(hipGraphExecDestroy(exec)); + cleanup(); +} + +/* -------------------------------------------------------------------------- + * Positive: a graph containing child graphs instantiated with + * hipGraphInstantiateFlagUseNodePriority executes correctly. + * + * Exercises the child-graph path of segment priority aggregation: one child + * graph holds a kernel set to high priority, the other a kernel left at the + * default. The parent runs both children in parallel and a final kernel that + * depends on both. + * -------------------------------------------------------------------------- */ +HIP_TEST_CASE(Unit_hipGraphKernelNodePriority_ChildGraphWithFlag) { + int lo, hi; + HIP_CHECK(hipDeviceGetStreamPriorityRange(&lo, &hi)); + if (lo == hi) { + HIP_SKIP_TEST("Device does not support stream priority"); + } + + constexpr int kElems = 1 << 16; + float* d_x = nullptr; + float* d_y_a = nullptr; + float* d_y_b = nullptr; + HIP_CHECK(hipMalloc(&d_x, kElems * sizeof(float))); + HIP_CHECK(hipMalloc(&d_y_a, kElems * sizeof(float))); + HIP_CHECK(hipMalloc(&d_y_b, kElems * sizeof(float))); + FillOnes(d_x, kElems); + HIP_CHECK(hipMemset(d_y_a, 0, kElems * sizeof(float))); + HIP_CHECK(hipMemset(d_y_b, 0, kElems * sizeof(float))); + + const float* dx = d_x; + float* dy_a = d_y_a; + float* dy_b = d_y_b; + float a_hi = 2.f, a_def = 3.f, a_tail = 4.f; + int n = kElems; + void* args_hi[] = {&dx, &dy_a, &a_hi, &n}; + void* args_def[] = {&dx, &dy_b, &a_def, &n}; + void* args_tail[] = {&dx, &dy_a, &a_tail, &n}; + + auto make_params = [&](void** args) { + hipKernelNodeParams p{}; + p.func = reinterpret_cast(saxpy_kernel); + p.gridDim = dim3((kElems + 255) / 256); + p.blockDim = dim3(256); + p.kernelParams = args; + return p; + }; + + // Child A: one high-priority kernel. + hipGraph_t child_a; + HIP_CHECK(hipGraphCreate(&child_a, 0)); + hipGraphNode_t k_hi; + hipKernelNodeParams p_hi = make_params(args_hi); + HIP_CHECK(hipGraphAddKernelNode(&k_hi, child_a, nullptr, 0, &p_hi)); + hipKernelNodeAttrValue prio{}; + prio.priority = hi; + HIP_CHECK(hipGraphKernelNodeSetAttribute(k_hi, hipLaunchAttributePriority, &prio)); + + // Child B: one kernel at default priority. + hipGraph_t child_b; + HIP_CHECK(hipGraphCreate(&child_b, 0)); + hipGraphNode_t k_def; + hipKernelNodeParams p_def = make_params(args_def); + HIP_CHECK(hipGraphAddKernelNode(&k_def, child_b, nullptr, 0, &p_def)); + + // Parent: child_b and child_a in parallel (low-priority branch added first), + // then a tail kernel depending on both. + hipGraph_t graph; + HIP_CHECK(hipGraphCreate(&graph, 0)); + hipGraphNode_t n_b, n_a, n_tail; + HIP_CHECK(hipGraphAddChildGraphNode(&n_b, graph, nullptr, 0, child_b)); + HIP_CHECK(hipGraphAddChildGraphNode(&n_a, graph, nullptr, 0, child_a)); + hipGraphNode_t deps[] = {n_a, n_b}; + hipKernelNodeParams p_tail = make_params(args_tail); + HIP_CHECK(hipGraphAddKernelNode(&n_tail, graph, deps, 2, &p_tail)); + + hipGraphExec_t exec; + hipError_t inst_status = hipGraphInstantiateWithFlags( + &exec, graph, hipGraphInstantiateFlagUseNodePriority); + + auto cleanup = [&]() { + HIP_CHECK(hipGraphDestroy(graph)); + HIP_CHECK(hipGraphDestroy(child_b)); + HIP_CHECK(hipGraphDestroy(child_a)); + HIP_CHECK(hipFree(d_y_b)); + HIP_CHECK(hipFree(d_y_a)); + HIP_CHECK(hipFree(d_x)); + }; + + if (inst_status == hipErrorNotSupported) { + (void)hipGetLastError(); + cleanup(); + HIP_SKIP_TEST("hipGraphInstantiateFlagUseNodePriority not supported on this backend"); + } + HIP_CHECK(inst_status); + + hipStream_t stream; + HIP_CHECK(hipStreamCreate(&stream)); + HIP_CHECK(hipGraphLaunch(exec, stream)); + HIP_CHECK(hipStreamSynchronize(stream)); + + // d_y_a = 2*1 + 0, then tail adds 4*1 -> 6. d_y_b = 3*1 + 0. + CHECK(AllEqual(d_y_a, kElems, 6.f)); + CHECK(AllEqual(d_y_b, kElems, 3.f)); + + HIP_CHECK(hipGraphExecDestroy(exec)); + HIP_CHECK(hipStreamDestroy(stream)); + cleanup(); +} diff --git a/projects/hip-tests/catch/unit/graph/hipStreamBeginCapture.cc b/projects/hip-tests/catch/unit/graph/hipStreamBeginCapture.cc index 40f3cd06d78d..503d91fca3c0 100644 --- a/projects/hip-tests/catch/unit/graph/hipStreamBeginCapture.cc +++ b/projects/hip-tests/catch/unit/graph/hipStreamBeginCapture.cc @@ -4,6 +4,10 @@ * SPDX-License-Identifier: MIT */ +#include +#include +#include + #include #include @@ -1541,8 +1545,12 @@ HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_SelfWaitOnForkedStream) { /** * Test Description * ------------------------ - * - Test to verify hipStreamSynchronize on a stream works when stream capture - * on another stream is ongoing. + * - Test to verify the self-wait shape where work is enqueued between the record and the + * wait. The two self-wait cases above record and wait back to back, which is the one + * shape where the wait cancels exactly. With a kernel in between, the event's recorded + * predecessor is re-added as a dependency of the next node, giving a fourth edge that + * the same chain would not otherwise have. That edge is implied transitively, so + * execution is unaffected. * Test source * ------------------------ * - catch\unit\graph\hipStreamBeginCapture.cc @@ -1550,92 +1558,72 @@ HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_SelfWaitOnForkedStream) { * ------------------------ * - HIP_VERSION >= 5.6 */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_SelfWaitWithInterveningWork) { + // Four kernels chained would give three edges; the self-wait adds one more. + constexpr size_t kExpectedNodes = 4; + constexpr size_t kExpectedEdges = 4; + constexpr int kExpectedIncrements = 4; -HIP_TEST_CASE(Unit_hipStreamBeginCapture_StreamSync_OngoingCapture) { - hipStreamCaptureMode flag = hipStreamCaptureModeRelaxed; - constexpr int GRIDSIZE = 1; - constexpr int BLOCKSIZE = 512; - constexpr int VALUE1 = 7, VALUE2 = 11; - hipGraph_t graph{nullptr}; - hipGraphExec_t graphExec{nullptr}; - // Allocate device memory - LinearAllocGuard Ah = LinearAllocGuard(LinearAllocs::malloc, BLOCKSIZE * sizeof(int)); - LinearAllocGuard Ad = - LinearAllocGuard(LinearAllocs::hipMalloc, BLOCKSIZE * sizeof(int)); - LinearAllocGuard Bh = LinearAllocGuard(LinearAllocs::malloc, BLOCKSIZE * sizeof(int)); - LinearAllocGuard Bd = - LinearAllocGuard(LinearAllocs::hipMalloc, BLOCKSIZE * sizeof(int)); - // Fill input data - std::fill_n(Ah.host_ptr(), BLOCKSIZE, VALUE1); - std::fill_n(Bh.host_ptr(), BLOCKSIZE, VALUE2); - // Stream create - StreamsGuard stream0(1); - // Capture streams into graph - SECTION("Stream Creation Before Capture") { - StreamsGuard stream1(1); - HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); - HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipStreamSynchronize(stream1[0])); - myadd<<>>(Ad.ptr(), Bd.ptr()); - HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture - } - SECTION("Synchronizing multiple streams during Capture") { - StreamsGuard stream1(1), stream2(1); - HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); - HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream2[0])); - HIP_CHECK(hipStreamSynchronize(stream1[0])); - HIP_CHECK(hipStreamSynchronize(stream2[0])); - myadd<<>>(Ad.ptr(), Bd.ptr()); - HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture - } - SECTION("Stream Creation After Capture") { - HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); - StreamsGuard stream1(1); - HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipStreamSynchronize(stream1[0])); - myadd<<>>(Ad.ptr(), Bd.ptr()); - HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture - } - SECTION("Stream Synchronize Before Capture") { - StreamsGuard stream1(1); - HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipStreamSynchronize(stream1[0])); - HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); - myadd<<>>(Ad.ptr(), Bd.ptr()); - HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture - } - SECTION("Stream Synchronize After Capture") { - HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); - myadd<<>>(Ad.ptr(), Bd.ptr()); - HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture - StreamsGuard stream1(1); - HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream1[0])); - HIP_CHECK(hipStreamSynchronize(stream1[0])); - } - // Execute and test the graph + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(2); + EventsGuard events(3); + + int* devMem = devMem_g.ptr(); + hipStream_t captureStream = streams[0]; + hipStream_t forkedStream = streams[1]; + hipEvent_t forkEvent = events[0]; + hipEvent_t selfEvent = events[1]; + hipEvent_t joinEvent = events[2]; + + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipStreamBeginCapture(captureStream, hipStreamCaptureModeThreadLocal)); + + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipEventRecord(forkEvent, captureStream)); + HIP_CHECK(hipStreamWaitEvent(forkedStream, forkEvent, 0)); + + incrementKernel<<<1, 1, 0, forkedStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + // Record, then enqueue work, and only then wait. The intervening kernel is what adds the extra + // edge. + HIP_CHECK(hipEventRecord(selfEvent, forkedStream)); + + incrementKernel<<<1, 1, 0, forkedStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipStreamWaitEvent(forkedStream, selfEvent, 0)); + + incrementKernel<<<1, 1, 0, forkedStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipEventRecord(joinEvent, forkedStream)); + HIP_CHECK(hipStreamWaitEvent(captureStream, joinEvent, 0)); + + hipGraph_t graph = nullptr; + HIP_CHECK(hipStreamEndCapture(captureStream, &graph)); + REQUIRE(graph != nullptr); + + size_t numNodes = 0; + size_t numEdges = 0; + HIP_CHECK(hipGraphGetNodes(graph, nullptr, &numNodes)); + HIP_CHECK(hipGraphGetEdges(graph, nullptr, nullptr, &numEdges)); + REQUIRE(numNodes == kExpectedNodes); + REQUIRE(numEdges == kExpectedEdges); + + hipGraphExec_t graphExec = nullptr; HIP_CHECK(hipGraphInstantiate(&graphExec, graph, nullptr, nullptr, 0)); - HIP_CHECK(hipGraphLaunch(graphExec, stream0[0])); - HIP_CHECK(hipStreamSynchronize(stream0[0])); - // Check output - HIP_CHECK(hipMemcpy(Ah.host_ptr(), Ad.ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDeviceToHost)); - for (int idx = 0; idx < BLOCKSIZE; idx++) { - REQUIRE(Ah.host_ptr()[idx] == (VALUE1 + VALUE2)); - } + HIP_CHECK(hipGraphLaunch(graphExec, captureStream)); + HIP_CHECK(hipStreamSynchronize(captureStream)); + + int increments = 0; + HIP_CHECK(hipMemcpy(&increments, devMem, sizeof(int), hipMemcpyDeviceToHost)); + REQUIRE(increments == kExpectedIncrements); + HIP_CHECK(hipGraphExecDestroy(graphExec)); HIP_CHECK(hipGraphDestroy(graph)); } @@ -1643,8 +1631,12 @@ HIP_TEST_CASE(Unit_hipStreamBeginCapture_StreamSync_OngoingCapture) { /** * Test Description * ------------------------ - * - Test to verify hipStreamSynchronize on a stream behavior when stream capture - * on another stream is ongoing in another thread. + * - Test to verify that a forked stream can be destroyed while the capture that pulled it + * in is still open. Captured nodes do not reference the stream they were captured on, so + * whether the capture can still be closed depends only on whether the forked stream's + * work was joined back: joined, the capture closes and yields a valid graph; unjoined, + * the graph has a dangling leaf and hipStreamEndCapture reports + * hipErrorStreamCaptureUnjoined. * Test source * ------------------------ * - catch\unit\graph\hipStreamBeginCapture.cc @@ -1652,93 +1644,936 @@ HIP_TEST_CASE(Unit_hipStreamBeginCapture_StreamSync_OngoingCapture) { * ------------------------ * - HIP_VERSION >= 5.6 */ -// Local function executed as thread -static void strmSyncThread(int* Ah, int* Ad, int* Bh, int* Bd, int BLOCKSIZE, hipError_t* error) { - StreamsGuard stream(1); - HIP_CHECK_THREAD(hipMemcpyAsync(Ad, Ah, BLOCKSIZE * sizeof(int), hipMemcpyDefault, stream[0])); - HIP_CHECK_THREAD(hipMemcpyAsync(Bd, Bh, BLOCKSIZE * sizeof(int), hipMemcpyDefault, stream[0])); - *error = hipStreamSynchronize(stream[0]); -} +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_DestroyForkedStreamDuringCapture) { + const bool joinBeforeDestroy = GENERATE(true, false); -// Local function executed as thread -static void captureStrmThread(hipGraph_t* graph, int* Ah, int* Ad, int* Bh, int* Bd, int BLOCKSIZE, - int GRIDSIZE, hipStreamCaptureMode flag, hipError_t* error) { - StreamsGuard stream(1); - // Capture streams into graph - HIP_CHECK(hipStreamBeginCapture(stream[0], flag)); - std::thread t1(strmSyncThread, Ah, Ad, Bh, Bd, BLOCKSIZE, error); - t1.join(); - HIP_CHECK_THREAD_FINALIZE(); - myadd<<>>(Ad, Bd); - if (flag == hipStreamCaptureModeGlobal) { - HIP_CHECK_ERROR(hipStreamEndCapture(stream[0], graph), - hipErrorStreamCaptureInvalidated); // End Capture + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(1); + EventsGuard events(2); + + int* devMem = devMem_g.ptr(); + hipStream_t captureStream = streams[0]; + hipEvent_t forkEvent = events[0]; + hipEvent_t joinEvent = events[1]; + + hipStream_t forkedStream = nullptr; + HIP_CHECK(hipStreamCreate(&forkedStream)); + + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipStreamBeginCapture(captureStream, hipStreamCaptureModeThreadLocal)); + + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipEventRecord(forkEvent, captureStream)); + HIP_CHECK(hipStreamWaitEvent(forkedStream, forkEvent, 0)); + + incrementKernel<<<1, 1, 0, forkedStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + if (joinBeforeDestroy) { + HIP_CHECK(hipEventRecord(joinEvent, forkedStream)); + HIP_CHECK(hipStreamWaitEvent(captureStream, joinEvent, 0)); + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + } + + HIP_CHECK(hipStreamDestroy(forkedStream)); + + // Destroying a participant does not end the capture on the origin. + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamIsCapturing(captureStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusActive); + + hipGraph_t graph = nullptr; + if (joinBeforeDestroy) { + HIP_CHECK(hipStreamEndCapture(captureStream, &graph)); + REQUIRE(graph != nullptr); + HIP_CHECK(hipGraphDestroy(graph)); } else { - HIP_CHECK(hipStreamEndCapture(stream[0], graph)); // End Capture + // An unjoined capture produces no graph. + HIP_CHECK_ERROR(hipStreamEndCapture(captureStream, &graph), hipErrorStreamCaptureUnjoined); + REQUIRE(graph == nullptr); + (void)hipGetLastError(); } } -HIP_TEST_CASE(Unit_hipStreamBeginCapture_StreamSync_OngoingCapture_MThread) { - constexpr int GRIDSIZE = 1; - constexpr int BLOCKSIZE = 512; - constexpr int VALUE1 = 7, VALUE2 = 11; - hipGraph_t graph{nullptr}; - // Allocate device memory - LinearAllocGuard Ah = LinearAllocGuard(LinearAllocs::malloc, BLOCKSIZE * sizeof(int)); - LinearAllocGuard Ad = - LinearAllocGuard(LinearAllocs::hipMalloc, BLOCKSIZE * sizeof(int)); - LinearAllocGuard Bh = LinearAllocGuard(LinearAllocs::malloc, BLOCKSIZE * sizeof(int)); - LinearAllocGuard Bd = - LinearAllocGuard(LinearAllocs::hipMalloc, BLOCKSIZE * sizeof(int)); - // Fill input data - std::fill_n(Ah.host_ptr(), BLOCKSIZE, VALUE1); - std::fill_n(Bh.host_ptr(), BLOCKSIZE, VALUE2); - // Stream create - hipError_t error = hipSuccess; - SECTION("Capture Flag = hipStreamCaptureModeGlobal Single Threaded") { - StreamsGuard stream(2); - // Capture streams into graph - HIP_CHECK(hipStreamBeginCapture(stream[0], hipStreamCaptureModeGlobal)); - HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream[1])); - HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream[1])); - error = hipStreamSynchronize(stream[1]); - REQUIRE(error == hipErrorStreamCaptureUnsupported); - hipGraph_t graph; - HIP_CHECK_ERROR(hipStreamEndCapture(stream[0], &graph), hipErrorStreamCaptureInvalidated); - } - SECTION("Capture Flag = hipStreamCaptureModeThreadLocal Single Threaded") { - StreamsGuard stream(2); - // Capture streams into graph - HIP_CHECK(hipStreamBeginCapture(stream[0], hipStreamCaptureModeThreadLocal)); - HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream[1])); - HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, - stream[1])); - error = hipStreamSynchronize(stream[1]); - REQUIRE(error == hipErrorStreamCaptureUnsupported); - hipGraph_t graph; - HIP_CHECK_ERROR(hipStreamEndCapture(stream[0], &graph), hipErrorStreamCaptureInvalidated); - } - SECTION("Capture Flag = hipStreamCaptureModeGlobal Multithreaded") { - captureStrmThread(&graph, Ah.host_ptr(), Ad.ptr(), Bh.host_ptr(), Bd.ptr(), BLOCKSIZE, GRIDSIZE, - hipStreamCaptureModeGlobal, &error); - REQUIRE(error == hipErrorStreamCaptureUnsupported); - } - SECTION("Capture Flag = hipStreamCaptureModeThreadLocal Multithreaded") { - captureStrmThread(&graph, Ah.host_ptr(), Ad.ptr(), Bh.host_ptr(), Bd.ptr(), BLOCKSIZE, GRIDSIZE, - hipStreamCaptureModeThreadLocal, &error); - REQUIRE(error == hipSuccess); - } - SECTION("Capture Flag = hipStreamCaptureModeRelaxed Multithreaded") { - captureStrmThread(&graph, Ah.host_ptr(), Ad.ptr(), Bh.host_ptr(), Bd.ptr(), BLOCKSIZE, GRIDSIZE, - hipStreamCaptureModeRelaxed, &error); - REQUIRE(error == hipSuccess); - } - if (graph != nullptr) { - hipGraphExec_t graphExec{nullptr}; - StreamsGuard stream(1); +// hipStreamGetCaptureInfo_v2 wraps cuStreamGetCaptureInfo_v2, which CUDA 13 removed. +#if HT_AMD || (defined(CUDA_VERSION) && CUDA_VERSION < 13000) +/** + * Test Description + * ------------------------ + * - Test to verify that destroying a capture's origin stream destroys the graph created + * internally by hipStreamBeginCapture. hipStreamGetCaptureInfo_v2 exposes a non-owning + * handle to that graph while capture is active; the handle must be invalid after the + * stream tears the capture down. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_DestroyOriginStreamDuringCapture) { + hipStream_t captureStream = nullptr; + HIP_CHECK(hipStreamCreate(&captureStream)); + HIP_CHECK(hipStreamBeginCapture(captureStream, hipStreamCaptureModeThreadLocal)); + + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + hipGraph_t captureGraph = nullptr; + HIP_CHECK(hipStreamGetCaptureInfo_v2(captureStream, &captureStatus, nullptr, &captureGraph, + nullptr, nullptr)); + REQUIRE(captureStatus == hipStreamCaptureStatusActive); + REQUIRE(captureGraph != nullptr); + + HIP_CHECK(hipStreamDestroy(captureStream)); + HIP_CHECK_ERROR(hipGraphDestroy(captureGraph), hipErrorInvalidValue); + (void)hipGetLastError(); +} +#endif + +/** + * Test Description + * ------------------------ + * - Test to verify that a stream whose capture has been invalidated refuses further + * capture work. Querying an event recorded inside a capture is illegal and invalidates + * the capture sequence it belongs to; a wait issued afterwards must report + * hipErrorStreamCaptureInvalidated rather than being accepted and silently ignored. + * - The illegal query names the forked stream, so the test also covers the scope of the + * invalidation: the origin is invalidated along with it and its teardown reports that + * rather than unjoined work. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Negative_SelfWaitOnInvalidatedForkedStream) { + + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(2); + EventsGuard events(2); + + int* devMem = devMem_g.ptr(); + hipStream_t captureStream = streams[0]; + hipStream_t forkedStream = streams[1]; + hipEvent_t forkEvent = events[0]; + hipEvent_t selfEvent = events[1]; + + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipStreamBeginCapture(captureStream, hipStreamCaptureModeThreadLocal)); + + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipEventRecord(forkEvent, captureStream)); + HIP_CHECK(hipStreamWaitEvent(forkedStream, forkEvent, 0)); + + incrementKernel<<<1, 1, 0, forkedStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + // Invalidate the capture with an illegal query on an event recorded on the forked stream. + HIP_CHECK(hipEventRecord(selfEvent, forkedStream)); + REQUIRE(hipEventQuery(selfEvent) != hipSuccess); + (void)hipGetLastError(); + + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamIsCapturing(forkedStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + + // An invalidated stream takes no further part in the capture, so the wait is refused + // rather than accepted and silently ignored. + HIP_CHECK_ERROR(hipStreamWaitEvent(forkedStream, selfEvent, 0), hipErrorStreamCaptureInvalidated); + + // The refusal leaves the status alone; it does not resurrect the stream or escalate it. + HIP_CHECK(hipStreamIsCapturing(forkedStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + + // The invalidation reached the origin, even though the illegal query named the participant. + HIP_CHECK(hipStreamIsCapturing(captureStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + + // Tear the capture down. The origin reports the invalidation rather than the forked + // stream's unjoined work, and produces no graph. + hipGraph_t graph = nullptr; + HIP_CHECK_ERROR(hipStreamEndCapture(captureStream, &graph), hipErrorStreamCaptureInvalidated); + REQUIRE(graph == nullptr); + (void)hipGetLastError(); +} + +/** + * Test Description + * ------------------------ + * - Test to verify that a cycle among forked streams does not break the capture. A chain + * of forked streams is built inside one capture and then the first of them waits on an + * event recorded by the last, which closes a cycle in the runtime's membership + * bookkeeping while leaving the node graph a legal chain. + * - Regression test: before the flat membership rewrite this crashed with a stack + * overflow, because the teardown walked the bookkeeping recursively with no cycle + * detection. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_CycleAmongForkedStreams) { + const int cycleLength = GENERATE(3, 4); + + // One kernel on the origin, one per forked stream, one more on the first forked stream + // after the cycle closes, and a trailing one on the origin. + const size_t expectedNodes = static_cast(cycleLength) + 3; + // The chain contributes one edge per forked stream, closing the cycle adds two into the + // node that follows it, and the join back to the origin adds two more. + const size_t expectedEdges = static_cast(cycleLength) + 4; + + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(cycleLength + 1); + EventsGuard events(cycleLength + 2); + + int* devMem = devMem_g.ptr(); + hipStream_t captureStream = streams[0]; + + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipStreamBeginCapture(captureStream, hipStreamCaptureModeThreadLocal)); + + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + // Fork the first stream off the origin. + HIP_CHECK(hipEventRecord(events[0], captureStream)); + HIP_CHECK(hipStreamWaitEvent(streams[1], events[0], 0)); + incrementKernel<<<1, 1, 0, streams[1]>>>(devMem); + HIP_CHECK(hipGetLastError()); + + // Extend the chain: each forked stream is pulled in by the one before it. + for (int i = 2; i <= cycleLength; ++i) { + HIP_CHECK(hipEventRecord(events[i - 1], streams[i - 1])); + HIP_CHECK(hipStreamWaitEvent(streams[i], events[i - 1], 0)); + incrementKernel<<<1, 1, 0, streams[i]>>>(devMem); + HIP_CHECK(hipGetLastError()); + } + + // Close the cycle: the first forked stream waits on the last one's event. Every existing + // guard passes here, because the two streams differ, neither is the origin, and the last + // stream's immediate predecessor is not the first one. + HIP_CHECK(hipEventRecord(events[cycleLength], streams[cycleLength])); + HIP_CHECK(hipStreamWaitEvent(streams[1], events[cycleLength], 0)); + incrementKernel<<<1, 1, 0, streams[1]>>>(devMem); + HIP_CHECK(hipGetLastError()); + + // Join back and add a trailing node so the node graph has a single leaf and the capture is + // closeable. Without this the unjoined-work check would reject it before teardown runs. + HIP_CHECK(hipEventRecord(events[cycleLength + 1], streams[1])); + HIP_CHECK(hipStreamWaitEvent(captureStream, events[cycleLength + 1], 0)); + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + hipGraph_t graph = nullptr; + HIP_CHECK(hipStreamEndCapture(captureStream, &graph)); + REQUIRE(graph != nullptr); + + size_t numNodes = 0; + HIP_CHECK(hipGraphGetNodes(graph, nullptr, &numNodes)); + REQUIRE(numNodes == expectedNodes); + + size_t numEdges = 0; + HIP_CHECK(hipGraphGetEdges(graph, nullptr, nullptr, &numEdges)); + REQUIRE(numEdges == expectedEdges); + + hipGraphExec_t graphExec = nullptr; + HIP_CHECK(hipGraphInstantiate(&graphExec, graph, nullptr, nullptr, 0)); + HIP_CHECK(hipGraphLaunch(graphExec, captureStream)); + HIP_CHECK(hipStreamSynchronize(captureStream)); + + int increments = 0; + HIP_CHECK(hipMemcpy(&increments, devMem, sizeof(int), hipMemcpyDeviceToHost)); + REQUIRE(increments == static_cast(expectedNodes)); + + HIP_CHECK(hipGraphExecDestroy(graphExec)); + HIP_CHECK(hipGraphDestroy(graph)); +} + +/** + * Test Description + * ------------------------ + * - Test to verify that waiting on an event recorded in a different capture is rejected + * rather than silently merging the two. A third stream is pulled into the first capture + * and then waits on an event from the second, which asks the runtime to splice two + * independent graphs together. That must return hipErrorStreamCaptureMerge and + * invalidate both sequences, so ending either one reports the invalidation. + * - Regression test: previously both waits succeeded, every stream stayed active, + * and one capture ended up silently missing the forked stream's work while the other failed as unjoined. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Negative_CrossCaptureWaitIsRejected) { + + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(3); + EventsGuard events(2); + + int* devMem = devMem_g.ptr(); + hipStream_t firstOrigin = streams[0]; + hipStream_t secondOrigin = streams[1]; + hipStream_t forkedStream = streams[2]; + hipEvent_t firstEvent = events[0]; + hipEvent_t secondEvent = events[1]; + + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + // Two independent captures, thread-local so they do not restrict each other globally. + HIP_CHECK(hipStreamBeginCapture(firstOrigin, hipStreamCaptureModeThreadLocal)); + HIP_CHECK(hipStreamBeginCapture(secondOrigin, hipStreamCaptureModeThreadLocal)); + + incrementKernel<<<1, 1, 0, firstOrigin>>>(devMem); + HIP_CHECK(hipGetLastError()); + incrementKernel<<<1, 1, 0, secondOrigin>>>(devMem); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipEventRecord(firstEvent, firstOrigin)); + HIP_CHECK(hipEventRecord(secondEvent, secondOrigin)); + + // Legitimate: the forked stream joins the first capture. + HIP_CHECK(hipStreamWaitEvent(forkedStream, firstEvent, 0)); + incrementKernel<<<1, 1, 0, forkedStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + // Illegal: it is already in the first capture, so waiting on the second capture's event + // would merge them. + HIP_CHECK_ERROR(hipStreamWaitEvent(forkedStream, secondEvent, 0), hipErrorStreamCaptureMerge); + + // Both sequences are invalidated, the forked stream included. + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamIsCapturing(firstOrigin, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + HIP_CHECK(hipStreamIsCapturing(secondOrigin, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + HIP_CHECK(hipStreamIsCapturing(forkedStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + + // An invalidated capture produces no graph. + hipGraph_t graph = nullptr; + HIP_CHECK_ERROR(hipStreamEndCapture(firstOrigin, &graph), hipErrorStreamCaptureInvalidated); + REQUIRE(graph == nullptr); + HIP_CHECK_ERROR(hipStreamEndCapture(secondOrigin, &graph), hipErrorStreamCaptureInvalidated); + REQUIRE(graph == nullptr); + (void)hipGetLastError(); +} + +/** + * Test Description + * ------------------------ + * - Test to verify that a stream whose capture has been invalidated cannot start a new + * one. An invalidated sequence has to be ended on the stream that began it before any + * of its streams can be reused, so hipStreamBeginCapture must reject both the origin + * and a stream that was pulled into the capture. + * - Regression test: previously both were accepted. On a participant that was the worse + * case: it became the origin of a second capture while still enrolled in the first, + * whose teardown then reported unjoined work. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Negative_BeginCaptureOnInvalidatedStream) { + const bool targetParticipant = GENERATE(false, true); + + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(2); + EventsGuard events(2); + + int* devMem = devMem_g.ptr(); + hipStream_t captureStream = streams[0]; + hipStream_t forkedStream = streams[1]; + hipEvent_t forkEvent = events[0]; + hipEvent_t probeEvent = events[1]; + + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipStreamBeginCapture(captureStream, hipStreamCaptureModeThreadLocal)); + + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + hipStream_t target = captureStream; + if (targetParticipant) { + HIP_CHECK(hipEventRecord(forkEvent, captureStream)); + HIP_CHECK(hipStreamWaitEvent(forkedStream, forkEvent, 0)); + incrementKernel<<<1, 1, 0, forkedStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + target = forkedStream; + } + + // Invalidate the target's capture with an illegal query on a captured event, and confirm + // it took effect, or the assertion below would prove nothing. + HIP_CHECK(hipEventRecord(probeEvent, target)); + REQUIRE(hipEventQuery(probeEvent) != hipSuccess); + (void)hipGetLastError(); + + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamIsCapturing(target, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + + HIP_CHECK_ERROR(hipStreamBeginCapture(target, hipStreamCaptureModeThreadLocal), + hipErrorIllegalState); + + // The invalidation covers the whole sequence whichever stream the query named, so + // the origin's teardown reports it and yields no graph in both cases. + hipGraph_t graph = nullptr; + HIP_CHECK_ERROR(hipStreamEndCapture(captureStream, &graph), hipErrorStreamCaptureInvalidated); + REQUIRE(graph == nullptr); + + // The two expected failures above leave a sticky thread-local error that only a query clears. + // The generator runs this body again, and its first hipGetLastError() would report that error + // as a failure of the kernel launch preceding it. + (void)hipGetLastError(); +} + +/** + * Test Description + * ------------------------ + * - Test to verify that destroying a stream mid-capture does not eject the streams that + * were forked from it. With nested forks, destroying the middle stream must leave the + * one it pulled in still enrolled, so that stream can be joined back to the origin and + * the capture closes normally. + * - Regression test: previously the grandchild was dropped out of the capture and the + * origin's teardown then failed with hipErrorStreamCaptureUnjoined. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_DestroyNestedForkDuringCapture) { + constexpr size_t kExpectedNodes = 4; + constexpr int kExpectedIncrements = 4; + + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(2); + EventsGuard events(3); + + int* devMem = devMem_g.ptr(); + hipStream_t captureStream = streams[0]; + hipStream_t grandchildStream = streams[1]; + hipEvent_t forkEvent = events[0]; + hipEvent_t nestedForkEvent = events[1]; + hipEvent_t joinEvent = events[2]; + + // Destroyed mid-test, so it is not held by a guard. + hipStream_t middleStream = nullptr; + HIP_CHECK(hipStreamCreate(&middleStream)); + + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipStreamBeginCapture(captureStream, hipStreamCaptureModeThreadLocal)); + + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + // origin -> middle + HIP_CHECK(hipEventRecord(forkEvent, captureStream)); + HIP_CHECK(hipStreamWaitEvent(middleStream, forkEvent, 0)); + incrementKernel<<<1, 1, 0, middleStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + // middle -> grandchild, so the grandchild was pulled in by the middle stream rather than + // by the origin. + HIP_CHECK(hipEventRecord(nestedForkEvent, middleStream)); + HIP_CHECK(hipStreamWaitEvent(grandchildStream, nestedForkEvent, 0)); + incrementKernel<<<1, 1, 0, grandchildStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + HIP_CHECK(hipStreamDestroy(middleStream)); + + // The grandchild must still be part of the capture. + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamIsCapturing(grandchildStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusActive); + + // So it can still be joined straight back to the origin. + HIP_CHECK(hipEventRecord(joinEvent, grandchildStream)); + HIP_CHECK(hipStreamWaitEvent(captureStream, joinEvent, 0)); + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + + hipGraph_t graph = nullptr; + HIP_CHECK(hipStreamEndCapture(captureStream, &graph)); + REQUIRE(graph != nullptr); + + size_t numNodes = 0; + HIP_CHECK(hipGraphGetNodes(graph, nullptr, &numNodes)); + REQUIRE(numNodes == kExpectedNodes); + + hipGraphExec_t graphExec = nullptr; + HIP_CHECK(hipGraphInstantiate(&graphExec, graph, nullptr, nullptr, 0)); + HIP_CHECK(hipGraphLaunch(graphExec, captureStream)); + HIP_CHECK(hipStreamSynchronize(captureStream)); + + int increments = 0; + HIP_CHECK(hipMemcpy(&increments, devMem, sizeof(int), hipMemcpyDeviceToHost)); + REQUIRE(increments == kExpectedIncrements); + + HIP_CHECK(hipGraphExecDestroy(graphExec)); + HIP_CHECK(hipGraphDestroy(graph)); +} + +namespace { +template +int CountConcurrentFailures(int threadCount, Operation operation) { + std::atomic ready{0}; + std::atomic failures{0}; + std::vector threads; + threads.reserve(threadCount); + + for (int t = 0; t < threadCount; ++t) { + threads.emplace_back([&, t]() { + ready.fetch_add(1); + while (ready.load() < threadCount) { + } + if (operation(t) != hipSuccess) { + failures.fetch_add(1); + } + }); + } + + for (auto& thread : threads) { + thread.join(); + } + return failures.load(); +} +} // namespace + +/** + * Test Description + * ------------------------ + * - Test to verify that several threads can fork their own stream into one capture at the + * same time. Enrolling a stream mutates a container on the origin, so concurrent + * hipStreamWaitEvent calls against one capture all write to it together. The threads are + * released from a spin barrier to land those calls as close together as possible, and + * the sequence repeats because the window is narrow. + * - Ending the capture resets every enrolled stream, so a forked stream still reporting + * a capture status afterwards was dropped. Before the origin's participant set was + * synchronised this lost roughly one enrollment in seventy and went on to fault outright. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_ConcurrentForkIntoOneCapture) { + constexpr int kThreads = 8; + // Measured against a build with the origin's participant set left unsynchronised: 5 + // iterations catch the loss in 8 runs out of 20 and 25 in 77 out of 80, while 50 and 100 + // both catch it in 80 out of 80. The count sits a stride past the point where detection + // first saturates rather than on it, because the window tracks how many of the threads + // genuinely run at once and the figures above come from a 256-core host. + constexpr int kIterations = 100; + + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + int* devMem = devMem_g.ptr(); + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + int lostEnrollments = 0; + int waitFailures = 0; + + for (int iter = 0; iter < kIterations; ++iter) { + StreamsGuard streams(kThreads + 1); + EventsGuard events(kThreads + 1); + + hipStream_t origin = streams[0]; + hipEvent_t forkEvent = events[0]; + + HIP_CHECK(hipStreamBeginCapture(origin, hipStreamCaptureModeThreadLocal)); + incrementKernel<<<1, 1, 0, origin>>>(devMem); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipEventRecord(forkEvent, origin)); + + waitFailures += CountConcurrentFailures( + kThreads, [&](int t) { return hipStreamWaitEvent(streams[t + 1], forkEvent, 0); }); + + // Join every fork back so the capture is closeable. + for (int t = 0; t < kThreads; ++t) { + incrementKernel<<<1, 1, 0, streams[t + 1]>>>(devMem); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipEventRecord(events[t + 1], streams[t + 1])); + HIP_CHECK(hipStreamWaitEvent(origin, events[t + 1], 0)); + } + + hipGraph_t graph = nullptr; + HIP_CHECK(hipStreamEndCapture(origin, &graph)); + REQUIRE(graph != nullptr); + HIP_CHECK(hipGraphDestroy(graph)); + + // Ending the capture resets every enrolled stream, so anything still carrying a capture + // status was never enrolled. + for (int t = 0; t < kThreads; ++t) { + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamIsCapturing(streams[t + 1], &captureStatus)); + if (captureStatus != hipStreamCaptureStatusNone) { + ++lostEnrollments; + } + } + } + + REQUIRE(waitFailures == 0); + REQUIRE(lostEnrollments == 0); +} + +/** + * Test Description + * ------------------------ + * - Test to verify that concurrent waits on one captured stream preserve every dependency. + * Each source stream records an event after a distinct kernel, then the threads make the + * origin wait on those events simultaneously. The origin's dependency set must retain its + * own predecessor and every source predecessor. + * - Every thread targets the same stream, so the calls concurrently update one + * last-captured-node set rather than only the origin's membership set. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Positive_ConcurrentWaitsAccumulateDependencies) { + constexpr int kThreads = 8; + // The overlap is scheduler-dependent. With 8 threads, 100 iterations aborted with vector + // corruption in the unlocked implementation on the first run. + constexpr int kIterations = 100; + constexpr size_t kExpectedNodes = kThreads + 2; + constexpr size_t kExpectedEdges = 2 * kThreads + 1; + + int waitFailures = 0; + for (int iter = 0; iter < kIterations; ++iter) { + StreamsGuard streams(kThreads + 1); + EventsGuard events(kThreads + 1); + + hipStream_t origin = streams[0]; + hipEvent_t forkEvent = events[0]; + + HIP_CHECK(hipStreamBeginCapture(origin, hipStreamCaptureModeRelaxed)); + dummyKernel<<<1, 1, 0, origin>>>(); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipEventRecord(forkEvent, origin)); + + for (int t = 0; t < kThreads; ++t) { + hipStream_t source = streams[t + 1]; + HIP_CHECK(hipStreamWaitEvent(source, forkEvent, 0)); + dummyKernel<<<1, 1, 0, source>>>(); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipEventRecord(events[t + 1], source)); + } + + waitFailures += CountConcurrentFailures( + kThreads, [&](int t) { return hipStreamWaitEvent(origin, events[t + 1], 0); }); + + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamGetCaptureInfo(origin, &captureStatus, nullptr)); + REQUIRE(captureStatus == hipStreamCaptureStatusActive); + + // Consume the complete dependency set so every source is joined back to the origin. + dummyKernel<<<1, 1, 0, origin>>>(); + HIP_CHECK(hipGetLastError()); + + hipGraph_t graph = nullptr; + HIP_CHECK(hipStreamEndCapture(origin, &graph)); + REQUIRE(graph != nullptr); + + size_t numNodes = 0; + size_t numEdges = 0; + HIP_CHECK(hipGraphGetNodes(graph, nullptr, &numNodes)); + HIP_CHECK(hipGraphGetEdges(graph, nullptr, nullptr, &numEdges)); + REQUIRE(numNodes == kExpectedNodes); + REQUIRE(numEdges == kExpectedEdges); + + HIP_CHECK(hipGraphDestroy(graph)); + } + + REQUIRE(waitFailures == 0); +} + +/** + * Test Description + * ------------------------ + * - Verifies that waiting on an event from an invalidated capture is rejected without + * enrolling the waiting stream. Previously, the wait enrolled the idle stream, marked it + * active, and reported success even though the capture could no longer produce a graph. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Negative_WaitDoesNotJoinInvalidatedCapture) { + + LinearAllocGuard devMem_g(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(3); + EventsGuard events(1); + + int* devMem = devMem_g.ptr(); + hipStream_t captureStream = streams[0]; + hipStream_t forkedStream = streams[1]; + hipStream_t outsider = streams[2]; + hipEvent_t originEvent = events[0]; + + HIP_CHECK(hipMemset(devMem, 0, sizeof(int))); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipStreamBeginCapture(captureStream, hipStreamCaptureModeThreadLocal)); + incrementKernel<<<1, 1, 0, captureStream>>>(devMem); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipEventRecord(originEvent, captureStream)); + + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamWaitEvent(forkedStream, originEvent, 0)); + HIP_CHECK(hipStreamIsCapturing(forkedStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusActive); + + // Querying an event recorded inside a capture is illegal and invalidates it. + REQUIRE(hipEventQuery(originEvent) != hipSuccess); + (void)hipGetLastError(); + + HIP_CHECK(hipStreamIsCapturing(captureStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + + HIP_CHECK_ERROR(hipStreamWaitEvent(outsider, originEvent, 0), hipErrorInvalidValue); + + // The refusal leaves the outsider outside the capture, and does not disturb the capture. + HIP_CHECK(hipStreamIsCapturing(outsider, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusNone); + HIP_CHECK(hipStreamIsCapturing(captureStream, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + + // An invalidated capture produces no graph. + hipGraph_t graph = nullptr; + HIP_CHECK_ERROR(hipStreamEndCapture(captureStream, &graph), hipErrorStreamCaptureInvalidated); + REQUIRE(graph == nullptr); + (void)hipGetLastError(); +} + +/** + * Test Description + * ------------------------ + * - Test to verify hipStreamSynchronize on a stream works when stream capture + * on another stream is ongoing. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ + +HIP_TEST_CASE(Unit_hipStreamBeginCapture_StreamSync_OngoingCapture) { + hipStreamCaptureMode flag = hipStreamCaptureModeRelaxed; + constexpr int GRIDSIZE = 1; + constexpr int BLOCKSIZE = 512; + constexpr int VALUE1 = 7, VALUE2 = 11; + hipGraph_t graph{nullptr}; + hipGraphExec_t graphExec{nullptr}; + // Allocate device memory + LinearAllocGuard Ah = LinearAllocGuard(LinearAllocs::malloc, BLOCKSIZE * sizeof(int)); + LinearAllocGuard Ad = + LinearAllocGuard(LinearAllocs::hipMalloc, BLOCKSIZE * sizeof(int)); + LinearAllocGuard Bh = LinearAllocGuard(LinearAllocs::malloc, BLOCKSIZE * sizeof(int)); + LinearAllocGuard Bd = + LinearAllocGuard(LinearAllocs::hipMalloc, BLOCKSIZE * sizeof(int)); + // Fill input data + std::fill_n(Ah.host_ptr(), BLOCKSIZE, VALUE1); + std::fill_n(Bh.host_ptr(), BLOCKSIZE, VALUE2); + // Stream create + StreamsGuard stream0(1); + // Capture streams into graph + SECTION("Stream Creation Before Capture") { + StreamsGuard stream1(1); + HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); + HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipStreamSynchronize(stream1[0])); + myadd<<>>(Ad.ptr(), Bd.ptr()); + HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture + } + SECTION("Synchronizing multiple streams during Capture") { + StreamsGuard stream1(1), stream2(1); + HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); + HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream2[0])); + HIP_CHECK(hipStreamSynchronize(stream1[0])); + HIP_CHECK(hipStreamSynchronize(stream2[0])); + myadd<<>>(Ad.ptr(), Bd.ptr()); + HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture + } + SECTION("Stream Creation After Capture") { + HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); + StreamsGuard stream1(1); + HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipStreamSynchronize(stream1[0])); + myadd<<>>(Ad.ptr(), Bd.ptr()); + HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture + } + SECTION("Stream Synchronize Before Capture") { + StreamsGuard stream1(1); + HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipStreamSynchronize(stream1[0])); + HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); + myadd<<>>(Ad.ptr(), Bd.ptr()); + HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture + } + SECTION("Stream Synchronize After Capture") { + HIP_CHECK(hipStreamBeginCapture(stream0[0], flag)); + myadd<<>>(Ad.ptr(), Bd.ptr()); + HIP_CHECK(hipStreamEndCapture(stream0[0], &graph)); // End Capture + StreamsGuard stream1(1); + HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream1[0])); + HIP_CHECK(hipStreamSynchronize(stream1[0])); + } + // Execute and test the graph + HIP_CHECK(hipGraphInstantiate(&graphExec, graph, nullptr, nullptr, 0)); + HIP_CHECK(hipGraphLaunch(graphExec, stream0[0])); + HIP_CHECK(hipStreamSynchronize(stream0[0])); + // Check output + HIP_CHECK(hipMemcpy(Ah.host_ptr(), Ad.ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDeviceToHost)); + for (int idx = 0; idx < BLOCKSIZE; idx++) { + REQUIRE(Ah.host_ptr()[idx] == (VALUE1 + VALUE2)); + } + HIP_CHECK(hipGraphExecDestroy(graphExec)); + HIP_CHECK(hipGraphDestroy(graph)); +} + +/** + * Test Description + * ------------------------ + * - Test to verify hipStreamSynchronize on a stream behavior when stream capture + * on another stream is ongoing in another thread. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +// Local function executed as thread +static void strmSyncThread(int* Ah, int* Ad, int* Bh, int* Bd, int BLOCKSIZE, hipError_t* error) { + StreamsGuard stream(1); + HIP_CHECK_THREAD(hipMemcpyAsync(Ad, Ah, BLOCKSIZE * sizeof(int), hipMemcpyDefault, stream[0])); + HIP_CHECK_THREAD(hipMemcpyAsync(Bd, Bh, BLOCKSIZE * sizeof(int), hipMemcpyDefault, stream[0])); + *error = hipStreamSynchronize(stream[0]); +} + +// Local function executed as thread +static void captureStrmThread(hipGraph_t* graph, int* Ah, int* Ad, int* Bh, int* Bd, int BLOCKSIZE, + int GRIDSIZE, hipStreamCaptureMode flag, hipError_t* error) { + StreamsGuard stream(1); + // Capture streams into graph + HIP_CHECK(hipStreamBeginCapture(stream[0], flag)); + std::thread t1(strmSyncThread, Ah, Ad, Bh, Bd, BLOCKSIZE, error); + t1.join(); + HIP_CHECK_THREAD_FINALIZE(); + myadd<<>>(Ad, Bd); + if (flag == hipStreamCaptureModeGlobal) { + HIP_CHECK_ERROR(hipStreamEndCapture(stream[0], graph), + hipErrorStreamCaptureInvalidated); // End Capture + } else { + HIP_CHECK(hipStreamEndCapture(stream[0], graph)); // End Capture + } +} + +HIP_TEST_CASE(Unit_hipStreamBeginCapture_StreamSync_OngoingCapture_MThread) { + constexpr int GRIDSIZE = 1; + constexpr int BLOCKSIZE = 512; + constexpr int VALUE1 = 7, VALUE2 = 11; + hipGraph_t graph{nullptr}; + // Allocate device memory + LinearAllocGuard Ah = LinearAllocGuard(LinearAllocs::malloc, BLOCKSIZE * sizeof(int)); + LinearAllocGuard Ad = + LinearAllocGuard(LinearAllocs::hipMalloc, BLOCKSIZE * sizeof(int)); + LinearAllocGuard Bh = LinearAllocGuard(LinearAllocs::malloc, BLOCKSIZE * sizeof(int)); + LinearAllocGuard Bd = + LinearAllocGuard(LinearAllocs::hipMalloc, BLOCKSIZE * sizeof(int)); + // Fill input data + std::fill_n(Ah.host_ptr(), BLOCKSIZE, VALUE1); + std::fill_n(Bh.host_ptr(), BLOCKSIZE, VALUE2); + // Stream create + hipError_t error = hipSuccess; + SECTION("Capture Flag = hipStreamCaptureModeGlobal Single Threaded") { + StreamsGuard stream(2); + // Capture streams into graph + HIP_CHECK(hipStreamBeginCapture(stream[0], hipStreamCaptureModeGlobal)); + HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream[1])); + HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream[1])); + error = hipStreamSynchronize(stream[1]); + REQUIRE(error == hipErrorStreamCaptureUnsupported); + hipGraph_t graph; + HIP_CHECK_ERROR(hipStreamEndCapture(stream[0], &graph), hipErrorStreamCaptureInvalidated); + } + SECTION("Capture Flag = hipStreamCaptureModeThreadLocal Single Threaded") { + StreamsGuard stream(2); + // Capture streams into graph + HIP_CHECK(hipStreamBeginCapture(stream[0], hipStreamCaptureModeThreadLocal)); + HIP_CHECK(hipMemcpyAsync(Ad.ptr(), Ah.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream[1])); + HIP_CHECK(hipMemcpyAsync(Bd.ptr(), Bh.host_ptr(), BLOCKSIZE * sizeof(int), hipMemcpyDefault, + stream[1])); + error = hipStreamSynchronize(stream[1]); + REQUIRE(error == hipErrorStreamCaptureUnsupported); + hipGraph_t graph; + HIP_CHECK_ERROR(hipStreamEndCapture(stream[0], &graph), hipErrorStreamCaptureInvalidated); + } + SECTION("Capture Flag = hipStreamCaptureModeGlobal Multithreaded") { + captureStrmThread(&graph, Ah.host_ptr(), Ad.ptr(), Bh.host_ptr(), Bd.ptr(), BLOCKSIZE, GRIDSIZE, + hipStreamCaptureModeGlobal, &error); + REQUIRE(error == hipErrorStreamCaptureUnsupported); + } + SECTION("Capture Flag = hipStreamCaptureModeThreadLocal Multithreaded") { + captureStrmThread(&graph, Ah.host_ptr(), Ad.ptr(), Bh.host_ptr(), Bd.ptr(), BLOCKSIZE, GRIDSIZE, + hipStreamCaptureModeThreadLocal, &error); + REQUIRE(error == hipSuccess); + } + SECTION("Capture Flag = hipStreamCaptureModeRelaxed Multithreaded") { + captureStrmThread(&graph, Ah.host_ptr(), Ad.ptr(), Bh.host_ptr(), Bd.ptr(), BLOCKSIZE, GRIDSIZE, + hipStreamCaptureModeRelaxed, &error); + REQUIRE(error == hipSuccess); + } + if (graph != nullptr) { + hipGraphExec_t graphExec{nullptr}; + StreamsGuard stream(1); // Execute and test the graph HIP_CHECK(hipGraphInstantiate(&graphExec, graph, nullptr, nullptr, 0)); HIP_CHECK(hipGraphLaunch(graphExec, stream[0])); @@ -1838,6 +2673,198 @@ HIP_TEST_CASE(Unit_hipStreamBeginCapture_MultipleStreams_ReuseEvent) { HIP_CHECK(hipStreamDestroy(str2)); } +namespace { +void emptyStreamCallback(hipStream_t, hipError_t, void*) {} + +// An operation that is not permitted while a stream is capturing. The first four name the stream +// they act on; the last two name none, so the runtime has to reach the capture without being told +// which one is meant. +enum class CaptureOffence { + AddCallback, + StreamQuery, + StreamSynchronize, + EventQuery, + Malloc, + DeviceSynchronize +}; + +// `stream` is ignored by the offences that name no stream. `event` must already be recorded +// inside the capture on `stream` for EventQuery, which is the one offence that reaches the +// capture through an event rather than through the stream itself. +hipError_t commitOffence(CaptureOffence offence, hipStream_t stream, hipEvent_t event) { + switch (offence) { + case CaptureOffence::AddCallback: + return hipStreamAddCallback(stream, emptyStreamCallback, nullptr, 0); + case CaptureOffence::StreamQuery: + return hipStreamQuery(stream); + case CaptureOffence::StreamSynchronize: + return hipStreamSynchronize(stream); + case CaptureOffence::EventQuery: + return hipEventQuery(event); + case CaptureOffence::Malloc: { + void* unused = nullptr; + const hipError_t status = hipMalloc(&unused, sizeof(int)); + if (status == hipSuccess) { + static_cast(hipFree(unused)); + } + return status; + } + case CaptureOffence::DeviceSynchronize: + return hipDeviceSynchronize(); + } + return hipSuccess; +} + +// Asserts that every stream taking part in the capture observes the invalidation and that the +// origin then yields no graph. +void requireWholeCaptureInvalidated(hipStream_t origin, const std::vector& forks) { + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + HIP_CHECK(hipStreamIsCapturing(origin, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + for (hipStream_t fork : forks) { + HIP_CHECK(hipStreamIsCapturing(fork, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusInvalidated); + } + + hipGraph_t graph = nullptr; + HIP_CHECK_ERROR(hipStreamEndCapture(origin, &graph), hipErrorStreamCaptureInvalidated); + REQUIRE(graph == nullptr); + + static_cast(hipGetLastError()); +} +} // namespace + +/** + * Test Description + * ------------------------ + * - Verifies that an unsafe call naming one stream of a capture invalidates the whole + * capture, not only the stream named. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Negative_StreamOffenceInvalidatesWholeCapture) { + const hipStreamCaptureMode captureMode = GENERATE( + hipStreamCaptureModeGlobal, hipStreamCaptureModeThreadLocal, hipStreamCaptureModeRelaxed); + const CaptureOffence offence = + GENERATE(CaptureOffence::AddCallback, CaptureOffence::StreamQuery, + CaptureOffence::StreamSynchronize, CaptureOffence::EventQuery); + const bool offendOnFork = GENERATE(false, true); + + StreamsGuard streams(2); + EventsGuard events(2); + hipStream_t origin = streams[0]; + hipStream_t forkedStream = streams[1]; + hipEvent_t forkEvent = events[0]; + + HIP_CHECK(hipStreamBeginCapture(origin, captureMode)); + dummyKernel<<<1, 1, 0, origin>>>(); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipEventRecord(forkEvent, origin)); + HIP_CHECK(hipStreamWaitEvent(forkedStream, forkEvent, 0)); + dummyKernel<<<1, 1, 0, forkedStream>>>(); + HIP_CHECK(hipGetLastError()); + + hipStream_t target = offendOnFork ? forkedStream : origin; + hipEvent_t offenceEvent = events[1]; + if (offence == CaptureOffence::EventQuery) { + HIP_CHECK(hipEventRecord(offenceEvent, target)); + } + + // Guards against the offence silently becoming legal, which would let the test pass without + // ever invalidating anything. + REQUIRE(commitOffence(offence, target, offenceEvent) != hipSuccess); + static_cast(hipGetLastError()); + + requireWholeCaptureInvalidated(origin, {forkedStream}); +} + +/** + * Test Description + * ------------------------ + * - Verifies that an unsafe call naming no stream invalidates every stream of a capture. + * Such a call identifies neither the origin nor any fork, so the runtime has to reach every + * stream of every live capture on its own. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Negative_GlobalOffenceInvalidatesWholeCapture) { + const hipStreamCaptureMode captureMode = + GENERATE(hipStreamCaptureModeGlobal, hipStreamCaptureModeThreadLocal); + const CaptureOffence offence = + GENERATE(CaptureOffence::Malloc, CaptureOffence::DeviceSynchronize); + + StreamsGuard streams(2); + EventsGuard events(1); + hipStream_t origin = streams[0]; + hipStream_t forkedStream = streams[1]; + hipEvent_t forkEvent = events[0]; + + HIP_CHECK(hipStreamBeginCapture(origin, captureMode)); + dummyKernel<<<1, 1, 0, origin>>>(); + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipEventRecord(forkEvent, origin)); + HIP_CHECK(hipStreamWaitEvent(forkedStream, forkEvent, 0)); + dummyKernel<<<1, 1, 0, forkedStream>>>(); + HIP_CHECK(hipGetLastError()); + + REQUIRE(commitOffence(offence, nullptr, nullptr) != hipSuccess); + static_cast(hipGetLastError()); + + requireWholeCaptureInvalidated(origin, {forkedStream}); +} + +/** + * Test Description + * ------------------------ + * - Verifies that invalidating a capture through one forked stream reaches its siblings. + * Test source + * ------------------------ + * - catch\unit\graph\hipStreamBeginCapture.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 5.6 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCapture_Negative_InvalidationReachesSiblingForks) { + constexpr int kForks = 3; + + StreamsGuard streams(kForks + 1); + EventsGuard events(kForks); + hipStream_t origin = streams[0]; + + HIP_CHECK(hipStreamBeginCapture(origin, hipStreamCaptureModeThreadLocal)); + dummyKernel<<<1, 1, 0, origin>>>(); + HIP_CHECK(hipGetLastError()); + + std::vector forks; + for (int i = 0; i < kForks; ++i) { + HIP_CHECK(hipEventRecord(events[i], origin)); + HIP_CHECK(hipStreamWaitEvent(streams[i + 1], events[i], 0)); + dummyKernel<<<1, 1, 0, streams[i + 1]>>>(); + HIP_CHECK(hipGetLastError()); + forks.push_back(streams[i + 1]); + } + + hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone; + for (hipStream_t fork : forks) { + HIP_CHECK(hipStreamIsCapturing(fork, &captureStatus)); + REQUIRE(captureStatus == hipStreamCaptureStatusActive); + } + + HIP_CHECK_ERROR(hipStreamAddCallback(forks[0], emptyStreamCallback, nullptr, 0), + hipErrorStreamCaptureUnsupported); + static_cast(hipGetLastError()); + + requireWholeCaptureInvalidated(origin, forks); +} + /** * End doxygen group GraphTest. * @} diff --git a/projects/hip-tests/catch/unit/graph/hipStreamBeginCaptureToGraph.cc b/projects/hip-tests/catch/unit/graph/hipStreamBeginCaptureToGraph.cc index 8b8d2470ff42..f819fe367eed 100644 --- a/projects/hip-tests/catch/unit/graph/hipStreamBeginCaptureToGraph.cc +++ b/projects/hip-tests/catch/unit/graph/hipStreamBeginCaptureToGraph.cc @@ -16,6 +16,7 @@ */ #include #include +#include #include #include #include @@ -1028,6 +1029,68 @@ HIP_TEST_CASE(Unit_hipStreamBeginCaptureToGraph_EndingWhileCaptureInProgress) { HIP_CHECK(hipStreamDestroy(stream1)); } +/** + * Test Description + * ------------------------ + * - Verifies that capture teardown destroys a graph supplied to hipStreamBeginCaptureToGraph + * when capture is invalidated, ends with unjoined work, or its origin stream is destroyed. + * hipStreamEndCapture returns a null graph on the failure paths, and the original handle is + * no longer valid. + * ------------------------ + * - catch\unit\graph\hipStreamBeginCaptureToGraph.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 6.2 + */ +HIP_TEST_CASE(Unit_hipStreamBeginCaptureToGraph_Positive_CaptureTeardownDestroysGraph) { + hipGraph_t graph{nullptr}; + HIP_CHECK(hipGraphCreate(&graph, 0)); + + SECTION("Invalidated capture") { + StreamsGuard streams(1); + EventsGuard events(1); + hipGraph_t captureResult = graph; + + HIP_CHECK(hipStreamBeginCaptureToGraph(streams[0], graph, nullptr, nullptr, 0, + hipStreamCaptureModeThreadLocal)); + HIP_CHECK(hipEventRecord(events[0], streams[0])); + REQUIRE(hipEventQuery(events[0]) != hipSuccess); + (void)hipGetLastError(); + + HIP_CHECK_ERROR(hipStreamEndCapture(streams[0], &captureResult), + hipErrorStreamCaptureInvalidated); + REQUIRE(captureResult == nullptr); + } + + SECTION("Unjoined capture") { + LinearAllocGuard devMem(LinearAllocs::hipMalloc, sizeof(int)); + StreamsGuard streams(2); + EventsGuard events(1); + hipGraph_t captureResult = graph; + + HIP_CHECK(hipStreamBeginCaptureToGraph(streams[0], graph, nullptr, nullptr, 0, + hipStreamCaptureModeThreadLocal)); + HIP_CHECK(hipEventRecord(events[0], streams[0])); + HIP_CHECK(hipStreamWaitEvent(streams[1], events[0], 0)); + HIP_CHECK(hipMemsetAsync(devMem.ptr(), 0, sizeof(int), streams[1])); + + HIP_CHECK_ERROR(hipStreamEndCapture(streams[0], &captureResult), + hipErrorStreamCaptureUnjoined); + REQUIRE(captureResult == nullptr); + } + + SECTION("Destroyed origin stream") { + hipStream_t stream = nullptr; + HIP_CHECK(hipStreamCreate(&stream)); + HIP_CHECK(hipStreamBeginCaptureToGraph(stream, graph, nullptr, nullptr, 0, + hipStreamCaptureModeThreadLocal)); + HIP_CHECK(hipStreamDestroy(stream)); + } + + HIP_CHECK_ERROR(hipGraphDestroy(graph), hipErrorInvalidValue); + (void)hipGetLastError(); +} + /** * Test Description * ------------------------ diff --git a/projects/hip-tests/catch/unit/memory/hipMemDiscardBatchAsync.cc b/projects/hip-tests/catch/unit/memory/hipMemDiscardBatchAsync.cc index bb37723c02e3..6f68aacd0ad4 100644 --- a/projects/hip-tests/catch/unit/memory/hipMemDiscardBatchAsync.cc +++ b/projects/hip-tests/catch/unit/memory/hipMemDiscardBatchAsync.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include /** @@ -411,6 +412,68 @@ HIP_TEST_CASE(Unit_hipMemDiscardBatchAsync_StreamOrdering) { HIP_CHECK(hipStreamDestroy(stream)); } +/** + * Test Description + * ------------------------ + * - Blocks the stream with a kernel that spins until the host sets a release flag. + * - Enqueues a discard behind the kernel and verifies that hipStreamQuery reports + * hipErrorNotReady until the host releases the kernel. + * Test source + * ------------------------ + * - catch/unit/memory/hipMemDiscardBatchAsync.cc + * Test requirements + * ------------------------ + * - HIP_VERSION >= 7.2 + * - Device supports managed memory, concurrent managed access and pageable memory access + */ +HIP_TEST_CASE(Unit_hipMemDiscardBatchAsync_Sync_Behavior) { + if (!HmmSupported()) { + HIP_SKIP_TEST("HMM/managed memory not supported"); + return; + } + // The discard is rejected at submission without XNACK, which completes the command at once + if (!DeviceAttributesSupport(0, hipDeviceAttributePageableMemoryAccess)) { + HIP_SKIP_TEST("PageableMemoryAccess not supported"); + return; + } + + constexpr size_t kSize = 4096; + hipStream_t stream; + HIP_CHECK(hipStreamCreate(&stream)); + + int* release = nullptr; + HIP_CHECK(hipHostMalloc(&release, sizeof(int))); + *release = 0; + void* managed_ptr = nullptr; + HIP_CHECK(hipMallocManaged(&managed_ptr, kSize)); + + void* ptrs[1] = {managed_ptr}; + size_t sizes[1] = {kSize}; + HIP_CHECK(hipMemDiscardBatchAsync(ptrs, sizes, 1, 0, stream)); + HIP_CHECK(hipStreamSynchronize(stream)); + + WaitForHostRelease<<<1, 1, 0, stream>>>(release); + HIP_CHECK(hipGetLastError()); + + const auto discard_error = hipMemDiscardBatchAsync(ptrs, sizes, 1, 0, stream); + + auto query_while_blocked = hipErrorNotReady; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds{100}; + while (query_while_blocked == hipErrorNotReady && std::chrono::steady_clock::now() < deadline) { + query_while_blocked = hipStreamQuery(stream); + } + + __atomic_store_n(release, 1, __ATOMIC_RELEASE); + HIP_CHECK(hipStreamSynchronize(stream)); + + HIP_CHECK(discard_error); + REQUIRE(query_while_blocked == hipErrorNotReady); + + HIP_CHECK(hipFree(managed_ptr)); + HIP_CHECK(hipHostFree(release)); + HIP_CHECK(hipStreamDestroy(stream)); +} + /** * Test Description * ------------------------ diff --git a/projects/hip-tests/catch/unit/memory/hipMemPrefetchAsync.cc b/projects/hip-tests/catch/unit/memory/hipMemPrefetchAsync.cc index 4a0ec11bd90c..d9342f72411c 100644 --- a/projects/hip-tests/catch/unit/memory/hipMemPrefetchAsync.cc +++ b/projects/hip-tests/catch/unit/memory/hipMemPrefetchAsync.cc @@ -4,6 +4,7 @@ * SPDX-License-Identifier: MIT */ +#include #include #include @@ -61,20 +62,40 @@ HIP_TEST_CASE(Unit_hipMemPrefetchAsync_Basic_AllDevices) { ArrayFindIfNot(alloc1.ptr(), fill_value, count); } +// The kernel spins until the host sets the release flag, so hipStreamQuery must report +// hipErrorNotReady until that happens. HIP_TEST_CASE(Unit_hipMemPrefetchAsync_Sync_Behavior) { const auto supported_devices = GetDevicesWithPrefetchSupport(); if (supported_devices.empty()) { HIP_SKIP_TEST(HipTest::SkipReason::kManagedMemoryUnsupported); } const auto device = supported_devices.front(); + HIP_CHECK(hipSetDevice(device)); const auto stream_type = GENERATE(Streams::nullstream, Streams::perThread, Streams::created); - StreamGuard sg(stream_type); - LinearAllocGuard alloc(LinearAllocs::hipMallocManaged, kPageSize); - LaunchDelayKernel(std::chrono::milliseconds{100}, sg.stream()); + LinearAllocGuard release(LinearAllocs::hipHostMalloc, sizeof(int)); + *release.ptr() = 0; + LinearAllocGuard alloc(LinearAllocs::hipMallocManaged, kPageSize); HIP_CHECK(hipMemPrefetchAsync(alloc.ptr(), kPageSize, device, sg.stream())); - HIP_CHECK_ERROR(hipStreamQuery(sg.stream()), hipErrorNotReady); HIP_CHECK(hipStreamSynchronize(sg.stream())); + + WaitForHostRelease<<<1, 1, 0, sg.stream()>>>(release.ptr()); + HIP_CHECK(hipGetLastError()); + + const auto prefetch_error = + hipMemPrefetchAsync(alloc.ptr(), kPageSize, hipCpuDeviceId, sg.stream()); + + auto query_while_blocked = hipErrorNotReady; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds{100}; + while (query_while_blocked == hipErrorNotReady && std::chrono::steady_clock::now() < deadline) { + query_while_blocked = hipStreamQuery(sg.stream()); + } + + __atomic_store_n(release.ptr(), 1, __ATOMIC_RELEASE); + HIP_CHECK(hipStreamSynchronize(sg.stream())); + + HIP_CHECK(prefetch_error); + REQUIRE(query_while_blocked == hipErrorNotReady); } HIP_TEST_CASE(Unit_hipMemPrefetchAsync_Rounding_Behavior) { diff --git a/projects/hip-tests/catch/unit/memory/hipMemPrefetchBatchAsync.cc b/projects/hip-tests/catch/unit/memory/hipMemPrefetchBatchAsync.cc index dbd27d623580..b211addc4094 100644 --- a/projects/hip-tests/catch/unit/memory/hipMemPrefetchBatchAsync.cc +++ b/projects/hip-tests/catch/unit/memory/hipMemPrefetchBatchAsync.cc @@ -24,6 +24,7 @@ THE SOFTWARE. #include #include +#include namespace { // Test value constant @@ -282,6 +283,60 @@ HIP_TEST_CASE(Unit_hipMemPrefetchBatchAsync_RoundTripDataIntegrity) { REQUIRE(last_prefetch_location == device); } +/** + * Test Description + * ------------------------ + * - Blocks the stream with a kernel that spins until the host sets a release flag + * - Enqueues a batch prefetch behind the kernel and verifies that hipStreamQuery reports + * hipErrorNotReady until the host releases the kernel + */ +HIP_TEST_CASE(Unit_hipMemPrefetchBatchAsync_Sync_Behavior) { + REQUIRE_MANAGED_ACCESS_DEVICE(device); + + const auto stream_type = GENERATE(Streams::perThread, Streams::created); + StreamGuard stream_guard(stream_type); + LinearAllocGuard release(LinearAllocs::hipHostMalloc, sizeof(int)); + *release.ptr() = 0; + LinearAllocGuard managed_memory(LinearAllocs::hipMallocManaged, kTestBufferBytes); + + std::array managed_ptrs = {managed_memory.ptr()}; + std::array buffer_sizes = {kTestBufferBytes}; + std::array location_indices = {0}; + constexpr unsigned long long flags = 0; + + std::array device_location; + device_location[0].type = hipMemLocationTypeDevice; + device_location[0].id = device; + + std::array host_location; + host_location[0].type = hipMemLocationTypeHost; + host_location[0].id = 0; + + HIP_CHECK(hipMemPrefetchBatchAsync(managed_ptrs.data(), buffer_sizes.data(), managed_ptrs.size(), + device_location.data(), location_indices.data(), + location_indices.size(), flags, stream_guard.stream())); + HIP_CHECK(hipStreamSynchronize(stream_guard.stream())); + + WaitForHostRelease<<<1, 1, 0, stream_guard.stream()>>>(release.ptr()); + HIP_CHECK(hipGetLastError()); + + const auto prefetch_error = hipMemPrefetchBatchAsync( + managed_ptrs.data(), buffer_sizes.data(), managed_ptrs.size(), host_location.data(), + location_indices.data(), location_indices.size(), flags, stream_guard.stream()); + + auto query_while_blocked = hipErrorNotReady; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds{100}; + while (query_while_blocked == hipErrorNotReady && std::chrono::steady_clock::now() < deadline) { + query_while_blocked = hipStreamQuery(stream_guard.stream()); + } + + __atomic_store_n(release.ptr(), 1, __ATOMIC_RELEASE); + HIP_CHECK(hipStreamSynchronize(stream_guard.stream())); + + HIP_CHECK(prefetch_error); + REQUIRE(query_while_blocked == hipErrorNotReady); +} + /** * Test Description * ------------------------ diff --git a/projects/hip-tests/catch/unit/module/CMakeLists.txt b/projects/hip-tests/catch/unit/module/CMakeLists.txt index 35f3ffe67b6e..3ddd71fe42f6 100644 --- a/projects/hip-tests/catch/unit/module/CMakeLists.txt +++ b/projects/hip-tests/catch/unit/module/CMakeLists.txt @@ -281,7 +281,8 @@ if(HIP_PLATFORM MATCHES "amd") --offload-arch=gfx9-generic --offload-arch=gfx9-4-generic:sramecc+:xnack- --offload-arch=gfx9-4-generic:sramecc-:xnack- - --offload-arch=gfx9-4-generic:xnack+ + --offload-arch=gfx9-4-generic:sramecc+:xnack+ + --offload-arch=gfx9-4-generic:sramecc-:xnack+ --offload-arch=gfx10-1-generic --offload-arch=gfx10-3-generic --offload-arch=gfx11-generic diff --git a/projects/hip-tests/catch/unit/module/anyOrderLaunch.cc b/projects/hip-tests/catch/unit/module/anyOrderLaunch.cc index 3bc43a4ab361..93026a70b0ca 100644 --- a/projects/hip-tests/catch/unit/module/anyOrderLaunch.cc +++ b/projects/hip-tests/catch/unit/module/anyOrderLaunch.cc @@ -24,7 +24,7 @@ THE SOFTWARE. __device__ int data = 0; extern "C" __global__ void first(int* res, int ticks_for_100us) { uint64_t start = wall_clock64(); - while (__hip_atomic_load(&data, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) != 1) { + while (__scoped_atomic_load_n(&data, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE) != 1) { __builtin_amdgcn_s_sleep(10); if (wall_clock64() - start >= ticks_for_100us) { *res = 2; diff --git a/projects/hip-tests/catch/unit/occupancy/CMakeLists.txt b/projects/hip-tests/catch/unit/occupancy/CMakeLists.txt index a0fde6cd3f3c..73ad72c8e5b1 100644 --- a/projects/hip-tests/catch/unit/occupancy/CMakeLists.txt +++ b/projects/hip-tests/catch/unit/occupancy/CMakeLists.txt @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT # Common Tests - Test independent of all platforms -set(TARGETS_SUPPORTING_CLUSTERS gfx1250 gfx1251) +set(TARGETS_SUPPORTING_CLUSTERS gfx1250) set(CLUSTERS_SUPPORTED -1) function(CheckRejectedArchs OFFLOAD_ARCH_STR_LOCAL) diff --git a/projects/hip-tests/catch/unit/synchronization/cache_coherency_cpu_gpu.cc b/projects/hip-tests/catch/unit/synchronization/cache_coherency_cpu_gpu.cc index 61164e20859a..fde3c95d5284 100644 --- a/projects/hip-tests/catch/unit/synchronization/cache_coherency_cpu_gpu.cc +++ b/projects/hip-tests/catch/unit/synchronization/cache_coherency_cpu_gpu.cc @@ -18,10 +18,9 @@ __device__ int gpu_spin_loop_or_abort_on_negative_one(unsigned int* address, uns bool check = false; do { compare = value; - check = - __hip_atomic_compare_exchange_strong(address, /*expected=*/&compare, - /*desired=*/value, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, - /*scope=*/__HIP_MEMORY_SCOPE_SYSTEM); + check = __scoped_atomic_compare_exchange_n(address, /*expected=*/&compare, /*desired=*/value, + /*weak=*/0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + /*scope=*/__MEMORY_SCOPE_SYSTEM); if (compare == -1) return -1; } while (!check); return 0; @@ -35,7 +34,7 @@ __global__ void gpu_kernel(int* A, int* B, int* X, int* Y, size_t N, unsigned in // Store data into A, system fence, and atomically mark flag. // This guarantees this global write is visible by device 1. A[i] = X[i]; - __hip_atomic_fetch_add(AA1, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_fetch_add(AA1, 1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); // Wait on device 1's global write to B. if (gpu_spin_loop_or_abort_on_negative_one(BA1, i + 1) == -1) { *dresult = -1; @@ -47,11 +46,11 @@ __global__ void gpu_kernel(int* A, int* B, int* X, int* Y, size_t N, unsigned in if (!stored_data_matches) { // If the data does not match, alert other thread and abort. printf("FAIL: at i=%zu, B[i]=%d, which does not match Y[i]=%d.\n", i, B[i], Y[i]); - __hip_atomic_exchange(AA2, -1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_exchange_n(AA2, -1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); *dresult = -1; } // Otherwise tell the other thread to continue. - __hip_atomic_fetch_add(AA2, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_fetch_add(AA2, 1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); // Wait on kernel gpu_cache1 to finish checking X is stored in A. if (gpu_spin_loop_or_abort_on_negative_one(BA2, i + 1) == -1) { *dresult = -1; diff --git a/projects/hip-tests/catch/unit/synchronization/cache_coherency_gpu_gpu.cc b/projects/hip-tests/catch/unit/synchronization/cache_coherency_gpu_gpu.cc index c9b4cbe0a9ed..d1b605e39577 100644 --- a/projects/hip-tests/catch/unit/synchronization/cache_coherency_gpu_gpu.cc +++ b/projects/hip-tests/catch/unit/synchronization/cache_coherency_gpu_gpu.cc @@ -16,10 +16,9 @@ __device__ int gpu_spin_loop_or_abort_on_negative_one(unsigned int* address, uns bool check = false; do { compare = value; - check = - __hip_atomic_compare_exchange_strong(address, /*expected=*/&compare, - /*desired=*/value, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, - /*scope=*/__HIP_MEMORY_SCOPE_SYSTEM); + check = __scoped_atomic_compare_exchange_n(address, /*expected=*/&compare, /*desired=*/value, + /*weak=*/0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + /*scope=*/__MEMORY_SCOPE_SYSTEM); if (compare == -1) return -1; } while (!check); return 0; @@ -33,7 +32,7 @@ __global__ void gpu_cache0(int* A, int* B, int* X, int* Y, size_t N, unsigned in // Store data into A, system fence, and atomically mark flag. // This guarantees this global write is visible by device 1. A[i] = X[i]; - __hip_atomic_fetch_add(AA1, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_fetch_add(AA1, 1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); // Wait on device 1's global write to B. if (gpu_spin_loop_or_abort_on_negative_one(BA1, i + 1) == -1) { *cache0_result = -1; @@ -45,11 +44,11 @@ __global__ void gpu_cache0(int* A, int* B, int* X, int* Y, size_t N, unsigned in if (!stored_data_matches) { // If the data does not match, alert other thread and abort. printf("FAIL: at i=%zu, B[i]=%d, which does not match Y[i]=%d.\n", i, B[i], Y[i]); - __hip_atomic_exchange(AA2, -1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_exchange_n(AA2, -1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); *cache0_result = -1; } // Otherwise tell the other thread to continue. - __hip_atomic_fetch_add(AA2, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_fetch_add(AA2, 1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); // Wait on kernel gpu_cache1 to finish checking X is stored in A. if (gpu_spin_loop_or_abort_on_negative_one(BA2, i + 1) == -1) { *cache0_result = -1; @@ -65,7 +64,7 @@ __global__ void gpu_cache1(int* A, int* B, int* X, int* Y, size_t N, unsigned in unsigned int* cache1_result) { for (size_t i = 0; i < N; i++) { B[i] = Y[i]; - __hip_atomic_fetch_add(BA1, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_fetch_add(BA1, 1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); if (gpu_spin_loop_or_abort_on_negative_one(AA1, i + 1) == -1) { *cache1_result = -1; break; @@ -74,10 +73,10 @@ __global__ void gpu_cache1(int* A, int* B, int* X, int* Y, size_t N, unsigned in bool stored_data_matches = (A[i] == X[i]); if (!stored_data_matches) { printf("FAIL: at i=%zu, A[i]=%d, which does not match X[i]=%d.\n", i, A[i], X[i]); - __hip_atomic_exchange(BA2, -1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_exchange_n(BA2, -1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); *cache1_result = -1; } - __hip_atomic_fetch_add(BA2, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + __scoped_atomic_fetch_add(BA2, 1, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM); if (gpu_spin_loop_or_abort_on_negative_one(AA2, i + 1) == -1) { *cache1_result = -1; break; diff --git a/projects/hip-tests/catch/unit/warp/warp_reduce.cc b/projects/hip-tests/catch/unit/warp/warp_reduce.cc index 55a7f015b9b2..18a4eb38570a 100644 --- a/projects/hip-tests/catch/unit/warp/warp_reduce.cc +++ b/projects/hip-tests/catch/unit/warp/warp_reduce.cc @@ -83,10 +83,8 @@ template void runTestMultipleMasks(unsigned long long masks[], int num LinearAllocGuard d_output(LinearAllocs::hipMalloc, wavefrontSize * sizeof(T)); std::plus op; std::mt19937_64 gen(123); - typename distribution::result_type a = std::is_same::value? std::numeric_limits::lowest() : - (std::is_signed::value? -1023 : 0); - typename distribution::result_type b = std::is_same::value? std::numeric_limits::max() : - 1023; + typename distribution::result_type a = TestRange::minimum; + typename distribution::result_type b = TestRange::maximum; distribution distInput(a, b); dim3 blkDim{wavefrontSize}; dim3 grdDim{1u}; diff --git a/projects/hip-tests/samples/0_Intro/bit_extract/CMakeLists.txt b/projects/hip-tests/samples/0_Intro/bit_extract/CMakeLists.txt index e4660772fba2..78fd05ef39bf 100644 --- a/projects/hip-tests/samples/0_Intro/bit_extract/CMakeLists.txt +++ b/projects/hip-tests/samples/0_Intro/bit_extract/CMakeLists.txt @@ -4,14 +4,6 @@ cmake_minimum_required(VERSION 3.21) -if(NOT WIN32 AND NOT DEFINED __HIP_ENABLE_PCH) - set(__HIP_ENABLE_PCH ON CACHE BOOL "enable/disable pre-compiled hip headers") -endif() - -if(${__HIP_ENABLE_PCH}) - add_definitions(-D__HIP_ENABLE_PCH) -endif() - if(UNIX) if(NOT DEFINED ROCM_PATH) if(DEFINED ENV{ROCM_PATH}) diff --git a/projects/hip-tests/samples/0_Intro/bit_extract/README.md b/projects/hip-tests/samples/0_Intro/bit_extract/README.md index c823f321656b..b4b7fbad9346 100644 --- a/projects/hip-tests/samples/0_Intro/bit_extract/README.md +++ b/projects/hip-tests/samples/0_Intro/bit_extract/README.md @@ -16,8 +16,6 @@ $ make ``` $ ./bit_extract -pch size: 11743288 -__hipGetPCH succeeded! info: running on device #0 info: allocate host mem ( 7.63 MB) info: allocate device mem ( 7.63 MB) diff --git a/projects/hip-tests/samples/0_Intro/bit_extract/bit_extract.cpp b/projects/hip-tests/samples/0_Intro/bit_extract/bit_extract.cpp index b0689bada299..b60593f2669e 100644 --- a/projects/hip-tests/samples/0_Intro/bit_extract/bit_extract.cpp +++ b/projects/hip-tests/samples/0_Intro/bit_extract/bit_extract.cpp @@ -29,7 +29,7 @@ int main(int argc, char* argv[]) { size_t N = 1000000; size_t Nbytes = N * sizeof(uint32_t); -#ifdef __HIP_ENABLE_PCH +#if __HIP_HAS_GET_PCH // Verify hip_pch.o const char* pch = nullptr; unsigned int size = 0; diff --git a/projects/hip-tests/samples/0_Intro/generic_target/CMakeLists.txt b/projects/hip-tests/samples/0_Intro/generic_target/CMakeLists.txt index 7af46853ee62..9433f814dbc9 100644 --- a/projects/hip-tests/samples/0_Intro/generic_target/CMakeLists.txt +++ b/projects/hip-tests/samples/0_Intro/generic_target/CMakeLists.txt @@ -53,7 +53,7 @@ target_compile_options(squareGenericTarget PRIVATE ) # Test generic target with features -set(OFFLOAD_ARCH_GENERIC_FEATURE_STR "--offload-arch=gfx9-generic --offload-arch=gfx9-4-generic:sramecc+:xnack- --offload-arch=gfx9-4-generic:sramecc-:xnack- --offload-arch=gfx9-4-generic:xnack+ --offload-arch=gfx10-1-generic --offload-arch=gfx10-3-generic --offload-arch=gfx11-generic --offload-arch=gfx12-generic") +set(OFFLOAD_ARCH_GENERIC_FEATURE_STR "--offload-arch=gfx9-generic --offload-arch=gfx9-4-generic:sramecc+:xnack- --offload-arch=gfx9-4-generic:sramecc-:xnack- --offload-arch=gfx9-4-generic:sramecc+:xnack+ --offload-arch=gfx9-4-generic:sramecc-:xnack+ --offload-arch=gfx10-1-generic --offload-arch=gfx10-3-generic --offload-arch=gfx11-generic --offload-arch=gfx12-generic") separate_arguments(OFFLOAD_ARCH_GENERIC_FEATURE_LIST UNIX_COMMAND "${OFFLOAD_ARCH_GENERIC_FEATURE_STR}") add_executable(squareGenericTargetWithFeatures ${EXCLUDE_OPTION} square.cpp ../../../catch/hipTestMain/hip_test_features.cc) diff --git a/projects/hip-tests/samples/0_Intro/generic_target/README.md b/projects/hip-tests/samples/0_Intro/generic_target/README.md index 10158ee41b98..34758fbfab59 100644 --- a/projects/hip-tests/samples/0_Intro/generic_target/README.md +++ b/projects/hip-tests/samples/0_Intro/generic_target/README.md @@ -27,7 +27,7 @@ $ cd ~/hip/samples/0_Intro/genericTarget /opt/rocm/bin/hipcc ../square.cpp ../../../../catch/hipTestMain/hip_test_features.cc -I../../../common -I../../../../catch/include --offload-arch=gfx9-generic --offload-arch=gfx9-4-generic --offload-arch=gfx10-1-generic --offload-arch=gfx10-3-generic --offload-arch=gfx11-generic --offload-arch=gfx12-generic -mcode-object-version=6 -w -o squareGenericTarget -/opt/rocm/bin/hipcc ../square1.cpp ../../../../catch/hipTestMain/hip_test_features.cc -I../../../common -I../../../../catch/include --offload-arch=gfx9-generic --offload-arch=gfx9-4-generic:sramecc+:xnack- --offload-arch=gfx9-4-generic:sramecc-:xnack- --offload-arch=gfx9-4-generic:xnack+ --offload-arch=gfx10-1-generic --offload-arch=gfx10-3-generic --offload-arch=gfx11-generic --offload-arch=gfx12-generic -mcode-object-version=6 -w -o squareGenericTargetWithFeatures +/opt/rocm/bin/hipcc ../square1.cpp ../../../../catch/hipTestMain/hip_test_features.cc -I../../../common -I../../../../catch/include --offload-arch=gfx9-generic --offload-arch=gfx9-4-generic:sramecc+:xnack- --offload-arch=gfx9-4-generic:sramecc-:xnack- --offload-arch=gfx9-4-generic:sramecc+:xnack+ --offload-arch=gfx9-4-generic:sramecc-:xnack+ --offload-arch=gfx10-1-generic --offload-arch=gfx10-3-generic --offload-arch=gfx11-generic --offload-arch=gfx12-generic -mcode-object-version=6 -w -o squareGenericTargetWithFeatures /opt/rocm/bin/hipcc ../saxpy.cpp ../../../../catch/hipTestMain/hip_test_features.cc -I../../../common -I../../../../catch/include -o saxpyGenericTarget ``` diff --git a/projects/hip-tests/samples/2_Cookbook/15_static_library/device_functions/CMakeLists.txt b/projects/hip-tests/samples/2_Cookbook/15_static_library/device_functions/CMakeLists.txt index 8747177434e8..537694834b4c 100644 --- a/projects/hip-tests/samples/2_Cookbook/15_static_library/device_functions/CMakeLists.txt +++ b/projects/hip-tests/samples/2_Cookbook/15_static_library/device_functions/CMakeLists.txt @@ -18,6 +18,10 @@ endif() project(static_lib LANGUAGES CXX HIP) +if(UNIX) + find_package(amd_comgr REQUIRED) +endif() + # For windows, AR is MS Librarian and that is picked by Visual Studio's command prompt. # If using LLVM/Clang toolchain, use llvm-ar instead. if(WIN32) diff --git a/projects/hip-tests/samples/2_Cookbook/15_static_library/host_functions/CMakeLists.txt b/projects/hip-tests/samples/2_Cookbook/15_static_library/host_functions/CMakeLists.txt index 67f417e6a4fb..9b67097bd0a8 100644 --- a/projects/hip-tests/samples/2_Cookbook/15_static_library/host_functions/CMakeLists.txt +++ b/projects/hip-tests/samples/2_Cookbook/15_static_library/host_functions/CMakeLists.txt @@ -18,6 +18,11 @@ endif() project(static_lib LANGUAGES CXX HIP) +if(UNIX) + find_package(amd_comgr REQUIRED) + find_package(hsa-runtime64 REQUIRED) +endif() + # For windows, AR is MS Librarian and that is picked by Visual Studio's command prompt. # If using LLVM/Clang toolchain, use llvm-ar instead. if(WIN32) @@ -137,7 +142,7 @@ if(WIN32) "${AMD_COMGR_LIB}" ) else() - target_link_libraries(test_opt_static PRIVATE amdhip64 amd_comgr hsa-runtime64) + target_link_libraries(test_opt_static PRIVATE amdhip64 amd_comgr hsa-runtime64::hsa-runtime64) endif() # Set RPATH so executable can find HIP libraries at runtime diff --git a/projects/hip-tests/utils/coverage/device_api_list.txt b/projects/hip-tests/utils/coverage/device_api_list.txt index 73adb4a3d2dc..f831b94d3084 100644 --- a/projects/hip-tests/utils/coverage/device_api_list.txt +++ b/projects/hip-tests/utils/coverage/device_api_list.txt @@ -27,20 +27,6 @@ Atomics [ atomicXor_system ] -Builtin atomics [ - __hip_atomic_load - __hip_atomic_store - __hip_atomic_compare_exchange_weak - __hip_atomic_compare_exchange_strong - __hip_atomic_exchange - __hip_atomic_fetch_add - __hip_atomic_fetch_and - __hip_atomic_fetch_or - __hip_atomic_fetch_xor - __hip_atomic_fetch_min - __hip_atomic_fetch_max -] - Cooperative groups [ File restriction: cooperative_groups Device groups: ( diff --git a/projects/hip/include/hip/hip_runtime_api.h b/projects/hip/include/hip/hip_runtime_api.h index 4525d6d42d03..9fb5231eb5e7 100644 --- a/projects/hip/include/hip/hip_runtime_api.h +++ b/projects/hip/include/hip/hip_runtime_api.h @@ -585,6 +585,15 @@ typedef enum hipDeviceAttribute_t { ///< management) hipDeviceAttributeHandleTypeFabricSupported, ///< Device supports exporting memory to a fabric handle hipDeviceAttributeHostAllocDmaBufSupported, ///< Device supports host-allocated DMABuf buffer sharing + hipDeviceAttributeGPUDirectRDMASupported, ///< Device supports GPUDirect RDMA APIs + hipDeviceAttributeGPUDirectRDMAFlushWritesOptions, ///< Bitmask of + ///< hipFlushGPUDirectRDMAWritesOptions + ///< describing the flush paths the device + ///< supports + hipDeviceAttributeGPUDirectRDMAWritesOrdering, ///< A hipGPUDirectRDMAWritesOrdering value + ///< giving the scope at which GPUDirect RDMA + ///< writes are naturally ordered, i.e. visible + ///< without an explicit flush hipDeviceAttributeCudaCompatibleEnd = 9999, hipDeviceAttributeAmdSpecificBegin = 10000, @@ -677,6 +686,25 @@ enum hipGPUDirectRDMAWritesOrdering { hipGPUDirectRDMAWritesOrderingAllDevices = 200 }; +/** + * The target of a hipDeviceFlushGPUDirectRDMAWrites operation. + */ +enum hipFlushGPUDirectRDMAWritesTarget { + hipFlushGPUDirectRDMAWritesTargetCurrentDevice = 0 ///< Memory of the current HIP device +}; + +/** + * The scope at which hipDeviceFlushGPUDirectRDMAWrites makes pending remote writes visible. + * + * The enumerator values match hipGPUDirectRDMAWritesOrdering, so a device whose + * hipDeviceAttributeGPUDirectRDMAWritesOrdering is greater than or equal to the requested + * scope already orders those writes and needs no explicit flush. + */ +enum hipFlushGPUDirectRDMAWritesScope { + hipFlushGPUDirectRDMAWritesToOwner = 100, ///< Visible to the device owning the memory + hipFlushGPUDirectRDMAWritesToAllDevices = 200 ///< Visible to all HIP devices +}; + #if defined(__HIP_PLATFORM_AMD__) && !defined(__HIP_PLATFORM_NVIDIA__) #ifndef GENERIC_GRID_LAUNCH @@ -1547,13 +1575,11 @@ typedef struct hipExternalSemaphoreWaitParams_st { unsigned int reserved[16]; } hipExternalSemaphoreWaitParams; -#if __HIP_HAS_GET_PCH /** * Internal use only. This API may change in the future * Pre-Compiled header for online compilation */ void __hipGetPCH(const char** pch, unsigned int* size); -#endif /** * HIP Access falgs for Interop resources. @@ -2505,6 +2531,29 @@ hipError_t hipGetDeviceCount(int* count); * @returns #hipSuccess, #hipErrorInvalidDevice, #hipErrorInvalidValue */ hipError_t hipDeviceGetAttribute(int* pi, hipDeviceAttribute_t attr, int deviceId); +/** + * @brief Blocks until remote writes are visible to the specified scope + * + * Blocks until GPUDirect RDMA writes to the target device, issued by a third-party device + * such as an RDMA-capable NIC, are visible to the specified scope. This is a host-ordered + * visibility barrier on inbound remote writes; it does not synchronize with any stream or + * kernel. + * + * If @p scope is at or within the scope reported by + * #hipDeviceAttributeGPUDirectRDMAWritesOrdering, the writes are already ordered by the + * hardware and the call is a no-op. + * + * Support is reported by #hipDeviceAttributeGPUDirectRDMAFlushWritesOptions. The call + * returns #hipErrorNotSupported when that bitmask does not contain + * #hipFlushGPUDirectRDMAWritesOptionHost. + * + * @param [in] target The target of the operation, see #hipFlushGPUDirectRDMAWritesTarget + * @param [in] scope The scope of the operation, see #hipFlushGPUDirectRDMAWritesScope + * + * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorNotSupported + */ +hipError_t hipDeviceFlushGPUDirectRDMAWrites(enum hipFlushGPUDirectRDMAWritesTarget target, + enum hipFlushGPUDirectRDMAWritesScope scope); /** * @brief Returns the default memory pool of the specified device * @@ -3231,6 +3280,7 @@ hipError_t hipStreamSynchronize(hipStream_t stream); * @param[in] flags Parameters to control the operation * * @returns #hipSuccess, #hipErrorInvalidHandle, #hipErrorInvalidValue, + * #hipErrorStreamCaptureInvalidated, #hipErrorStreamCaptureMerge, * #hipErrorStreamCaptureIsolation * * This function inserts a wait operation into the specified stream. @@ -5404,8 +5454,10 @@ hipError_t hipMemcpyHtoAAsync(hipArray_t dstArray, size_t dstOffset, const void* * @ingroup Module * * Returns in *dptr and *bytes the pointer and size of the global of name name located in module - * hmod. If no variable of that name exists, it returns hipErrorNotFound. Both parameters dptr and - * bytes are optional. If one of them is NULL, it is ignored and hipSuccess is returned. + * hmod. If no variable of that name exists, it returns hipErrorNotFound. A registered + * `__device__` global that the compiler dropped from the loaded code object is also reported as + * hipErrorNotFound (the runtime no longer aborts). Both parameters dptr and bytes are optional. + * If one of them is NULL, it is ignored and hipSuccess is returned. * * @param[out] dptr Returns global device pointer * @param[out] bytes Returns global size in bytes @@ -5424,7 +5476,7 @@ hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes, hipModule_t h * @param[out] devPtr pointer to the device associated the symbole * @param[in] symbol pointer to the symbole of the device * - * @returns #hipSuccess, #hipErrorInvalidValue + * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorInvalidSymbol * */ hipError_t hipGetSymbolAddress(void** devPtr, const void* symbol); @@ -5985,7 +6037,7 @@ hipError_t hipArray3DGetDescriptor(HIP_ARRAY3D_DESCRIPTOR* pArrayDescriptor, hip * @param[in] src Source memory address * @param[in] spitch Pitch size in bytes of source memory * @param[in] width Width size in bytes of matrix transfer (columns) - * @param[in] height Height size in bytes of matrix transfer (rows) + * @param[in] height Height of matrix transfer (rows) * @param[in] kind Type of transfer * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorInvalidPitchValue, * #hipErrorInvalidDevicePointer, #hipErrorInvalidMemcpyDirection @@ -7221,7 +7273,7 @@ hipError_t hipLibraryGetKernelCount(unsigned int *count, hipLibrary_t library); * @param [in] library Input hip library handle. * @param [in] name Name of the global symbol to look up. * @return #hipSuccess, #hipErrorInvalidValue, #hipErrorInvalidResourceHandle, - * #hipErrorNotFound + * #hipErrorNotFound, #hipErrorInvalidSymbol */ hipError_t hipLibraryGetGlobal(void** dptr, size_t* bytes, hipLibrary_t library, const char* name); @@ -8811,9 +8863,10 @@ hipError_t hipStreamBeginCaptureToGraph(hipStream_t stream, hipGraph_t graph, * @brief Ends capture on a stream, returning the captured graph. * * @param [in] stream - Stream to end capture. - * @param [out] pGraph - Captured graph. + * @param [out] pGraph - Captured graph. Set to NULL on every error. * - * @returns #hipSuccess, #hipErrorInvalidValue + * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorStreamCaptureInvalidated, + * #hipErrorStreamCaptureUnjoined * */ hipError_t hipStreamEndCapture(hipStream_t stream, hipGraph_t* pGraph); @@ -8870,7 +8923,8 @@ hipError_t hipStreamIsCapturing(hipStream_t stream, hipStreamCaptureStatus* pCap * @param [in] numDependencies Size of the dependencies array. * @param [in] flags Flag to update dependency set. Should be one of the values * in enum #hipStreamUpdateCaptureDependenciesFlags. - * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorIllegalState + * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorIllegalState, + * #hipErrorStreamCaptureInvalidated * */ hipError_t hipStreamUpdateCaptureDependencies(hipStream_t stream, hipGraphNode_t* dependencies, diff --git a/projects/hipfile/.clang-tidy b/projects/hipfile/.clang-tidy index 9ac8bc4f9e24..d4d0a69d6e2d 100644 --- a/projects/hipfile/.clang-tidy +++ b/projects/hipfile/.clang-tidy @@ -15,9 +15,9 @@ # # If clang-tidy reports "Error: no checks enabled." with this config, the wrong # clang-tidy was selected -- a ROCm-built one, which has no clang-analyzer-* -# checks compiled in. Set AIS_CLANG_TIDY_EXECUTABLE to an upstream clang-tidy -# instead. Do NOT "fix" it with --allow-no-checks or by loosening the globs -# below; that silently lints nothing. +# checks compiled in. Set HIPFILE_CLANG_TIDY_EXECUTABLE to an upstream +# clang-tidy instead. Do NOT "fix" it with --allow-no-checks or by loosening the +# globs below; that silently lints nothing. # `clang-diagnostic-*` is silently prepended to this list. It re-reports compiler # warnings that get flagged in the normal build process. `-*` suppresses these diff --git a/projects/hipfile/CHANGELOG.md b/projects/hipfile/CHANGELOG.md index 87e7c52b1a0b..b8aec084c8f8 100644 --- a/projects/hipfile/CHANGELOG.md +++ b/projects/hipfile/CHANGELOG.md @@ -6,8 +6,13 @@ ### Changed +* The hipFile CMake options have been renamed from `AIS_*` to `HIPFILE_*` (e.g., `AIS_CXX_STANDARD` is now `HIPFILE_CXX_STANDARD`). The old `AIS_*` names still work but are deprecated and emit a CMake deprecation warning. + ### Fixed +* hipFile now returns `hipFileGetNewFDFailed` instead of `hipFileInternalError` when the process or system runs out of file descriptors. API calls that need a new file descriptor, such as `hipFileHandleRegister()`, can return this error. + + ### Removed ### Known issues diff --git a/projects/hipfile/CMakeLists.txt b/projects/hipfile/CMakeLists.txt index 1d5f1c781043..bf02118b97fc 100644 --- a/projects/hipfile/CMakeLists.txt +++ b/projects/hipfile/CMakeLists.txt @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: MIT -# Top-level CMake file for AIS +# Top-level CMake file for hipFile # CMake version according to latest ROCm platform requirements cmake_minimum_required(VERSION 3.21) @@ -16,20 +16,20 @@ endif() # Set the library version in a variable so we can use # it in *.in files and avoid version mismatch. -set(AIS_LIBRARY_MAJOR 0) -set(AIS_LIBRARY_MINOR 6) -set(AIS_LIBRARY_PATCH 0) -set(AIS_LIBRARY_VERSION "${AIS_LIBRARY_MAJOR}.${AIS_LIBRARY_MINOR}.${AIS_LIBRARY_PATCH}") +set(HIPFILE_LIBRARY_MAJOR 0) +set(HIPFILE_LIBRARY_MINOR 6) +set(HIPFILE_LIBRARY_PATCH 0) +set(HIPFILE_LIBRARY_VERSION "${HIPFILE_LIBRARY_MAJOR}.${HIPFILE_LIBRARY_MINOR}.${HIPFILE_LIBRARY_PATCH}") # Set the SOVERSION to the major library version (this is the default # but we set it explicitly to convey intent). # # Changes to minor and patch releases must NEVER break the ABI. This # includes things like reordering struct fields that don't break the API. -set(AIS_LIBRARY_SOVERSION "${AIS_LIBRARY_MAJOR}") +set(HIPFILE_LIBRARY_SOVERSION "${HIPFILE_LIBRARY_MAJOR}") project("hipfile" - VERSION ${AIS_LIBRARY_VERSION} + VERSION ${HIPFILE_LIBRARY_VERSION} DESCRIPTION "Direct-to-GPU IO for AMD's ROCm platform" LANGUAGES C CXX HIP ) @@ -48,8 +48,13 @@ set(HIPFILE_TEST_COMMON_PATH "${CMAKE_CURRENT_SOURCE_DIR}/test/common") set(HIPFILE_AMD_TEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}/test/amd_detail") set(HIPFILE_THIRD_PARTY_PATH "${CMAKE_CURRENT_SOURCE_DIR}/third-party") -set(AIS_CXX_STANDARD "20" CACHE STRING "C++ standard to build with") -set_property(CACHE AIS_CXX_STANDARD PROPERTY STRINGS "20" "23" "26") +# Map the deprecated AIS_* options to their HIPFILE_* replacements +# +# NOTE: This has to be done BEFORE any of the HIPFILE_* options are declared! +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/HipFileDeprecatedAISOptions.cmake") + +set(HIPFILE_CXX_STANDARD "20" CACHE STRING "C++ standard to build with") +set_property(CACHE HIPFILE_CXX_STANDARD PROPERTY STRINGS "20" "23" "26") # Set the list of build types for ccmake/CMake GUI # @@ -90,11 +95,11 @@ endif() # Set the platform variables (internal only) if(CMAKE_HIP_PLATFORM STREQUAL "nvidia") - set(AIS_BUILD_AMD_DETAIL OFF) - set(AIS_BUILD_NVIDIA_DETAIL ON) + set(HIPFILE_BUILD_AMD_DETAIL OFF) + set(HIPFILE_BUILD_NVIDIA_DETAIL ON) else() - set(AIS_BUILD_AMD_DETAIL ON) - set(AIS_BUILD_NVIDIA_DETAIL OFF) + set(HIPFILE_BUILD_AMD_DETAIL ON) + set(HIPFILE_BUILD_NVIDIA_DETAIL OFF) endif() #----------------------------------------------------------------------------- @@ -168,16 +173,16 @@ message(STATUS "ROCM_PATH set to: ${ROCM_PATH}") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") execute_process( COMMAND "${CMAKE_CXX_COMPILER}" -v -x c++ /dev/null -fsyntax-only - ERROR_VARIABLE _ais_cxx_verbose + ERROR_VARIABLE _hipfile_cxx_verbose OUTPUT_QUIET ERROR_STRIP_TRAILING_WHITESPACE ) - if(_ais_cxx_verbose MATCHES "Selected GCC installation: ([^\n]+)") + if(_hipfile_cxx_verbose MATCHES "Selected GCC installation: ([^\n]+)") message(STATUS "Selected GCC installation (libstdc++): ${CMAKE_MATCH_1}") else() message(STATUS "Selected GCC installation (libstdc++): ") endif() - unset(_ais_cxx_verbose) + unset(_hipfile_cxx_verbose) endif() # Add ROCm search paths @@ -200,7 +205,7 @@ set(CMAKE_INSTALL_LIBDIR "lib" CACHE STRING "Directory name for installed ROCm l #----------------------------------------------------------------------------- # Import useful things for later #----------------------------------------------------------------------------- -include(AISAddLibraries) +include(HipFileAddLibraries) #----------------------------------------------------------------------------- # Options @@ -210,7 +215,7 @@ include(AISAddLibraries) # Code coverage (via llvm-cov) # # To use with the hipFile library: -# - Build with AIS_USE_CODE_COVERAGE set to ON +# - Build with HIPFILE_USE_CODE_COVERAGE set to ON # - Run `ctest .` # # This should create a default.profraw file in each test directory @@ -224,37 +229,37 @@ include(AISAddLibraries) # You can export the data to lcov or json # - Run `llvm-cov export -instr-profile=ais.profdata -format= > coverage.` #----------------------------------------------------------------------------- -option(AIS_USE_CODE_COVERAGE "Build with code coverage flags (llvm C++ only)" OFF) +option(HIPFILE_USE_CODE_COVERAGE "Build with code coverage flags (llvm C++ only)" OFF) #---------------------------------------------- # Clang "safe buffer" warnings and suggestions #---------------------------------------------- -include(AISClangSafeBuffers) +include(HipFileClangSafeBuffers) #---------------- # Run clang-tidy #---------------- -include(AISClangTidy) +include(HipFileClangTidy) #--------------------------------- # Run include-what-you-use (IWYU) #--------------------------------- -include(AISIWYU) +include(HipFileIWYU) #------------------- # Sanitizer options #------------------- -include(AISSanitizers) +include(HipFileSanitizers) #------------------ # Example programs #------------------ -option(AIS_INSTALL_EXAMPLES "Install example programs" ON) +option(HIPFILE_INSTALL_EXAMPLES "Install example programs" ON) #------------------ # Tool programs #------------------ -option(AIS_INSTALL_TOOLS "Install tool programs" ON) +option(HIPFILE_INSTALL_TOOLS "Install tool programs" ON) #------------------ # rocprofiler-register integration @@ -270,15 +275,15 @@ option(HIPFILE_ROCPROFILER_REGISTER "Enable rocprofiler-register API table regis # from the build of the tests, matching the convention used by sibling projects # (e.g. rocprofiler-systems' ROCPROFSYS_INSTALL_TESTING). Installing the tests # requires building them, so enabling this forces BUILD_TESTING on. -option(AIS_INSTALL_TESTS "Install unit-test programs" OFF) -if(AIS_INSTALL_TESTS) - set(BUILD_TESTING ON CACHE BOOL "Build testing (forced on by AIS_INSTALL_TESTS)" FORCE) +option(HIPFILE_INSTALL_TESTS "Install unit-test programs" OFF) +if(HIPFILE_INSTALL_TESTS) + set(BUILD_TESTING ON CACHE BOOL "Build testing (forced on by HIPFILE_INSTALL_TESTS)" FORCE) endif() #------ # docs #------ -include(AISDocumentation) +include(HipFileDocumentation) #----------------------------------------------------------------------------- # Find important packages for building the software @@ -298,7 +303,7 @@ else() message(NOTICE "find_package did NOT find hip") endif() -if(HIPFILE_ROCPROFILER_REGISTER AND AIS_BUILD_AMD_DETAIL) +if(HIPFILE_ROCPROFILER_REGISTER AND HIPFILE_BUILD_AMD_DETAIL) find_package( rocprofiler-register REQUIRED @@ -311,7 +316,7 @@ if(HIPFILE_ROCPROFILER_REGISTER AND AIS_BUILD_AMD_DETAIL) endif() # Find the cuFile library and headers on NVIDIA -if(AIS_BUILD_NVIDIA_DETAIL) +if(HIPFILE_BUILD_NVIDIA_DETAIL) find_package(CUDAToolkit REQUIRED) if(CMAKE_VERSION VERSION_LESS "3.28") @@ -329,8 +334,8 @@ if(AIS_BUILD_NVIDIA_DETAIL) endif() endif() -if(NOT AIS_CAPABLE_DIR) - set(AIS_CAPABLE_DIR "/tmp" CACHE STRING "Directory to use for e2e tests.") +if(NOT HIPFILE_CAPABLE_DIR) + set(HIPFILE_CAPABLE_DIR "/tmp" CACHE STRING "Directory to use for e2e tests.") endif() option(HIPFILE_ALLOW_SKIP_FASTPATH_TESTS @@ -341,7 +346,7 @@ option(HIPFILE_ALLOW_SKIP_FASTPATH_TESTS #----------------------------------------------------------------------------- # Add the src directory depending on the HIP platform -if(AIS_BUILD_NVIDIA_DETAIL) +if(HIPFILE_BUILD_NVIDIA_DETAIL) add_subdirectory(src/nvidia_detail) else() add_subdirectory(src/amd_detail) @@ -349,24 +354,25 @@ endif() # Add the testing directories if(BUILD_TESTING) - include(AISUseGTest) + include(HipFileUseGTest) add_subdirectory(test) endif() -if(BUILD_TESTING AND AIS_BUILD_AMD_DETAIL) +if(BUILD_TESTING AND HIPFILE_BUILD_AMD_DETAIL) add_subdirectory(test/amd_detail) endif() # Add the example directory -if(AIS_INSTALL_EXAMPLES) +if(HIPFILE_INSTALL_EXAMPLES) add_subdirectory(examples/aiscp) add_subdirectory(examples/api) add_subdirectory(examples/common) add_subdirectory(examples/basics) + add_subdirectory(examples/batch) add_subdirectory(examples/async) endif() # Add tools directory -if(AIS_INSTALL_TOOLS AND AIS_BUILD_AMD_DETAIL) +if(HIPFILE_INSTALL_TOOLS AND HIPFILE_BUILD_AMD_DETAIL) add_subdirectory(tools/ais-stats) endif() @@ -374,13 +380,13 @@ endif() #----------------------------------------------------------------------------- # Set up installs (must come AFTER target creation) #----------------------------------------------------------------------------- -include(AISUseROCmCMake) +include(HipFileUseROCmCMake) # Install the relocatable unit-test assets (binaries + a self-locating # CTestTestfile.cmake) so the suite can run from an installed/packaged tree. -# Gated on AIS_INSTALL_TESTS (which forces BUILD_TESTING on above, so the test +# Gated on HIPFILE_INSTALL_TESTS (which forces BUILD_TESTING on above, so the test # targets exist). Must come after the test targets are created # (add_subdirectory(test) above). -if(AIS_INSTALL_TESTS) - include(AISInstallTests) +if(HIPFILE_INSTALL_TESTS) + include(HipFileInstallTests) endif() -include(AISInstall) +include(HipFileInstall) diff --git a/projects/hipfile/cmake/AISAddExecutable.cmake b/projects/hipfile/cmake/HipFileAddExecutable.cmake similarity index 83% rename from projects/hipfile/cmake/AISAddExecutable.cmake rename to projects/hipfile/cmake/HipFileAddExecutable.cmake index 99e1148c0967..a88c2ebe9c50 100644 --- a/projects/hipfile/cmake/AISAddExecutable.cmake +++ b/projects/hipfile/cmake/HipFileAddExecutable.cmake @@ -2,23 +2,23 @@ # # SPDX-License-Identifier: MIT -include(AISCompilerOptions) +include(HipFileCompilerOptions) -# Add an executable program using AIS build conventions +# Add an executable program using hipFile build conventions # # Parameters: # NAME The name of the executable program to create -# DEPS [dependency1 [dependency2 ...]] List of AIS target library dependencies +# DEPS [dependency1 [dependency2 ...]] List of hipFile target library dependencies # SRCS [src1 [src2 ...]] The source files # SYSINCLS [path1 [path2 ...]] Paths to include dirs # -# NOTE: Assumes AIS target libraries are named _(static|shared) +# NOTE: Assumes hipFile target libraries are named _(static|shared) # # NOTE: This isn't the most robust function. It's mainly intended # to reduce code duplication. For example, DEPS is for passing # the hipFile shared/static dependency and won't work for passing # general library dependencies. -function(ais_add_executable) +function(hipfile_add_executable) # Parse arguments set(options) # None at this time @@ -27,10 +27,10 @@ function(ais_add_executable) cmake_parse_arguments(arg "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${arg_NAME} ${arg_SRCS}) - ais_set_compiler_flags(${arg_NAME}) + hipfile_set_compiler_flags(${arg_NAME}) # Set C++ standard - target_compile_features(${arg_NAME} PRIVATE cxx_std_${AIS_CXX_STANDARD}) + target_compile_features(${arg_NAME} PRIVATE cxx_std_${HIPFILE_CXX_STANDARD}) set_target_properties(${arg_NAME} PROPERTIES CXX_EXTENSIONS OFF) # Set position-independent code @@ -71,17 +71,17 @@ function(ais_add_executable) target_include_directories(${arg_NAME} PRIVATE "${HIPFILE_ROOT_PATH}/shared") endfunction() -# Add an executable test program using AIS build conventions +# Add an executable test program using hipFile build conventions # # Parameters: # NAME The name of the executable program to create -# DEPS [dependency1 [dependency2 ...]] List of AIS target library dependencies +# DEPS [dependency1 [dependency2 ...]] List of hipFile target library dependencies # SRCS [src1 [src2 ...]] The source files # SYSINCLS [path1 [path2 ...]] Paths to include dirs # # NOTE: Simply a pass-through to add -UNDEBUG to test programs so they # always have assert() available. -function(ais_add_test_executable) +function(hipfile_add_test_executable) # Parse arguments set(options) # None at this time @@ -89,7 +89,7 @@ function(ais_add_test_executable) set(multiValueArgs SRCS DEPS SYSINCLS) cmake_parse_arguments(arg "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - ais_add_executable( + hipfile_add_executable( NAME ${arg_NAME} DEPS ${arg_DEPS} SRCS ${arg_SRCS} diff --git a/projects/hipfile/cmake/AISAddLibraries.cmake b/projects/hipfile/cmake/HipFileAddLibraries.cmake similarity index 90% rename from projects/hipfile/cmake/AISAddLibraries.cmake rename to projects/hipfile/cmake/HipFileAddLibraries.cmake index 758981bb4b53..c096c4c7eea0 100644 --- a/projects/hipfile/cmake/AISAddLibraries.cmake +++ b/projects/hipfile/cmake/HipFileAddLibraries.cmake @@ -2,10 +2,10 @@ # # SPDX-License-Identifier: MIT -include(AISCompilerOptions) +include(HipFileCompilerOptions) include(CheckLinkerFlag) -# Add a library using AIS build conventions. Shared vs. static +# Add a library using hipFile build conventions. Shared vs. static # is decided by the BUILD_SHARED_LIBS option and only one can # be built at a time (like the rest of ROCm). # @@ -16,7 +16,7 @@ include(CheckLinkerFlag) # SYSINCLS [path1 [path2 ...]] Paths to other include dirs # SRCS [src1 [src2 ...]] The source files # LIBS [lib1, [lib2 ...]] List of external library dependencies -function(ais_add_libraries) +function(hipfile_add_libraries) # Parse arguments set(options) # None at this time @@ -28,16 +28,16 @@ function(ais_add_libraries) add_library(${arg_NAME} ${arg_SRCS}) # Set C++ standard - target_compile_features(${arg_NAME} PUBLIC cxx_std_${AIS_CXX_STANDARD}) + target_compile_features(${arg_NAME} PUBLIC cxx_std_${HIPFILE_CXX_STANDARD}) set_target_properties(${arg_NAME} PROPERTIES CXX_EXTENSIONS OFF) # Set position-independent code set_target_properties(${arg_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON) # Add version numbers - set_target_properties(${arg_NAME} PROPERTIES VERSION ${AIS_LIBRARY_VERSION}) + set_target_properties(${arg_NAME} PROPERTIES VERSION ${HIPFILE_LIBRARY_VERSION}) if(BUILD_SHARED_LIBS) - set_target_properties(${arg_NAME} PROPERTIES SOVERSION ${AIS_LIBRARY_SOVERSION}) + set_target_properties(${arg_NAME} PROPERTIES SOVERSION ${HIPFILE_LIBRARY_SOVERSION}) endif() # Add dependencies on external libraries @@ -96,5 +96,5 @@ function(ais_add_libraries) target_include_directories(${arg_NAME} PRIVATE "${HIPFILE_ROOT_PATH}/shared") # Set compiler flags - ais_set_compiler_flags(${arg_NAME}) + hipfile_set_compiler_flags(${arg_NAME}) endfunction() diff --git a/projects/hipfile/cmake/AISClangCompilerOptions.cmake b/projects/hipfile/cmake/HipFileClangCompilerOptions.cmake similarity index 99% rename from projects/hipfile/cmake/AISClangCompilerOptions.cmake rename to projects/hipfile/cmake/HipFileClangCompilerOptions.cmake index a33578ab12eb..49a426ab92ca 100644 --- a/projects/hipfile/cmake/AISClangCompilerOptions.cmake +++ b/projects/hipfile/cmake/HipFileClangCompilerOptions.cmake @@ -6,7 +6,7 @@ # # https://clang.llvm.org/docs/DiagnosticsReference.html -function(get_ais_clang_warning_flags outvar compiler_version) +function(get_hipfile_clang_warning_flags outvar compiler_version) # Warning flags for clang 17 and earlier set(flags diff --git a/projects/hipfile/cmake/AISClangSafeBuffers.cmake b/projects/hipfile/cmake/HipFileClangSafeBuffers.cmake similarity index 78% rename from projects/hipfile/cmake/AISClangSafeBuffers.cmake rename to projects/hipfile/cmake/HipFileClangSafeBuffers.cmake index c56ec43d318b..b649a334ae80 100644 --- a/projects/hipfile/cmake/AISClangSafeBuffers.cmake +++ b/projects/hipfile/cmake/HipFileClangSafeBuffers.cmake @@ -16,14 +16,14 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") else() set(USE_UBO OFF) endif() -option(AIS_WARN_UNSAFE_BUFFER_OPS "Warn about unsafe buffer operations (llvm C++ only)" ${USE_UBO}) +option(HIPFILE_WARN_UNSAFE_BUFFER_OPS "Warn about unsafe buffer operations (llvm C++ only)" ${USE_UBO}) -if(AIS_WARN_UNSAFE_BUFFER_OPS) +if(HIPFILE_WARN_UNSAFE_BUFFER_OPS) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(AIS_CLANG_WARNING_FLAGS + set(HIPFILE_CLANG_WARNING_FLAGS -Wunsafe-buffer-usage -fsafe-buffer-usage-suggestions - ${AIS_CLANG_WARNING_FLAGS} + ${HIPFILE_CLANG_WARNING_FLAGS} ) else() message(FATAL_ERROR "Unsafe buffer warnings are only useful for clang/llvm") diff --git a/projects/hipfile/cmake/AISClangTidy.cmake b/projects/hipfile/cmake/HipFileClangTidy.cmake similarity index 60% rename from projects/hipfile/cmake/AISClangTidy.cmake rename to projects/hipfile/cmake/HipFileClangTidy.cmake index 38bb79df955a..e4865c07e66b 100644 --- a/projects/hipfile/cmake/AISClangTidy.cmake +++ b/projects/hipfile/cmake/HipFileClangTidy.cmake @@ -9,11 +9,11 @@ include_guard(GLOBAL) # # When this option is enabled, compilation will emit clang-tidy suggestions. # -# AIS_CLANG_TIDY_EXECUTABLE selects which clang-tidy to run. It is deliberately -# decoupled from CMAKE_CXX_COMPILER: the ROCm LLVM toolchain is built with -# CLANG_TIDY_ENABLE_STATIC_ANALYZER=OFF (see ROCm/TheRock compiler/CMakeLists.txt), -# so its clang-tidy ships with zero clang-analyzer-* checks and cannot run the -# Clang Static Analyzer at all. An upstream clang-tidy of a matching LLVM major +# HIPFILE_CLANG_TIDY_EXECUTABLE selects which clang-tidy to run. It is +# deliberately decoupled from CMAKE_CXX_COMPILER: the ROCm LLVM toolchain is +# built with CLANG_TIDY_ENABLE_STATIC_ANALYZER=OFF (see ROCm/TheRock +# compiler/CMakeLists.txt), so its clang-tidy ships with zero clang-analyzer-* +# checks and cannot run the Clang Static Analyzer at all. An upstream clang-tidy of a matching LLVM major # version is used instead. Keep that version aligned with the ROCm LLVM major # version -- mismatched clang-tidy versions report conflicting diagnostics. # @@ -21,13 +21,13 @@ include_guard(GLOBAL) # than relying on clang-tidy's built-in defaults, because those defaults are not # stable across LLVM versions (see the comment in that file). #----------------------------------------------------------------------------- -option(AIS_USE_CLANG_TIDY "Run clang-tidy when compiling" OFF) -set(AIS_CLANG_TIDY_EXECUTABLE "" CACHE STRING +option(HIPFILE_USE_CLANG_TIDY "Run clang-tidy when compiling" OFF) +set(HIPFILE_CLANG_TIDY_EXECUTABLE "" CACHE STRING "clang-tidy executable to use (name or absolute path); empty searches PATH for clang-tidy") -if(AIS_USE_CLANG_TIDY) - if(AIS_CLANG_TIDY_EXECUTABLE) - find_program(CLANG_TIDY_EXE NAMES "${AIS_CLANG_TIDY_EXECUTABLE}" REQUIRED) +if(HIPFILE_USE_CLANG_TIDY) + if(HIPFILE_CLANG_TIDY_EXECUTABLE) + find_program(CLANG_TIDY_EXE NAMES "${HIPFILE_CLANG_TIDY_EXECUTABLE}" REQUIRED) else() find_program(CLANG_TIDY_EXE NAMES clang-tidy REQUIRED) endif() diff --git a/projects/hipfile/cmake/AISCompilerOptions.cmake b/projects/hipfile/cmake/HipFileCompilerOptions.cmake similarity index 75% rename from projects/hipfile/cmake/AISCompilerOptions.cmake rename to projects/hipfile/cmake/HipFileCompilerOptions.cmake index b539ecb4a91b..23c181b648a0 100644 --- a/projects/hipfile/cmake/AISCompilerOptions.cmake +++ b/projects/hipfile/cmake/HipFileCompilerOptions.cmake @@ -4,16 +4,16 @@ # Set compiler flags on target based on compilers being used on sources -include(AISClangCompilerOptions) -include(AISGNUCompilerOptions) -include(AISSanitizers) +include(HipFileClangCompilerOptions) +include(HipFileGNUCompilerOptions) +include(HipFileSanitizers) -function(ais_set_compiler_flags target) +function(hipfile_set_compiler_flags target) get_target_property(sources ${target} SOURCES) foreach(source IN LISTS sources) get_source_file_property(language ${source} LANGUAGE) # CMake HIP language prior to 3.28 only supports AMD, so change LANGUAGE to CUDA for HIP files - if(AIS_BUILD_NVIDIA_DETAIL AND CMAKE_VERSION VERSION_LESS "3.28" AND language STREQUAL HIP) + if(HIPFILE_BUILD_NVIDIA_DETAIL AND CMAKE_VERSION VERSION_LESS "3.28" AND language STREQUAL HIP) set_source_files_properties(${source} PROPERTIES LANGUAGE "CUDA") set(language CUDA) message(STATUS "Setting ${source} LANGUAGE to CUDA because CMake < 3.28") @@ -24,23 +24,23 @@ function(ais_set_compiler_flags target) # Only use default flags with include-what-you-use. Otherwise you'll # clutter the output with a lot of "unrecognized flags" warnings if # there's mismatch between IWYU's clang and the compiler you are using. - if(NOT AIS_USE_IWYU) + if(NOT HIPFILE_USE_IWYU) if(compiler_id STREQUAL "GNU" OR compiler_id STREQUAL "NVIDIA") - get_ais_gnu_warning_flags(compiler_flags ${compiler_version}) + get_hipfile_gnu_warning_flags(compiler_flags ${compiler_version}) elseif(compiler_id STREQUAL "Clang") - get_ais_clang_warning_flags(compiler_flags ${compiler_version}) + get_hipfile_clang_warning_flags(compiler_flags ${compiler_version}) endif() endif() target_compile_options(${target} PRIVATE $<$:${compiler_flags}>) - if(AIS_USE_CODE_COVERAGE) + if(HIPFILE_USE_CODE_COVERAGE) target_compile_options(${target} PRIVATE $<$:-fprofile-instr-generate -fcoverage-mapping>) target_link_options(${target} PRIVATE $<$:-fprofile-instr-generate>) target_link_options(${target} PRIVATE $<$:-fprofile-instr-generate>) endif() endforeach() - if(AIS_USE_SANITIZERS OR AIS_USE_THREAD_SANITIZER) - ais_add_sanitizers(${target}) + if(HIPFILE_USE_SANITIZERS OR HIPFILE_USE_THREAD_SANITIZER) + hipfile_add_sanitizers(${target}) endif() if(NOT BUILD_TESTING) diff --git a/projects/hipfile/cmake/HipFileDeprecatedAISOptions.cmake b/projects/hipfile/cmake/HipFileDeprecatedAISOptions.cmake new file mode 100644 index 000000000000..b2da8224a327 --- /dev/null +++ b/projects/hipfile/cmake/HipFileDeprecatedAISOptions.cmake @@ -0,0 +1,69 @@ +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# +# SPDX-License-Identifier: MIT + +include_guard(GLOBAL) + +#----------------------------------------------------------------------------- +# Backward compatibility for the deprecated AIS_* CMake options +# +# The public hipFile CMake options were renamed from AIS_ to +# HIPFILE_. This module keeps the old names working while downstream +# projects (e.g., TheRock) and build scripts switch over. +# +# For each deprecated option that is defined, either on the command line or in +# an existing CMakeCache.txt, the value is copied to the new option and the old +# cache entry is removed. Since the old entry is removed on every configure, a +# defined AIS_ is always a fresh request, so it overrides any +# HIPFILE_ value that is already in the cache. +# +# The new cache entry is given the UNINITIALIZED type, which makes it look +# exactly like -DHIPFILE_= on the command line. The option() or +# set(... CACHE ...) that declares the option later fills in the real type and +# help string. +# +# This must be included BEFORE any of the HIPFILE_ options are declared. +# +# To remove this compatibility layer, delete: +# - This file +# - The include() of this file in the top-level CMakeLists.txt +# - The test/cmake/ais-compat directory +# - The hipfile_ais_compat test in test/CMakeLists.txt +#----------------------------------------------------------------------------- + +# The deprecated options. Each new name is the old name with the AIS_ prefix +# replaced by HIPFILE_. +set(HIPFILE_DEPRECATED_AIS_OPTIONS + AIS_BUILD_DOCS + AIS_CAPABLE_DIR + AIS_CLANG_TIDY_EXECUTABLE + AIS_CXX_STANDARD + AIS_INSTALL_EXAMPLES + AIS_INSTALL_TESTS + AIS_INSTALL_TOOLS + AIS_USE_CLANG_TIDY + AIS_USE_CODE_COVERAGE + AIS_USE_IWYU + AIS_USE_SANITIZERS + AIS_USE_THREAD_SANITIZER + AIS_WARN_UNSAFE_BUFFER_OPS +) + +# A function is used so the loop variables don't leak into the caller's scope. +# The cache is global, so the cache operations are still visible to the caller. +function(hipfile_map_deprecated_ais_options) + foreach(old_name IN LISTS HIPFILE_DEPRECATED_AIS_OPTIONS) + if(NOT DEFINED ${old_name}) + continue() + endif() + string(REGEX REPLACE "^AIS_" "HIPFILE_" new_name "${old_name}") + message(DEPRECATION + "${old_name} is deprecated, use ${new_name} instead. " + "Setting ${new_name} to '${${old_name}}'.") + set(${new_name} "${${old_name}}" CACHE STRING "" FORCE) + set_property(CACHE ${new_name} PROPERTY TYPE UNINITIALIZED) + unset(${old_name} CACHE) + endforeach() +endfunction() + +hipfile_map_deprecated_ais_options() diff --git a/projects/hipfile/cmake/AISDocumentation.cmake b/projects/hipfile/cmake/HipFileDocumentation.cmake similarity index 60% rename from projects/hipfile/cmake/AISDocumentation.cmake rename to projects/hipfile/cmake/HipFileDocumentation.cmake index 41de29a83dd4..1397e2890585 100644 --- a/projects/hipfile/cmake/AISDocumentation.cmake +++ b/projects/hipfile/cmake/HipFileDocumentation.cmake @@ -7,30 +7,30 @@ include_guard(GLOBAL) #----------------------------------------------------------------------------- # Option to build the hipFile API documentation #----------------------------------------------------------------------------- -option(AIS_BUILD_DOCS "Build hipFile API docs (requires Doxygen)" OFF) +option(HIPFILE_BUILD_DOCS "Build hipFile API docs (requires Doxygen)" OFF) -if(AIS_BUILD_DOCS) +if(HIPFILE_BUILD_DOCS) find_package(Doxygen REQUIRED) # Set Doxygen input (pasted into Doxyfile.in) - set(AIS_DOXYFILE_INPUT "${HIPFILE_ROOT_PATH}/include") + set(HIPFILE_DOXYFILE_INPUT "${HIPFILE_ROOT_PATH}/include") # Set the path to the documentation - set(AIS_DOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/docs") + set(HIPFILE_DOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/docs") # Set the Doxyfile install location - set(AIS_DOXYFILE ${AIS_DOC_PATH}/Doxyfile) + set(HIPFILE_DOXYFILE ${HIPFILE_DOC_PATH}/Doxyfile) # Create the Doxyfile from the input file - configure_file("docs/doxygen/Doxyfile.in" ${AIS_DOXYFILE}) + configure_file("docs/doxygen/Doxyfile.in" ${HIPFILE_DOXYFILE}) # Set the output directory - set(DOXYGEN_OUT ${AIS_DOC_PATH}) + set(DOXYGEN_OUT ${HIPFILE_DOC_PATH}) # Configure the documentation build add_custom_target("doc" - COMMAND ${DOXYGEN_EXECUTABLE} ${AIS_DOXYFILE} - WORKING_DIRECTORY ${AIS_DOC_PATH} + COMMAND ${DOXYGEN_EXECUTABLE} ${HIPFILE_DOXYFILE} + WORKING_DIRECTORY ${HIPFILE_DOC_PATH} COMMENT "Generating hipFile API documentation with Doxygen" VERBATIM ) diff --git a/projects/hipfile/cmake/AISGNUCompilerOptions.cmake b/projects/hipfile/cmake/HipFileGNUCompilerOptions.cmake similarity index 98% rename from projects/hipfile/cmake/AISGNUCompilerOptions.cmake rename to projects/hipfile/cmake/HipFileGNUCompilerOptions.cmake index 683b488ab0ca..07b12ffe28ba 100644 --- a/projects/hipfile/cmake/AISGNUCompilerOptions.cmake +++ b/projects/hipfile/cmake/HipFileGNUCompilerOptions.cmake @@ -6,7 +6,7 @@ # # https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html -function(get_ais_gnu_warning_flags outvar compiler_version) +function(get_hipfile_gnu_warning_flags outvar compiler_version) # Warning flags for g++ 9 and earlier set(flags diff --git a/projects/hipfile/cmake/AISIWYU.cmake b/projects/hipfile/cmake/HipFileIWYU.cmake similarity index 94% rename from projects/hipfile/cmake/AISIWYU.cmake rename to projects/hipfile/cmake/HipFileIWYU.cmake index db7652d9dce6..77713a9ea40f 100644 --- a/projects/hipfile/cmake/AISIWYU.cmake +++ b/projects/hipfile/cmake/HipFileIWYU.cmake @@ -9,9 +9,9 @@ include_guard(GLOBAL) # # When this option is enabled, compilation will emit IWYU suggestions. #----------------------------------------------------------------------------- -option(AIS_USE_IWYU "Run include-what-you-use when compiling" OFF) +option(HIPFILE_USE_IWYU "Run include-what-you-use when compiling" OFF) -if(AIS_USE_IWYU) +if(HIPFILE_USE_IWYU) find_program(IWYU_EXE NAMES include-what-you-use REQUIRED) # GoogleTest/GoogleMock expose their public API through umbrella headers diff --git a/projects/hipfile/cmake/AISInstall.cmake b/projects/hipfile/cmake/HipFileInstall.cmake similarity index 98% rename from projects/hipfile/cmake/AISInstall.cmake rename to projects/hipfile/cmake/HipFileInstall.cmake index f79a6563a9b5..35c891674472 100644 --- a/projects/hipfile/cmake/AISInstall.cmake +++ b/projects/hipfile/cmake/HipFileInstall.cmake @@ -20,7 +20,7 @@ rocm_install( ) # Install AIS tools -if(AIS_INSTALL_TOOLS) +if(HIPFILE_INSTALL_TOOLS) install(PROGRAMS tools/ais-check/ais-check DESTINATION bin) if(TARGET ais-stats) install(TARGETS ais-stats DESTINATION bin) @@ -30,7 +30,7 @@ endif() # Install example code # Since the input DIRECTORY is `examples` don't include it in # the DESTINATION path or you'll get `examples/examples/*` output -if(AIS_INSTALL_EXAMPLES) +if(HIPFILE_INSTALL_EXAMPLES) install( DIRECTORY ${HIPFILE_ROOT_PATH}/examples DESTINATION share/doc/${CMAKE_PROJECT_NAME} @@ -44,14 +44,15 @@ if(AIS_INSTALL_EXAMPLES) # The CMakeLists.txt files in the examples tree use our build # flags and infrastructure so we can ensure they are well-vetted # and these can't be installed. - set(AIS_EXAMPLE_DIRS + set(HIPFILE_EXAMPLE_DIRS aiscp api common basics + batch async ) - foreach(example_dir ${AIS_EXAMPLE_DIRS}) + foreach(example_dir ${HIPFILE_EXAMPLE_DIRS}) configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/examples/${example_dir}/CMakeLists.install.cmake" "examples/${example_dir}/CMakeLists.txt" diff --git a/projects/hipfile/cmake/AISInstallTests.cmake b/projects/hipfile/cmake/HipFileInstallTests.cmake similarity index 98% rename from projects/hipfile/cmake/AISInstallTests.cmake rename to projects/hipfile/cmake/HipFileInstallTests.cmake index 20b6aeea7158..ca215721d883 100644 --- a/projects/hipfile/cmake/AISInstallTests.cmake +++ b/projects/hipfile/cmake/HipFileInstallTests.cmake @@ -114,7 +114,7 @@ install( COMPONENT tests ) file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/hipfile_test_CTestTestfile.cmake" - "# hipFile unit tests -- relocatable. Generated by AISInstallTests.cmake.\n" + "# hipFile unit tests -- relocatable. Generated by HipFileInstallTests.cmake.\n" "include(script/hipfile_discover.cmake)\n" ) install( diff --git a/projects/hipfile/cmake/AISSanitizers.cmake b/projects/hipfile/cmake/HipFileSanitizers.cmake similarity index 78% rename from projects/hipfile/cmake/AISSanitizers.cmake rename to projects/hipfile/cmake/HipFileSanitizers.cmake index 36a2a2f78429..c35660ba3e93 100644 --- a/projects/hipfile/cmake/AISSanitizers.cmake +++ b/projects/hipfile/cmake/HipFileSanitizers.cmake @@ -4,20 +4,20 @@ include_guard(GLOBAL) -option(AIS_USE_SANITIZERS "Build with sanitizers (clang only - see docs for a list)" OFF) -option(AIS_USE_THREAD_SANITIZER "Build with -fsanitize=thread (clang only - not compatible with AIS_USE_SANITIZERS)" OFF) +option(HIPFILE_USE_SANITIZERS "Build with sanitizers (clang only - see docs for a list)" OFF) +option(HIPFILE_USE_THREAD_SANITIZER "Build with -fsanitize=thread (clang only - not compatible with HIPFILE_USE_SANITIZERS)" OFF) -if(AIS_USE_THREAD_SANITIZER) +if(HIPFILE_USE_THREAD_SANITIZER) message(WARNING "TSAN has known problems with higher levels of entropy, try using `sudo sysctl vm.mmap_rnd_bits=28` if you encounter errors concerning unexpected memory mappings.") endif() -function(ais_add_sanitizers target) +function(hipfile_add_sanitizers target) if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") message(FATAL_ERROR "The sanitizers are only available on clang") endif() - if(AIS_USE_SANITIZERS AND AIS_USE_THREAD_SANITIZER) - message(FATAL_ERROR "AIS_USE_SANITIZERS is not compatible with AIS_USE_THREAD_SANITIZER") + if(HIPFILE_USE_SANITIZERS AND HIPFILE_USE_THREAD_SANITIZER) + message(FATAL_ERROR "HIPFILE_USE_SANITIZERS is not compatible with HIPFILE_USE_THREAD_SANITIZER") endif() get_target_property(aliased_target "${target}" ALIASED_TARGET) @@ -25,7 +25,7 @@ function(ais_add_sanitizers target) set(target "${aliased_target}") endif() - if(AIS_USE_SANITIZERS) + if(HIPFILE_USE_SANITIZERS) set(SANITIZER_LIST address,float-divide-by-zero,integer,leak,local-bounds,nullability,undefined,vptr) # unsigned-integer-overflow and unsigned-shift-base are not UB. # Keep -fsanitize=integer enabled, but disable both checks. @@ -43,7 +43,7 @@ function(ais_add_sanitizers target) ) endif() - if(AIS_USE_THREAD_SANITIZER) + if(HIPFILE_USE_THREAD_SANITIZER) target_compile_options(${target} PRIVATE $<$:-fsanitize=thread> ) diff --git a/projects/hipfile/cmake/AISSetCoverageFile.cmake b/projects/hipfile/cmake/HipFileSetCoverageFile.cmake similarity index 88% rename from projects/hipfile/cmake/AISSetCoverageFile.cmake rename to projects/hipfile/cmake/HipFileSetCoverageFile.cmake index deb963fa2dd3..400d613d16cc 100644 --- a/projects/hipfile/cmake/AISSetCoverageFile.cmake +++ b/projects/hipfile/cmake/HipFileSetCoverageFile.cmake @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: MIT -function(ais_set_coverage_file test_list_name directory) +function(hipfile_set_coverage_file test_list_name directory) set(test_counter 0) set(test_list "${${test_list_name}}") foreach(test IN LISTS test_list) diff --git a/projects/hipfile/cmake/AISUseGTest.cmake b/projects/hipfile/cmake/HipFileUseGTest.cmake similarity index 85% rename from projects/hipfile/cmake/AISUseGTest.cmake rename to projects/hipfile/cmake/HipFileUseGTest.cmake index 8b385ac53a3b..6e859410e1bf 100644 --- a/projects/hipfile/cmake/AISUseGTest.cmake +++ b/projects/hipfile/cmake/HipFileUseGTest.cmake @@ -6,7 +6,7 @@ # run exactly once. include_guard(GLOBAL) -include(AISSanitizers) +include(HipFileSanitizers) include(FetchContent) # GoogleTest's ThreadLocal implementation uses POSIX thread-key APIs @@ -18,10 +18,10 @@ find_package(Threads REQUIRED) # Decide whether to check for a system GoogleTest install first. When building # with the sanitizers, we HAVE to build GoogleTest from source. -if(AIS_USE_SANITIZERS OR AIS_USE_THREAD_SANITIZER) - set(AIS_GTEST_TRY_SYSTEM FALSE) +if(HIPFILE_USE_SANITIZERS OR HIPFILE_USE_THREAD_SANITIZER) + set(HIPFILE_GTEST_TRY_SYSTEM FALSE) else() - set(AIS_GTEST_TRY_SYSTEM TRUE) + set(HIPFILE_GTEST_TRY_SYSTEM TRUE) endif() set(INSTALL_GTEST OFF CACHE BOOL "Don't install GoogleTest") @@ -31,14 +31,14 @@ set(GTEST_HAS_ABSL OFF CACHE BOOL "Don't use Abseil for GoogleTest") # FetchContent_Declare, since FIND_PACKAGE_ARGS was introduced in CMake # version 3.24, which is unavailable on some operating systems: # https://cmake.org/cmake/help/latest/module/FetchContent.html -if(AIS_GTEST_TRY_SYSTEM) +if(HIPFILE_GTEST_TRY_SYSTEM) find_package(GTest QUIET) endif() if(NOT GTest_FOUND) # CMAKE_CXX_CLANG_TIDY runs on all CMake targets after it is set, including those # fetched via FetchContent. clang-tidy should not run on 3rd party code at all. - set(_ais_saved_cxx_clang_tidy "${CMAKE_CXX_CLANG_TIDY}") + set(_hipfile_saved_cxx_clang_tidy "${CMAKE_CXX_CLANG_TIDY}") unset(CMAKE_CXX_CLANG_TIDY) # lint_cmake: -readability/wonkycase FetchContent_Declare( @@ -50,8 +50,8 @@ if(NOT GTest_FOUND) ) FetchContent_MakeAvailable(googletest) # lint_cmake: +readability/wonkycase - set(CMAKE_CXX_CLANG_TIDY "${_ais_saved_cxx_clang_tidy}") - unset(_ais_saved_cxx_clang_tidy) + set(CMAKE_CXX_CLANG_TIDY "${_hipfile_saved_cxx_clang_tidy}") + unset(_hipfile_saved_cxx_clang_tidy) endif() if(googletest_SOURCE_DIR) @@ -60,9 +60,9 @@ else() message(STATUS "Using system GoogleTest") endif() -if(AIS_USE_SANITIZERS OR AIS_USE_THREAD_SANITIZER) - ais_add_sanitizers(GTest::gtest) - ais_add_sanitizers(GTest::gmock) +if(HIPFILE_USE_SANITIZERS OR HIPFILE_USE_THREAD_SANITIZER) + hipfile_add_sanitizers(GTest::gtest) + hipfile_add_sanitizers(GTest::gmock) endif() # Propagate pthread linkage to anything that links a GTest target, so the @@ -106,17 +106,17 @@ include(GoogleTest) # Absolute path to the LeakSanitizer suppression file. Only applied when # building with the (non-TSAN) sanitizers. -set(AIS_LSAN_SUPPRESSIONS_FILE "${HIPFILE_ROOT_PATH}/cmake/lsan-suppressions.supp") +set(HIPFILE_LSAN_SUPPRESSIONS_FILE "${HIPFILE_ROOT_PATH}/cmake/lsan-suppressions.supp") -function(ais_gtest_discover_tests target) +function(hipfile_gtest_discover_tests target) set(forwarded_args ${ARGV}) # When the address sanitizer is on, force every discovered test to load the # LSan suppression file via LSAN_OPTIONS. We splice an ENVIRONMENT entry into # the PROPERTIES list that gtest_discover_tests applies to each test case, so # the suppression travels with the test regardless of how ctest is invoked. - if(AIS_USE_SANITIZERS) - set(lsan_env "LSAN_OPTIONS=suppressions=${AIS_LSAN_SUPPRESSIONS_FILE}") + if(HIPFILE_USE_SANITIZERS) + set(lsan_env "LSAN_OPTIONS=suppressions=${HIPFILE_LSAN_SUPPRESSIONS_FILE}") list(FIND forwarded_args "PROPERTIES" props_idx) if(props_idx EQUAL -1) list(APPEND forwarded_args PROPERTIES ENVIRONMENT "${lsan_env}") @@ -130,7 +130,7 @@ function(ais_gtest_discover_tests target) cmake_language(CALL gtest_discover_tests ${forwarded_args}) - if(AIS_USE_CODE_COVERAGE) + if(HIPFILE_USE_CODE_COVERAGE) set(options) set(oneValueArgs TEST_LIST) set(multiValueArgs) @@ -146,12 +146,12 @@ function(ais_gtest_discover_tests target) set_property( DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES - "${HIPFILE_ROOT_PATH}/cmake/AISSetCoverageFile.cmake" + "${HIPFILE_ROOT_PATH}/cmake/HipFileSetCoverageFile.cmake" "${coverage_include_file}" ) file(WRITE "${coverage_include_file}" - "ais_set_coverage_file(\"${arg_TEST_LIST}\" \"${CMAKE_CURRENT_BINARY_DIR}\")" + "hipfile_set_coverage_file(\"${arg_TEST_LIST}\" \"${CMAKE_CURRENT_BINARY_DIR}\")" ) endif() endfunction() diff --git a/projects/hipfile/cmake/AISUseROCmCMake.cmake b/projects/hipfile/cmake/HipFileUseROCmCMake.cmake similarity index 100% rename from projects/hipfile/cmake/AISUseROCmCMake.cmake rename to projects/hipfile/cmake/HipFileUseROCmCMake.cmake diff --git a/projects/hipfile/cmake/hipfile_discover.cmake.in b/projects/hipfile/cmake/hipfile_discover.cmake.in index 1c3cdff3d6ab..1d7c391ec356 100644 --- a/projects/hipfile/cmake/hipfile_discover.cmake.in +++ b/projects/hipfile/cmake/hipfile_discover.cmake.in @@ -10,7 +10,7 @@ # honors add_test/subdirs/include/set_tests_properties). # # All paths are resolved RELATIVE to this script's own location, so the tree is -# fully relocatable. Generated from hipfile_discover.cmake.in by AISInstallTests. +# fully relocatable. Generated from hipfile_discover.cmake.in by HipFileInstallTests. # Locate ourselves: /script/hipfile_discover.cmake -> . file(REAL_PATH "${CMAKE_CURRENT_LIST_FILE}" _hipfile_self) @@ -93,5 +93,5 @@ macro(_hipfile_discover_binary _bin_name _label_suffix) endforeach() endmacro() -# Per-binary discovery calls, baked in by AISInstallTests at configure time. +# Per-binary discovery calls, baked in by HipFileInstallTests at configure time. @_discover_blocks@ diff --git a/projects/hipfile/docs/conf.py b/projects/hipfile/docs/conf.py index 92ff37847db5..9d978b912912 100644 --- a/projects/hipfile/docs/conf.py +++ b/projects/hipfile/docs/conf.py @@ -20,9 +20,9 @@ def _cmake_version_part(part): - match = re.search(rf"set\(AIS_LIBRARY_{part}\s+(\d+)\)", _cmake) + match = re.search(rf"set\(HIPFILE_LIBRARY_{part}\s+(\d+)\)", _cmake) if not match: - raise ValueError(f"AIS_LIBRARY_{part} not found in CMakeLists.txt") + raise ValueError(f"HIPFILE_LIBRARY_{part} not found in CMakeLists.txt") return match.group(1) diff --git a/projects/hipfile/docs/doxygen/Doxyfile.in b/projects/hipfile/docs/doxygen/Doxyfile.in index 92383278a7df..7c0637bec311 100644 --- a/projects/hipfile/docs/doxygen/Doxyfile.in +++ b/projects/hipfile/docs/doxygen/Doxyfile.in @@ -48,7 +48,7 @@ PROJECT_NAME = "hipFile" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @AIS_LIBRARY_VERSION@ +PROJECT_NUMBER = @HIPFILE_LIBRARY_VERSION@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -951,7 +951,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = "@AIS_DOXYFILE_INPUT@" +INPUT = "@HIPFILE_DOXYFILE_INPUT@" # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/projects/hipfile/docs/how-to/checking-system-compatibility.rst b/projects/hipfile/docs/how-to/checking-system-compatibility.rst index 1b83b8ceef9f..dbb1caa7762a 100644 --- a/projects/hipfile/docs/how-to/checking-system-compatibility.rst +++ b/projects/hipfile/docs/how-to/checking-system-compatibility.rst @@ -10,7 +10,7 @@ The ``ais-check`` tool reports whether the kernel, driver, runtime, and mounted meet hipFile fastpath requirements. If any component is missing or if no volume qualifies, hipFile uses the fallback path instead. -``ais-check`` is located under ``tools/ais-check`` in the hipFile source tree and is installed in the ``bin`` directory of your ROCm installation unless hipFile was built with ``AIS_INSTALL_TOOLS=OFF``. +``ais-check`` is located under ``tools/ais-check`` in the hipFile source tree and is installed in the ``bin`` directory of your ROCm installation unless hipFile was built with ``HIPFILE_INSTALL_TOOLS=OFF``. ``ais-check`` must be run as root to produce accurate information. diff --git a/projects/hipfile/docs/how-to/use-python-api.rst b/projects/hipfile/docs/how-to/use-python-api.rst index 6784bb3580e1..ef35e762ef5e 100644 --- a/projects/hipfile/docs/how-to/use-python-api.rst +++ b/projects/hipfile/docs/how-to/use-python-api.rst @@ -14,7 +14,7 @@ .. - python/hipfile/error.py .. - python/hipfile/enums.py .. - python/hipfile/properties.py -.. - python/main.py +.. - python/tests/main.py ******************* Use the Python API diff --git a/projects/hipfile/docs/index.rst b/projects/hipfile/docs/index.rst index cec1135781a7..fc7c94f78276 100644 --- a/projects/hipfile/docs/index.rst +++ b/projects/hipfile/docs/index.rst @@ -38,6 +38,7 @@ hipFile is delivered as part of `TheRock `_. Th * :doc:`Copy a file via GPU memory using hipFile ` * :doc:`Async multistream I/O ` + * :doc:`Batch I/O ` * :doc:`Query the hipFile version ` * :doc:`Perform GPU I/O with the Python bindings ` diff --git a/projects/hipfile/docs/install/build-from-source.rst b/projects/hipfile/docs/install/build-from-source.rst index 4d33cf8cd7c0..0c9acbbc1053 100644 --- a/projects/hipfile/docs/install/build-from-source.rst +++ b/projects/hipfile/docs/install/build-from-source.rst @@ -19,7 +19,7 @@ hipFile targets Linux with ROCm HIP. For supported GPUs and stack components, see :ref:`ROCm Core SDK components `. - CMake 3.21 or later -- A C++ compiler that matches the ``AIS_CXX_STANDARD`` value you select. The +- A C++ compiler that matches the ``HIPFILE_CXX_STANDARD`` value you select. The default is C++20. - HIP and HSA packages from ROCm so CMake can locate ``hip`` and ``hsa-runtime64`` - ``libmount`` from ``util-linux`` for mount metadata parsing @@ -98,19 +98,19 @@ exist for sanitizers, clang-tidy, and documentation generation. Inspect * - ``BUILD_SHARED_LIBS`` - ``ON`` - Builds shared libraries when ``ON`` and static libraries when ``OFF``. - * - ``AIS_CXX_STANDARD`` + * - ``HIPFILE_CXX_STANDARD`` - ``20`` - C++ dialect. Allowed values are ``20``, ``23``, and ``26``. - * - ``AIS_INSTALL_EXAMPLES`` + * - ``HIPFILE_INSTALL_EXAMPLES`` - ``ON`` - Installs sample binaries such as ``aiscp`` when enabled. - * - ``AIS_INSTALL_TOOLS`` + * - ``HIPFILE_INSTALL_TOOLS`` - ``ON`` - Installs host tools such as ``ais-stats`` when enabled on AMD builds. - * - ``AIS_BUILD_DOCS`` + * - ``HIPFILE_BUILD_DOCS`` - ``OFF`` - Enables the ``doc`` target when ``ON``. Requires Doxygen in your environment. - * - ``AIS_USE_CODE_COVERAGE`` + * - ``HIPFILE_USE_CODE_COVERAGE`` - ``OFF`` - Adds LLVM coverage instrumentation for Clang builds. * - ``BUILD_TESTING`` @@ -120,3 +120,9 @@ exist for sanitizers, clang-tidy, and documentation generation. Inspect - ``RelWithDebInfo`` - CMake build flavor. Other common values are ``Debug``, ``Release``, ``MinSizeRel``, and ``None``. + +.. note:: + + The ``HIPFILE_`` options were previously named with an ``AIS_`` prefix (for + example, ``AIS_CXX_STANDARD``). The ``AIS_`` names still work but are + deprecated, and CMake prints a deprecation warning when you use them. diff --git a/projects/hipfile/docs/sphinx/_toc.yml.in b/projects/hipfile/docs/sphinx/_toc.yml.in index 8a53bce47f61..819eeb835aef 100644 --- a/projects/hipfile/docs/sphinx/_toc.yml.in +++ b/projects/hipfile/docs/sphinx/_toc.yml.in @@ -30,6 +30,8 @@ subtrees: title: Copying a file - file: tutorials/async-multistream-io title: Asynchronous multistream I/O + - file: tutorials/batch-io + title: Batch I/O - file: tutorials/get-version title: Query the hipFile version - file: tutorials/python-gpu-io diff --git a/projects/hipfile/docs/troubleshooting/limitations.rst b/projects/hipfile/docs/troubleshooting/limitations.rst index b57d9989d399..7f74dbc29efd 100644 --- a/projects/hipfile/docs/troubleshooting/limitations.rst +++ b/projects/hipfile/docs/troubleshooting/limitations.rst @@ -25,3 +25,32 @@ LVM (Logical Volume Manager) volumes are supported for the fastpath when their underlying physical volumes are all local NVMe devices. A volume whose physical volumes include any non-NVMe or multipath device falls back to compatibility mode. + +File descriptor limits +====================== + +hipFile opens file descriptors of its own, which count toward the process's +open file limit (``RLIMIT_NOFILE``): + +- ``hipFileHandleRegister()`` opens a second file descriptor for each + registered file, one with ``O_DIRECT`` and one without. It is closed by + ``hipFileHandleDeregister()``. For more information, see + :doc:`/reference/hipFile-file-registration`. +- hipFile also opens file descriptors internally, some for the lifetime of + the process and some only temporarily. + +If the process reaches its open file limit, or the system-wide limit is +reached, hipFile API calls that need a new file descriptor, such as +``hipFileHandleRegister()``, return ``hipFileGetNewFDFailed``. + +To register a large number of files, allow about two file descriptors per +registered file plus the descriptors the application uses itself. Check the +current limits with: + +.. code-block:: none + + $ ulimit -Sn + $ ulimit -Hn + +The soft limit can be raised up to the hard limit, for example with +``ulimit -n `` before starting the application. diff --git a/projects/hipfile/docs/tutorials/async-multistream-io.rst b/projects/hipfile/docs/tutorials/async-multistream-io.rst index 5bee51319d61..13db68f12f73 100644 --- a/projects/hipfile/docs/tutorials/async-multistream-io.rst +++ b/projects/hipfile/docs/tutorials/async-multistream-io.rst @@ -33,96 +33,95 @@ Verify you have: Step-by-step walkthrough =========================== -Select the GPU and seed the input file --------------------------------------- +The walkthrough follows a complete file round trip, from seeding the input +file and registering per-stream GPU buffers and HIP streams to verifying the +output and cleaning up. Each stream reads and then writes a separate slice of +the file, allowing I/O on different slices to overlap. After synchronizing the +streams, the example checks byte counts and compares file hashes before +releasing the buffers and streams. -.. code-block:: cpp +#. **Select the GPU and seed the input file** - hip_err = hipSetDevice(gpu_id); + .. code-block:: cpp - if (seed_read_file(read_path, TOTAL_SIZE)) - return EXIT_FAILURE; + hip_err = hipSetDevice(gpu_id); -``seed_read_file()`` allocates a buffer where byte ``i`` is ``i & 0xFF``. It -writes that buffer to ``read_path`` with POSIX ``write()``, replacing any prior -file contents. The output size is ``NUM_STREAMS * SLICE_SIZE``. The defaults -use four streams and 1 MiB per slice, so 4 MiB total. + if (seed_read_file(read_path, TOTAL_SIZE)) + return EXIT_FAILURE; -Allocate per-stream GPU buffers and register them -------------------------------------------------- + ``seed_read_file()`` allocates a buffer where byte ``i`` is ``i & 0xFF``. It + writes that buffer to ``read_path`` with POSIX ``write()``, replacing any prior + file contents. The output size is ``NUM_STREAMS * SLICE_SIZE``. The defaults + use four streams and 1 MiB per slice, so 4 MiB total. -For each of the ``NUM_STREAMS`` streams, the example calls ``hipMalloc()`` to -allocate a GPU buffer. It registers that buffer with hipFile, creates a -non-blocking HIP stream, zeros the buffer on that stream, and registers the -stream with the driver. +#. **Allocate per-stream GPU buffers and register them** -Each buffer is registered separately with ``hipFileBufRegister()`` with ``flags`` -set to ``0``. Streams are created with the ``hipStreamNonBlocking`` flag so they -don't implicitly synchronize with the legacy default stream. + For each of the ``NUM_STREAMS`` streams, the example calls ``hipMalloc()`` to + allocate a GPU buffer. It registers that buffer with hipFile, creates a + non-blocking HIP stream, zeros the buffer on that stream, and registers the + stream with the driver. -``hipFileStreamRegister()`` takes flags that mark the properties that stay fixed -for every later submission on that stream: + Each buffer is registered separately with ``hipFileBufRegister()`` with ``flags`` + set to ``0``. Streams are created with the ``hipStreamNonBlocking`` flag so they + don't implicitly synchronize with the legacy default stream. -- ``HIPFILE_STREAM_FIXED_BUF_OFFSET``: the buffer offset cannot be changed after submission. -- ``HIPFILE_STREAM_FIXED_FILE_OFFSET``: the file offset cannot be changed after submission. -- ``HIPFILE_STREAM_FIXED_FILE_SIZE``: the transfer size cannot be changed after submission. -- ``HIPFILE_STREAM_PAGE_ALIGNED_INPUTS``: all offsets and sizes are 4 KiB - aligned. + ``hipFileStreamRegister()`` takes flags that mark the properties that stay fixed + for every later submission on that stream: -The ``slice_state`` struct holds sizes and offsets as named fields. The -asynchronous API reads those values by pointer and dereferences them at -completion time. You must keep them valid until ``hipStreamSynchronize()`` -finishes. + - ``HIPFILE_STREAM_FIXED_BUF_OFFSET``: the buffer offset cannot be changed after submission. + - ``HIPFILE_STREAM_FIXED_FILE_OFFSET``: the file offset cannot be changed after submission. + - ``HIPFILE_STREAM_FIXED_FILE_SIZE``: the transfer size cannot be changed after submission. + - ``HIPFILE_STREAM_PAGE_ALIGNED_INPUTS``: all offsets and sizes are 4 KiB + aligned. -Open input and output files ---------------------------- + The ``slice_state`` struct holds sizes and offsets as named fields. The + asynchronous API reads those values by pointer and dereferences them at + completion time. You must keep them valid until ``hipStreamSynchronize()`` + finishes. -The ``open_file()`` helper from ``examples_common`` calls ``open()`` and -then ``hipFileHandleRegister()``. One pair of file descriptors is shared across -all streams. +#. **Open input and output files** -Submit reads on all streams ---------------------------- + The ``open_file()`` helper from ``examples_common`` calls ``open()`` and + then ``hipFileHandleRegister()``. One pair of file descriptors is shared across + all streams. -Each ``hipFileReadAsync()`` call enqueues a read of ``SLICE_SIZE`` bytes at file -offset ``i * SLICE_SIZE`` into the matching GPU buffer. The streams are -independent, so those reads can run concurrently. +#. **Submit reads on all streams** -Submit writes on all streams ----------------------------- + Each ``hipFileReadAsync()`` call enqueues a read of ``SLICE_SIZE`` bytes at file + offset ``i * SLICE_SIZE`` into the matching GPU buffer. The streams are + independent, so those reads can run concurrently. -Each write uses the same stream as its matching read. HIP stream semantics run -operations on one stream in submission order. The write for slice ``i`` sees -the read for slice ``i`` finish without an extra host synchronization call -between them. Across streams, reads and writes can overlap freely. +#. **Submit writes on all streams** -Synchronize and verify byte counts ----------------------------------- + Each write uses the same stream as its matching read. HIP stream semantics run + operations on one stream in submission order. The write for slice ``i`` sees + the read for slice ``i`` finish without an extra host synchronization call + between them. Across streams, reads and writes can overlap freely. -After synchronization, each ``slice_state`` instance reports ``bytes_read`` and -``bytes_written``. The example checks that every slice moved ``SLICE_SIZE`` -bytes. +#. **Synchronize and verify byte counts** -Verify the output file hash ---------------------------- + After synchronization, each ``slice_state`` instance reports ``bytes_read`` and + ``bytes_written``. The example checks that every slice moved ``SLICE_SIZE`` + bytes. -``verify_files_match()`` hashes the first ``TOTAL_SIZE`` bytes of both files -with FNV-1a and compares the digests. Matching hashes mean the round trip was -lossless. +#. **Verify the output file hash** -Clean up resources ------------------- + ``verify_files_match()`` hashes the first ``TOTAL_SIZE`` bytes of both files + with FNV-1a and compares the digests. Matching hashes mean the round trip was + lossless. -Teardown reverses setup for each slice: +#. **Clean up resources** -1. ``hipFileStreamDeregister()``: remove the stream from hipFile. -2. ``hipStreamDestroy()``: destroy the HIP stream. -3. ``hipFileBufDeregister()``: remove the GPU buffer from hipFile. -4. ``hipFree()``: free the device memory. + Teardown reverses setup for each slice: -The booleans ``stream_registered``, ``stream_created``, and ``buf_registered`` -track partial setup. If the setup loop fails partway through, only resources -that were created get torn down. + 1. ``hipFileStreamDeregister()``: remove the stream from hipFile. + 2. ``hipStreamDestroy()``: destroy the HIP stream. + 3. ``hipFileBufDeregister()``: remove the GPU buffer from hipFile. + 4. ``hipFree()``: free the device memory. + + The booleans ``stream_registered``, ``stream_created``, and ``buf_registered`` + track partial setup. If the setup loop fails partway through, only resources + that were created get torn down. Stream ordering ================= diff --git a/projects/hipfile/docs/tutorials/batch-io.rst b/projects/hipfile/docs/tutorials/batch-io.rst new file mode 100644 index 000000000000..933e865bbd39 --- /dev/null +++ b/projects/hipfile/docs/tutorials/batch-io.rst @@ -0,0 +1,220 @@ +.. meta:: + :description: Tutorial walking through batch I/O using hipFile. + :keywords: hipFile, ROCm, batch, GPU I/O, hipFileBatchIOSubmit, hipFileBatchIOGetStatus, hipFileBatchIOSetUp, tutorial + +********* +Batch I/O +********* + +`batch-roundtrip.cpp +`_ +reads a file into a registered GPU buffer using batch IO operations, writes +the buffer back out in a second batch phase, and verifies that the output matches +the input. 4KiB IOs are used and the batch queue is refilled by submitting +additional batch IO operations as IO completions are received. + +When to use this pattern +========================= + +Use batch I/O when you need to: + +- Amortize submission overhead across many small transfers instead of having + per-request overhead. +- Keep the storage device busy with a bounded number of outstanding requests. +- Process completions as they arrive, in whatever order the device returns them. + +Prerequisites +=============== + +Verify you have: + +- A working hipFile installation. See :doc:`/install/install`. +- An AMD GPU with ROCm and hipFile support. +- A file system that supports ``O_DIRECT``, for example, ext4 or XFS. +- The ``examples_common`` helper library shipped with hipFile under + ``examples/common/``. + +Step-by-step walkthrough +=========================== + +The walkthrough follows a complete file round trip, from creating the input +file and registering resources to verifying the output and cleaning up. Reads +and writes run in separate phases, each using a bounded window of 4 KiB +requests that is refilled as completions arrive. + +#. **Parse arguments and seed the input file** + + The example takes the payload size and the batch capacity on the command line: + + .. code-block:: cpp + + payload_size = parse_integral(argv[3]); + batch_capacity = parse_integral(argv[4]); + + ``batch_capacity`` must be between ``1`` and ``128``. + + ``seed_read_file()`` then writes ``payload_size`` bytes where byte ``i`` is + ``i & 0xFF`` to ``read_path`` with POSIX ``write()``, replacing any prior file + contents. + +#. **Allocate and register one GPU buffer** + + .. code-block:: cpp + + const size_t buffer_size = align_up(payload_size, BATCH_IO_SIZE); + + hip_err = hipMalloc(&device_buffer, buffer_size); + hipfile_err = hipFileBufRegister(device_buffer, buffer_size, 0); + + The buffer holds the whole file rounded up to a 4 KiB boundary, so every request + transfers a full ``BATCH_IO_SIZE`` block without a partial tail. All requests in + both phases share this one registered buffer and address disjoint regions of it + through ``devPtr_offset``. + +#. **Open input and output files** + + The ``open_file()`` helper from ``examples_common`` calls ``open()`` with + ``O_DIRECT`` and then ``hipFileHandleRegister()``. Each phase uses one file + handle for all of its requests. + +#. **Create the batch context** + + .. code-block:: cpp + + hipfile_err = hipFileBatchIOSetUp(&batch_handle, batch_capacity); + + ``batch_capacity`` is the maximum number of requests that can be in flight on + this handle at one time. The same handle is reused for the read phase and the + write phase. Each phase drains completely before the next one starts. + +#. **Describe a request** + + ``configure_request()`` fills in one ``hipFileIOParams_t`` for chunk + ``chunk_index``: + + .. code-block:: cpp + + request.operation.mode = hipFileBatch; + request.operation.u.batch.devPtr_base = device_buffer; + request.operation.u.batch.file_offset = static_cast(operation_offset); + request.operation.u.batch.devPtr_offset = static_cast(operation_offset); + request.operation.u.batch.size = BATCH_IO_SIZE; + request.operation.fh = file_handle; + request.operation.opcode = opcode; + request.operation.cookie = request.cookie.get(); + + ``mode`` must be ``hipFileBatch``. The ``cookie`` is an opaque pointer that + hipFile hands back in the completion event. The example passes a pointer to + the ``BatchCookie`` holding the chunk index and the byte count that chunk + should transfer. Because the cookie is the only link between an event and + its request, it must stay valid until that request completes. + + Reads at the tail of the file expect fewer than ``BATCH_IO_SIZE`` bytes, so + ``expected_bytes`` is clamped to the bytes remaining in the payload. Writes + always expect a full block because the padded buffer is written out in full and + truncated afterwards. + +#. **Fill the window and submit** + + ``run_batch_phase()`` starts with up to ``batch_capacity`` configured requests + and submits every request that isn't already in flight: + + .. code-block:: cpp + + hipfile_err = hipFileBatchIOSubmit(batch_handle, submissions.size(), + submissions.data(), /*flags=*/0); + + ``hipFileBatchIOSubmit()`` takes an array of parameters, so one call can enqueue + many requests. The number of in-flight requests plus the number being submitted + must not exceed the capacity passed to ``hipFileBatchIOSetUp()``. + +#. **Wait for completions** + + .. code-block:: cpp + + unsigned nr = batch_capacity; + hipfile_err = hipFileBatchIOGetStatus(batch_handle, /*min_nr=*/1, &nr, + events.data(), /*timeout=*/nullptr); + + ``min_nr`` is the minimum number of events to wait for. ``nr`` is an in/out + parameter that carries the event array capacity in and the number of events + returned out. A ``nullptr`` timeout blocks until at least ``min_nr`` events are + available. Passing ``min_nr`` of ``1`` returns as soon as any request finishes, + which is what lets the example refill the window promptly. + +#. **Check each event and refill the window** + + For every returned event, the example finds the matching request by cookie, then + checks both the status and the transferred byte count: + + .. code-block:: cpp + + if (event.status != hipFileComplete) { /* request failed */ } + if (event.ret != cookie->expected_bytes) { /* short transfer */ } + + The ``hipFileComplete`` status will be returned for successful requests. + ``hipFileFailed`` or ``hipFileCanceled`` will be returned for failed or + canceled events. ``event.ret`` is the byte count for that request, which + can be short even when the status is ``hipFileComplete``. + + If chunks remain, the completed slot is reconfigured for the next chunk index + and resubmitted on the following loop iteration. Otherwise the slot is swapped + to the end of the request vector and popped, shrinking the window. The phase + ends when no request slots are left. + +#. **Finish the round trip** + + All reads complete before any writes are submitted, so the write phase always + sees a fully populated buffer. After the write phase, ``ftruncate()`` cuts the + output file back from the 4 KiB-aligned size to ``payload_size``, and + ``verify_files_match()`` hashes the first ``payload_size`` bytes of both files + with FNV-1a and compares the digests. Matching hashes mean the round trip was + lossless. + +#. **Clean up resources** + + Teardown reverses setup: + + 1. ``hipFileBatchIODestroy()``: destroy the batch context. + 2. ``close_file()``: deregister and close each file handle. + 3. ``hipFileBufDeregister()``: remove the GPU buffer from hipFile. + 4. ``hipFree()``: free the device memory. + + The flags ``read_handle_open``, ``write_handle_open``, and ``buffer_registered`` + track partial setup. If setup fails partway through, only resources that were + created get torn down. + +Completion ordering +==================== + +Batch requests carry no ordering guarantees: + +- Events can come back in any order, and one ``hipFileBatchIOGetStatus()`` call + can return anywhere from ``min_nr`` to ``nr`` events. +- Requests in a batch can overlap in time, so two requests must not modify the + same file or buffer region. +- Ordering between phases has to be enforced by the application. This example + drains the read phase entirely before submitting any writes. + +Increasing ``BATCH_SIZE`` widens the window and raises the number of concurrent +requests. Each request touches a disjoint 4 KiB region of the file and of the +buffer, so no cross-request coordination is needed. + +Running the example +======================= + +.. code:: shell + + ./batch-roundtrip /path/to/readfile /path/to/writefile FILE_SIZE BATCH_SIZE [GPUID] + +For example, a 4 MiB payload with 16 requests in flight: + +.. code:: shell + + ./batch-roundtrip /path/to/readfile /path/to/writefile 4194304 16 + +On success, the program prints: + +.. code-block:: text + + OK /path/to/readfile == /path/to/writefile (4194304 bytes, hash 0x...) diff --git a/projects/hipfile/docs/tutorials/copy-a-file.rst b/projects/hipfile/docs/tutorials/copy-a-file.rst index 54bee2a2b008..dfe6be9a3086 100644 --- a/projects/hipfile/docs/tutorials/copy-a-file.rst +++ b/projects/hipfile/docs/tutorials/copy-a-file.rst @@ -56,195 +56,194 @@ Both paths should sit on a file system that supports ``O_DIRECT``. If the source Step-by-step walkthrough =========================== -Define the chunk size limit ---------------------------- +The walkthrough follows a synchronous file copy through GPU memory, from +setting the chunk size and registering file handles to copying data and +releasing resources. It covers buffer allocation, a read/write loop with error +handling, and trimming the destination to the source size after block-aligned +writes. -``AISCP_CHUNK_SIZE`` defaults to ``0x7ffff000``, roughly 2 GiB minus one page. That value matches the Linux kernel ``MAX_RW_COUNT`` cap on a single ``read()`` or ``write()``-sized transfer. Each loop iteration moves at most that many bytes. +#. **Define the chunk size limit** -.. code-block:: cpp + ``AISCP_CHUNK_SIZE`` defaults to ``0x7ffff000``, roughly 2 GiB minus one page. That value matches the Linux kernel ``MAX_RW_COUNT`` cap on a single ``read()`` or ``write()``-sized transfer. Each loop iteration moves at most that many bytes. - #ifndef AISCP_CHUNK_SIZE - #define AISCP_CHUNK_SIZE 0x7ffff000LU - #endif + .. code-block:: cpp -You can override the macro at compile time if smaller chunks help limit GPU buffer size. + #ifndef AISCP_CHUNK_SIZE + #define AISCP_CHUNK_SIZE 0x7ffff000LU + #endif -Open and register files ------------------------ + You can override the macro at compile time if smaller chunks help limit GPU buffer size. -Both paths use ``O_DIRECT`` so hipFile can use its direct GPU I/O fast path. Each POSIX descriptor is wrapped in a ``hipFileDescr_t`` and registered with ``hipFileHandleRegister()``. On success the call fills in an opaque ``hipFileHandle_t`` for all later hipFile I/O on that file. +#. **Open and register files** -.. code-block:: cpp + Both paths use ``O_DIRECT`` so hipFile can use its direct GPU I/O fast path. Each POSIX descriptor is wrapped in a ``hipFileDescr_t`` and registered with ``hipFileHandleRegister()``. On success the call fills in an opaque ``hipFileHandle_t`` for all later hipFile I/O on that file. - static int - open_file(const char *path, int flags, mode_t mode, int *fd, hipFileHandle_t *handle) - { - hipFileError_t hipfile_err; - hipFileDescr_t descr; + .. code-block:: cpp - *fd = open(path, flags | O_DIRECT, mode); - if (-1 == *fd) { - fprintf(stderr, "Could not open %s (%s)\n", path, strerror(errno)); - return 1; - } + static int + open_file(const char *path, int flags, mode_t mode, int *fd, hipFileHandle_t *handle) + { + hipFileError_t hipfile_err; + hipFileDescr_t descr; - descr.type = hipFileHandleTypeOpaqueFD; - descr.handle.fd = *fd; + *fd = open(path, flags | O_DIRECT, mode); + if (-1 == *fd) { + fprintf(stderr, "Could not open %s (%s)\n", path, strerror(errno)); + return 1; + } - hipfile_err = hipFileHandleRegister(handle, &descr); - if (hipFileSuccess != hipfile_err.err) { - fprintf(stderr, "Could not register %s (%s)\n", path, - hipFileGetOpErrorString(hipfile_err.err)); - close(*fd); - return 1; - } + descr.type = hipFileHandleTypeOpaqueFD; + descr.handle.fd = *fd; - return 0; - } + hipfile_err = hipFileHandleRegister(handle, &descr); + if (hipFileSuccess != hipfile_err.err) { + fprintf(stderr, "Could not register %s (%s)\n", path, + hipFileGetOpErrorString(hipfile_err.err)); + close(*fd); + return 1; + } -Important details: + return 0; + } -- ``hipFileDescr_t.type`` is ``hipFileHandleTypeOpaqueFD`` for a POSIX file descriptor. -- ``hipFileHandleRegister()`` opens the hipFile driver automatically if it was not already opened with ``hipFileDriverOpen()``. See :doc:`/reference/hipFile-reference-count` for reference counting and lifecycle rules. -- The error path compares ``hipfile_err.err`` to ``hipFileSuccess`` and prints with ``hipFileGetOpErrorString()`` when registration fails. + Important details: -The destination opens with ``O_WRONLY | O_CREAT``. The source opens with ``O_RDONLY``: + - ``hipFileDescr_t.type`` is ``hipFileHandleTypeOpaqueFD`` for a POSIX file descriptor. + - ``hipFileHandleRegister()`` opens the hipFile driver automatically if it was not already opened with ``hipFileDriverOpen()``. See :doc:`/reference/hipFile-reference-count` for reference counting and lifecycle rules. + - The error path compares ``hipfile_err.err`` to ``hipFileSuccess`` and prints with ``hipFileGetOpErrorString()`` when registration fails. -.. code-block:: cpp + The destination opens with ``O_WRONLY | O_CREAT``. The source opens with ``O_RDONLY``: - if (open_file(dst_path, O_WRONLY | O_CREAT, S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH, - &dst_fd, &dst_handle)) { - goto program_exit; - } + .. code-block:: cpp - if (open_file(src_path, O_RDONLY, 0, &src_fd, &src_handle)) { - goto close_dst; - } + if (open_file(dst_path, O_WRONLY | O_CREAT, S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH, + &dst_fd, &dst_handle)) { + goto program_exit; + } -For a detailed procedure on registering files and GPU buffers, see :doc:`/how-to/register-file-and-buffer`. + if (open_file(src_path, O_RDONLY, 0, &src_fd, &src_handle)) { + goto close_dst; + } -Determine file size and block size ----------------------------------- + For a detailed procedure on registering files and GPU buffers, see :doc:`/how-to/register-file-and-buffer`. -.. code-block:: cpp +#. **Determine file size and block size** - struct stat statbuf; - stat(src_path, &statbuf); - file_size = static_cast(statbuf.st_size); - block_size = static_cast(statbuf.st_blksize); + .. code-block:: cpp -The program uses ``stat()`` to obtain the source file's total size and the file system block size. The block size is critical because writes performed with ``O_DIRECT`` must be aligned to the block size. + struct stat statbuf; + stat(src_path, &statbuf); + file_size = static_cast(statbuf.st_size); + block_size = static_cast(statbuf.st_blksize); -If the source size is zero, the program skips the read loop and exits after closing the destination handle. + The program uses ``stat()`` to obtain the source file's total size and the file system block size. The block size is critical because writes performed with ``O_DIRECT`` must be aligned to the block size. -Allocate a GPU buffer ---------------------- + If the source size is zero, the program skips the read loop and exits after closing the destination handle. -``hipMalloc()`` receives a buffer sized to the minimum of the source file size and ``AISCP_CHUNK_SIZE``, rounded up to the file system block size so ``O_DIRECT`` alignment rules hold. +#. **Allocate a GPU buffer** -.. code-block:: cpp + ``hipMalloc()`` receives a buffer sized to the minimum of the source file size and ``AISCP_CHUNK_SIZE``, rounded up to the file system block size so ``O_DIRECT`` alignment rules hold. - buffer_size = align_up(std::min(file_size, AISCP_CHUNK_SIZE), block_size); - hip_err = hipMalloc(&devbuf, buffer_size); - if (hipSuccess != hip_err) { - fprintf(stderr, "Could not allocate device buffer (%d)", hip_err); - goto close_src; - } + .. code-block:: cpp -``align_up`` is a helper in ``aiscp`` to round up to the next multiple of a power-of-two alignment: + buffer_size = align_up(std::min(file_size, AISCP_CHUNK_SIZE), block_size); + hip_err = hipMalloc(&devbuf, buffer_size); + if (hipSuccess != hip_err) { + fprintf(stderr, "Could not allocate device buffer (%d)", hip_err); + goto close_src; + } -.. code-block:: cpp + ``align_up`` is a helper in ``aiscp`` to round up to the next multiple of a power-of-two alignment: - static inline size_t - align_up(size_t value, size_t align) - { - return (value + align - 1) & ~(align - 1); - } + .. code-block:: cpp -``aiscp`` does not call ``hipFileBufRegister()``. Registration is optional. When you skip it, hipFile may use an internal bounce buffer. Registering with ``hipFileBufRegister()`` can increase throughput when you reuse the same buffer for many operations. See :doc:`/reference/api-file-and-buffer` for registration and I/O entry points. + static inline size_t + align_up(size_t value, size_t align) + { + return (value + align - 1) & ~(align - 1); + } -Read and write in a chunk loop ------------------------------- + ``aiscp`` does not call ``hipFileBufRegister()``. Registration is optional. When you skip it, hipFile may use an internal bounce buffer. Registering with ``hipFileBufRegister()`` can increase throughput when you reuse the same buffer for many operations. See :doc:`/reference/api-file-and-buffer` for registration and I/O entry points. -The copy is a ``do`` / ``while`` loop that reads from the source into device memory, then writes from device memory to the destination. ``file_offset`` advances by the number of bytes read each iteration. +#. **Read and write in a chunk loop** -.. code-block:: cpp + The copy is a ``do`` / ``while`` loop that reads from the source into device memory, then writes from device memory to the destination. ``file_offset`` advances by the number of bytes read each iteration. - do { - nread = hipFileRead(src_handle, devbuf, buffer_size, file_offset, 0); - if (nread < 0) { - fprintf(stderr, "Could not read from %s (%zd) (%s)\n", src_path, nread, - IS_HIPFILE_ERR(nread) ? HIPFILE_ERRSTR(nread) : strerror(errno)); - goto free_devbuf; - } + .. code-block:: cpp - nwrite = 0; - while (nwrite < nread) { - nbytes = - hipFileWrite(dst_handle, devbuf, align_up(static_cast(nread - nwrite), block_size), - file_offset + nwrite, nwrite); - if (nbytes < 0) { - fprintf(stderr, "Could not write to %s (%zd) (%s)\n", dst_path, nbytes, - IS_HIPFILE_ERR(nbytes) ? HIPFILE_ERRSTR(nbytes) : strerror(errno)); - goto free_devbuf; - } - nwrite += nbytes; - } - file_offset += nread; - } while (nread > 0); + do { + nread = hipFileRead(src_handle, devbuf, buffer_size, file_offset, 0); + if (nread < 0) { + fprintf(stderr, "Could not read from %s (%zd) (%s)\n", src_path, nread, + IS_HIPFILE_ERR(nread) ? HIPFILE_ERRSTR(nread) : strerror(errno)); + goto free_devbuf; + } -This is the core copy logic: + nwrite = 0; + while (nwrite < nread) { + nbytes = + hipFileWrite(dst_handle, devbuf, align_up(static_cast(nread - nwrite), block_size), + file_offset + nwrite, nwrite); + if (nbytes < 0) { + fprintf(stderr, "Could not write to %s (%zd) (%s)\n", dst_path, nbytes, + IS_HIPFILE_ERR(nbytes) ? HIPFILE_ERRSTR(nbytes) : strerror(errno)); + goto free_devbuf; + } + nwrite += nbytes; + } + file_offset += nread; + } while (nread > 0); -- Outer loop: reads a chunk of up to ``buffer_size`` bytes from the source file into the GPU buffer. When ``hipFileRead()`` returns ``0``, all data has been read. -- Inner loop: writes the chunk from the GPU buffer to the destination file. A partial write is possible, so the inner loop continues until every byte from the current chunk is written. + This is the core copy logic: -``hipFileRead()`` and ``hipFileWrite()`` return: + - Outer loop: reads a chunk of up to ``buffer_size`` bytes from the source file into the GPU buffer. When ``hipFileRead()`` returns ``0``, all data has been read. + - Inner loop: writes the chunk from the GPU buffer to the destination file. A partial write is possible, so the inner loop continues until every byte from the current chunk is written. -- A non-negative byte count on success. -- ``-1`` for a POSIX error. Check ``errno``. -- A negative hipFile error code, the negated ``hipFileOpError_t`` value. + ``hipFileRead()`` and ``hipFileWrite()`` return: -The sample uses ``IS_HIPFILE_ERR()`` to pick between ``HIPFILE_ERRSTR()`` and ``strerror()`` for logging: + - A non-negative byte count on success. + - ``-1`` for a POSIX error. Check ``errno``. + - A negative hipFile error code, the negated ``hipFileOpError_t`` value. -.. code-block:: cpp + The sample uses ``IS_HIPFILE_ERR()`` to pick between ``HIPFILE_ERRSTR()`` and ``strerror()`` for logging: - IS_HIPFILE_ERR(nread) ? HIPFILE_ERRSTR(nread) : strerror(errno) + .. code-block:: cpp -The ``IS_HIPFILE_ERR`` macro tests whether the absolute value of the return code falls in the hipFile error range at or above ``HIPFILE_BASE_ERR``. In the public headers, ``HIPFILE_BASE_ERR`` is 5000. When the macro matches, ``HIPFILE_ERRSTR`` converts the code to a human-readable string via ``hipFileGetOpErrorString()``. Otherwise, the program falls back to ``strerror(errno)`` for standard system errors. + IS_HIPFILE_ERR(nread) ? HIPFILE_ERRSTR(nread) : strerror(errno) -The inner ``while`` loop handles partial writes. Each ``hipFileWrite()`` call uses ``align_up`` on the remaining byte count and passes ``nwrite`` as ``buffer_offset`` so the write starts at the correct offset inside the GPU buffer. + The ``IS_HIPFILE_ERR`` macro tests whether the absolute value of the return code falls in the hipFile error range at or above ``HIPFILE_BASE_ERR``. In the public headers, ``HIPFILE_BASE_ERR`` is 5000. When the macro matches, ``HIPFILE_ERRSTR`` converts the code to a human-readable string via ``hipFileGetOpErrorString()``. Otherwise, the program falls back to ``strerror(errno)`` for standard system errors. -Two alignment details: + The inner ``while`` loop handles partial writes. Each ``hipFileWrite()`` call uses ``align_up`` on the remaining byte count and passes ``nwrite`` as ``buffer_offset`` so the write starts at the correct offset inside the GPU buffer. -1. Maximum I/O size per call: because each ``hipFileRead()`` or ``hipFileWrite()`` call can transfer at most ``0x7ffff000`` bytes, the buffer is sized accordingly, and larger files are handled by iterating. -2. Block-size alignment on writes: the write size is rounded up to the file system block size with ``align_up()``. This is required because the file is opened with ``O_DIRECT``, which mandates that offsets and sizes are aligned to the block size. + Two alignment details: -For a full list of error codes, see :doc:`/reference/api-errors`. + 1. Maximum I/O size per call: because each ``hipFileRead()`` or ``hipFileWrite()`` call can transfer at most ``0x7ffff000`` bytes, the buffer is sized accordingly, and larger files are handled by iterating. + 2. Block-size alignment on writes: the write size is rounded up to the file system block size with ``align_up()``. This is required because the file is opened with ``O_DIRECT``, which mandates that offsets and sizes are aligned to the block size. -Truncate to the exact size --------------------------- + For a full list of error codes, see :doc:`/reference/api-errors`. -``O_DIRECT`` forces block-aligned transfer sizes. The last write can overshoot the true file size. ``ftruncate()`` trims the destination to match the source. +#. **Truncate to the exact size** -.. code-block:: cpp + ``O_DIRECT`` forces block-aligned transfer sizes. The last write can overshoot the true file size. ``ftruncate()`` trims the destination to match the source. - if (-1 == ftruncate(dst_fd, static_cast(file_size))) { - fprintf(stderr, "Could not truncate %s (%zu) (%s)\n", dst_path, file_size, - strerror(errno)); - } + .. code-block:: cpp -Clean up resources ------------------- + if (-1 == ftruncate(dst_fd, static_cast(file_size))) { + fprintf(stderr, "Could not truncate %s (%zu) (%s)\n", dst_path, file_size, + strerror(errno)); + } -Teardown reverses acquisition order: ``hipFree()`` releases device memory, ``hipFileHandleDeregister()`` drops hipFile tracking for each handle, then ``close()`` closes the POSIX descriptors. +#. **Clean up resources** -.. code-block:: cpp + Teardown reverses acquisition order: ``hipFree()`` releases device memory, ``hipFileHandleDeregister()`` drops hipFile tracking for each handle, then ``close()`` closes the POSIX descriptors. - hipFree(devbuf); + .. code-block:: cpp - hipFileHandleDeregister(handle); - close(fd); + hipFree(devbuf); -``hipFileHandleDeregister()`` takes the ``hipFileHandle_t`` from registration. The POSIX ``close()`` call is separate. + hipFileHandleDeregister(handle); + close(fd); -There is no need to call ``hipFileDriverClose()`` explicitly. The hipFile library cleans up its internal state automatically at program exit. + ``hipFileHandleDeregister()`` takes the ``hipFileHandle_t`` from registration. The POSIX ``close()`` call is separate. + + There is no need to call ``hipFileDriverClose()`` explicitly. The hipFile library cleans up its internal state automatically at program exit. diff --git a/projects/hipfile/docs/tutorials/get-version.rst b/projects/hipfile/docs/tutorials/get-version.rst index 08a0916e695d..35da93bd7a48 100644 --- a/projects/hipfile/docs/tutorials/get-version.rst +++ b/projects/hipfile/docs/tutorials/get-version.rst @@ -71,51 +71,53 @@ The following program prints the hipFile version using both methods. This code i Step-by-step walkthrough ======================== -Include the header ------------------- +The walkthrough queries the hipFile version at compile time and runtime. It +starts with the API header and version macros, then calls +``hipFileGetVersion()`` and checks both error fields before using the returned +version components. -.. code-block:: cpp +#. **Include the header** - #include + .. code-block:: cpp -The ``hipfile.h`` header declares the version macros, ``hipFileGetVersion()``, error types, and all other hipFile API functions. Including this single header gives you access to everything in the hipFile C API. + #include -Print the compile-time version ------------------------------- + The ``hipfile.h`` header declares the version macros, ``hipFileGetVersion()``, error types, and all other hipFile API functions. Including this single header gives you access to everything in the hipFile C API. -.. code-block:: cpp +#. **Print the compile-time version** - printf("Version from the header symbols (major.minor.patch): %d.%d.%d\n", - HIPFILE_VERSION_MAJOR, - HIPFILE_VERSION_MINOR, - HIPFILE_VERSION_PATCH); + .. code-block:: cpp -``HIPFILE_VERSION_MAJOR``, ``HIPFILE_VERSION_MINOR``, and ``HIPFILE_VERSION_PATCH`` are integer-valued preprocessor macros. Because they are resolved at compile time, you can also use them in preprocessor conditionals: + printf("Version from the header symbols (major.minor.patch): %d.%d.%d\n", + HIPFILE_VERSION_MAJOR, + HIPFILE_VERSION_MINOR, + HIPFILE_VERSION_PATCH); -.. code-block:: cpp + ``HIPFILE_VERSION_MAJOR``, ``HIPFILE_VERSION_MINOR``, and ``HIPFILE_VERSION_PATCH`` are integer-valued preprocessor macros. Because they are resolved at compile time, you can also use them in preprocessor conditionals: - #if HIPFILE_VERSION_MAJOR >= 1 - // Use a feature introduced in version 1.0.0 - #endif + .. code-block:: cpp -Query the runtime version -------------------------- + #if HIPFILE_VERSION_MAJOR >= 1 + // Use a feature introduced in version 1.0.0 + #endif -.. code-block:: cpp +#. **Query the runtime version** + + .. code-block:: cpp - hipFileError_t err; - unsigned major = 0, minor = 0, patch = 0; + hipFileError_t err; + unsigned major = 0, minor = 0, patch = 0; - err = hipFileGetVersion(&major, &minor, &patch); - if (err.err != hipFileSuccess || err.hip_drv_err != hipSuccess) - return EXIT_FAILURE; + err = hipFileGetVersion(&major, &minor, &patch); + if (err.err != hipFileSuccess || err.hip_drv_err != hipSuccess) + return EXIT_FAILURE; -``hipFileGetVersion()`` writes the major, minor, and patch components of the running library's version into the provided output parameters. It returns a ``hipFileError_t`` struct containing two fields: + ``hipFileGetVersion()`` writes the major, minor, and patch components of the running library's version into the provided output parameters. It returns a ``hipFileError_t`` struct containing two fields: -- ``err``: a ``hipFileOpError_t`` value indicating a hipFile-specific error. -- ``hip_drv_err``: a ``hipError_t`` value indicating a GPU driver error. + - ``err``: a ``hipFileOpError_t`` value indicating a hipFile-specific error. + - ``hip_drv_err``: a ``hipError_t`` value indicating a GPU driver error. -Always check both fields before using the output values. + Always check both fields before using the output values. Build the example =================== diff --git a/projects/hipfile/docs/tutorials/python-gpu-io.rst b/projects/hipfile/docs/tutorials/python-gpu-io.rst index 97d7357d8fe9..d0dee99a8dc6 100644 --- a/projects/hipfile/docs/tutorials/python-gpu-io.rst +++ b/projects/hipfile/docs/tutorials/python-gpu-io.rst @@ -37,7 +37,7 @@ Complete example The following script reads a source file into GPU memory and writes its contents to a destination file. It uses the same API sequence as -``python/main.py`` in the hipFile source tree, adapted for chunked copying +``python/tests/main.py`` in the hipFile source tree, adapted for chunked copying with CLI arguments. Save it as ``gpu_copy.py``: .. code-block:: python @@ -89,92 +89,89 @@ with CLI arguments. Save it as ``gpu_copy.py``: Step-by-step walkthrough ========================== -Import modules --------------- +The walkthrough follows a file copy through GPU memory using the Python +bindings. It covers importing the API, setting up the driver, file handles, +and GPU buffer, and performing synchronous reads and writes. It finishes with +error handling, cleanup through context managers, and freeing GPU memory. -.. code-block:: python +#. **Import modules** - import os - from hipfile import Buffer, Driver, FileHandle, HipFileException + .. code-block:: python -The Python package name is ``hipfile``. For the full API, see -:doc:`/reference/api-python`. + import os + from hipfile import Buffer, Driver, FileHandle, HipFileException -Initialize the driver ---------------------- + The Python package name is ``hipfile``. For the full API, see + :doc:`/reference/api-python`. -.. code-block:: python +#. **Initialize the driver** - with Driver(): - ... + .. code-block:: python -``Driver`` is a context manager that calls ``hipFileDriverOpen()`` on entry and -``hipFileDriverClose()`` on exit. + with Driver(): + ... -Open files with FileHandle --------------------------- + ``Driver`` is a context manager that calls ``hipFileDriverOpen()`` on entry and + ``hipFileDriverClose()`` on exit. -.. code-block:: python +#. **Open files with FileHandle** - with FileHandle(src_path, os.O_RDONLY | os.O_DIRECT) as src, \ - FileHandle(dst_path, os.O_WRONLY | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as dst: - ... + .. code-block:: python -``FileHandle`` takes a file system path and ``os.open()`` flags. Include -``os.O_DIRECT`` for direct I/O. It opens the file, registers the handle with -hipFile, and deregisters on exit. + with FileHandle(src_path, os.O_RDONLY | os.O_DIRECT) as src, \ + FileHandle(dst_path, os.O_WRONLY | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as dst: + ... -Allocate and register GPU memory --------------------------------- + ``FileHandle`` takes a file system path and ``os.open()`` flags. Include + ``os.O_DIRECT`` for direct I/O. It opens the file, registers the handle with + hipFile, and deregisters on exit. -.. code-block:: python +#. **Allocate and register GPU memory** - gpu_ptr = hipMalloc(CHUNK_SIZE) - with Buffer.from_ctypes_void_p(gpu_ptr, CHUNK_SIZE, 0) as buf: - ... + .. code-block:: python -``hipMalloc()`` allocates device memory. ``Buffer.from_ctypes_void_p()`` wraps the -pointer for registration with hipFile. ``Buffer`` doesn't free the allocation. -Call ``hipFree()`` after the buffer is deregistered. + gpu_ptr = hipMalloc(CHUNK_SIZE) + with Buffer.from_ctypes_void_p(gpu_ptr, CHUNK_SIZE, 0) as buf: + ... -Read data into the GPU buffer ------------------------------ + ``hipMalloc()`` allocates device memory. ``Buffer.from_ctypes_void_p()`` wraps the + pointer for registration with hipFile. ``Buffer`` doesn't free the allocation. + Call ``hipFree()`` after the buffer is deregistered. -.. code-block:: python +#. **Read data into the GPU buffer** - bytes_read = src.read(buf, CHUNK_SIZE, file_offset, 0) + .. code-block:: python -``read()`` performs a synchronous read from the registered file into device -memory. It returns the number of bytes read. A return value of ``0`` indicates -end-of-file. + bytes_read = src.read(buf, CHUNK_SIZE, file_offset, 0) -Write data from the GPU buffer ------------------------------- + ``read()`` performs a synchronous read from the registered file into device + memory. It returns the number of bytes read. A return value of ``0`` indicates + end-of-file. -.. code-block:: python +#. **Write data from the GPU buffer** - dst.write(buf, bytes_read, file_offset, 0) + .. code-block:: python -``write()`` performs a synchronous write from device memory to the registered -file. It returns the number of bytes written. + dst.write(buf, bytes_read, file_offset, 0) -Handle errors -------------- + ``write()`` performs a synchronous write from device memory to the registered + file. It returns the number of bytes written. -.. code-block:: python +#. **Handle errors** + + .. code-block:: python - except HipFileException as exc: - print(f"hipFile error: {exc}", file=sys.stderr) + except HipFileException as exc: + print(f"hipFile error: {exc}", file=sys.stderr) -hipFile C API failures raise ``HipFileException``. ``FileHandle.read()`` and -``FileHandle.write()`` may also raise ``RuntimeError`` when the handle isn't open -or ``OSError`` when the platform sets ``errno``. + hipFile C API failures raise ``HipFileException``. ``FileHandle.read()`` and + ``FileHandle.write()`` may also raise ``RuntimeError`` when the handle isn't open + or ``OSError`` when the platform sets ``errno``. -Clean up resources ------------------- +#. **Clean up resources** -``Driver``, ``FileHandle``, and ``Buffer`` are context managers. Call -``hipFree()`` to release the GPU allocation after the ``Buffer`` context exits. + ``Driver``, ``FileHandle``, and ``Buffer`` are context managers. Call + ``hipFree()`` to release the GPU allocation after the ``Buffer`` context exits. Running the example ******************* diff --git a/projects/hipfile/examples/README.md b/projects/hipfile/examples/README.md index 1c228379d585..29081c3ca4b3 100644 --- a/projects/hipfile/examples/README.md +++ b/projects/hipfile/examples/README.md @@ -1,8 +1,8 @@ # hipFile examples This directory contains working examples of the hipFile API, grouped by what -they demonstrate. The `basics` and `async` programs verify their results with -an FNV-1a hash and print `OK …` on success. +they demonstrate. The `basics`, `batch`, and `async` programs verify their +results with an FNV-1a hash and print `OK …` on success. Most examples move data through the GPU on hipFile's fast path, which opens files with `O_DIRECT`; running them as written therefore requires an AMD GPU @@ -19,9 +19,10 @@ query the library and need neither a GPU nor an `O_DIRECT` filesystem. | --- | --- | | [`api`](api) | Minimal examples of the non-I/O API — calls that query or configure the library (e.g. `get-version`). No `O_DIRECT` filesystem or file arguments needed. | | [`basics`](basics) | Small, single-purpose programs that each exercise one facet of the synchronous API: buffer registration, the `O_DIRECT` fast path vs. compat fallback, chunked reads, device-buffer offsets, sub-region writes, GPU memory types, and a full round trip. | +| [`batch`](batch) | A registered-buffer round trip using 4 KiB batch requests and the submit/status APIs. | | [`async`](async) | Examples of the asynchronous, stream-based API (`hipFileReadAsync` / `hipFileWriteAsync`), including single-stream, non-blocking-stream, and concurrent multi-stream round trips. | | [`aiscp`](aiscp) | A standalone `cp`-like utility built on hipFile (`aiscp SOURCE DEST`). | -| [`common`](common) | Not an example — a small static library (`examples_common`) of helpers shared by `basics` and `async` (alignment math, pattern fill, hashing, file open/register). | +| [`common`](common) | Not an example — a small static library (`examples_common`) of helpers shared by `basics`, `batch`, and `async` (alignment math, pattern fill, hashing, file open/register). | ## Building @@ -40,12 +41,12 @@ run snippets below assume you have `cd build`'d into that directory first. Drop the `-DCMAKE_PREFIX_PATH` argument if ROCm and hipFile are installed in standard locations. If the install tree is read-only, copy the directory (and, -for `basics`/`async`, the sibling [`common`](common) directory) somewhere -writable first. +for `basics`/`batch`/`async`, the sibling [`common`](common) directory) +somewhere writable first. -`basics` and `async` link the shared helpers in [`common`](common) and pull it -in via `add_subdirectory(../common …)`, so keep the `common` directory alongside -them when you build. +`basics`, `batch`, and `async` link the shared helpers in [`common`](common) and +pull it in via `add_subdirectory(../common …)`, so keep the `common` directory +alongside them when you build. ### Finding the library at runtime @@ -144,6 +145,39 @@ filesystem (ext4 mounted `data=ordered`, or xfs); verify with --- +## `batch` + +The batch example creates or truncates and seeds an input file, then copies it +through a whole-file GPU buffer registered with `hipFileBufRegister`. It divides +the file into 4 KiB requests and initially submits up to `BATCH_SIZE` of them with +`hipFileBatchIOSubmit`. Each call to `hipFileBatchIOGetStatus` requests at least +one completion, and the program immediately submits one replacement per event +to keep the batch full. All reads complete before the separately submitted +write phase begins. After the writes finish, the program truncates any final +alignment padding and verifies the output by FNV-1a hash. + +The first four arguments are required: + +```text +batch-roundtrip READ_FILE WRITE_FILE FILE_SIZE BATCH_SIZE [GPUID] +``` + +`FILE_SIZE` is a positive base-10 byte count. `READ_FILE` is created if it does +not exist; an existing file is truncated and reseeded to exactly that size. +`WRITE_FILE` must identify a different file and is created or truncated. Both +paths must be on an `O_DIRECT`-capable local filesystem. `BATCH_SIZE` is the +number of requests kept in flight and must be between `1` and `128`, the +largest capacity `hipFileBatchIOSetUp` accepts. `GPUID` is optional and +defaults to `0`. The registered GPU buffer holds the whole file rounded up to +the next 4 KiB boundary. + +```bash +cd build +./batch-roundtrip input.bin output.bin 1048577 128 +``` + +--- + ## `async` Examples of hipFile's asynchronous, stream-based API @@ -199,14 +233,15 @@ aiscp SOURCE DEST ## `common` Not an example — a small static library (`examples_common`) of helpers shared -by the [`basics`](basics) and [`async`](async) examples. It was pulled out to -remove verbatim duplication; each example still drives the hipFile API directly -in its own `main()` so the example flow stays readable top-to-bottom. +by the [`basics`](basics), [`batch`](batch), and [`async`](async) examples. It +was pulled out to remove verbatim duplication; each example still drives the +hipFile API directly in its own `main()` so the example flow stays readable +top-to-bottom. -There is nothing to run or build here on its own. The `basics`/`async` +There is nothing to run or build here on its own. The `basics`/`batch`/`async` directories pull it in via `add_subdirectory(../common …)`, so the helper code is compiled once and shared rather than duplicated into each example. Keep this -directory alongside `basics`/`async` when building them. +directory alongside them when building. ### What's in it diff --git a/projects/hipfile/examples/aiscp/CMakeLists.txt b/projects/hipfile/examples/aiscp/CMakeLists.txt index d9439acf629a..9d6900a59459 100644 --- a/projects/hipfile/examples/aiscp/CMakeLists.txt +++ b/projects/hipfile/examples/aiscp/CMakeLists.txt @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: MIT -include(AISAddExecutable) +include(HipFileAddExecutable) -ais_add_executable( +hipfile_add_executable( NAME aiscp DEPS hipfile SRCS "aiscp.cpp" diff --git a/projects/hipfile/examples/api/CMakeLists.txt b/projects/hipfile/examples/api/CMakeLists.txt index d49989f38a3b..4247373e7bc2 100644 --- a/projects/hipfile/examples/api/CMakeLists.txt +++ b/projects/hipfile/examples/api/CMakeLists.txt @@ -2,14 +2,14 @@ # # SPDX-License-Identifier: MIT -include(AISAddExecutable) +include(HipFileAddExecutable) set(API_EXAMPLES get-version ) foreach(example ${API_EXAMPLES}) - ais_add_executable( + hipfile_add_executable( NAME ${example} DEPS hipfile SRCS "${example}.cpp" diff --git a/projects/hipfile/examples/async/CMakeLists.txt b/projects/hipfile/examples/async/CMakeLists.txt index 00b2a03e1983..4ae00472f3a7 100644 --- a/projects/hipfile/examples/async/CMakeLists.txt +++ b/projects/hipfile/examples/async/CMakeLists.txt @@ -2,14 +2,14 @@ # # SPDX-License-Identifier: MIT -# Async examples for the hipFile C API. They build with the AIS infrastructure -# (ais_add_executable) and link the shared helpers from the examples_common -# target defined in examples/common; the parent CMakeLists adds examples/common -# before examples/async, so the target is in scope here. examples_common is -# PUBLIC-linked against hipfile and exports its own include dir, so -# DEPS examples_common transitively supplies the library and headers. +# Async examples for the hipFile C API. They build with the hipFile CMake +# infrastructure (hipfile_add_executable) and link the shared helpers from the +# examples_common target defined in examples/common; the parent CMakeLists adds +# examples/common before examples/async, so the target is in scope here. +# examples_common is PUBLIC-linked against hipfile and exports its own include +# dir, so DEPS examples_common transitively supplies the library and headers. -include(AISAddExecutable) +include(HipFileAddExecutable) set(ASYNC_EXAMPLES roundtrip-async @@ -19,7 +19,7 @@ set(ASYNC_EXAMPLES ) foreach(example ${ASYNC_EXAMPLES}) - ais_add_executable( + hipfile_add_executable( NAME ${example} DEPS examples_common SRCS "${example}.cpp" @@ -28,11 +28,11 @@ foreach(example ${ASYNC_EXAMPLES}) endforeach() # CTest wrappers: run each async example as a hipFile system test. -# Gated on BUILD_TESTING; AIS_INSTALL_EXAMPLES is already enforced by the +# Gated on BUILD_TESTING; HIPFILE_INSTALL_EXAMPLES is already enforced by the # parent CMakeLists which only adds this subdirectory when examples are built. if(BUILD_TESTING) # Unique-per-configure scratch dir so concurrent ctest runs / multiple - # checkouts don't collide on a shared path. Kept under AIS_CAPABLE_DIR + # checkouts don't collide on a shared path. Kept under HIPFILE_CAPABLE_DIR # because the e2e tests need an O_DIRECT-capable local FS — the build # tree may live on overlay/nfs and isn't a safe default. execute_process( @@ -41,7 +41,7 @@ if(BUILD_TESTING) OUTPUT_STRIP_TRAILING_WHITESPACE ) set(ASYNC_TEST_DIR - "${AIS_CAPABLE_DIR}/hipfile_async_tests_${_async_scratch_token}" + "${HIPFILE_CAPABLE_DIR}/hipfile_async_tests_${_async_scratch_token}" ) add_test( diff --git a/projects/hipfile/examples/basics/CMakeLists.txt b/projects/hipfile/examples/basics/CMakeLists.txt index 9d7945164737..2dc76cd43a43 100644 --- a/projects/hipfile/examples/basics/CMakeLists.txt +++ b/projects/hipfile/examples/basics/CMakeLists.txt @@ -3,13 +3,14 @@ # SPDX-License-Identifier: MIT # These examples use the hipFile C API directly from C++. They build with the -# AIS infrastructure (ais_add_executable) and link the shared helpers from the -# examples_common target defined in examples/common; the parent CMakeLists adds -# examples/common before examples/basics, so the target is in scope here. +# hipFile CMake infrastructure (hipfile_add_executable) and link the shared +# helpers from the examples_common target defined in examples/common; the parent +# CMakeLists adds examples/common before examples/basics, so the target is in +# scope here. # examples_common is PUBLIC-linked against hipfile and exports its own include # dir, so DEPS examples_common transitively supplies the library and headers. -include(AISAddExecutable) +include(HipFileAddExecutable) set(BASICS_EXAMPLES bufregister-write @@ -23,7 +24,7 @@ set(BASICS_EXAMPLES ) foreach(example ${BASICS_EXAMPLES}) - ais_add_executable( + hipfile_add_executable( NAME ${example} DEPS examples_common SRCS "${example}.cpp" @@ -32,11 +33,11 @@ foreach(example ${BASICS_EXAMPLES}) endforeach() # CTest wrappers: run each example as a hipFile system test. -# Gated on BUILD_TESTING; AIS_INSTALL_EXAMPLES is already enforced by the +# Gated on BUILD_TESTING; HIPFILE_INSTALL_EXAMPLES is already enforced by the # parent CMakeLists which only adds this subdirectory when examples are built. if(BUILD_TESTING) # Unique-per-configure scratch dir so concurrent ctest runs / multiple - # checkouts don't collide on a shared path. Kept under AIS_CAPABLE_DIR + # checkouts don't collide on a shared path. Kept under HIPFILE_CAPABLE_DIR # because the e2e tests need an O_DIRECT-capable local FS — the build # tree may live on overlay/nfs and isn't a safe default. execute_process( @@ -45,7 +46,7 @@ if(BUILD_TESTING) OUTPUT_STRIP_TRAILING_WHITESPACE ) set(BASICS_TEST_DIR - "${AIS_CAPABLE_DIR}/hipfile_basics_tests_${_basics_scratch_token}" + "${HIPFILE_CAPABLE_DIR}/hipfile_basics_tests_${_basics_scratch_token}" ) set(BASICS_INPUT "${BASICS_TEST_DIR}/input.bin") diff --git a/projects/hipfile/examples/batch/CMakeLists.install.cmake b/projects/hipfile/examples/batch/CMakeLists.install.cmake new file mode 100644 index 000000000000..ccafecf52b8f --- /dev/null +++ b/projects/hipfile/examples/batch/CMakeLists.install.cmake @@ -0,0 +1,24 @@ +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# +# SPDX-License-Identifier: MIT + +# You may need to add -DCMAKE_PREFIX_PATH=/path/to/hipfile +# if hipFile has been installed to a non-standard location. That same path is +# recorded in the example binaries (via CMake's default RPATH handling), so they +# find libhipfile.so at runtime without LD_LIBRARY_PATH even from a non-standard +# install location. + +cmake_minimum_required(VERSION 3.21) + +project(batch_examples LANGUAGES CXX) + +find_package(hip REQUIRED CONFIG) +find_package(hipfile REQUIRED CONFIG) + +add_subdirectory(../common "${CMAKE_CURRENT_BINARY_DIR}/common") + +add_executable(batch-roundtrip batch-roundtrip.cpp) +target_compile_features(batch-roundtrip PRIVATE cxx_std_20) +set_target_properties(batch-roundtrip PROPERTIES CXX_EXTENSIONS OFF) +target_compile_options(batch-roundtrip PRIVATE -Wall -Wextra) +target_link_libraries(batch-roundtrip PRIVATE examples_common) diff --git a/projects/hipfile/examples/batch/CMakeLists.txt b/projects/hipfile/examples/batch/CMakeLists.txt new file mode 100644 index 000000000000..352b64c91ba9 --- /dev/null +++ b/projects/hipfile/examples/batch/CMakeLists.txt @@ -0,0 +1,53 @@ +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# +# SPDX-License-Identifier: MIT + +# Batch examples for the hipFile C API. examples_common supplies shared file +# and verification helpers and transitively links hipFile and HIP. + +include(HipFileAddExecutable) + +hipfile_add_executable( + NAME batch-roundtrip + DEPS examples_common + SRCS "batch-roundtrip.cpp" + SYSINCLS ${HIPFILE_INCLUDE_PATH} +) + +if(BUILD_TESTING) + execute_process( + COMMAND cat /proc/sys/kernel/random/uuid + OUTPUT_VARIABLE _batch_scratch_token + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + set(BATCH_TEST_DIR + "${HIPFILE_CAPABLE_DIR}/hipfile_batch_tests_${_batch_scratch_token}" + ) + set(BATCH_INPUT "${BATCH_TEST_DIR}/input.bin") + set(BATCH_OUTPUT "${BATCH_TEST_DIR}/output.bin") + + add_test( + NAME batch_setup_dir + COMMAND ${CMAKE_COMMAND} -E make_directory ${BATCH_TEST_DIR} + ) + add_test( + NAME batch_cleanup + COMMAND ${CMAKE_COMMAND} -E rm -rf ${BATCH_TEST_DIR} + ) + set_tests_properties(batch_setup_dir PROPERTIES + FIXTURES_SETUP batch_scratch + ) + set_tests_properties(batch_cleanup PROPERTIES + FIXTURES_CLEANUP batch_scratch + ) + + add_test( + NAME batch_roundtrip + COMMAND $ ${BATCH_INPUT} ${BATCH_OUTPUT} 1048577 128 + WORKING_DIRECTORY ${BATCH_TEST_DIR} + ) + set_tests_properties(batch_roundtrip PROPERTIES + FIXTURES_REQUIRED batch_scratch + LABELS "system;hipfile;batch" + ) +endif() diff --git a/projects/hipfile/examples/batch/batch-roundtrip.cpp b/projects/hipfile/examples/batch/batch-roundtrip.cpp new file mode 100644 index 000000000000..2b05a5044cbf --- /dev/null +++ b/projects/hipfile/examples/batch/batch-roundtrip.cpp @@ -0,0 +1,399 @@ +/* Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + * + * SPDX-License-Identifier: MIT + */ + +/* batch-roundtrip - Seed an input file, read it into a registered GPU buffer + * using a continuously refilled batch, write it in a second batch phase, and + * verify the output matches the input. + * + * Usage: ./batch-roundtrip READ_FILE WRITE_FILE FILE_SIZE BATCH_SIZE [GPUID] + * + * READ_FILE Input path to create or truncate and seed with test data. + * WRITE_FILE Distinct output path to create or truncate. + * FILE_SIZE Positive base-10 payload size in bytes. + * BATCH_SIZE Number of requests kept in flight, 1 to 128. + * GPUID GPU device index (optional, default 0). + * + * The GPU buffer holds the whole file rounded up to a 4 KiB boundary. Each + * phase starts with up to BATCH_SIZE requests, waits for at least one + * completion, then submits one replacement for each returned event until all + * 4 KiB requests have completed. All reads finish before any writes are + * submitted. The final padded write is truncated to FILE_SIZE. + */ + +#include "examples_common.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// Size to copy with each IO +constexpr size_t BATCH_IO_SIZE{BLOCK_ALIGN}; + +// Maximum queue size for a batch context +constexpr unsigned MAX_BATCH_SIZE{128}; + +struct BatchCookie { + size_t chunk_index{}; + size_t expected_bytes{}; +}; + +struct BatchRequest { + hipFileIOParams_t operation{}; + std::unique_ptr cookie; + bool submitted{}; +}; + +template + requires std::is_integral_v +Integral +parse_integral(std::string_view text) +{ + Integral value{}; + const char *end{text.data() + text.size()}; + const auto result = std::from_chars(text.data(), end, value, 10); + if (result.ec != std::errc{} || result.ptr != end) { + throw std::invalid_argument{"Invalid integral"}; + } + return value; +} + +const char * +operation_name(hipFileOpcode_t opcode) +{ + switch (opcode) { + case hipFileBatchRead: + return "read"; + case hipFileBatchWrite: + return "write"; + default: + return "unknown"; + } +} + +void +configure_request(BatchRequest &request, size_t chunk_index, hipFileHandle_t file_handle, + hipFileOpcode_t opcode, void *device_buffer, size_t payload_size) +{ + const size_t operation_offset{chunk_index * BATCH_IO_SIZE}; + const size_t bytes_remaining{payload_size - std::min(payload_size, operation_offset)}; + const size_t expected_bytes{opcode == hipFileBatchRead ? std::min(bytes_remaining, BATCH_IO_SIZE) + : BATCH_IO_SIZE}; + + if (request.cookie == nullptr) + request.cookie = std::make_unique(); + request.cookie->chunk_index = chunk_index; + request.cookie->expected_bytes = expected_bytes; + + request.operation = {}; + request.operation.mode = hipFileBatch; + request.operation.u.batch.devPtr_base = device_buffer; + request.operation.u.batch.file_offset = static_cast(operation_offset); + request.operation.u.batch.devPtr_offset = static_cast(operation_offset); + request.operation.u.batch.size = BATCH_IO_SIZE; + request.operation.fh = file_handle; + request.operation.opcode = opcode; + request.operation.cookie = request.cookie.get(); + request.submitted = false; +} + +int +run_batch_phase(hipFileBatchHandle_t batch_handle, hipFileHandle_t file_handle, hipFileOpcode_t opcode, + void *device_buffer, size_t payload_size, unsigned batch_capacity) +{ + const char *op_name{operation_name(opcode)}; + const size_t operation_count{align_up(payload_size, BATCH_IO_SIZE) / BATCH_IO_SIZE}; + const size_t request_count{std::min(operation_count, static_cast(batch_capacity))}; + + std::vector requests(request_count); + size_t next_chunk{0}; + for (auto &request : requests) { + configure_request(request, next_chunk, file_handle, opcode, device_buffer, payload_size); + ++next_chunk; + } + + std::vector events(batch_capacity); + std::vector submissions; + submissions.reserve(batch_capacity); + size_t in_flight{0}; + + while (!requests.empty()) { + submissions.clear(); + const size_t available{static_cast(batch_capacity) - in_flight}; + for (auto &request : requests) { + if (submissions.size() == available) + break; + if (request.submitted) + continue; + submissions.push_back(request.operation); + request.submitted = true; + } + + if (!submissions.empty()) { + const hipFileError_t hipfile_err = hipFileBatchIOSubmit( + batch_handle, static_cast(submissions.size()), submissions.data(), /*flags=*/0); + if (hipFileSuccess != hipfile_err.err) { + fprintf(stderr, "Could not submit %s batch (%s)\n", op_name, + hipFileGetOpErrorString(hipfile_err.err)); + return 1; + } + in_flight += submissions.size(); + } + + unsigned nr{batch_capacity}; + const hipFileError_t hipfile_err = + hipFileBatchIOGetStatus(batch_handle, /*min_nr=*/1, &nr, events.data(), /*timeout=*/nullptr); + if (hipFileSuccess != hipfile_err.err) { + fprintf(stderr, "Could not get %s batch status (%s)\n", op_name, + hipFileGetOpErrorString(hipfile_err.err)); + return 1; + } + if (nr == 0 || nr > in_flight) { + fprintf(stderr, "Invalid number of %s batch events: %u\n", op_name, nr); + return 1; + } + + for (unsigned i{0}; i < nr; ++i) { + const hipFileIOEvents_t &event = events[i]; + const auto request = std::find_if(requests.begin(), requests.end(), [&event](const auto &entry) { + return event.cookie == entry.cookie.get(); + }); + if (request == requests.end()) { + fprintf(stderr, "%s batch returned an unknown cookie\n", op_name); + return 1; + } + const BatchCookie *cookie{request->cookie.get()}; + if (event.status != hipFileComplete) { + fprintf(stderr, "%s batch request %zu failed: status=%d ret=%zd\n", op_name, + cookie->chunk_index, static_cast(event.status), static_cast(event.ret)); + return 1; + } + if (event.ret != cookie->expected_bytes) { + fprintf(stderr, "Short %s for batch request %zu: %zu of %zu bytes\n", op_name, + cookie->chunk_index, event.ret, cookie->expected_bytes); + return 1; + } + + if (next_chunk < operation_count) { + configure_request(*request, next_chunk, file_handle, opcode, device_buffer, payload_size); + ++next_chunk; + } + else { + if (auto back{std::prev(requests.end())}; request != back) + std::iter_swap(request, back); + requests.pop_back(); + } + } + in_flight -= nr; + } + + return 0; +} + +} // namespace + +int +main(int argc, char *argv[]) +{ + if (argc != 5 && argc != 6) { + fprintf(stderr, "Usage: %s READ_FILE WRITE_FILE FILE_SIZE BATCH_SIZE [GPUID]\n", argv[0]); + return EXIT_FAILURE; + } + + const char *read_path{argv[1]}; + const char *write_path{argv[2]}; + + size_t payload_size{}; + try { + payload_size = parse_integral(argv[3]); + } + catch (const std::invalid_argument &) { + fprintf(stderr, "Invalid FILE_SIZE: %s\n", argv[3]); + return EXIT_FAILURE; + } + if (payload_size == 0) { + fprintf(stderr, "FILE_SIZE must be greater than zero\n"); + return EXIT_FAILURE; + } + + unsigned batch_capacity{}; + try { + batch_capacity = parse_integral(argv[4]); + } + catch (const std::invalid_argument &) { + fprintf(stderr, "Invalid BATCH_SIZE: %s\n", argv[4]); + return EXIT_FAILURE; + } + if (batch_capacity == 0 || batch_capacity > MAX_BATCH_SIZE) { + fprintf(stderr, "BATCH_SIZE must be between 1 and %u\n", MAX_BATCH_SIZE); + return EXIT_FAILURE; + } + + unsigned gpu_id{0}; + if (argc == 6) { + try { + gpu_id = parse_integral(argv[5]); + } + catch (const std::invalid_argument &) { + fprintf(stderr, "Invalid GPU ID: %s\n", argv[5]); + return EXIT_FAILURE; + } + } + + if (seed_read_file(read_path, payload_size)) + return EXIT_FAILURE; + + struct stat read_stat; + if (stat(read_path, &read_stat) || read_stat.st_size != static_cast(payload_size)) { + fprintf(stderr, "Could not create %s with size %zu\n", read_path, payload_size); + return EXIT_FAILURE; + } + + const size_t buffer_size{align_up(payload_size, BATCH_IO_SIZE)}; + + int read_fd{-1}, write_fd{-1}; + hipFileHandle_t read_handle{nullptr}, write_handle{nullptr}; + hipFileBatchHandle_t batch_handle{nullptr}; + bool read_handle_open{false}, write_handle_open{false}; + void *device_buffer{nullptr}; + bool buffer_registered{false}; + int exit_status{EXIT_FAILURE}; + hipFileError_t hipfile_err{}; + + hipError_t hip_err{hipSetDevice(static_cast(gpu_id))}; + if (hipSuccess != hip_err) { + fprintf(stderr, "Could not select GPU %u (%d)\n", gpu_id, hip_err); + return EXIT_FAILURE; + } + + hip_err = hipMalloc(&device_buffer, buffer_size); + if (hipSuccess != hip_err) { + fprintf(stderr, "Could not allocate %zu bytes on GPU %u (%d)\n", buffer_size, gpu_id, hip_err); + return EXIT_FAILURE; + } + + hip_err = hipMemset(device_buffer, 0, buffer_size); + if (hipSuccess != hip_err) { + fprintf(stderr, "Could not zero device buffer (%d)\n", hip_err); + goto cleanup; + } + hip_err = hipDeviceSynchronize(); + if (hipSuccess != hip_err) { + fprintf(stderr, "Could not synchronize after memset (%d)\n", hip_err); + goto cleanup; + } + + hipfile_err = hipFileBufRegister(device_buffer, buffer_size, 0); + if (hipFileSuccess != hipfile_err.err) { + fprintf(stderr, "Buffer register failed (%s)\n", hipFileGetOpErrorString(hipfile_err.err)); + goto cleanup; + } + buffer_registered = true; + + if (open_file(read_path, O_RDONLY | O_DIRECT, 0, &read_fd, &read_handle)) + goto cleanup; + read_handle_open = true; + + if (open_file(write_path, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH, + &write_fd, &write_handle)) { + goto cleanup; + } + write_handle_open = true; + + hipfile_err = hipFileBatchIOSetUp(&batch_handle, batch_capacity); + if (hipFileSuccess != hipfile_err.err) { + fprintf(stderr, "Could not set up batch handle (%s)\n", hipFileGetOpErrorString(hipfile_err.err)); + goto cleanup; + } + + if (run_batch_phase(batch_handle, read_handle, hipFileBatchRead, device_buffer, payload_size, + batch_capacity)) { + goto cleanup; + } + if (run_batch_phase(batch_handle, write_handle, hipFileBatchWrite, device_buffer, payload_size, + batch_capacity)) { + goto cleanup; + } + + if (ftruncate(write_fd, static_cast(payload_size))) { + fprintf(stderr, "Could not truncate %s (%s)\n", write_path, strerror(errno)); + goto cleanup; + } + + { + const int close_status{close_file(write_path, write_fd, write_handle)}; + write_fd = -1; + write_handle = nullptr; + write_handle_open = false; + if (close_status) + goto cleanup; + } + { + const int close_status{close_file(read_path, read_fd, read_handle)}; + read_fd = -1; + read_handle = nullptr; + read_handle_open = false; + if (close_status) + goto cleanup; + } + + { + uint64_t hash{}; + if (verify_files_match(read_path, write_path, payload_size, &hash)) + goto cleanup; + printf("OK %s == %s (%zu bytes, hash 0x%016" PRIx64 ")\n", read_path, write_path, payload_size, + hash); + } + + exit_status = EXIT_SUCCESS; + +cleanup: + if (batch_handle != nullptr) + hipFileBatchIODestroy(batch_handle); + + if (write_handle_open) { + if (close_file(write_path, write_fd, write_handle)) + exit_status = EXIT_FAILURE; + } + if (read_handle_open) { + if (close_file(read_path, read_fd, read_handle)) + exit_status = EXIT_FAILURE; + } + + if (buffer_registered) { + hipfile_err = hipFileBufDeregister(device_buffer); + if (hipFileSuccess != hipfile_err.err) { + fprintf(stderr, "Buffer deregister failed (%s)\n", hipFileGetOpErrorString(hipfile_err.err)); + exit_status = EXIT_FAILURE; + } + } + + hip_err = hipFree(device_buffer); + if (hipSuccess != hip_err) { + fprintf(stderr, "Could not free device buffer (%d)\n", hip_err); + exit_status = EXIT_FAILURE; + } + + return exit_status; +} diff --git a/projects/hipfile/examples/common/CMakeLists.txt b/projects/hipfile/examples/common/CMakeLists.txt index c3c3b98eaa26..8155f4315bcf 100644 --- a/projects/hipfile/examples/common/CMakeLists.txt +++ b/projects/hipfile/examples/common/CMakeLists.txt @@ -2,12 +2,12 @@ # # SPDX-License-Identifier: MIT -# Shared helpers (open/close/hash/etc.) used by the basics/ and async/ -# examples. Pulled out of the per-example sources to remove verbatim +# Shared helpers (open/close/hash/etc.) used by the basics/, batch/, and +# async/ examples. Pulled out of the per-example sources to remove verbatim # duplication; each example still drives the hipFile API directly in main() # so the example flow stays readable. add_library(examples_common STATIC examples_common.cpp) -target_compile_features(examples_common PRIVATE cxx_std_${AIS_CXX_STANDARD}) +target_compile_features(examples_common PRIVATE cxx_std_${HIPFILE_CXX_STANDARD}) set_target_properties(examples_common PROPERTIES CXX_EXTENSIONS OFF) target_compile_options(examples_common PRIVATE -Wall -Wextra) target_include_directories(examples_common PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/projects/hipfile/examples/common/examples_common.h b/projects/hipfile/examples/common/examples_common.h index 42b1e4a8e463..069e1b17ba5e 100644 --- a/projects/hipfile/examples/common/examples_common.h +++ b/projects/hipfile/examples/common/examples_common.h @@ -3,10 +3,10 @@ * SPDX-License-Identifier: MIT */ -/* Shared helpers for the basics/ and async/ examples. Pulled out of the - * per-example .cpp files to remove verbatim duplication; each example still - * drives the hipFile API directly in its main() so the example flow stays - * readable top-to-bottom. */ +/* Shared helpers for the basics/, batch/, and async/ examples. Pulled out of + * the per-example .cpp files to remove verbatim duplication; each example + * still drives the hipFile API directly in its main() so the example flow + * stays readable top-to-bottom. */ #pragma once diff --git a/projects/hipfile/python/main.py b/projects/hipfile/python/main.py deleted file mode 100644 index da89a2fee836..000000000000 --- a/projects/hipfile/python/main.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# SPDX-License-Identifier: MIT - -""" -A quick & rough script for testing the Cython bindings to the -hipFile C library. Reads a given file and copies it to an -output file, and then prints the hashes of the files. -""" - -import hashlib -import os -import pathlib - -from hipfile.hipMalloc import hipFree, hipMalloc - -from hipfile import ( - Driver, - FileHandle, - Buffer, - FileHandleType, - get_version, -) - -hipfile_version = get_version() - -input_path = pathlib.Path("/mnt/ais/ext4/random_2MiB.bin") -output_path = pathlib.Path("/mnt/ais/ext4/output.bin") - -print(f"hipFile Version: {hipfile_version}") -print(f"Driver Use Count Before: {Driver.use_count()}") - -# Max to a 2GiB - 4KiB Buffer -# Note: Max IO in a single transaction is 2GiB - 4KiB as set by the Linux Kernel -# Larger IOs will be quietly truncated. -size = min(input_path.stat().st_size, 2 * 1024 * 1024 * 1024 - 4 * 1024) -buffer = hipMalloc(size) -buffer_ptr = buffer.value # pylint: disable=C0103 # False Positive -print(f"Buffer located at: {buffer_ptr} | {hex(buffer_ptr)}") - -with Driver() as hipfile_driver: - print(f"Driver Use Count After: {hipfile_driver.use_count()}") - with Buffer.from_ctypes_void_p(buffer, size, 0) as registered_buffer: - with FileHandle( - input_path, - os.O_RDWR | os.O_DIRECT | os.O_CREAT, - handle_type=FileHandleType.OPAQUE_FD, - ) as fh_input: - with FileHandle( - output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC - ) as fh_output: - print(f"Transferring {size} bytes...") - bytes_read = fh_input.read(registered_buffer, size, 0, 0) - print(f"Bytes Read: {bytes_read}") - bytes_written = fh_output.write(registered_buffer, size, 0, 0) - print(f"Bytes Written: {bytes_written}") - -hipFree(buffer) - -with open(input_path, "br") as file_in: - hash_in = hashlib.sha256() - chunk = file_in.read(1 * 1024 * 1024) # 1 MiB - while len(chunk) != 0: - hash_in.update(chunk) - chunk = file_in.read(1 * 1024 * 1024) # 1 MiB - print(f"Input File Hash: {hash_in.hexdigest()}") - -with open(output_path, "br") as file_out: - hash_out = hashlib.sha256() - chunk = file_out.read(1 * 1024 * 1024) # 1 MiB - while len(chunk) != 0: - hash_out.update(chunk) - chunk = file_out.read(1 * 1024 * 1024) # 1 MiB - print(f"Output File Hash: {hash_out.hexdigest()}") diff --git a/projects/hipfile/python/tests/README.md b/projects/hipfile/python/tests/README.md index 31ef0ec3de0d..d386985f77b7 100644 --- a/projects/hipfile/python/tests/README.md +++ b/projects/hipfile/python/tests/README.md @@ -2,7 +2,7 @@ Unit tests for the high-level `hipfile` Python API. -They are fully hermetic: the compiled Cython extension `hipfile._hipfile` is +The pytest suite is fully hermetic: the compiled Cython extension `hipfile._hipfile` is replaced with a pure-Python fake injected into `sys.modules` before `hipfile` is imported, so the suite runs on any machine — no ROCm install, GPU, AIS-capable storage, or build step required. Filesystem access in the `FileHandle` tests is @@ -18,6 +18,14 @@ cd projects/hipfile/python/tests python3 -m pytest ``` +`main.py` is *not* part of that suite — `python_files = test_*.py` keeps pytest +from collecting it. It is a standalone hardware check, run against an installed +`hipfile` from the repository root: + +``` +python3 python/tests/main.py INPUT OUTPUT +``` + ## Layout | File | Covers | @@ -29,11 +37,12 @@ python3 -m pytest | `test_buffer.py` | `Buffer` — null rejection, register/deregister success and error, no-op deregister, context manager. | | `test_file.py` | `FileHandle` — `handle_type` setter guards, fd cleanup on registration failure, idempotent close, and the parametrized read/write return-code contract. | | `test_properties.py` | `get_version` / `driver_get_properties` — success and error paths. | +| `main.py` | Standalone end-to-end hardware check (not collected by pytest): copies a file through GPU memory with the *real* extension and compares SHA-256 digests. Needs a GPU and AIS-capable storage. | ## Notes -- This is a **unit** suite only — it verifies the Python wrapper's contract, not - real GPU/driver/filesystem I/O. +- Everything except `main.py` is a **unit** suite — it verifies the Python + wrapper's contract, not real GPU/driver/filesystem I/O. - Per-test overrides use `unittest.mock.patch.object` on the *consuming* module (e.g. `hipfile.driver.hipFileDriverOpen`), since each module does `from hipfile._hipfile import ...` and holds its own reference. diff --git a/projects/hipfile/python/tests/main.py b/projects/hipfile/python/tests/main.py new file mode 100644 index 000000000000..680f797df8f5 --- /dev/null +++ b/projects/hipfile/python/tests/main.py @@ -0,0 +1,185 @@ +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +A quick & rough script for testing the Cython bindings to the +hipFile C library. Reads a given file and copies it to an +output file, and then compares the hashes of the files. + +Exits non-zero if the round-trip did not reproduce the input byte for byte, +so CI can use this as a pass/fail hardware test. +""" + +import argparse +import hashlib +import os +import pathlib +import sys + +from hipfile.hipMalloc import hipFree, hipMalloc + +from hipfile import ( + Driver, + FileHandle, + Buffer, + FileHandleType, + get_version, +) + +# Size of a single read/write transaction. Keep it a multiple of the 4 KiB +# O_DIRECT block so every transfer but a trailing partial one stays eligible for +# the fast path -- hipFile silently falls back to the buffered backend when the +# offset or length is unaligned, which would defeat the point of this check. +# Must also stay under the kernel's 2 GiB - 4 KiB per-transaction ceiling, above +# which hipFile clamps the request and returns short. +CHUNK_SIZE = 1 * 1024 * 1024 # 1 MiB + + +def is_same_file(input_path, output_path): + """Return whether both paths name the same file on disk. + + Compares the resolved paths, which normalizes relative spellings, ``..`` + segments and symlinks, and additionally compares inode identity when both + already exist, which catches hard links and bind mounts that resolve to + different paths. + """ + if input_path.resolve() == output_path.resolve(): + return True + try: + return input_path.samefile(output_path) + except OSError: + # At least one does not exist yet, so they cannot be the same file. + return False + + +def parse_args(): + """Parse the input & output paths from the command line.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input", + type=pathlib.Path, + help="File to read through hipFile. Must live on an AIS-capable filesystem.", + ) + parser.add_argument( + "output", + type=pathlib.Path, + help="File to write through hipFile. Must live on an AIS-capable filesystem.", + ) + args = parser.parse_args() + if is_same_file(args.input, args.output): + # The output is opened O_TRUNC, so a shared path destroys the input + # before it is ever read and leaves both digests covering the same + # wreckage -- which the round trip would then report as a pass. + parser.error( + f"input and output must name different files, but '{args.input}' " + f"and '{args.output}' are the same file. Writing would destroy the " + f"input before it is read." + ) + return args + + +def sha256(path): + """Return the hex SHA-256 digest of *path*, read in CHUNK_SIZE pieces.""" + digest = hashlib.sha256() + with open(path, "br") as file: + chunk = file.read(CHUNK_SIZE) + while len(chunk) != 0: + digest.update(chunk) + chunk = file.read(CHUNK_SIZE) + return digest.hexdigest() + + +def drain_write(fh_output, buffer, count, file_offset): + """Write *count* bytes of *buffer* to *fh_output* at *file_offset*. + + ``hipFileWrite`` may transfer fewer bytes than asked for, so keep issuing + writes -- advancing both the file and buffer offsets -- until the whole + range is out. + """ + written = 0 + while written < count: + put = fh_output.write(buffer, count - written, file_offset + written, written) + if put == 0: + raise RuntimeError( + f"Write to {fh_output.path} stalled at offset " + f"{file_offset + written}: {written} of {count} bytes written." + ) + written += put + + +def copy(fh_input, fh_output, buffer, size): + """Copy *size* bytes from *fh_input* to *fh_output* through *buffer*. + + Both sides are driven by the byte counts the calls return rather than the + counts requested: a short read must not cause the untouched tail of the + buffer -- which holds whatever ``hipMalloc`` handed back, not file data -- + to be written out, and a short write must be finished rather than accepted. + """ + offset = 0 + while offset < size: + want = min(CHUNK_SIZE, size - offset) + got = fh_input.read(buffer, want, offset, 0) + if got == 0: + raise RuntimeError( + f"Unexpected EOF reading {fh_input.path} at offset {offset}: " + f"{offset} of {size} bytes read." + ) + drain_write(fh_output, buffer, got, offset) + offset += got + return offset + + +def transfer(input_path, output_path): + """Copy *input_path* to *output_path* via GPU memory using hipFile.""" + print(f"hipFile Version: {get_version()}") + print(f"Driver Use Count Before: {Driver.use_count()}") + + size = input_path.stat().st_size + buffer = hipMalloc(CHUNK_SIZE) + buffer_ptr = buffer.value # pylint: disable=C0103 # False Positive + print(f"Buffer located at: {buffer_ptr} | {hex(buffer_ptr)}") + + with Driver() as hipfile_driver: + print(f"Driver Use Count After: {hipfile_driver.use_count()}") + with Buffer.from_ctypes_void_p(buffer, CHUNK_SIZE, 0) as registered_buffer: + with FileHandle( + input_path, + os.O_RDONLY | os.O_DIRECT, + handle_type=FileHandleType.OPAQUE_FD, + ) as fh_input: + with FileHandle( + output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC + ) as fh_output: + print(f"Transferring {size} bytes in {CHUNK_SIZE} byte chunks...") + transferred = copy(fh_input, fh_output, registered_buffer, size) + print(f"Bytes Transferred: {transferred}") + + hipFree(buffer) + + +def main(): + """Run the round-trip and report whether the copy is faithful.""" + args = parse_args() + + transfer(args.input, args.output) + + hash_in = sha256(args.input) + hash_out = sha256(args.output) + print(f"Input File Hash: {hash_in}") + print(f"Output File Hash: {hash_out}") + + if hash_in != hash_out: + print( + f"Files differ! Test failed. " + f"SHA-256 of {args.input} and {args.output} do not match.", + file=sys.stderr, + ) + return 1 + + print("Hashes match. Test passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/hipfile/src/amd_detail/CMakeLists.txt b/projects/hipfile/src/amd_detail/CMakeLists.txt index 71d74bd432ac..4039abca17ba 100644 --- a/projects/hipfile/src/amd_detail/CMakeLists.txt +++ b/projects/hipfile/src/amd_detail/CMakeLists.txt @@ -34,7 +34,7 @@ set(HIPFILE_SOURCES ) # Create hipfile target -ais_add_libraries( +hipfile_add_libraries( NAME hipfile DETAIL amd LIBS mount @@ -60,9 +60,9 @@ if(HIPFILE_ROCPROFILER_REGISTER) hipfile PRIVATE HIPFILE_ROCPROFILER_REGISTER=1 - HIPFILE_ROCP_REG_VERSION_MAJOR=${AIS_LIBRARY_MAJOR} - HIPFILE_ROCP_REG_VERSION_MINOR=${AIS_LIBRARY_MINOR} - HIPFILE_ROCP_REG_VERSION_PATCH=${AIS_LIBRARY_PATCH}) + HIPFILE_ROCP_REG_VERSION_MAJOR=${HIPFILE_LIBRARY_MAJOR} + HIPFILE_ROCP_REG_VERSION_MINOR=${HIPFILE_LIBRARY_MINOR} + HIPFILE_ROCP_REG_VERSION_PATCH=${HIPFILE_LIBRARY_PATCH}) target_link_libraries(hipfile PRIVATE rocprofiler-register::rocprofiler-register) set_target_properties(hipfile PROPERTIES INSTALL_RPATH "\$ORIGIN") endif() diff --git a/projects/hipfile/src/amd_detail/backend/fallback.cpp b/projects/hipfile/src/amd_detail/backend/fallback.cpp index 96d337051751..d669882f319e 100644 --- a/projects/hipfile/src/amd_detail/backend/fallback.cpp +++ b/projects/hipfile/src/amd_detail/backend/fallback.cpp @@ -39,8 +39,6 @@ using std::min; using std::shared_ptr; using std::unique_ptr; -static const size_t DefaultChunkSize = 16 * 1024 * 1024; - namespace { // Makes the buffer's GPU current for the lifetime of the guard (hipMemcpy operates on the current diff --git a/projects/hipfile/src/amd_detail/backend/fallback.h b/projects/hipfile/src/amd_detail/backend/fallback.h index d9b897344794..7fcae87f5d78 100644 --- a/projects/hipfile/src/amd_detail/backend/fallback.h +++ b/projects/hipfile/src/amd_detail/backend/fallback.h @@ -29,6 +29,8 @@ struct Fallback : public Backend { using Backend::io; virtual ~Fallback() override = default; + static constexpr size_t DefaultChunkSize = 16 * 1024 * 1024; + int score(const std::shared_ptr &file, const std::shared_ptr &buffer, size_t size, hoff_t file_offset, hoff_t buffer_offset) const override; diff --git a/projects/hipfile/src/amd_detail/hipfile.cpp b/projects/hipfile/src/amd_detail/hipfile.cpp index f3f7823ae0ca..102fa0406c5b 100644 --- a/projects/hipfile/src/amd_detail/hipfile.cpp +++ b/projects/hipfile/src/amd_detail/hipfile.cpp @@ -49,6 +49,13 @@ catch (hipFileError_t e) { catch (const Hip::RuntimeError &e) { return {hipFileHipDriverError, e.error}; } +catch (const std::system_error &err) { + int err_no = err.code().value(); + if (err_no == EMFILE || err_no == ENFILE) { + return {hipFileGetNewFDFailed, hipSuccess}; + } + return {hipFileInternalError, hipSuccess}; +} catch (...) { return {hipFileInternalError, hipSuccess}; } diff --git a/projects/hipfile/src/amd_detail/mountinfo.cpp b/projects/hipfile/src/amd_detail/mountinfo.cpp index f2fcadf00be6..4ab0468dff6c 100644 --- a/projects/hipfile/src/amd_detail/mountinfo.cpp +++ b/projects/hipfile/src/amd_detail/mountinfo.cpp @@ -9,6 +9,7 @@ #include #include #include +#include struct libmnt_context; struct libmnt_fs; @@ -64,6 +65,7 @@ std::optional LibMountHelper::getMountInfo(dev_t dev) const { auto libmount{Context::get()}; + int libmount_err; auto mnt_ctx{libmount->mnt_new_context()}; if (!mnt_ctx) { @@ -71,9 +73,11 @@ LibMountHelper::getMountInfo(dev_t dev) const } struct libmnt_table *mnt_tbl{}; - if (libmount->mnt_context_get_mtab(mnt_ctx, &mnt_tbl) != 0) { + libmount_err = libmount->mnt_context_get_mtab(mnt_ctx, &mnt_tbl); + if (libmount_err) { libmount->mnt_free_context(mnt_ctx); - throw std::runtime_error("libmount: Could not get mount table"); + throw std::system_error(-libmount_err, std::generic_category(), + "libmount: Could not get mount table"); } struct libmnt_fs *mnt_fs{libmount->mnt_table_find_devno(mnt_tbl, dev, MNT_ITER_FORWARD)}; diff --git a/projects/hipfile/src/amd_detail/thread-pool.cpp b/projects/hipfile/src/amd_detail/thread-pool.cpp index 7b052f4d187e..dd78f56316fd 100644 --- a/projects/hipfile/src/amd_detail/thread-pool.cpp +++ b/projects/hipfile/src/amd_detail/thread-pool.cpp @@ -19,8 +19,7 @@ namespace { class TaskflowTaskGroup final : public ITaskGroup { public: - explicit TaskflowTaskGroup(std::shared_ptr _executor) - : executor{std::move(_executor)}, state{std::make_shared()} + explicit TaskflowTaskGroup(std::shared_ptr _executor) : executor{std::move(_executor)} { } @@ -33,7 +32,7 @@ namespace { void run(std::function task) override { uint64_t task_generation = 0; - auto task_state = state; + auto task_state = &state; { std::lock_guard lock{task_state->mutex}; @@ -66,16 +65,16 @@ namespace { void cancel() override { - std::lock_guard lock{state->mutex}; + std::lock_guard lock{state.mutex}; - state->generation++; + state.generation++; } void wait() override { - std::unique_lock lock{state->mutex}; + std::unique_lock lock{state.mutex}; - state->cv.wait(lock, [this]() { return state->outstanding == 0; }); + state.cv.wait(lock, [this]() { return state.outstanding == 0; }); } private: @@ -88,7 +87,7 @@ namespace { class Completion { public: - explicit Completion(std::shared_ptr _state) : state{std::move(_state)} + explicit Completion(State *_state) : state{_state} { } @@ -104,21 +103,21 @@ namespace { Completion &operator=(Completion &&) = delete; private: - std::shared_ptr state; + State *state; }; - static void finish(const std::shared_ptr &state) + static void finish(State *state) { { std::lock_guard lock{state->mutex}; state->outstanding--; + state->cv.notify_all(); } - state->cv.notify_all(); } std::shared_ptr executor; - std::shared_ptr state; + State state; }; } diff --git a/projects/hipfile/src/nvidia_detail/CMakeLists.txt b/projects/hipfile/src/nvidia_detail/CMakeLists.txt index 1bef11be1fda..0c6a8182b8d3 100644 --- a/projects/hipfile/src/nvidia_detail/CMakeLists.txt +++ b/projects/hipfile/src/nvidia_detail/CMakeLists.txt @@ -11,7 +11,7 @@ set(HIPFILE_SOURCES ) # Create hipfile target -ais_add_libraries( +hipfile_add_libraries( NAME hipfile DETAIL nvidia SRCS ${HIPFILE_SOURCES} diff --git a/projects/hipfile/test/CMakeLists.txt b/projects/hipfile/test/CMakeLists.txt index 905fe5eafbdb..f3cf7d4b57a0 100644 --- a/projects/hipfile/test/CMakeLists.txt +++ b/projects/hipfile/test/CMakeLists.txt @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: MIT -include(AISAddExecutable) +include(HipFileAddExecutable) set(SHARED_SOURCE_FILES common/magic-word.cpp @@ -32,11 +32,16 @@ set(TEST_SYSINCLS ${HIPFILE_THIRD_PARTY_PATH} ) -if(AIS_BUILD_NVIDIA_DETAIL) +if(HIPFILE_BUILD_NVIDIA_DETAIL) list(APPEND UNIT_TEST_SOURCE_FILES nvidia_detail/cufile-api-compat.cpp) list(APPEND TEST_SYSINCLS ${HIPFILE_NVIDIA_SOURCE_PATH}) else() - list(APPEND SYSTEM_TEST_SOURCE_FILES system/amd/io.cpp) + list(APPEND SYSTEM_TEST_SOURCE_FILES + system/amd/fd-limit.cpp + system/amd/io.cpp + system/amd/io-data-modification.cpp + system/amd/io-data-modification-kernel.hip + ) list(APPEND SHARED_SOURCE_FILES common/ais-capability.cpp) list(APPEND TEST_SYSINCLS ${HIPFILE_AMD_SOURCE_PATH}) endif() @@ -48,7 +53,7 @@ endif() # (rather than the platform) means the target reappears automatically if a # portable unit test is later added to UNIT_TEST_SOURCE_FILES. if(UNIT_TEST_SOURCE_FILES) - ais_add_test_executable( + hipfile_add_test_executable( NAME hipfile_tests DEPS hipfile SRCS ${UNIT_TEST_SOURCE_FILES} ${SHARED_SOURCE_FILES} @@ -61,14 +66,14 @@ if(UNIT_TEST_SOURCE_FILES) target_compile_options(hipfile_tests PRIVATE -pthread) target_link_options(hipfile_tests PRIVATE -pthread) - ais_gtest_discover_tests( + hipfile_gtest_discover_tests( hipfile_tests PROPERTIES "LABELS;unit;LABELS;hipfile" TEST_LIST hipfile_unit_tests ) endif() -ais_add_test_executable( +hipfile_add_test_executable( NAME hipfile_system_tests DEPS hipfile SRCS ${SYSTEM_TEST_SOURCE_FILES} ${SHARED_SOURCE_FILES} @@ -85,23 +90,57 @@ if(HIPFILE_ALLOW_SKIP_FASTPATH_TESTS) set(_allow_skip_arg --allow-skip-fastpath) endif() -ais_gtest_discover_tests( +hipfile_gtest_discover_tests( hipfile_system_tests - EXTRA_ARGS --ais-capable-dir ${AIS_CAPABLE_DIR} ${_allow_skip_arg} + EXTRA_ARGS --ais-capable-dir ${HIPFILE_CAPABLE_DIR} ${_allow_skip_arg} # Keep this marker synchronized with the GTEST_SKIP message in system/amd/io.cpp. PROPERTIES "LABELS;system;LABELS;hipfile;SKIP_REGULAR_EXPRESSION;fastpath not available" TEST_LIST hipfile_system_tests ) +# Deprecated AIS_* CMake options +# +# Configures a small project with every deprecated AIS_* option set and checks +# that each one was mapped to its HIPFILE_* replacement. Remove these tests +# along with cmake/HipFileDeprecatedAISOptions.cmake. +# ---------------------------------------- +set(_ais_compat_build_dir "${CMAKE_CURRENT_BINARY_DIR}/ais-compat") +set(_ais_compat_args) +foreach(old_name IN LISTS HIPFILE_DEPRECATED_AIS_OPTIONS) + list(APPEND _ais_compat_args "-D${old_name}=ais-compat-${old_name}") +endforeach() + +# Start from an empty build directory each time so values left in the cache +# by an earlier run can't hide a failure +add_test(NAME hipfile_ais_compat_clean + COMMAND ${CMAKE_COMMAND} -E rm -rf "${_ais_compat_build_dir}" +) +add_test(NAME hipfile_ais_compat + COMMAND ${CMAKE_COMMAND} + -G "${CMAKE_GENERATOR}" + -S "${CMAKE_CURRENT_SOURCE_DIR}/cmake/ais-compat" + -B "${_ais_compat_build_dir}" + ${_ais_compat_args} +) +set_tests_properties(hipfile_ais_compat_clean PROPERTIES + FIXTURES_SETUP hipfile_ais_compat + LABELS "unit;hipfile" +) +set_tests_properties(hipfile_ais_compat PROPERTIES + FIXTURES_REQUIRED hipfile_ais_compat + LABELS "unit;hipfile" +) +# ---------------------------------------- + # Temporary Test Files # Can be deleted once proper testing added # ---------------------------------------- -ais_add_test_executable( +hipfile_add_test_executable( NAME test_hipfile_basic SRCS legacy/test-hipfile.cpp DEPS hipfile ) -ais_add_test_executable( +hipfile_add_test_executable( NAME test_hipfile_batch SRCS legacy/test-hipfile-batch.cpp DEPS hipfile diff --git a/projects/hipfile/test/amd_detail/CMakeLists.txt b/projects/hipfile/test/amd_detail/CMakeLists.txt index 35fe2d01088e..a12b8c26d724 100644 --- a/projects/hipfile/test/amd_detail/CMakeLists.txt +++ b/projects/hipfile/test/amd_detail/CMakeLists.txt @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: MIT -include(AISAddExecutable) +include(HipFileAddExecutable) find_package(Threads REQUIRED) set(SHARED_SOURCE_FILES @@ -41,7 +41,7 @@ set(TEST_SYSINCLS ${HIPFILE_THIRD_PARTY_PATH} ) -ais_add_test_executable( +hipfile_add_test_executable( NAME internal_tests DEPS hipfile SRCS ${TEST_SOURCE_FILES} ${SHARED_SOURCE_FILES} @@ -54,7 +54,7 @@ target_link_libraries(internal_tests PRIVATE GTest::gtest) target_compile_options(internal_tests PRIVATE -pthread) target_link_options(internal_tests PRIVATE -pthread) -ais_gtest_discover_tests( +hipfile_gtest_discover_tests( internal_tests PROPERTIES "LABELS;unit;LABELS;internal" TEST_LIST internal_unit_tests @@ -66,7 +66,7 @@ ais_gtest_discover_tests( # so test failures may not be due to the commit in which you # observe the failure. -ais_add_test_executable( +hipfile_add_test_executable( NAME state_mt DEPS hipfile SRCS "state_mt.cpp" @@ -81,7 +81,7 @@ add_test( ) set_tests_properties(state_mt_test PROPERTIES LABELS "stress;internal") -ais_add_test_executable( +hipfile_add_test_executable( NAME batch_mt DEPS hipfile SRCS "batch/batch_mt.cpp" diff --git a/projects/hipfile/test/amd_detail/handle.cpp b/projects/hipfile/test/amd_detail/handle.cpp index f98571201eea..ed40771d830f 100644 --- a/projects/hipfile/test/amd_detail/handle.cpp +++ b/projects/hipfile/test/amd_detail/handle.cpp @@ -279,6 +279,42 @@ TEST_F(HipFileHandle, RocfileHandleRegisterLibMountError) ASSERT_EQ(hipFileHandleRegister(&fh, &rfd), HipFileOpError(hipFileInternalError)); } +struct ErrnoParam { + int value; + const char *name; +}; + +// Readable test parameter in gtest/ctest output +static void +PrintTo(const ErrnoParam ¶m, std::ostream *os) +{ + *os << param.name; +} + +struct HipFileHandleOutOfFds : public HipFileHandle, public WithParamInterface {}; + +// If reopening the file during registration fails because the process or +// system is out of file descriptors return hipFileGetNewFDFailed +TEST_P(HipFileHandleOutOfFds, hipfileHandleRegisterReopenError) +{ + hipFileHandle_t fh{}; + hipFileDescr_t rfd{}; + rfd.type = hipFileHandleTypeOpaqueFD; + rfd.handle.fd = 0xBADF00D; + + // Deliberately test the registering non-O_DIRECT path as it catches std::system_error. + ExpectUnregisteredFileBuilder(msys, mlibmounthelper) + .fd_flags(~O_DIRECT) + .open_throws(GetParam().value) + .build(); + + ASSERT_EQ(hipFileHandleRegister(&fh, &rfd), HipFileOpError(hipFileGetNewFDFailed)); +} + +INSTANTIATE_TEST_SUITE_P(, HipFileHandleOutOfFds, + Values(ErrnoParam{EMFILE, "EMFILE"}, ErrnoParam{ENFILE, "ENFILE"}), + PrintToStringParamName()); + TEST_F(HipFileHandle, register_handle_linux_fd_already_registered) { hipFileHandle_t fh{}; diff --git a/projects/hipfile/test/amd_detail/mountinfo.cpp b/projects/hipfile/test/amd_detail/mountinfo.cpp index 671559fb51fc..cbd33a033046 100644 --- a/projects/hipfile/test/amd_detail/mountinfo.cpp +++ b/projects/hipfile/test/amd_detail/mountinfo.cpp @@ -13,9 +13,11 @@ #include #include #include +#include #include #include #include +#include #include struct libmnt_context; @@ -27,16 +29,37 @@ using namespace testing; HIPFILE_WARN_NO_GLOBAL_CTOR_OFF +// A simpler version of GTest's output. +#define CATCH_OTHER_EXCEPTIONS(expected) \ + catch (const std::exception &err) \ + { \ + /* Note: the typeid shown here is mangled. */ \ + FAIL() << "Expected " #expected " to be thrown.\n Actual: it throws " << typeid(err).name() \ + << " with a description \"" << err.what() << "\"."; \ + } \ + catch (...) \ + { \ + FAIL() << "Expected " #expected " to be thrown.\n " \ + "Actual: Something other than a std::exception was thrown."; \ + } + TEST(LibMountHelper, GetMountInfoThrowsOnContextCreationFailure) { StrictMock mlibmount; auto dev{makedev(123, 456)}; - EXPECT_CALL(mlibmount, mnt_new_context).Times(1).WillOnce(Return(nullptr)); - - ASSERT_THAT(([=] { LibMountHelper().getMountInfo(dev); }), - ThrowsMessage(StrEq("libmount: Could not create context"))); + EXPECT_CALL(mlibmount, mnt_new_context).WillOnce(Return(nullptr)); + + try { + LibMountHelper().getMountInfo(dev); + FAIL() << "Expected std::runtime_error to be thrown.\n " + "Actual: Nothing thrown."; + } + catch (const std::runtime_error &err) { + ASSERT_STREQ(err.what(), "libmount: Could not create context"); + } + CATCH_OTHER_EXCEPTIONS(std::runtime_error) } TEST(LibMountHelper, GetMountInfoThrowsOnGetMountTableFailure) @@ -50,11 +73,22 @@ TEST(LibMountHelper, GetMountInfoThrowsOnGetMountTableFailure) EXPECT_CALL(mlibmount, mnt_context_get_mtab(cxt, NotNull())).WillOnce(Return(-1)); EXPECT_CALL(mlibmount, mnt_free_context(cxt)); - ASSERT_THAT(([=] { LibMountHelper().getMountInfo(dev); }), - ThrowsMessage(StrEq("libmount: Could not get mount table"))); + try { + LibMountHelper().getMountInfo(dev); + FAIL() << "Expected std::system_error to be thrown.\n " + "Actual: Nothing thrown."; + } + catch (const std::system_error &err) { + // Additionally assert that the errno is being passed through. + // Note that the sign of the errno value is negated. + ASSERT_EQ(err.code().value(), 1); + // Don't check the message std::system_error appends to the error message. + ASSERT_THAT(err.what(), StartsWith("libmount: Could not get mount table")); + } + CATCH_OTHER_EXCEPTIONS(std::system_error) } -TEST(LibMountHelper, GetMountInfoThowsOnGetFilesystemOptionFailure) +TEST(LibMountHelper, GetMountInfoThrowsOnGetFilesystemOptionFailure) { StrictMock mlibmount; @@ -71,8 +105,15 @@ TEST(LibMountHelper, GetMountInfoThowsOnGetFilesystemOptionFailure) EXPECT_CALL(mlibmount, mnt_fs_get_option(fs, StrEq("data"), NotNull(), _)).WillOnce(Return(-1)); EXPECT_CALL(mlibmount, mnt_free_context(cxt)); - ASSERT_THAT(([=] { LibMountHelper().getMountInfo(dev); }), - ThrowsMessage(StrEq("libmount: Could not get mount option: data"))); + try { + LibMountHelper().getMountInfo(dev); + FAIL() << "Expected std::runtime_error to be thrown.\n " + "Actual: Nothing thrown."; + } + catch (const std::runtime_error &err) { + ASSERT_STREQ(err.what(), "libmount: Could not get mount option: data"); + } + CATCH_OTHER_EXCEPTIONS(std::runtime_error) } TEST(LibMountHelper, GetMountInfoReturnsEmptyOptionIfNoInfoForDevFound) diff --git a/projects/hipfile/test/amd_detail/thread-pool.cpp b/projects/hipfile/test/amd_detail/thread-pool.cpp index 601d47d5c4e1..581a746d9e5c 100644 --- a/projects/hipfile/test/amd_detail/thread-pool.cpp +++ b/projects/hipfile/test/amd_detail/thread-pool.cpp @@ -6,10 +6,12 @@ #include "hipfile-warnings.h" #include "thread-pool.h" +#include #include #include #include #include +#include #include @@ -140,4 +142,132 @@ TEST(HipFileThreadPool, DestructorCancelsAndWaits) ASSERT_FALSE(pending_ran); } +// Repeatedly publish captures to active workers and wait before reading their +// results. This also exercises task-group state teardown immediately after the +// last completion, which must finish notifying waiters before releasing state. +TEST(HipFileThreadPool, RepeatedPublicationAndWait) +{ + ThreadPool pool{4}; + for (int iteration = 0; iteration < 64; ++iteration) { + SCOPED_TRACE(iteration); + std::array results{}; + auto group = pool.makeTaskGroup(); + for (size_t index = 0; index < results.size(); ++index) { + const auto expected = static_cast(index) + iteration + 1; + group->run([&results, index, expected]() { results[index] = expected; }); + } + group->wait(); + group.reset(); + for (size_t index = 0; index < results.size(); ++index) { + EXPECT_EQ(results[index], static_cast(index) + iteration + 1); + } + } +} + +// Keep one task running while cancelling queued work, then destroy the group +// as the running task completes. Reuse the pool to exercise task-node reuse. +TEST(HipFileThreadPool, RepeatedCancelAndDestroy) +{ + ThreadPool pool{1}; + for (int iteration = 0; iteration < 64; ++iteration) { + SCOPED_TRACE(iteration); + std::atomic started{false}; + std::atomic release{false}; + std::atomic completed{false}; + std::atomic pending_count{0}; + auto group = pool.makeTaskGroup(); + group->run([&]() { + started.store(true); + started.notify_all(); + release.wait(false); + completed.store(true); + }); + started.wait(false); + for (int index = 0; index < 32; ++index) { + group->run([&]() { pending_count.fetch_add(1); }); + } + group->cancel(); + release.store(true); + release.notify_all(); + group.reset(); + EXPECT_TRUE(completed.load()); + EXPECT_EQ(pending_count.load(), 0); + } +} + +// Submissions from a Taskflow worker use a different queue from submissions +// from external threads. Keep the submitting worker occupied until all children +// finish so another worker must acquire their published captures. +TEST(HipFileThreadPool, WorkerPublishesToOtherWorkers) +{ + ThreadPool pool{4}; + for (int iteration = 0; iteration < 32; ++iteration) { + SCOPED_TRACE(iteration); + std::array results{}; + std::atomic remaining{32}; + std::atomic other_worker{false}; + auto group = pool.makeTaskGroup(); + group->run([&]() { + const auto publisher = std::this_thread::get_id(); + for (size_t index = 0; index < results.size(); ++index) { + const int expected = iteration + static_cast(index) + 1; + group->run([&, index, expected, publisher]() { + results[index] = expected; + if (std::this_thread::get_id() != publisher) { + other_worker.store(true); + } + if (remaining.fetch_sub(1) == 1) { + remaining.notify_all(); + } + }); + } + for (int count = remaining.load(); count != 0; count = remaining.load()) { + remaining.wait(count); + } + }); + group->wait(); + EXPECT_TRUE(other_worker.load()); + for (size_t index = 0; index < results.size(); ++index) { + EXPECT_EQ(results[index], iteration + static_cast(index) + 1); + } + } +} + +// Different external submitters and task groups share an executor's spill +// queues. Every task writes a distinct result and waits finish before reads. +TEST(HipFileThreadPool, ConcurrentExternalSubmitters) +{ + ThreadPool pool{4}; + for (int iteration = 0; iteration < 16; ++iteration) { + SCOPED_TRACE(iteration); + std::array, 4> results{}; + std::array, 4> groups; + std::array producers; + std::atomic ready{0}; + for (size_t producer = 0; producer < producers.size(); ++producer) { + groups[producer] = pool.makeTaskGroup(); + producers[producer] = std::thread([&, producer]() { + ready.fetch_add(1); + while (ready.load() != static_cast(producers.size())) { + std::this_thread::yield(); + } + for (size_t index = 0; index < results[producer].size(); ++index) { + const int expected = iteration + static_cast(producer + index) + 1; + groups[producer]->run( + [&, producer, index, expected]() { results[producer][index] = expected; }); + } + }); + } + for (auto &producer : producers) { + producer.join(); + } + for (size_t producer = 0; producer < groups.size(); ++producer) { + groups[producer]->wait(); + for (size_t index = 0; index < results[producer].size(); ++index) { + EXPECT_EQ(results[producer][index], iteration + static_cast(producer + index) + 1); + } + } + } +} + HIPFILE_WARN_NO_GLOBAL_CTOR_ON diff --git a/projects/hipfile/test/cmake/ais-compat/CMakeLists.txt b/projects/hipfile/test/cmake/ais-compat/CMakeLists.txt new file mode 100644 index 000000000000..79c54ac7e1e3 --- /dev/null +++ b/projects/hipfile/test/cmake/ais-compat/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# +# SPDX-License-Identifier: MIT + +# Configure-only test for cmake/HipFileDeprecatedAISOptions.cmake +# +# The hipfile_ais_compat test in test/CMakeLists.txt configures this project +# with every deprecated option set as -DAIS_=ais-compat-AIS_. The +# values are unique so a mix-up between options would be detected. This +# project doesn't declare any options, so HIPFILE_ will only be set if +# the compatibility layer mapped it. +# +# Remove this along with cmake/HipFileDeprecatedAISOptions.cmake. + +cmake_minimum_required(VERSION 3.21) + +project(hipfile_ais_compat_test LANGUAGES NONE) + +include("${CMAKE_CURRENT_LIST_DIR}/../../../cmake/HipFileDeprecatedAISOptions.cmake") + +foreach(old_name IN LISTS HIPFILE_DEPRECATED_AIS_OPTIONS) + string(REGEX REPLACE "^AIS_" "HIPFILE_" new_name "${old_name}") + set(expected "ais-compat-${old_name}") + if(NOT "${${new_name}}" STREQUAL "${expected}") + message(FATAL_ERROR "${new_name} is '${${new_name}}', expected '${expected}'") + endif() + if(DEFINED CACHE{${old_name}}) + message(FATAL_ERROR "${old_name} was not removed from the cache") + endif() +endforeach() diff --git a/projects/hipfile/test/system/amd/fd-limit.cpp b/projects/hipfile/test/system/amd/fd-limit.cpp new file mode 100644 index 000000000000..946bdc3efc86 --- /dev/null +++ b/projects/hipfile/test/system/amd/fd-limit.cpp @@ -0,0 +1,143 @@ +/* Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + * + * SPDX-License-Identifier: MIT + */ + +/* Verify hipFile reports hipFileGetNewFDFailed when the process runs out of + * file descriptors. + * + * These tests lower this process's soft RLIMIT_NOFILE, restoring it when each + * test finishes. They assume each test runs in its own process, as it does + * under ctest: running the whole binary directly lets earlier tests initialize + * hipFile first, which invalidates the fd limits chosen below. + */ + +#include "hipfile-warnings.h" +#include "hipfile.h" + +#include "test-common.h" +#include "test-options.h" +#include "test-shared-fixtures.h" + +#include +#include +#include +#include +#include +#include + +extern SystemTestOptions test_env; + +namespace { + +// Lowers the soft RLIMIT_NOFILE, restoring the original limits on destruction +class ScopedFdLimit { +public: + explicit ScopedFdLimit(rlim_t soft_limit) + { + if (getrlimit(RLIMIT_NOFILE, &m_saved) != 0) { + throw std::system_error(errno, std::generic_category(), "getrlimit"); + } + rlimit lowered{m_saved}; + lowered.rlim_cur = soft_limit; + if (setrlimit(RLIMIT_NOFILE, &lowered) != 0) { + throw std::system_error(errno, std::generic_category(), "setrlimit"); + } + } + + ~ScopedFdLimit() + { + setrlimit(RLIMIT_NOFILE, &m_saved); + } + + ScopedFdLimit(const ScopedFdLimit &) = delete; + ScopedFdLimit &operator=(const ScopedFdLimit &) = delete; + +private: + rlimit m_saved{}; +}; + +struct FdLimitParams { + const char *name; + int file_count; + rlim_t fd_limit; + hipFileError_t expected_err; +}; + +// Readable test parameter in gtest/ctest output, also used as the test name +void +PrintTo(const FdLimitParams ¶ms, std::ostream *os) +{ + *os << params.name; +} + +} + +HIPFILE_WARN_NO_GLOBAL_CTOR_OFF + +struct HipFileFdLimit : public DriverInit, public testing::WithParamInterface {}; + +// Registration needs an extra fd per file (hipFile reopens each file), so the +// parameters are chosen to be well clear of the exact point of exhaustion. +TEST_P(HipFileFdLimit, RegisterFiles) +{ + const FdLimitParams ¶ms{GetParam()}; + ScopedFdLimit fd_limit{params.fd_limit}; + + std::vector> files; + for (int i{0}; i < params.file_count; i++) { + files.push_back(std::make_unique(test_env.ais_capable_dir)); + } + + ASSERT_EQ(hipFileDriverOpen(), HIPFILE_SUCCESS); + + std::vector handles; + hipFileError_t err{HIPFILE_SUCCESS}; + for (const auto &file : files) { + hipFileDescr_t descr{}; + descr.type = hipFileHandleTypeOpaqueFD; + descr.handle.fd = file->fd; + + hipFileHandle_t handle{}; + err = hipFileHandleRegister(&handle, &descr); + if (!(err == HIPFILE_SUCCESS)) { + break; + } + handles.push_back(handle); + } + + for (hipFileHandle_t handle : handles) { + hipFileHandleDeregister(handle); + } + + EXPECT_EQ(err, params.expected_err); +} + +// On file registration, hipFile will open up a new fd to the same file that +// is being registered, just with slightly different flags. As such, to +// register N files with hipfile, there must be at least another N fds +// available. +// This does not consider any other file descriptors that hipFile may have +// opened internally. +const FdLimitParams fd_limit_params[] = { + {"Files10Limit50", 10, 50, HIPFILE_SUCCESS}, + {"Files30Limit50", 30, 50, HipFileOpError(hipFileGetNewFDFailed)}, +}; + +INSTANTIATE_TEST_SUITE_P(, HipFileFdLimit, testing::ValuesIn(fd_limit_params), + testing::PrintToStringParamName()); + +// A limit of 0 leaves no fds for the stats server hipFile creates on first use +TEST(HipFileFdLimitDriverOpen, DriverOpenOutOfFds) +{ + { + ScopedFdLimit fd_limit{0}; + ASSERT_EQ(hipFileDriverOpen(), HipFileOpError(hipFileGetNewFDFailed)); + } + + // With fds available again the driver opens normally + ASSERT_EQ(hipFileDriverOpen(), HIPFILE_SUCCESS); + ASSERT_EQ(hipFileDriverClose(), HIPFILE_SUCCESS); +} + +HIPFILE_WARN_NO_GLOBAL_CTOR_ON diff --git a/projects/hipfile/test/system/amd/io-data-modification-kernel.h b/projects/hipfile/test/system/amd/io-data-modification-kernel.h new file mode 100644 index 000000000000..0ac8e7a4ab9f --- /dev/null +++ b/projects/hipfile/test/system/amd/io-data-modification-kernel.h @@ -0,0 +1,53 @@ +/* Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + * + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include +#include +#include + +namespace hipFileTest { + +// Launches the grid-stride integer verify+modify kernel and synchronizes the device. +// +// device_buffer_start: start of the whole device allocation. +// device_buffer_size: element count of the whole allocation. +// data_start: element index of the start of the data within the allocation; must satisfy +// data_start + n <= device_buffer_size. +// n: data element count. +// expect_base: value the kernel expects at data index 0. +// first_bad_idx: device int, init -1; first data index not matching the pattern. +// slack_sentinel: value every non-data allocation element must still hold. +// first_bad_slack_idx: device int, init -1; first non-data allocation index != sentinel. +// modify_stride: only data elements whose index is a multiple of this are +// doubled; verify-in still covers the whole data region. Must be +// >= 1; pass 1 to double every element. +hipError_t launchVerifyAndModify(int32_t *device_buffer_start, size_t device_buffer_size, size_t data_start, + size_t n, int32_t expect_base, int32_t *first_bad_idx, + int32_t slack_sentinel, int32_t *first_bad_slack_idx, dim3 grid, + dim3 workgroup, size_t modify_stride); + +// Launches the grid-stride byte-granular verify+modify kernel and synchronizes the device. +// +// buf: start of the whole device allocation (in BYTES). +// buf_size: byte count of the whole allocation. +// data_start: byte index of the start of the Data within the allocation; must satisfy +// data_start + n <= buf_size. +// n: Data byte count. +// entry: value every Data byte must hold on entry. +// modified: value qualifying Data bytes are set to. +// first_bad_idx: device int32, init -1; first Data index not equal to `entry`. +// slack_sentinel: value every device sentinel region byte must still hold. +// first_bad_slack_idx: device int32, init -1; first device sentinel region allocation index != sentinel. +// modify_stride: only Data bytes whose data-index is a multiple of this are set to +// `modified`; verify-in still covers the whole Data region. Must be +// >= 1; pass 1 to modify every byte. +hipError_t launchVerifyAndModifyBytes(uint8_t *buf, size_t buf_size, size_t data_start, size_t n, + uint8_t entry, uint8_t modified, int32_t *first_bad_idx, + uint8_t slack_sentinel, int32_t *first_bad_slack_idx, dim3 grid, + dim3 workgroup, size_t modify_stride); + +} // namespace hipFileTest diff --git a/projects/hipfile/test/system/amd/io-data-modification-kernel.hip b/projects/hipfile/test/system/amd/io-data-modification-kernel.hip new file mode 100644 index 000000000000..097909e8f6b8 --- /dev/null +++ b/projects/hipfile/test/system/amd/io-data-modification-kernel.hip @@ -0,0 +1,120 @@ +/* Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + * + * SPDX-License-Identifier: MIT + */ + +#include "io-data-modification-kernel.h" + +#include + +namespace hipFileTest { + +// Grid-stride integer verify+modify kernel. +// +// This kernel verifies: +// - bytes within [device_buffer_start, device_buffer_start + device_buffer_size) +// but outside of [device_buffer_start + data_start, device_buffer_start + data_start + n) +// are untouched (remain equal to slack_sentinel this buffer was initialized +// with by hipMemcpy). +// - bytes (interpreted as integers) within [device_buffer_start + data_start, +// device_buffer_start + data_start + n) are initially equal to the expected +// pattern (buffer + data_start)[i] = expect_base + i. +// +// This kernel then modifies the data by: +// - (buffer + data_start)[i] = expect_base + i iff i % modify_stride != 0. +// = (expect_base + i)*2 iff i % modify_stride == 0. +static __global__ void +verifyAndModify(int32_t *device_buffer_start, size_t device_buffer_size, size_t data_start, size_t n, + int32_t expect_base, int32_t *first_bad_idx, int32_t slack_sentinel, + int32_t *first_bad_slack_idx, size_t modify_stride) +{ + const size_t stride = static_cast(blockDim.x) * gridDim.x; + const size_t start = blockIdx.x * blockDim.x + threadIdx.x; + + for (size_t i = start; i < device_buffer_size; i += stride) { + const bool in_data = (data_start <= i) && (i < data_start + n); + if (in_data) { + const size_t d = i - data_start; + if (device_buffer_start[i] != expect_base + static_cast(d)) { + atomicCAS(first_bad_idx, -1, static_cast(d)); + } + if (d % modify_stride == 0) { + device_buffer_start[i] = device_buffer_start[i] * 2; + } + } + else { + if (device_buffer_start[i] != slack_sentinel) { + atomicCAS(first_bad_slack_idx, -1, static_cast(i)); + } + } + } +} + +hipError_t +launchVerifyAndModify(int32_t *device_buffer_start, size_t device_buffer_size, size_t data_start, size_t n, + int32_t expect_base, int32_t *first_bad_idx, int32_t slack_sentinel, + int32_t *first_bad_slack_idx, dim3 grid, dim3 workgroup, size_t modify_stride) +{ + verifyAndModify<<>>(device_buffer_start, device_buffer_size, data_start, n, expect_base, + first_bad_idx, slack_sentinel, first_bad_slack_idx, modify_stride); + hipError_t launch_err = hipGetLastError(); + if (launch_err != hipSuccess) { + return launch_err; + } + return hipDeviceSynchronize(); +} + +// Grid-stride byte-wise verify+modify kernel. +// +// This kernel verifies: +// - bytes within [buf, buf + buf_size) but outside of [buf + data_start, buf + data_start + n) +// are untouched (remain equal to slack_sentinel this buffer was initialized +// with by hipMemcpy). +// - bytes within [device_buffer_start + data_start, device_buffer_start + data_start + n) +// are initially equal to the expected pattern (buf + data_start)[i] = entry. +// +// This kernel then modifies the data by: +// - (buf + data_start)[i] = entry iff i % modify_stride != 0. +// = modified iff i % modify_stride == 0. +static __global__ void +verifyAndModifyBytes(uint8_t *buf, size_t buf_size, size_t data_start, size_t n, uint8_t entry, + uint8_t modified, int32_t *first_bad_idx, uint8_t slack_sentinel, + int32_t *first_bad_slack_idx, size_t modify_stride) +{ + const size_t stride = static_cast(blockDim.x) * gridDim.x; + const size_t start = blockIdx.x * blockDim.x + threadIdx.x; + + for (size_t i = start; i < buf_size; i += stride) { + const bool in_data = (data_start <= i) && (i < data_start + n); + if (in_data) { + const size_t d = i - data_start; + if (buf[i] != entry) { + atomicCAS(first_bad_idx, -1, static_cast(d)); + } + if (d % modify_stride == 0) { + buf[i] = modified; + } + } + else { + if (buf[i] != slack_sentinel) { + atomicCAS(first_bad_slack_idx, -1, static_cast(i)); + } + } + } +} + +hipError_t +launchVerifyAndModifyBytes(uint8_t *buf, size_t buf_size, size_t data_start, size_t n, uint8_t entry, + uint8_t modified, int32_t *first_bad_idx, uint8_t slack_sentinel, + int32_t *first_bad_slack_idx, dim3 grid, dim3 workgroup, size_t modify_stride) +{ + verifyAndModifyBytes<<>>(buf, buf_size, data_start, n, entry, modified, first_bad_idx, + slack_sentinel, first_bad_slack_idx, modify_stride); + hipError_t launch_err = hipGetLastError(); + if (launch_err != hipSuccess) { + return launch_err; + } + return hipDeviceSynchronize(); +} + +} // namespace hipFileTest diff --git a/projects/hipfile/test/system/amd/io-data-modification.cpp b/projects/hipfile/test/system/amd/io-data-modification.cpp new file mode 100644 index 000000000000..4b7774c910bc --- /dev/null +++ b/projects/hipfile/test/system/amd/io-data-modification.cpp @@ -0,0 +1,1142 @@ +/* Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + * + * SPDX-License-Identifier: MIT + */ + +#include "backend/fallback.h" +#include "configuration.h" +#include "context.h" +#include "hipfile-literals.h" +#include "hipfile-warnings.h" +#include "hipfile.h" + +#include "io-data-modification-kernel.h" +#include "io-test.h" +#include "test-common.h" +#include "test-options.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern SystemTestOptions test_env; + +namespace hipFileTest { + +// =========================================================================================================== +// Constants and helper functions for setting up sentinel regions of files and device buffers, as well as +// verifying that the GPU correctly modified the targeted data without clobbering sentinel regions. +// =========================================================================================================== +inline constexpr int32_t kPatternBase = 1; + +// --------------------------------------------------------------------------- +// Device sentinel region: values and sizing. +// +// Sentinel values for writing to device buffer regions to ensure that hipFile does not transfer data to +// device memory regions where it is not supposed to. +// --------------------------------------------------------------------------- +constexpr int32_t kSentinel = -1; +constexpr uint8_t kSentinelByte = 0xFF; + +// Number of elements in a sentinel region (sentinel region always 4KiB). +inline constexpr size_t +slackElems() +{ + return 4_KiB / sizeof(int32_t); +} + +// --------------------------------------------------------------------------- +// Host-side file preparation. +// --------------------------------------------------------------------------- +inline void +fillIndexPattern(int32_t *arr, size_t n) +{ + for (size_t i = 0; i < n; ++i) { + arr[i] = kPatternBase + static_cast(i); + } +} + +inline void +seedFilePattern(int fd, hoff_t byte_offset, size_t n) +{ + std::vector host(n); + fillIndexPattern(host.data(), n); // host[i] == i+1 + ssize_t rv = pwrite(fd, host.data(), n * sizeof(int32_t), byte_offset); + ASSERT_EQ(static_cast(n * sizeof(int32_t)), rv); +} + +inline void +seedFileConstant(int fd, hoff_t byte_offset, size_t n, int32_t value) +{ + std::vector host(n, value); + ssize_t rv = pwrite(fd, host.data(), n * sizeof(int32_t), byte_offset); + ASSERT_EQ(static_cast(n * sizeof(int32_t)), rv); +} + +inline void +seedFileBytesConstant(int fd, hoff_t byte_offset, size_t n, uint8_t value) +{ + std::vector host(n, value); + ssize_t rv = pwrite(fd, host.data(), n, byte_offset); + ASSERT_EQ(static_cast(n), rv); +} + +// --------------------------------------------------------------------------- +// Host-side file inspection. +// --------------------------------------------------------------------------- +inline hoff_t +fileSize(int fd) +{ + struct stat st {}; + EXPECT_EQ(0, fstat(fd, &st)); + return static_cast(st.st_size); +} + +inline std::vector +readFileInts(int fd, hoff_t byte_offset, size_t n) +{ + std::vector file(n); + ssize_t rv = pread(fd, file.data(), n * sizeof(int32_t), byte_offset); + EXPECT_EQ(static_cast(n * sizeof(int32_t)), rv); + return file; +} + +inline std::vector +readFileBytes(int fd, hoff_t byte_offset, size_t n) +{ + std::vector file(n); + ssize_t rv = pread(fd, file.data(), n, byte_offset); + EXPECT_EQ(static_cast(n), rv); + return file; +} + +// --------------------------------------------------------------------------- +// Host-side memory verification. Can be memory read from either a file or device buffer. +// --------------------------------------------------------------------------- +inline void +assertHoleZero(int fd, hoff_t from, hoff_t to) +{ + ASSERT_LE(from, to); + const size_t n = static_cast(to - from); + if (n == 0) { + return; + } + std::vector hole(n, 0xEE); // pre-fill non-zero so a short pread is caught + ssize_t rv = pread(fd, hole.data(), n, from); + ASSERT_EQ(static_cast(n), rv); + for (size_t i = 0; i < n; ++i) { + ASSERT_EQ(0, hole[i]) << "hole byte non-zero at file offset " << (from + static_cast(i)); + } +} + +// Asserts every element was doubled: arr[i] == 2 * (kPatternBase + i). +inline void +assertDoubledPattern(const int32_t *arr, size_t n) +{ + for (size_t i = 0; i < n; ++i) { + ASSERT_EQ(2 * (kPatternBase + static_cast(i)), arr[i]) << "doubled mismatch at index " << i; + } +} + +// Asserts the every-Nth-element modify policy: element i is doubled iff +// (i % modify_stride == 0), otherwise it still holds the un-doubled seed. +inline void +assertModifiedPattern(const int32_t *arr, size_t n, size_t modify_stride) +{ + ASSERT_NE(0U, modify_stride) << "modify_stride must be >= 1"; + for (size_t i = 0; i < n; ++i) { + const int32_t seed = kPatternBase + static_cast(i); + const int32_t want = (i % modify_stride == 0) ? 2 * seed : seed; + ASSERT_EQ(want, arr[i]) << "modify-policy mismatch at index " << i; + } +} + +inline void +assertConstant(const int32_t *arr, size_t from, size_t to, int32_t value) +{ + ASSERT_LE(from, to); + for (size_t i = from; i < to; ++i) { + ASSERT_EQ(value, arr[i]) << "sentinel changed at index " << i; + } +} + +inline constexpr uint8_t kByteEntry = 0xFF; // Initial value +inline constexpr uint8_t kByteModified = 0x22; +inline constexpr uint8_t kByteDevSlack = 0xAA; +inline constexpr uint8_t kByteFileSlack = 0x55; + +// Asserts the every-Nth-byte modify policy over n Data bytes: byte i equals +// kByteModified iff (i % modify_stride == 0), otherwise it still holds kByteEntry. +inline void +assertBytesModified(const uint8_t *arr, size_t n, size_t modify_stride) +{ + ASSERT_NE(0U, modify_stride) << "modify_stride must be >= 1"; + for (size_t i = 0; i < n; ++i) { + const uint8_t want = (i % modify_stride == 0) ? kByteModified : kByteEntry; + ASSERT_EQ(want, arr[i]) << "byte modify-policy mismatch at index " << i; + } +} + +// Asserts bytes in [from, to) all equal `value` to verify untouched data was truly untouched. +inline void +assertBytesConstant(const uint8_t *arr, size_t from, size_t to, uint8_t value) +{ + ASSERT_LE(from, to); + for (size_t i = from; i < to; ++i) { + ASSERT_EQ(value, arr[i]) << "byte constant region changed at index " << i; + } +} + +// --------------------------------------------------------------------------- +// Device-side buffer inspection. +// --------------------------------------------------------------------------- +inline std::vector +readbackInts(void *device_buffer, hoff_t byte_offset, size_t n) +{ + std::vector host(n); + EXPECT_EQ(hipSuccess, hipMemcpy(host.data(), + reinterpret_cast(reinterpret_cast(device_buffer) + + static_cast(byte_offset)), + n * sizeof(int32_t), hipMemcpyDeviceToHost)); + return host; +} + +// --------------------------------------------------------------------------- +// Device-side buffer preparation. +// --------------------------------------------------------------------------- +inline void +seedDevicePattern(void *device_buffer, hoff_t byte_offset, size_t n) +{ + std::vector host(n); + fillIndexPattern(host.data(), n); // host[i] == i+1 + ASSERT_EQ(hipSuccess, hipMemcpy(reinterpret_cast(reinterpret_cast(device_buffer) + + static_cast(byte_offset)), + host.data(), n * sizeof(int32_t), hipMemcpyHostToDevice)); +} + +// =========================================================================================================== +// Default or frequently used sizes and offsets. +// =========================================================================================================== +constexpr size_t kChunkBytes = hipFile::Fallback::DefaultChunkSize; +constexpr int32_t kDefaultWorkgroupSize = 256; +constexpr hoff_t kChunkOff = static_cast(kChunkBytes); + +// Where a suite places its data to be modified in a file or buffer by default. +constexpr hoff_t kFileOffBase = static_cast(kChunkBytes + 4_KiB); +constexpr hoff_t kBufOffBase = static_cast(4_KiB); + +// Matrix for unaligned test cases. +constexpr size_t kSmallAligned = 4_KiB; +constexpr size_t kSmallUnaligned = 4_KiB + 1; +constexpr size_t kLargeAligned = kChunkBytes + 4_KiB; +constexpr size_t kLargeUnaligned = kChunkBytes + 4_KiB + 1; + +// =========================================================================================================== +// Named cases where a test is extending the length of a file by using hipFileWrite. +// =========================================================================================================== +// An extending write covers [file_off, file_off + io_bytes) of a file whose length before the write is +// base_len. hipFileWrite should be able to extend the length of a file, and when hipFileWrite targets a +// file_off that is beyond the existing end of the file (file_end), then [file_end, file_off) should be filled +// with zeros. +struct ExtendCase { + hoff_t base_len; // file length before the extending write + hoff_t file_off; // where the extending write starts +}; + +// Append to a zero-length file. +inline constexpr ExtendCase +appendFromEmpty() +{ + return {0, 0}; +} + +// Append exactly at the EOF of a file. +inline constexpr ExtendCase +appendAt(hoff_t eof) +{ + return {eof, eof}; +} + +// Append at `gap` many bytes past the EOF of a file. [base_len, base_len + gap) should be filled with zeros. +inline constexpr ExtendCase +holeAfter(hoff_t base_len, hoff_t gap) +{ + return {base_len, base_len + gap}; +} + +// Write to a `file_off` that appears before the EOF, but also extend the file with this write. +inline constexpr ExtendCase +overwriteAppend(hoff_t file_off, hoff_t overlap) +{ + return {file_off + overlap, file_off}; +} + +// =========================================================================================================== +// Common kernel launch configurations, helpers for launching kernels and checking results for errors. +// =========================================================================================================== +enum class GridMode { Default, OneWorkgroup, ManyWorkgroups }; + +constexpr unsigned kManyWorkgroups = 300; + +inline dim3 +defaultGrid(size_t n) +{ + constexpr unsigned wg_size = kDefaultWorkgroupSize; + const size_t workgroups = (n + wg_size - 1) / wg_size; + const unsigned grid = static_cast(std::min(workgroups, 65535)); + return dim3(grid == 0 ? 1 : grid); +} + +inline dim3 +gridFor(GridMode mode, size_t n) +{ + switch (mode) { + case GridMode::OneWorkgroup: + return dim3(1); + case GridMode::ManyWorkgroups: + return dim3(kManyWorkgroups); + case GridMode::Default: + default: + return defaultGrid(n); + } +} + +// RAII for flag in device memory the kernel uses to report the first bad index. +struct BadIdxFlag { + int32_t *dev{nullptr}; + + BadIdxFlag() + { + if (hipMalloc(&dev, sizeof(int32_t)) != hipSuccess) { + dev = nullptr; + ADD_FAILURE() << "hipMalloc for the kernel bad-index flag failed"; + return; + } + reset(); + } + ~BadIdxFlag() + { + if (dev != nullptr) { + (void)hipFree(dev); + } + } + // Callers must check this before launching a kernel that dereferences dev. + bool allocated() const + { + return dev != nullptr; + } + void reset() + { + if (dev == nullptr) { + return; + } + int32_t init = -1; + EXPECT_EQ(hipSuccess, hipMemcpy(dev, &init, sizeof(int32_t), hipMemcpyHostToDevice)); + } + int32_t value() const + { + int32_t v = -2; + if (dev == nullptr) { + return v; + } + EXPECT_EQ(hipSuccess, hipMemcpy(&v, dev, sizeof(int32_t), hipMemcpyDeviceToHost)); + return v; + } +}; + +// --------------------------------------------------------------------------- +// Kernel launch and error checking helpers. +// --------------------------------------------------------------------------- +// Launches the int32 verify+modify kernel and asserts neither the payload pattern nor the +// device sentinel region was corrupted. +inline void +assertVerifyAndModify(int32_t *start, size_t alloc_n, size_t data_start, size_t n, dim3 grid, dim3 workgroup, + size_t modify_stride = 1) +{ + ASSERT_NE(0U, modify_stride) << "modify_stride must be >= 1"; + BadIdxFlag bad; + BadIdxFlag bad_slack; + ASSERT_TRUE(bad.allocated() && bad_slack.allocated()) << "kernel bad-index flag allocation failed"; + ASSERT_EQ(hipSuccess, launchVerifyAndModify(start, alloc_n, data_start, n, kPatternBase, bad.dev, + kSentinel, bad_slack.dev, grid, workgroup, modify_stride)); + ASSERT_EQ(-1, bad.value()) << "payload pattern corrupted"; + ASSERT_EQ(-1, bad_slack.value()) << "untouched device sentinel region was clobbered"; +} + +// Launches the byte verify+modify kernel and asserts neither the payload bytes nor the +// device sentinel region was corrupted. +inline void +assertVerifyAndModifyBytes(uint8_t *start, size_t alloc_bytes, size_t data_start, size_t n, dim3 grid, + dim3 workgroup, size_t modify_stride) +{ + ASSERT_NE(0U, modify_stride) << "modify_stride must be >= 1"; + BadIdxFlag bad; + BadIdxFlag bad_slack; + ASSERT_TRUE(bad.allocated() && bad_slack.allocated()) << "kernel bad-index flag allocation failed"; + ASSERT_EQ(hipSuccess, launchVerifyAndModifyBytes(start, alloc_bytes, data_start, n, kByteEntry, + kByteModified, bad.dev, kByteDevSlack, bad_slack.dev, + grid, workgroup, modify_stride)); + ASSERT_EQ(-1, bad.value()) << "payload bytes corrupted"; + ASSERT_EQ(-1, bad_slack.value()) << "untouched device sentinel region was clobbered"; +} + +// =========================================================================================================== +// Common struct containing test parameters that will be reused by all tests with helpers for creating +// complex matrices of tests. +// =========================================================================================================== +struct IoTestScenario { + std::string name{}; // the full gtest case name + IoTestBackend backend = IoTestBackend::Fallback; // backend fulfilling the I/O request + size_t io_bytes = 4_KiB; // bytes transferred through the hipFile API + hoff_t file_off = 0; // where the data sits in the file + hoff_t buf_off = 0; // where the data sits in the device buffer + size_t stride = 1; // 1 == modify every element + GridMode grid = GridMode::Default; // workgroup count the kernel launches with + std::optional ext = std::nullopt; // set only for extending writes +}; + +// A value of one of the parameters of an `IoTestScenario` and the string to reflect +// it with in the test name. +template struct Axis { + T value; + std::string name; +}; + +// --------------------------------------------------------------------------- +// This class is responsible for building a series of IoTestScenarios. Allowing you +// to use a single class/object to represent all of the parameters of a test instance, +// as opposed to many small classes. +// +// This class contains functions that mimic functionality of GTest's `param_generator`s +// (e.g., `over` for `testing::Combine`, `add` for `testing::Values`). +// --------------------------------------------------------------------------- +class IoTestScenarioSet { +public: + explicit IoTestScenarioSet(const IoTestScenario &base) : rows_{base} + { + } + + // This function is like GTest's `testing::Combine`. You specify a `field` of `IoTestScenario` + // that you would like to vary over a range of values defined by `axis`, and `over` + // multiplies each existing `IoTestScenario` in `rows_` to have an instance that takes on + // each of the values in `axis`. + template IoTestScenarioSet &over(F IoTestScenario::*field, const Range &axis) + { + std::vector expanded; + for (const IoTestScenario &row : rows_) { + for (const auto &a : axis) { + IoTestScenario next = row; + next.*field = a.value; + next.name = next.name.empty() ? a.name : next.name + "_" + a.name; + expanded.push_back(next); + } + } + rows_ = expanded; + return *this; + } + + // If you were to repeatedly use this function, this function is like GTest's `testing::Values`, + // except you can append to an existing set of `IoTestScenario`. + // This function should be used for adding bespoke test parameter scenarios that cannot be + // expressed by a matrix of individual parameters that are combined by `over`. + IoTestScenarioSet &add(const IoTestScenario &one) + { + rows_.push_back(one); + return *this; + } + + // This function combines two sets of `IoTestScenario`, which typically isn't possible in GTest, + // unless you used multiple `INSTANTIATE_TEST_SUITE_P` calls. + IoTestScenarioSet &add(const IoTestScenarioSet &other) + { + rows_.insert(rows_.end(), other.rows_.begin(), other.rows_.end()); + return *this; + } + + std::vector build() const + { + return rows_; + } + +private: + std::vector rows_; +}; + +inline std::string +ioTestScenarioName(const testing::TestParamInfo &info) +{ + return info.param.name; +} + +// --------------------------------------------------------------------------- +// Axis values that are common between many test suites. +// --------------------------------------------------------------------------- +HIPFILE_WARN_NO_EXIT_DTOR_OFF +inline const std::array, 2> kBackends{{ + {IoTestBackend::Fastpath, "Fastpath"}, + {IoTestBackend::Fallback, "Fallback"}, +}}; + +inline const std::array, 3> kCombinedSizes{{ + {4_KiB, "sub_chunk"}, + {kChunkBytes + 4_KiB, "cross_chunk"}, + {2 * kChunkBytes, "multi_chunk"}, +}}; +HIPFILE_WARN_NO_EXIT_DTOR_ON + +// =========================================================================================================== +// Common helper functions used in the tests, specialized for given data types. +// =========================================================================================================== +// Element policies that determine the datatype that the GPU kernels will operate on. +struct IntElementPolicy { + using Elem = int32_t; + + static constexpr uint8_t kDevFill = kSentinelByte; + + static constexpr size_t elems(size_t bytes) + { + return bytes / sizeof(Elem); + } + + static void seedFileData(int fd, hoff_t off, size_t n) + { + seedFilePattern(fd, off, n); + } + + static void seedFileSlack(int fd, hoff_t off, size_t n) + { + seedFileConstant(fd, off, n, kSentinel); + } + + static void seedDeviceData(void *buf, hoff_t off, size_t n) + { + seedDevicePattern(buf, off, n); + } + + static std::vector readFile(int fd, hoff_t off, size_t n) + { + return readFileInts(fd, off, n); + } + + static void assertFileSlack(const Elem *arr, size_t from, size_t to) + { + assertConstant(arr, from, to, kSentinel); + } + + static void assertModified(const Elem *arr, size_t n, size_t stride) + { + assertModifiedPattern(arr, n, stride); + } + + static void verifyAndModify(Elem *start, size_t alloc_n, size_t data_start, size_t n, dim3 grid, + dim3 workgroup, size_t stride) + { + assertVerifyAndModify(start, alloc_n, data_start, n, grid, workgroup, stride); + } +}; + +struct ByteElementPolicy { + using Elem = uint8_t; + + static constexpr uint8_t kDevFill = kByteDevSlack; + + static constexpr size_t elems(size_t bytes) + { + return bytes; + } + + static void seedFileData(int fd, hoff_t off, size_t n) + { + seedFileBytesConstant(fd, off, n, kByteEntry); + } + + static void seedFileSlack(int fd, hoff_t off, size_t n) + { + seedFileBytesConstant(fd, off, n, kByteFileSlack); + } + + static void seedDeviceData(void *buf, hoff_t off, size_t n) + { + ASSERT_EQ(hipSuccess, + hipMemset(static_cast(buf) + static_cast(off), kByteEntry, n)); + } + + static std::vector readFile(int fd, hoff_t off, size_t n) + { + return readFileBytes(fd, off, n); + } + + static void assertFileSlack(const Elem *arr, size_t from, size_t to) + { + assertBytesConstant(arr, from, to, kByteFileSlack); + } + + static void assertModified(const Elem *arr, size_t n, size_t stride) + { + assertBytesModified(arr, n, stride); + } + + static void verifyAndModify(Elem *start, size_t alloc_bytes, size_t data_start, size_t n, dim3 grid, + dim3 workgroup, size_t stride) + { + assertVerifyAndModifyBytes(start, alloc_bytes, data_start, n, grid, workgroup, stride); + } +}; + +// =========================================================================================================== +// Base test fixture for common file and buffer allocation and modification for test suites to reuse. +// =========================================================================================================== +// Individual suites should use ftruncate to size the file's initial length before +// calling this SetUp. +template struct DataModificationBase : public testing::TestWithParam { + + const IoTestBackend backend{GetParam().backend}; // backend fulfilling the I/O request + const size_t io_bytes{GetParam().io_bytes}; // bytes transferred using the hipFile API + // Over-allocate a device sentinel region on each side of the data. + // Device layout (each sentinel region 4_KiB, data io_bytes): + // [head device sentinel region][data][tail device sentinel region]. + const size_t buffer_bytes{io_bytes + 2 * 4_KiB}; // total device buffer size + const size_t io_elems{Policy::elems(io_bytes)}; // transferred elements + const size_t buffer_elems{Policy::elems(buffer_bytes)}; // elements spanning the whole buffer + + Tmpfile tmpfile; + hipFileHandle_t tmpfile_handle{nullptr}; + void *device_buffer{nullptr}; // allocated by SetUp + typename Policy::Elem *buffer_start{nullptr}; // typed view of device_buffer + + DataModificationBase() : tmpfile{test_env.ais_capable_dir} + { + } + + void SetUp() override + { + hipFile::Context::get()->fastpath(false); + hipFile::Context::get()->fallback(false); + + switch (backend) { + case IoTestBackend::Fastpath: + hipFile::Context::get()->fastpath(true); + break; + case IoTestBackend::Fallback: + hipFile::Context::get()->fallback(true); + break; + default: + FAIL() << "Unsupported IoTestBackend"; + } + + hipFileDescr_t descr{}; + descr.type = hipFileHandleTypeOpaqueFD; + descr.handle.fd = tmpfile.fd; + ASSERT_EQ(HIPFILE_SUCCESS, hipFileHandleRegister(&tmpfile_handle, &descr)); + + ASSERT_EQ(hipSuccess, hipMalloc(&device_buffer, buffer_bytes)); + buffer_start = static_cast(device_buffer); + ASSERT_EQ(hipSuccess, hipMemset(device_buffer, Policy::kDevFill, buffer_bytes)); + // hipMemset is not synchronous w.r.t. the host, and hipFileRead is not ordered w.r.t. the stream. + ASSERT_EQ(hipSuccess, hipDeviceSynchronize()); + + if (backend == IoTestBackend::Fastpath) { + enforceFastpathGate(tmpfile_handle, device_buffer); + } + } + + void TearDown() override + { + if (device_buffer) { + ASSERT_EQ(hipSuccess, hipFree(device_buffer)); + } + hipFileHandleDeregister(tmpfile_handle); + } +}; + +// =========================================================================================================== +// Shared test bodies that are reused across multiple test suites. +// =========================================================================================================== +// These bodies need to be called with ASSERT_NO_FATAL_FAILURE from a TEST_P in order for +// failures to propagate. + +// Round trip with both the device sentinel regions and the file sentinel regions guarded +// (checked to make sure they are not erroneously modified). +template +void +runAllRegionsTest(DataModificationBase &f) +{ + using Elem = typename Policy::Elem; + + const IoTestScenario &s = f.GetParam(); + + const size_t n = f.io_elems; + const size_t slack_n = Policy::elems(4_KiB); + const hoff_t buf_off = s.buf_off; + const hoff_t file_off = s.file_off; + const hoff_t head_off = file_off - static_cast(4_KiB); // head file sentinel region + const hoff_t tail_off = file_off + static_cast(f.io_bytes); + + // File layout (each sentinel region slack_n elements, data n elements; data begins at + // file_off past the chunk boundary): + // [unwritten hole = 0][head file sentinel region][data][tail file sentinel region]. + Policy::seedFileSlack(f.tmpfile.fd, head_off, slack_n); + Policy::seedFileData(f.tmpfile.fd, file_off, n); + Policy::seedFileSlack(f.tmpfile.fd, tail_off, slack_n); + + ASSERT_EQ(static_cast(f.io_bytes), + hipFileRead(f.tmpfile_handle, f.device_buffer, f.io_bytes, file_off, buf_off)); + + // Device layout (each sentinel region slack_n elements, data n elements): + // [head device sentinel region][data][tail device sentinel region]. + Policy::verifyAndModify(f.buffer_start, f.buffer_elems, Policy::elems(static_cast(buf_off)), n, + gridFor(s.grid, n), dim3(kDefaultWorkgroupSize), s.stride); + + ASSERT_EQ(static_cast(f.io_bytes), + hipFileWrite(f.tmpfile_handle, f.device_buffer, f.io_bytes, file_off, buf_off)); + + // Start of file that was supposed to be untouched remains untouched. + assertHoleZero(f.tmpfile.fd, 0, head_off); + // Existing data before write is fully intact. + std::vector head = Policy::readFile(f.tmpfile.fd, head_off, slack_n); + Policy::assertFileSlack(head.data(), 0, slack_n); + // Modified data is correctly modified. + std::vector body = Policy::readFile(f.tmpfile.fd, file_off, n); + Policy::assertModified(body.data(), n, s.stride); + // Existing data after write is fully intact. + std::vector tail = Policy::readFile(f.tmpfile.fd, tail_off, slack_n); + Policy::assertFileSlack(tail.data(), 0, slack_n); +} + +// A write that extends the file size. If hipFileWrite is made past the current EOF, that +// gap should be zero-filled. +template +void +runExtendTest(DataModificationBase &f) +{ + using Elem = typename Policy::Elem; + + const IoTestScenario &s = f.GetParam(); + ASSERT_TRUE(s.ext.has_value()) << "runExtendTest requires IoTestScenario::ext to be set"; + + const size_t n = f.io_elems; + const hoff_t base_len = s.ext->base_len; + const hoff_t file_off = s.ext->file_off; + const hoff_t buf_off = s.buf_off; + const size_t data_start = Policy::elems(static_cast(buf_off)); + + // Device layout (data begins at buf_off, after the head sentinel region): + // [head device sentinel region][data][tail device sentinel region]. + Policy::seedDeviceData(f.device_buffer, buf_off, n); + + Policy::verifyAndModify(f.buffer_start, f.buffer_elems, data_start, n, gridFor(s.grid, n), + dim3(kDefaultWorkgroupSize), s.stride); + + // File layout after the write (the hole spans [base_len, file_off) and is empty when + // we append to the file): + // [preserved][hole = 0][data = stride-modified], when file_off >= base_len. + // [preserved][data = stride-modified], when file_off < base_len. + const hoff_t preserved_end = (file_off < base_len) ? file_off : base_len; + const size_t preserved_n = Policy::elems(static_cast(preserved_end)); + if (preserved_n > 0) { + Policy::seedFileSlack(f.tmpfile.fd, 0, preserved_n); + } + + ASSERT_EQ(static_cast(f.io_bytes), + hipFileWrite(f.tmpfile_handle, f.device_buffer, f.io_bytes, file_off, buf_off)); + + std::vector body = Policy::readFile(f.tmpfile.fd, file_off, n); + Policy::assertModified(body.data(), n, s.stride); + + // Hole was zero-filled. + if (file_off > base_len) { + assertHoleZero(f.tmpfile.fd, base_len, file_off); + } + + // Final size correct. + ASSERT_EQ(file_off + static_cast(f.io_bytes), fileSize(f.tmpfile.fd)); + + // Existing data below the write is fully intact. + if (preserved_n > 0) { + std::vector head = Policy::readFile(f.tmpfile.fd, 0, preserved_n); + Policy::assertFileSlack(head.data(), 0, preserved_n); + } +} + +} // namespace hipFileTest + +using namespace hipFile; +using namespace hipFileTest; + +HIPFILE_WARN_NO_GLOBAL_CTOR_OFF + +// =========================================================================================================== +// Beginning of test suites. +// =========================================================================================================== + +// This test suite exercises the ability of hipFileRead and hipFileWrite while varying the size of I/O and the +// backend that is used to fulfill the I/O request. +// +// We verify hipFileRead and hipFileWrite behave as expected by guarding the targeted regions of the +// device memory allocation and the region of the file that data will be read from and written to, surrounding +// them with poisoned memory containing sentinel values that would tell us if hipFile ever read or wrote data +// to a location that the user did not specify. +struct HipFileVerify : public DataModificationBase { + void SetUp() override + { + // Size the file to hold io_bytes of data plus a leading empty chunk. + ASSERT_EQ(0, ftruncate(tmpfile.fd, static_cast(io_bytes + kChunkBytes))); + DataModificationBase::SetUp(); + } +}; + +// Isolated hipFileWrite test. +TEST_P(HipFileVerify, WritePersistsDoubledData) +{ + const size_t n = io_elems; + seedDevicePattern(device_buffer, 0, n); + + assertVerifyAndModify(buffer_start, buffer_elems, 0, n, defaultGrid(n), dim3(kDefaultWorkgroupSize)); + + ASSERT_EQ(static_cast(io_bytes), hipFileWrite(tmpfile_handle, device_buffer, io_bytes, 0, 0)); + + std::vector file = readFileInts(tmpfile.fd, 0, n); + assertDoubledPattern(file.data(), n); +} + +// Isolated hipFileRead test. +TEST_P(HipFileVerify, ReadDeliversDoubledData) +{ + const size_t n = io_elems; + seedFilePattern(tmpfile.fd, 0, n); + + ASSERT_EQ(static_cast(io_bytes), hipFileRead(tmpfile_handle, device_buffer, io_bytes, 0, 0)); + + assertVerifyAndModify(buffer_start, buffer_elems, 0, n, defaultGrid(n), dim3(kDefaultWorkgroupSize)); + + std::vector buf = readbackInts(device_buffer, 0, n); + assertDoubledPattern(buf.data(), n); +} + +// hipFileRead + hipFileWrite at offset into device buffer (places sentinels before and after data). +// i.e., verify that data from hipFileRead does not clobber outside specified device buffer. +TEST_P(HipFileVerify, RoundTripGuardsDeviceSlack) +{ + const size_t n = io_elems; + const hoff_t buf_off = static_cast(4_KiB); // one head device sentinel region + + seedFilePattern(tmpfile.fd, 0, n); + ASSERT_EQ(static_cast(io_bytes), + hipFileRead(tmpfile_handle, device_buffer, io_bytes, 0, buf_off)); + + // Device layout (each sentinel region slackElems() ints, data n ints): + // [head device sentinel region][data][tail device sentinel region] + assertVerifyAndModify(buffer_start, buffer_elems, slackElems(), n, defaultGrid(n), + dim3(kDefaultWorkgroupSize)); + + ASSERT_EQ(static_cast(io_bytes), + hipFileWrite(tmpfile_handle, device_buffer, io_bytes, 0, buf_off)); + + std::vector file = readFileInts(tmpfile.fd, 0, n); + assertDoubledPattern(file.data(), n); +} + +// hipFileRead + hipFileWrite at offset into file buffer (places sentinels before and after data). +// i.e., verify that data from hipFileWrite does not clobber outside specified file buffer. +TEST_P(HipFileVerify, RoundTripGuardsFileSlack) +{ + const size_t n = io_elems; + const size_t bracket_n = 4_KiB / sizeof(int32_t); // file sentinel region head/tail each + const hoff_t file_off = static_cast(4_KiB); // data starts after head + const size_t total_n = bracket_n + n + bracket_n; // head + data + tail + const hoff_t tail_off = static_cast((bracket_n + n) * sizeof(int32_t)); + + // File layout (each sentinel region bracket_n ints, data n ints): + // [head file sentinel region = -1][data = i+1][tail file sentinel region = -1]. + seedFileConstant(tmpfile.fd, 0, bracket_n, kSentinel); + seedFilePattern(tmpfile.fd, file_off, n); + seedFileConstant(tmpfile.fd, tail_off, bracket_n, kSentinel); + + ASSERT_EQ(static_cast(io_bytes), + hipFileRead(tmpfile_handle, device_buffer, io_bytes, file_off, 0)); + + assertVerifyAndModify(buffer_start, buffer_elems, 0, n, defaultGrid(n), dim3(kDefaultWorkgroupSize)); + + ASSERT_EQ(static_cast(io_bytes), + hipFileWrite(tmpfile_handle, device_buffer, io_bytes, file_off, 0)); + + std::vector file = readFileInts(tmpfile.fd, 0, total_n); + assertConstant(file.data(), 0, bracket_n, kSentinel); + assertDoubledPattern(file.data() + bracket_n, n); + assertConstant(file.data(), bracket_n + n, total_n, kSentinel); +} + +HIPFILE_WARN_NO_EXIT_DTOR_OFF +const std::array, 5> kVerifySizes{{ + {4_KiB, "sub_chunk"}, + {kChunkBytes - 4_KiB, "near_chunk"}, + {kChunkBytes, "exact_chunk"}, + {kChunkBytes + 4_KiB, "cross_chunk"}, + {2 * kChunkBytes, "multi_chunk"}, +}}; +HIPFILE_WARN_NO_EXIT_DTOR_ON + +INSTANTIATE_TEST_SUITE_P(, HipFileVerify, + testing::ValuesIn(IoTestScenarioSet{IoTestScenario{}} + .over(&IoTestScenario::backend, kBackends) + .over(&IoTestScenario::io_bytes, kVerifySizes) + .build()), + ioTestScenarioName); + +// This test suite performs the same combined hipFileRead + hipFileWrite test that the hipFileVerify test +// suite exercises. We use the same poisoned memory region mechanism to verify hipFile transferred the +// specified data. We also varying the following new parameters: +// - the stride between cache lines that the modified data touches. +// - the number of workgroups that modify the data (this can also vary the number of different CUs or XCDs +// on the GPU that touch the data). +// Additionally, this test suite places the targeted data in the file at an offset past the fallback path's +// chunking boundary to ensure the fallback path operates on the specified chunks. +struct HipFileVerifyCombined : public DataModificationBase { + void SetUp() override + { + // File layout (each sentinel region 4_KiB, data io_bytes; data begins at file + // offset kFileOffBase past the chunk boundary): + // [head file sentinel region][data][tail file sentinel region]. + const hoff_t tail_off = GetParam().file_off + static_cast(io_bytes); + ASSERT_EQ(0, ftruncate(tmpfile.fd, tail_off + static_cast(4_KiB))); + DataModificationBase::SetUp(); + } +}; + +TEST_P(HipFileVerifyCombined, RoundTripGuardsAllRegions) +{ + ASSERT_NO_FATAL_FAILURE(runAllRegionsTest(*this)); +} + +HIPFILE_WARN_NO_EXIT_DTOR_OFF +const std::array, 2> kGridSweep{{ + {GridMode::OneWorkgroup, "1wg"}, + {GridMode::ManyWorkgroups, "300wg"}, +}}; + +const std::array, 3> kStrides{{{2, "stride2"}, {32, "stride32"}, {64, "stride64"}}}; +const std::array, 2> kStridesWide{{{2, "stride2"}, {512, "stride512"}}}; +const std::array, 1> kMultiChunkOnly{{{2 * kChunkBytes, "multi_chunk"}}}; +HIPFILE_WARN_NO_EXIT_DTOR_ON + +INSTANTIATE_TEST_SUITE_P(, HipFileVerifyCombined, + testing::ValuesIn(IoTestScenarioSet{ + IoTestScenario{.file_off = kFileOffBase, .buf_off = kBufOffBase}} + .over(&IoTestScenario::backend, kBackends) + .over(&IoTestScenario::io_bytes, kCombinedSizes) + .over(&IoTestScenario::stride, kStrides) + // A large sweep with many workgroups, exercising the behaviour + // of only some CUs modifying the data. + .add(IoTestScenarioSet{IoTestScenario{.file_off = kFileOffBase, + .buf_off = kBufOffBase}} + .over(&IoTestScenario::backend, kBackends) + .over(&IoTestScenario::io_bytes, kMultiChunkOnly) + .over(&IoTestScenario::grid, kGridSweep) + .over(&IoTestScenario::stride, kStridesWide)) + .build()), + ioTestScenarioName); + +// This test suite exercises the same hipFileRead + hipFileWrite behaviour, in combination with the behaviour +// of extending the file length to ensure that untouched regions of a file that is extended as a result of +// hipFileWrite are either the previously present data, or a hole of 0-initialized data, matching the POSIX +// behaviour. +struct HipFileExtend : public DataModificationBase { + void SetUp() override + { + // Every scenario must actually extend the file; the final-size assertion in the + // body depends on it. + ASSERT_TRUE(GetParam().ext.has_value()); + ASSERT_GT(GetParam().ext->file_off + static_cast(io_bytes), GetParam().ext->base_len); + + // Size file to base_len to be extended. + ASSERT_EQ(0, ftruncate(tmpfile.fd, static_cast(GetParam().ext->base_len))); + DataModificationBase::SetUp(); + } +}; + +TEST_P(HipFileExtend, Extends) +{ + ASSERT_NO_FATAL_FAILURE(runExtendTest(*this)); +} + +HIPFILE_WARN_NO_EXIT_DTOR_OFF +const std::array, 5> kExtendCases{{ + {appendFromEmpty(), "extend_empty_contiguous"}, + {appendAt(kChunkOff), "append_aligned"}, + {holeAfter(0, kChunkOff), "hole_from_empty"}, + {holeAfter(kChunkOff, kChunkOff), "hole_from_aligned"}, + {overwriteAppend(kChunkOff, static_cast(4_KiB) / 2), "overwrite_and_append"}, +}}; + +const std::array, 2> kExtendSizes{{{4_KiB, "small"}, {kChunkBytes + 4_KiB, "large"}}}; +HIPFILE_WARN_NO_EXIT_DTOR_ON + +INSTANTIATE_TEST_SUITE_P(, HipFileExtend, + testing::ValuesIn(IoTestScenarioSet{ + IoTestScenario{.buf_off = kBufOffBase, .stride = 2}} + .over(&IoTestScenario::backend, kBackends) + .over(&IoTestScenario::ext, kExtendCases) + .over(&IoTestScenario::io_bytes, kExtendSizes) + .build()), + ioTestScenarioName); + +// This test suite exercises the ability of hipFileRead and hipFileWrite while varying the size of I/O and the +// backend that is used to fulfill the I/O request. Additionally, we test the ability of hipFileRead and +// hipFileWrite to correctly transfer data when data is modified at a byte granularity. It additionally +// exercises unaligned I/O, varying the source of unaligned-ness (device buffer offset vs. file offset vs. +// size). +// +// NOTE: unaligned I/O only uses the fallback path. +// +// We verify hipFileRead and hipFileWrite behave as expected by guarding the targeted regions of the +// device memory allocation and the region of the file that data will be read from and written to, surrounding +// them with poisoned memory containing sentinel values that would tell us if hipFile ever read or wrote data +// to a location that the user did not specify. +struct HipFileVerifyBytes : public DataModificationBase { + void SetUp() override + { + // File layout (each sentinel region 4_KiB, data io_bytes; data begins at file + // offset GetParam().file_off past the chunk boundary). Size through the tail bracket: + // [head file sentinel region][data][tail file sentinel region] + const hoff_t tail_off = GetParam().file_off + static_cast(io_bytes); + ASSERT_EQ(0, ftruncate(tmpfile.fd, tail_off + static_cast(4_KiB))); + DataModificationBase::SetUp(); + } +}; + +TEST_P(HipFileVerifyBytes, RoundTripGuardsAllRegions) +{ + ASSERT_NO_FATAL_FAILURE(runAllRegionsTest(*this)); +} + +HIPFILE_WARN_NO_EXIT_DTOR_OFF +const std::array, 2> kFileOffs{{ + {kFileOffBase, "file_aligned"}, + {kFileOffBase + 1, "file_unaligned"}, +}}; + +const std::array, 2> kBufOffs{{ + {kBufOffBase, "buffer_aligned"}, + {kBufOffBase + 1, "buffer_unaligned"}, +}}; + +const std::array, 4> kUnalignedSizes{{ + {kSmallAligned, "small_aligned"}, + {kSmallUnaligned, "small_unaligned"}, + {kLargeAligned, "large_aligned"}, + {kLargeUnaligned, "large_unaligned"}, +}}; +HIPFILE_WARN_NO_EXIT_DTOR_ON + +INSTANTIATE_TEST_SUITE_P(, HipFileVerifyBytes, + testing::ValuesIn(IoTestScenarioSet{ + IoTestScenario{.file_off = kFileOffBase, .buf_off = kBufOffBase, .stride = 2}} + .over(&IoTestScenario::backend, kBackends) + .over(&IoTestScenario::io_bytes, kCombinedSizes) + .build()), + ioTestScenarioName); + +// Roughly a matrix of {device buffer aligned, unaligned} x {file offset aligned, unaligned} x {size +// aligned, unaligned}. Unaligned I/O only uses the fallback path, which is the IoTestScenario default. +INSTANTIATE_TEST_SUITE_P(Unaligned, HipFileVerifyBytes, + testing::ValuesIn(IoTestScenarioSet{IoTestScenario{.stride = 2}} + .over(&IoTestScenario::file_off, kFileOffs) + .over(&IoTestScenario::buf_off, kBufOffs) + .over(&IoTestScenario::io_bytes, kUnalignedSizes) + .build()), + ioTestScenarioName); + +// This test suite exercises the behaviour of extending the length of a file with an unaligned hipFileWrite, +// to ensure that untouched regions of a file that is extended are either the previously present data, or a +// hole of 0-initialized data, matching the POSIX behaviour. The length of the file before the write, the file +// offset of the write, the size of the write, and the device buffer offset each vary between aligned and one +// byte past a boundary. +// +// NOTE: unaligned I/O only uses the fallback path. +struct HipFileExtendUnaligned : public DataModificationBase { + void SetUp() override + { + // Every scenario must actually extend the file. + ASSERT_TRUE(GetParam().ext.has_value()); + ASSERT_GT(GetParam().ext->file_off + static_cast(io_bytes), GetParam().ext->base_len); + + ASSERT_EQ(0, ftruncate(tmpfile.fd, static_cast(GetParam().ext->base_len))); + DataModificationBase::SetUp(); + } +}; + +TEST_P(HipFileExtendUnaligned, Extends) +{ + ASSERT_NO_FATAL_FAILURE(runExtendTest(*this)); +} + +// The axis values every scenario is built from. Each unaligned value is its aligned counterpart +// pushed one byte past the boundary, so a scenario's name reads as the list of axes it misaligns. +constexpr hoff_t kBaseEmpty = 0; // file length before the extending write +constexpr hoff_t kBaseAligned = kChunkOff; +constexpr hoff_t kBaseUnaligned = kChunkOff + 1; + +constexpr hoff_t kGapAligned = + static_cast(4_KiB); // distance from EOF to the start of a write that leaves a hole +constexpr hoff_t kGapUnaligned = static_cast(4_KiB) + 1; + +// The bytes from the file that a write that also extends the file will overlap. +constexpr hoff_t kOverlap = static_cast(4_KiB) / 2; + +constexpr hoff_t kBufAligned = 0; // where the data starts within the device buffer +constexpr hoff_t kBufUnaligned = 1; + +constexpr size_t kUnalignedStride = 2; // every scenario modifies every other element + +static IoTestScenario +extendScenario(const char *name, ExtendCase ext, size_t io_bytes = kSmallAligned, + hoff_t buf_off = kBufAligned) +{ + return IoTestScenario{ + .name = name, .io_bytes = io_bytes, .buf_off = buf_off, .stride = kUnalignedStride, .ext = ext}; +} + +// These are hard to express using `testing::Combine` or `over`, so we use a helper function. +HIPFILE_WARN_NO_EXIT_DTOR_OFF +const std::array kExtendUnalignedScenarios{{ + // Append to empty file. + extendScenario("empty_contiguous_aligned_size", appendFromEmpty()), + extendScenario("empty_contiguous_unaligned_size", appendFromEmpty(), kSmallUnaligned), + // Contiguous append onto a non-empty file. + extendScenario("append_unaligned_base", appendAt(kBaseUnaligned)), + extendScenario("append_aligned_base_unaligned_size", appendAt(kBaseAligned), kSmallUnaligned), + // Write past EOF, creating a hole from the existing EOF of the file to the start of the write. + extendScenario("hole_from_empty_aligned_off", holeAfter(kBaseEmpty, kGapAligned)), + extendScenario("hole_from_empty_unaligned_off", holeAfter(kBaseEmpty, kGapUnaligned)), + extendScenario("hole_from_unaligned_base", holeAfter(kBaseUnaligned, kGapUnaligned)), + extendScenario("hole_unaligned_off_unaligned_size", holeAfter(kBaseAligned, kGapUnaligned), + kSmallUnaligned), + // Unaligned device buffer. + extendScenario("append_unaligned_buffer", appendAt(kBaseAligned), kSmallAligned, kBufUnaligned), + extendScenario("hole_unaligned_buffer", holeAfter(kBaseEmpty, kGapAligned), kSmallAligned, kBufUnaligned), + extendScenario("append_unaligned_base_unaligned_buffer", appendAt(kBaseUnaligned), kSmallUnaligned, + kBufUnaligned), + extendScenario("hole_all_unaligned", holeAfter(kBaseUnaligned, kGapUnaligned), kSmallUnaligned, + kBufUnaligned), + // Transfers larger than the fallback chunking size. + extendScenario("large_hole_cross_chunk", holeAfter(kBaseEmpty, kGapAligned), kLargeAligned), + extendScenario("large_append_unaligned_base", appendAt(kBaseUnaligned), kLargeUnaligned), + // Write before the existing EOF, but extend the length of the file with the same write. + extendScenario("overwrite_append_aligned_base", overwriteAppend(kBaseAligned, kOverlap)), + extendScenario("overwrite_append_all_unaligned", overwriteAppend(kBaseUnaligned, kOverlap), + kSmallUnaligned, kBufUnaligned), +}}; +HIPFILE_WARN_NO_EXIT_DTOR_ON + +INSTANTIATE_TEST_SUITE_P(, HipFileExtendUnaligned, testing::ValuesIn(kExtendUnalignedScenarios), + ioTestScenarioName); + +HIPFILE_WARN_NO_GLOBAL_CTOR_ON diff --git a/projects/hipfile/test/system/amd/io-test.h b/projects/hipfile/test/system/amd/io-test.h new file mode 100644 index 000000000000..3c14fd624469 --- /dev/null +++ b/projects/hipfile/test/system/amd/io-test.h @@ -0,0 +1,53 @@ +/* Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + * + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include "hipfile-warnings.h" +#include "hipfile.h" + +#include "ais-capability.h" +#include "test-options.h" + +#include +#include +#include + +extern SystemTestOptions test_env; + +enum class IoTestBackend { + Fastpath, + Fallback, +}; + +struct IoTestParam { + IoTestBackend backend; + std::string name; +}; + +HIPFILE_WARN_NO_EXIT_DTOR_OFF +inline const std::array io_test_params{ + {{IoTestBackend::Fastpath, "Fastpath"}, {IoTestBackend::Fallback, "Fallback"}}}; +HIPFILE_WARN_NO_EXIT_DTOR_ON + +// Gate fastpath-only tests on AIS capability. +inline void +enforceFastpathGate(hipFileHandle_t handle, void *device_buffer) +{ + hipFile::test::AisCapability ais_capability{test_env.allow_skip_fastpath}; + + const auto decision = ais_capability.populate(handle, device_buffer); + + if (decision == hipFile::test::AisCapability::GateDecision::Run) { + return; + } + + if (decision == hipFile::test::AisCapability::GateDecision::Skip) { + // Keep this marker synchronized with test/CMakeLists.txt SKIP_REGULAR_EXPRESSION. + GTEST_SKIP() << "fastpath not available in this environment\n" << ais_capability.report(); + } + + FAIL() << "Fastpath Validation Failed!\n" << ais_capability.report() << "\n" << ais_capability.skipHint(); +} diff --git a/projects/hipfile/test/system/amd/io.cpp b/projects/hipfile/test/system/amd/io.cpp index 8265a72d14bc..2ff614ea3c6a 100644 --- a/projects/hipfile/test/system/amd/io.cpp +++ b/projects/hipfile/test/system/amd/io.cpp @@ -9,7 +9,7 @@ #include "hipfile-warnings.h" #include "hipfile.h" -#include "ais-capability.h" +#include "io-test.h" #include "test-common.h" #include "test-options.h" @@ -29,47 +29,8 @@ extern SystemTestOptions test_env; using namespace hipFile; -namespace { - -// Gate fastpath-only tests on AIS capability. -void -enforceFastpathGate(hipFileHandle_t handle, void *device_buffer) -{ - hipFile::test::AisCapability ais_capability{test_env.allow_skip_fastpath}; - - const auto decision = ais_capability.populate(handle, device_buffer); - - if (decision == hipFile::test::AisCapability::GateDecision::Run) { - return; - } - - if (decision == hipFile::test::AisCapability::GateDecision::Skip) { - // Keep this marker synchronized with test/CMakeLists.txt SKIP_REGULAR_EXPRESSION. - GTEST_SKIP() << "fastpath not available in this environment\n" << ais_capability.report(); - } - - FAIL() << "Fastpath Validation Failed!\n" << ais_capability.report() << "\n" << ais_capability.skipHint(); -} - -} - HIPFILE_WARN_NO_GLOBAL_CTOR_OFF -enum class IoTestBackend { - Fastpath, - Fallback, -}; - -struct IoTestParam { - IoTestBackend backend; - std::string name; -}; - -HIPFILE_WARN_NO_EXIT_DTOR_OFF -static std::array io_test_params{ - {{IoTestBackend::Fastpath, "Fastpath"}, {IoTestBackend::Fallback, "Fallback"}}}; -HIPFILE_WARN_NO_EXIT_DTOR_ON - struct HipFileIo : public testing::TestWithParam { Tmpfile tmpfile; diff --git a/projects/hipfile/tools/ais-stats/CMakeLists.txt b/projects/hipfile/tools/ais-stats/CMakeLists.txt index 4d268b510abb..38742bfbb069 100644 --- a/projects/hipfile/tools/ais-stats/CMakeLists.txt +++ b/projects/hipfile/tools/ais-stats/CMakeLists.txt @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: MIT -include(AISAddExecutable) +include(HipFileAddExecutable) -ais_add_executable( +hipfile_add_executable( NAME ais-stats DEPS hipfile SRCS "ais-stats.cpp" diff --git a/projects/hipother/hipnv/include/hip/nvidia_detail/nvidia_hip_runtime_api.h b/projects/hipother/hipnv/include/hip/nvidia_detail/nvidia_hip_runtime_api.h index 38fb8cfc5dd3..a93b8a71f3f8 100644 --- a/projects/hipother/hipnv/include/hip/nvidia_detail/nvidia_hip_runtime_api.h +++ b/projects/hipother/hipnv/include/hip/nvidia_detail/nvidia_hip_runtime_api.h @@ -19,6 +19,7 @@ #define CUDA_10000 10000 #define CUDA_10010 10010 #define CUDA_10020 10020 +#define CUDA_11000 11000 #define CUDA_11010 11010 #define CUDA_11020 11020 #define CUDA_11030 11030 @@ -3122,6 +3123,17 @@ inline static hipError_t hipDeviceGetAttribute(int* pi, hipDeviceAttribute_t att case hipDeviceAttributeGPUDirectRDMAWithHipVMMSupported: return hipCUResultTohipError(cuDeviceGetAttribute( pi, CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED, device)); +#if CUDA_VERSION >= CUDA_11000 + case hipDeviceAttributeGPUDirectRDMASupported: + cdattr = cudaDevAttrGPUDirectRDMASupported; + break; + case hipDeviceAttributeGPUDirectRDMAFlushWritesOptions: + cdattr = cudaDevAttrGPUDirectRDMAFlushWritesOptions; + break; + case hipDeviceAttributeGPUDirectRDMAWritesOrdering: + cdattr = cudaDevAttrGPUDirectRDMAWritesOrdering; + break; +#endif // CUDA_VERSION >= CUDA_11000 #if CUDA_VERSION >= CUDA_12040 case hipDeviceAttributeHandleTypeFabricSupported: return hipCUResultTohipError(cuDeviceGetAttribute( @@ -3834,6 +3846,40 @@ inline static hipError_t hipDeviceGetByPCIBusId(int* device, const char* pciBusI return hipCUDAErrorTohipError(cudaDeviceGetByPCIBusId(device, pciBusId)); } +inline static hipError_t hipDeviceFlushGPUDirectRDMAWrites( + enum hipFlushGPUDirectRDMAWritesTarget target, enum hipFlushGPUDirectRDMAWritesScope scope) { + // Validate here rather than deferring to CUDA: cudaDeviceFlushGPUDirectRDMAWrites documents + // only cudaSuccess and cudaErrorNotSupported, so it is not specified to reject an + // out-of-domain target or scope. Checking first keeps the hipErrorInvalidValue contract + // identical on both backends. + if (target != hipFlushGPUDirectRDMAWritesTargetCurrentDevice) { + return hipErrorInvalidValue; + } + +#if CUDA_VERSION >= CUDA_11000 + enum cudaFlushGPUDirectRDMAWritesScope cudaScope; + switch (scope) { + case hipFlushGPUDirectRDMAWritesToOwner: + cudaScope = cudaFlushGPUDirectRDMAWritesToOwner; + break; + case hipFlushGPUDirectRDMAWritesToAllDevices: + cudaScope = cudaFlushGPUDirectRDMAWritesToAllDevices; + break; + default: + return hipErrorInvalidValue; + } + + return hipCUDAErrorTohipError(cudaDeviceFlushGPUDirectRDMAWrites( + cudaFlushGPUDirectRDMAWritesTargetCurrentDevice, cudaScope)); +#else + if (scope != hipFlushGPUDirectRDMAWritesToOwner && + scope != hipFlushGPUDirectRDMAWritesToAllDevices) { + return hipErrorInvalidValue; + } + return hipErrorNotSupported; +#endif // CUDA_VERSION >= CUDA_11000 +} + inline static hipError_t hipDeviceGetSharedMemConfig(hipSharedMemConfig* config) { return hipCUDAErrorTohipError(cudaDeviceGetSharedMemConfig(config)); } diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 5500a0f42b34..bff44dd5fd36 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -48,7 +48,7 @@ options in [README.md](README.md). ### Capture Layer (`projects/clr/hipamd/src/hrr/`) At HIP init time the capture layer snapshots the real `HipDispatchTable` and installs a parallel capture table with ~529 shim function pointers. Every HIP API call flows through one of these shims, which records a fixed-size `hrr_event_header` (32 bytes: type, sequence ID, timestamp, thread ID) followed by a variable-length payload of serialised arguments into a streaming `events.bin` file. -Twenty-seven APIs require hand-written shims (`MANUAL_CAPTURE_APIS`): kernel launches (4 variants), host-to-device memcpy (4 variants), module load (3 variants), `__hipRegisterFatBinary`, host memory registration (`hipHostRegister`, `hipHostUnregister`), all `hipMemcpy3D` variants (4), array creation, VMM, stream/memory attribute APIs, and others where struct pointer serialisation requires custom handling. All others are code-generated by `gen_hrr_api_args.py` from `hip_api_trace.hpp`. Fat binaries registered before shim install (C++ static-init) are recovered retroactively via `ForEachFatBinaryBlob` at init. +Thirty-six APIs require hand-written shims (`MANUAL_CAPTURE_APIS`): kernel launches (4 variants), host-to-device memcpy (4 variants), module load (3 variants), `__hipRegisterFatBinary`, host memory registration (`hipHostRegister`, `hipHostUnregister`), all `hipMemcpy3D` variants (4), array creation, VMM, stream/memory attribute APIs, and others where struct pointer serialisation requires custom handling. All others are code-generated by `gen_hrr_api_args.py` from `hip_api_trace.hpp`. Fat binaries registered before shim install (C++ static-init) are recovered retroactively via `ForEachFatBinaryBlob` at init. H2D memcpy captures the host source buffer as a content-addressed blob (FNV-1a-128 hash). D2H memcpy captures the expected output blob after a forced stream sync. Kernel argument pointers are detected via `kernel->signature()` (in-tree access to `amd::KernelSignature`) — no ELF parsing needed. @@ -131,7 +131,7 @@ precisely the ones left without a trailer and absent from the root index, while the parent that exited cleanly needs no repair. Sub-archives that already carry a clean trailer are skipped without being read. -### Archive Format (v5) +### Archive Format (v6) ``` capture.hrr/ manifest.json { version, capture_mode, owner_pid, processes[] } @@ -151,7 +151,7 @@ The reader (`hrr_reader`) loads all events sorted by sequence ID. One CPU thread Handle translation maps (one per HIP handle type) remap captured raw pointers to live playback handles. Device pointer translation supports sub-allocation offsets via range search through `alloc_map`. D2H memcpy validation compares live GPU output against captured expected blobs, reporting pass/fail counts. -Fifty-six APIs have hand-written playback handlers (`MANUAL_PLAYBACK_APIS`): everything in the capture manual set plus all malloc/free variants, stream/event create/destroy, `hipModuleGetFunction`, host memory registration, the full HIP graph chain (`hipStreamBeginCapture` → `hipStreamEndCapture` → `hipGraphInstantiate` → `hipGraphLaunch`), VMM APIs (`hipMemAddressReserve/Free`, `hipMemCreate/Release`, `hipMemMap/Unmap`), and others requiring handle translation or non-trivial replay logic. All others are code-generated. +Sixty-six APIs have hand-written playback handlers (`MANUAL_PLAYBACK_APIS`): everything in the capture manual set plus all malloc/free variants, stream/event create/destroy, `hipModuleGetFunction`, host memory registration, the full HIP graph chain (`hipStreamBeginCapture` → `hipStreamEndCapture` → `hipGraphInstantiate` → `hipGraphLaunch`), VMM APIs (`hipMemAddressReserve/Free`, `hipMemCreate/Release`, `hipMemMap/Unmap`), and others requiring handle translation or non-trivial replay logic. All others are code-generated. ### Ordering — `needs_ordering` Handle-creating and handle-destroying events (malloc, free, stream/event/module create/destroy, graph instantiate, etc.) must execute in global capture order so that handle maps are populated before any dependent thread reads them. Kernel launches and syncs are excluded — GPU stream ordering handles those. The `SeqAdvance` RAII guard in `dispatch_event` releases `next_seq` either immediately (non-ordering events) or on scope exit (ordering events), using a lookahead at the next event to decide. @@ -168,7 +168,9 @@ Handle-creating and handle-destroying events (malloc, free, stream/event/module | `projects/clr/hipamd/src/hrr/hip_capture_generated.cpp` | ~517 capture shims + `hip_capture_build_table()` | | `projects/hrr/playback/hip_playback_generated.cpp` | ~504 playback shims + `hrr_playback_dispatch[]` | -Run from any directory: `python3 projects/hrr/tools/gen_hrr_api_args.py` (the generator resolves its output paths relative to its own location). +The checked-in generated files above are the compilation inputs. During CLR builds, `projects/clr/cmake/HrrCodegen.cmake` generates temporary copies in the binary directory, runs `--check-hrr-coverage`, and updates any checked-in generated file that differs from its temporary counterpart. The build prints a warning listing every updated file. + +When a build updates generated files, review them with `git diff`. The generator can also be run directly from the repository root with `python3 projects/hrr/tools/gen_hrr_api_args.py --check-hrr-coverage` to refresh all checked-in generated files. ### Build ```bash @@ -240,7 +242,7 @@ projects/hrr/ — standalone HRR project (portable layer) Not used by capture — it is the format an out-of-tree producer writes and playback reads playback/ - hrr_reader.h/.cpp — archive loader; record framing + hrr_reader.h/.cpp — archive loader, v6 format; record framing (read_raw_record / open_record_stream) shared with the region sidecars hrr_region_map.h/.cpp — region timeline: merge, cursor, live block set, @@ -250,7 +252,7 @@ projects/hrr/ — standalone HRR project (portable layer) stream/event create/destroy, hipModuleGetFunction, hipHostRegister/Unregister, hipMemcpy3D variants, hipArrayCreate, VMM APIs, and more) - hip_playback_generated.cpp — AUTO-GENERATED: ~201 playback shims + dispatch table + hip_playback_generated.cpp — AUTO-GENERATED: ~245 playback shims + dispatch table hrr_playback.cpp — hrr-playback tool (replay + D2H validation + --info) CMakeLists.txt — hrr_reader lib + hrr-playback exe producers/ @@ -271,7 +273,7 @@ projects/clr/hipamd/src/hrr/ — in-tree capture layer (compiled into amdhi + dispatch table install/uninstall hip_capture_metadata.h/.cpp — capture-side metadata helpers hip_capture_writer.h/.cpp — streaming event serialization to events.bin / blobs/ - hip_capture_generated.cpp — AUTO-GENERATED: ~502 capture shims + build_table() + hip_capture_generated.cpp — AUTO-GENERATED: ~517 capture shims + build_table() CMakeLists.txt — adds capture sources to the amdhip64 target; the shared hrr_api_args.h wire format is included from projects/hrr/include ``` @@ -304,12 +306,12 @@ A single Python script produces three files from `hip_api_trace.hpp`: | Output | Description | |--------|-------------| | `projects/hrr/include/hrr/hrr_api_args.h` | Shared header: format constants, `hrr_event_header`, one `hrr_args_*` struct per API, `hrr_api_id_t` enum, `hrr_api_names[]` | -| `projects/clr/hipamd/src/hrr/hip_capture_generated.cpp` | ~502 capture shims for APIs not in `MANUAL_CAPTURE_APIS` | -| `projects/hrr/playback/hip_playback_generated.cpp` | ~201 playback shims + dispatch table for APIs not in `MANUAL_PLAYBACK_APIS` | +| `projects/clr/hipamd/src/hrr/hip_capture_generated.cpp` | ~517 capture shims for APIs not in `MANUAL_CAPTURE_APIS` | +| `projects/hrr/playback/hip_playback_generated.cpp` | ~245 playback shims + dispatch table for APIs not in `MANUAL_PLAYBACK_APIS` | Script location: `projects/hrr/tools/gen_hrr_api_args.py` -Run from any directory: `python3 projects/hrr/tools/gen_hrr_api_args.py` (output paths resolve relative to the script location). +Normal builds generate temporary files and verify them against the checked-in compilation inputs. Run `python3 projects/hrr/tools/gen_hrr_api_args.py --check-hrr-coverage` from the repository root only when intentionally refreshing those checked-in files; use the temporary-output command above for inspection. The generator classifies each API: - **`MANUAL_CAPTURE_APIS`** (27): kernel launches ×4, memcpy H2D+D2H ×8, module load ×3, `__hipRegisterFatBinary`, `hipHostRegister/Unregister`, `hipMemcpy3D` variants ×4, array creation ×2, VMM, stream/memory attribute APIs, `hipMemcpyWithStream` @@ -317,14 +319,15 @@ The generator classifies each API: - **`NOOP_PLAYBACK_APIS`** (~271): APIs where replay is not meaningful (stale handles, context-destroying calls, missing in ROCm SDK 6.4) — emit `return hipSuccess;` without calling the real function. These are **not replay-tested**; tests that exercise only these APIs verify capture does not crash, not that replay works. Generated capture shims for manual APIs are pass-throughs (no `write_event()`). +When adding HIP API support, update this script to classify the API in the appropriate capture and playback policy sets. APIs requiring non-trivial serialization or replay belong in `MANUAL_CAPTURE_APIS` and/or `MANUAL_PLAYBACK_APIS`; intentionally unsupported replay APIs belong in `NOOP_PLAYBACK_APIS`. -## Archive Format (v5) +## Archive Format (v6) Single-authority definition in `hrr_api_args.h` (auto-generated): ``` HRR_MAGIC = 0x52524845 ("HRRE") -HRR_VERSION = 5 +HRR_VERSION = 6 ``` Version history, so an archive written by an older runtime can be placed: @@ -354,7 +357,7 @@ Version history, so an archive written by an older runtime can be placed: / manifest.json { version, capture_mode, owner_pid, processes[] } (version here is the manifest schema = 1, distinct from the - events.bin HRR_VERSION = 5) + events.bin HRR_VERSION = 6) pid-/ manifest.json { pid, parent_pid, complete, event_count, blob_count } writer_state.json checkpoint cursor (next_seq, event/blob counts, events file @@ -657,20 +660,40 @@ with `hipEventRecord` to accumulate elapsed time into `total_graph_ms`. ## Init / Shutdown `hip_capture_init()` is called from `hip_context.cpp` at HIP init (after `amd::Runtime` -and the live `HipDispatchTable` are ready). If `HIP_HRR_CAPTURE_OUTPUT` is set it +and the live `HipDispatchTable` are ready). If capture is enabled (see [Enable +Flag](#enable-flag)) it snapshots the runtime dispatch table, installs runtime capture shims, opens the writer, -recovers pre-init fat binaries (compiler-table shims + retroactive sweep), and -registers `hip_capture_shutdown` via `atexit`. Runtime shims are **not** installed at +recovers pre-init fat binaries (compiler-table shims + retroactive sweep), +registers `hip_capture_shutdown` via `atexit`, and prints the capture notice. Runtime +shims are **not** installed at `libamdhip64` static-init time: that pulled every HIP call through capture from DSO load before `hip::init()` completed and disturbed host stacks that load HIP early (e.g. Python + `spawn`). Events before `writer::open()` were never persisted anyway. Shutdown uninstalls shims and flushes `events.bin` + `manifest.json`. +The capture notice is one line on stderr, printed with `fprintf` rather than through the +CLR log so that `AMD_LOG_LEVEL` cannot hide it. It names this process's archive +directory, `/pid-`, and says that child processes record to their own `pid-*` +directories in ``. It appears only after the writer has opened the archive, and at +most once per process, since `hip::init()` runs under `std::call_once`. A child created +with `fork()` does not run `hip::init()` again, so it records without a line of its own; +a child started with `exec` initialises HIP again and prints its own. + ## Enable Flag -Capture is enabled when `HIP_HRR_CAPTURE_OUTPUT` is set to a non-empty directory -(see [README.md](README.md#capture-environment)). Defined as a `cstring` release flag in -`rocclr/utils/flags.hpp`. +Capture is enabled when `HIP_HRR_CAPTURE_OUTPUT` is set to a directory and the process +was not started in secure-execution mode (see +[README.md](README.md#capture-environment)). Defined as a `cstring` release flag in +`rocclr/utils/flags.hpp`. An empty or blank value leaves capture off: the flag parser +stores an exported empty variable as a single space, and `hrr_capture_requested()` treats +a value made only of whitespace as unset. + +On Linux the kernel sets `AT_SECURE` in the auxiliary vector for a set-user-ID, +set-group-ID or file-capability exec, and for an LSM transition. Such a process can hold +privileges that whoever set its environment does not, so it ignores the variable, as +`secure_getenv()` would, and `hip_capture_init()` prints one line on stderr saying so. +The check is `hrr_cap::metadata::secure_exec()` in `hip_capture_metadata.cpp`, and it is +always false off Linux. ## Playback Tools @@ -971,24 +994,20 @@ There is no practical fix: capturing general CPU synchronisation would require instrumentation of the application binary, OS synchronisation primitives, or a language runtime — all of which are out of scope for a HIP API trace. -### `__hipRegisterVar` / `__hipRegisterManagedVar` — Not Replayed +### `__hipRegisterManagedVar` — Not Replayed These compiler-generated functions populate the runtime's internal host-symbol → device-symbol table used by `hipMemcpyToSymbol`, `hipMemcpyFromSymbol`, `hipGetSymbolAddress`, and `hipGetSymbolSize`. -The capture shims record events for `__hipRegisterVar` and `__hipRegisterManagedVar`, -but the symbol name is stored only as a raw `uint64_t` host pointer (a capture-time -address, useless at replay) rather than the name string, and the playback shims are -no-ops. The host-symbol → device-symbol table is therefore never rebuilt. +`__hipRegisterVar` is handled — see "Symbol Registry" below: the name string is recorded +and replay rebuilds the table through `hipModuleGetGlobal`, so the `__device__` global +families replay. `__hipRegisterManagedVar` is not: a `__managed__` global is never +registered in a way the sweep sees, so its symbol never reaches `symbol_map`. -**Impact:** workloads that use `hipMemcpyToSymbol` / `hipMemcpyFromSymbol` will fail -at replay because the symbol table is never populated. Kernel launches and all -pointer-based memcpy APIs are unaffected. - -**Workaround:** none currently. A fix would require recording the variable name and -host pointer, then at playback calling `hipModuleGetGlobal` on the loaded module to -resolve the device address and rebuilding the symbol table entry. +**Impact:** a workload whose `hipMemcpy*Symbol` subject is a `__managed__` rather than a +`__device__` global fails loudly at replay, naming the symbol it could not resolve. +Kernel launches and all pointer-based memcpy APIs are unaffected. ### `co_hash` Kernel → Code-Object Binding @@ -1281,31 +1300,47 @@ Other HIP features with similar complications include device-side enqueue, coope groups with CPU-side barriers, and persistent kernels with CPU-side steering. These are not currently handled. -### Explicit Graph Construction — Not Supported (fails loudly) - -Only the **stream-capture** graph chain is supported (`hipStreamBeginCapture` → -`hipStreamEndCapture` → `hipGraphInstantiate` → `hipGraphLaunch`). Graphs built -explicitly through the node API are not — and replay no longer silently produces an -empty graph with skipped launches (finding H1). The failure is split into two levels: - -- **Hard fail-loud at instantiate.** `hipGraphInstantiate` / `hipGraphInstantiateWithFlags` - (manual handlers) return `hipErrorNotSupported` — which the dispatcher treats as - fatal — when the graph handle is absent from `graph_map`. A miss can only happen for - a node-API-built graph, since the stream-capture chain always records its graph via - `hipStreamEndCapture`. This is the point that actually matters: a non-replayable - graph can never be launched as an empty graph. -- **Attributable warning at construction (non-fatal).** `hipGraphCreate` / - `hipGraphClone` / every `hipGraphAdd*Node` / `hipGraphInstantiateWithParams` are - `ERROR_STUB_PLAYBACK_APIS`: they log a loud, per-API "explicit (node-API) graph - construction is NOT supported" message and return `hipSuccess`. They are deliberately - non-fatal so a program that merely *creates/clones* a graph (or exercises these APIs - for coverage) without instantiating it in an unsupported way still replays; if such a - graph is later instantiated, the instantiate gate above fires. -- Node-parameter struct pointers are still stored as raw `uint64_t` without - dereferencing (full node-API replay remains future work). - -The supported stream-capture path is unaffected: it never emits `hipGraphCreate` / -`hipGraphAdd*Node` events, and its graph is in `graph_map` so instantiate succeeds. +### Explicit Graph Construction — Supported, With One Refusal + +Both graph chains replay: **stream capture** (`hipStreamBeginCapture` → +`hipStreamEndCapture` → `hipGraphInstantiate` → `hipGraphLaunch`) and **explicit +node-API construction** (`hipGraphCreate` → `hipGraphAdd*Node` → `hipGraphInstantiate`). +A node-API graph used to replay as an empty graph with its launches skipped (finding +H1); it is now rebuilt node by node: + +- `graph_node_map` in `PlaybackContext` maps each recorded `hipGraphNode_t` to the node + replay created, which is what lets dependency arrays, `*NodeGetParams` and the + `*NodeSetParams` / `*ExecNodeSetParams` mutation family refer to earlier nodes. +- Every `hipGraphAdd*Node` construction API has a real handler. Node-parameter structs + are deep-copied at capture through `DEREF_FIELDS` (kernel node params reuse the + kernel-argument encoder), and at replay the pointers, streams, events and symbols + inside them are translated before the node is created. +- The mutation spellings are real handlers rather than no-ops, so a graph whose + parameters change between launches replays with the parameters the recording used + instead of every launch repeating the first one. + +The fail-loud gate at instantiate stays as the backstop. `hipGraphInstantiate` returns +`hipErrorNotSupported` when the graph handle is absent from `graph_map`, or when the +graph contains a node HRR cannot reproduce — a host-function node, since host callbacks +are a declared scope exclusion. A graph missing work the recording had is refused rather +than launched. + +### Symbol Registry — `__device__` Globals Only + +Replay loads code objects with `hipModuleLoadData` and so has no host shadow variables, +while the `hipMemcpy*Symbol`, `hipGetSymbolAddress/Size` and graph symbol-node families +all take a host shadow address as their subject. Capture bridges the gap by sweeping the +registered `__device__` globals (`StatCO::ForEachGlobalVar`) and recording each one's +*name* alongside the recorded host address; replay resolves the name through +`hipModuleGetGlobal` and keeps the result in `symbol_map`. The graph symbol nodes are +then spelled as 1D memcpy nodes against the resolved device address, which is +semantically the same node without needing a host shadow to point at. + +The sweep runs from `hip_capture_init()`, which is itself called from `hip::init()`, so +it must not call back into a public HIP entry point: doing so re-enters the +initialisation once-guard and deadlocks. Resolution therefore happens inside +`ForEachGlobalVar` while it already holds the code-object lock, falling back to device 0 +when no device is current yet. ### Wire-Format Size Limits @@ -1316,7 +1351,7 @@ The event wire format (finding H5): shrinking `reserved` to 2 bytes), so kernel launches with large serialized payloads (many args / long mangled names / large by-value structs) up to ~4 GiB are recorded normally instead of being dropped at 65535 bytes. This is the change that bumped - `HRR_VERSION` to 4; the current version is 5, see Archive Format above. The + `HRR_VERSION` to 4; the current version is 6, see Archive Format above. The writer's single-record buffer path now writes any oversized record straight through. - **Per-argument size limit (64 KiB) now fails loudly.** Each kernel arg's size is still a `uint16_t`. A by-value struct argument ≥ 64 KiB cannot be represented, so the launch @@ -1325,6 +1360,10 @@ The event wire format (finding H5): `manifest.complete=false`, so replay/validation cannot mistake a capture missing a GPU launch for a faithful one. (Previously the size wrapped mod 65536, slipping past the total-payload guard and writing a corrupt event.) +- **Kernel-name length limit (64 KiB) fails loudly too.** The kernel name's length is + also a `uint16_t` on the wire. A launch whose name is longer than 65,535 bytes is + dropped and the archive marked incomplete in the same way, rather than recorded with a + truncated name that matches no symbol at replay. - **Pointer-translation size precondition.** Whole-arg pointer translation requires the recorded `arg_size >= 8`; a smaller pointer descriptor is copied through untranslated, passing the stale capture-time VA to the kernel. @@ -1395,15 +1434,53 @@ resolver scans **all** loaded modules and binds (and caches) the first same-name function from any code object, with no identity check — potentially binding the wrong kernel. -### Host Callbacks and Struct-Pointer Inputs — Not Captured - -- `hipStreamAddCallback` / `hipLaunchHostFunc` record the callback pointer as 0; the - host-side work and any device-staging side effects it performs are never captured or - replayed. This is distinct from the app-thread CPU-sync limitation above. -- By default the code generator stores an input struct-pointer parameter as a raw - `uint64_t` without serialising the pointed-to struct; only hand-written manual shims - copy struct bytes. A newly-captured, non-manual struct-input API therefore loses its - payload silently. +### Struct-Pointer Inputs — Declared, Not Silent + +The generator no longer drops a pointed-to struct by default. `DEREF_FIELDS` declares, +per API and parameter, how much the pointer points at — a fixed struct, `count` +elements of a fixed size, a NUL-terminated string, or an array whose elements are +themselves device addresses — and the generator emits both the capture-side copy and +the playback-side reconstruct-and-pass. The recorded capture-time address is carried for +comparison only; the value handed back to the runtime is always a local of the pointee's +own type. + +What is left is the honest residue: an API whose pointer argument has no `DEREF_FIELDS` +entry still records only the address. That is now visible rather than silent — the +manifest classifies it as payload loss and `derive_manifest.py` fails the build if the +count drifts from its recorded baseline, so a newly-captured struct-input API cannot +lose its payload unnoticed. + +### Fail-Loud Scope Exclusions + +Some calls cannot be reproduced in a different process no matter how much of the +argument is recorded. Rather than pass a null or a stale value and let the runtime +report something unattributable, these are `UNREPLAYABLE_PLAYBACK_APIS`: the handler +returns `hipErrorNotSupported` (fatal unless `--continue-on-error`), names itself and +its reason on stderr, and the replay summary lists the archive as incomplete. Capture +warns once at record time as well, so the incompleteness is visible when the recording +is made and not only when someone tries to replay it. The exclusions are: + +- **Host callbacks** — `hipLaunchHostFunc`, `hipLaunchHostFunc_spt`, + `hipStreamAddCallback`, `hipStreamAddCallback_spt`. The callback is a function pointer + in the recording process; there is nothing to call at replay. +- **Host nodes in a graph** — `hipGraphAddHostNode`, `hipGraphHostNodeSetParams`, + `hipGraphExecHostNodeSetParams`: the same function pointer reached through the graph + API. Like every exclusion these stop the replay at the call itself. Under + `--continue-on-error` the replay goes on, so the handler also marks the owning graph + incomplete, and instantiating that graph is then refused rather than run short a node. +- **Cross-process handle import** — `hipMemImportFromShareableHandle`, + `hipMemPoolImportFromShareableHandle`. The exported fd or HANDLE is meaningful only + inside the exporting process and its peers, which a later replay is not. +- **Multi-device launch by host function address** — + `hipLaunchCooperativeKernelMultiDevice`, `hipExtLaunchMultiKernelMultiDevice`. + `hipLaunchParams` names each kernel by a host function address and, unlike the + single-device spellings, has no entry point that takes a `hipFunction_t` instead. +- **Host-object lifetime callbacks** — `hipUserObjectCreate`, whose destructor is a host + function pointer with the same problem as a stream callback. + +A recorded call that *failed* is a separate case and is not an exclusion: when the +archived return is non-zero and replay reproduces the same error, that is the faithful +outcome and the dispatcher reports it as such instead of as a handler failure. ## Relationship to Original HRR Code diff --git a/projects/hrr/README.md b/projects/hrr/README.md index 491bb82ee05c..a08efc2a2a92 100644 --- a/projects/hrr/README.md +++ b/projects/hrr/README.md @@ -19,6 +19,8 @@ HRR Capture is now disabled by default on AMD platforms. The feature will be sec HIP_HRR_CAPTURE_OUTPUT=./my_capture.hrr ./my_hip_app ``` +A process that initialises HIP with capture on says so once on stderr, whatever `AMD_LOG_LEVEL` is set to: the line begins with `[HRR capture] Recording` and names the process's own directory in the archive, such as `./my_capture.hrr/pid-4242`. Child processes record to their own `pid-*` directories in the same archive. A child started with `exec` prints a line of its own; a child created with `fork()` after HIP has started does not. + Use the in-tree `libamdhip64` from the **same source commit** when testing capture changes: ```bash @@ -88,7 +90,7 @@ User-facing capture, replay, and validation knobs. Implementation details can be | Variable | Default | Purpose | |----------|---------|---------| -| `HIP_HRR_CAPTURE_OUTPUT` | *(unset)* | Enable capture; path to the `.hrr` archive directory | +| `HIP_HRR_CAPTURE_OUTPUT` | *(unset)* | Enable capture; path to the `.hrr` archive directory. An empty or blank value leaves capture off. Ignored, with one line on stderr, when Linux starts the program in secure-execution mode (set-user-ID, set-group-ID, file capabilities or an LSM transition) | | `HIP_HRR_DEBUG_ARGS` | off | Dump every captured kernel arg to the log (debug / provenance) | ### `hrr-playback` CLI options @@ -162,7 +164,13 @@ the next event, so the divergence is confined to the launch under examination. | `HIP_HRR_REPLAY_NO_RESCAN` | off | Disable suballoc pointer rescan at replay | | `HIP_HRR_PTR_RELAX` | off | Disable replay-side stale-pointer guard (debug only) | | `HIP_HRR_REPLAY_FORCE_EXT_CIJK` | off | Force external Cijk kernel binding workaround (debug) | -| `HIP_HRR_REPLAY_DUMP_PTRS_ORDINAL` | `0` | Dump pointer translation map at event ordinal `N` (debug) | +| `HIP_HRR_REPLAY_DUMP_PTRS_ORDINAL` | `0` | Dump pointer translation map, and the allocation each argument lands in, at kernel ordinal `N` (debug) | +| `HIP_HRR_REPLAY_SCAN_ARGS_ORDINAL` | `0` | Before kernel `N`, read back each pointer argument's allocation and report words that are recorded addresses (debug) | +| `HIP_HRR_REPLAY_SCAN_ARGS_BYTES` | `4096` | Per-allocation cap for the argument scan | +| `HIP_HRR_REPLAY_SCAN_H2D` | off | Report recorded addresses inside replayed H2D payloads (debug) | +| `HIP_HRR_REPLAY_AUDIT_HOST_ARGS` | off | Report kernels taking a pointer into host memory, whose contents replay cannot restore (debug) | +| `HIP_HRR_REPLAY_FILL_BYTE` | `0` | Byte to fill fresh allocations with; set it to e.g. `0xa5` to expose kernels reading memory nothing wrote (debug) | +| `HIP_HRR_REPLAY_EXPLAIN_ADDR` | unset | At the scan ordinal, report whether an address is recorded, live, or neither, and whether the GPU can read it (debug) | ### D2H validation @@ -182,6 +190,7 @@ capture.hrr/ pid-/ events.bin blobs/ + code_objects/ regions/ (optional) manifest.json ``` @@ -191,8 +200,16 @@ capture.hrr/ dispatch table never saw. Written by code outside the runtime, never by capture itself - **blobs/** — host payloads referenced by the trace +- **code_objects/** — code objects, content-addressed by hash (device images + extracted by the runtime, for `hipModuleLoad` too); a launch + records the hash so kernels sharing a name (Triton emits many `triton_`) + still resolve to the code object they came from - **Complete: NO** — original run crashed before clean shutdown; reader still recovers complete events +Code objects read back from the runtime must start with `\x7fELF`, +`__CLANG_OFFLOAD_BUNDLE__` or `CCOB`; capture warns and records none rather +than storing an image that fails only at replay with HIP error 200. + Capture wire version must match the `hrr-playback` reader (see DESIGN.md wire-format notes). ## Agent tooling @@ -203,6 +220,11 @@ Optional Cursor/agent skill: [skills/decode-and-triage/SKILL.md](skills/decode-a requires Linux or WSL2. Docker replay uses the image HRR stack by default; set `HRR_DOCKER_MOUNT_CLR=1` to overlay a host dev build (`CLR_BUILD` / `HRR_PLAYBACK`). +For the other end, producing an archive in the first place: +[skills/enable-recording/SKILL.md](skills/enable-recording/SKILL.md) +checks that the runtime a workload loads can capture at all, runs it with capture +enabled, and reports whether the archive holds anything. + ## Copyright AMD SPDX MIT — see individual source files. diff --git a/projects/hrr/include/hrr/hrr_api_args.h b/projects/hrr/include/hrr/hrr_api_args.h index 5a0fc8bf316b..5ed6e4e78c76 100644 --- a/projects/hrr/include/hrr/hrr_api_args.h +++ b/projects/hrr/include/hrr/hrr_api_args.h @@ -13,7 +13,7 @@ * One packed struct per HIP API covering both HipDispatchTable (runtime) and * HipCompilerDispatchTable (compiler stubs). * - * Archive format (v5): + * Archive format (v6): * events.bin: * [0..7] hrr_file_header { magic, version, reserved } * [8..] hrr_event_header (32 bytes) + payload bytes, repeated per event @@ -38,6 +38,14 @@ * Extra fields on selected APIs (blob hashes, module_id) are appended AFTER * the normal parameters — see EXTRA_FIELDS in gen_hrr_api_args.py. * + * Dereferenced pointer arguments (DEREF_FIELDS in gen_hrr_api_args.py) add, + * for a pointer argument whose pointee must survive into the archive: + * - uint8_t _bytes[N] the pointee's bytes, copied at capture + * - uint8_t _present 1 when the argument was non-null + * - uint32_t _n element count, for array arguments only + * Without them a pointer argument reaches the archive as a capture-time host + * address and nothing else, which is the payload-loss class of section 8.3. + * * The structs use #pragma pack(1) so layout is identical on all platforms. * ============================================================================ */ @@ -61,8 +69,12 @@ * HipDispatchTable member takes the first compiler ID and shifts the whole * compiler tail up by one. Compiler APIs do write events, so that second * case still needs a version bump. A retired dispatch-table slot (nulled - * void*) still occupies an ID. */ -#define HRR_VERSION ((uint16_t)5u) + * void*) still occupies an ID. + * v6: pointer arguments whose pointee used to be dropped now carry it inline + * (DEREF_FIELDS). Event payloads grew for ~50 APIs, so an archive written + * before v6 cannot be read by a v6 reader: re-capture rather than replay an + * old recording. */ +#define HRR_VERSION ((uint16_t)6u) /* Written once at byte 0 of events.bin. */ #pragma pack(push, 1) @@ -228,7 +240,13 @@ typedef struct { uint64_t size; int32_t constant; int32_t global; + uint8_t deviceVar_bytes[256]; /* char inline copy */ + uint8_t deviceVar_present; /* 1 when deviceVar was non-null */ + uint64_t dev_addr; /* capture-time device address of the symbol, 0 if unresolved */ } hrr_args___hipRegisterVar; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(char) <= 1, "hrr_args___hipRegisterVar::deviceVar_bytes too small for char"); +#endif /* void __hipUnregisterFatBinary(void** modules) */ typedef struct { @@ -350,7 +368,12 @@ typedef struct { int32_t ret; uint64_t device; uint64_t prop; + uint8_t prop_bytes[1472]; /* hipDeviceProp_t inline copy */ + uint8_t prop_present; /* 1 when prop was non-null */ } hrr_args_hipChooseDevice; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipDeviceProp_t) <= 1472, "hrr_args_hipChooseDevice::prop_bytes too small for hipDeviceProp_t"); +#endif /* hipError_t hipChooseDeviceR0000(int* device, const hipDeviceProp_tR0000* properties) */ typedef struct { @@ -358,7 +381,12 @@ typedef struct { int32_t ret; uint64_t device; uint64_t properties; + uint8_t properties_bytes[792]; /* hipDeviceProp_tR0000 inline copy */ + uint8_t properties_present; /* 1 when properties was non-null */ } hrr_args_hipChooseDeviceR0000; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipDeviceProp_tR0000) <= 792, "hrr_args_hipChooseDeviceR0000::properties_bytes too small for hipDeviceProp_tR0000"); +#endif /* hipError_t hipConfigureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream) */ typedef struct { @@ -613,7 +641,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; int32_t device; - uint64_t /* hipGraphMemAttributeType */ attr; + int32_t attr; uint64_t value; } hrr_args_hipDeviceGetGraphMemAttribute; @@ -748,7 +776,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; int32_t device; - uint64_t /* hipGraphMemAttributeType */ attr; + int32_t attr; uint64_t value; } hrr_args_hipDeviceSetGraphMemAttribute; @@ -822,7 +850,16 @@ typedef struct { uint64_t numDependencies; uint64_t copyParams; uint64_t ctx; + uint8_t dependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t dependencies_present; /* 1 when dependencies was non-null */ + uint32_t dependencies_n; /* elements copied from dependencies */ + uint8_t copyParams_bytes[184]; /* HIP_MEMCPY3D inline copy */ + uint8_t copyParams_present; /* 1 when copyParams was non-null */ } hrr_args_hipDrvGraphAddMemcpyNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipDrvGraphAddMemcpyNode::dependencies_bytes too small for hipGraphNode_t"); +static_assert(sizeof(HIP_MEMCPY3D) <= 184, "hrr_args_hipDrvGraphAddMemcpyNode::copyParams_bytes too small for HIP_MEMCPY3D"); +#endif /* hipError_t hipDrvMemcpy2DUnaligned(const hip_Memcpy2D* pCopy) */ typedef struct { @@ -1048,7 +1085,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t value; - uint64_t /* hipFunction_attribute */ attrib; + int32_t attrib; uint64_t hfunc; } hrr_args_hipFuncGetAttribute; @@ -1092,7 +1129,7 @@ typedef struct { uint64_t pHipDeviceCount; uint64_t pHipDevices; uint32_t hipDeviceCount; - uint64_t /* hipGLDeviceList */ deviceList; + int32_t deviceList; } hrr_args_hipGLGetDevices; /* hipError_t hipGetChannelDesc(hipChannelFormatDesc* desc, hipArray_const_t array) */ @@ -1234,7 +1271,13 @@ typedef struct { uint64_t pDependencies; uint64_t numDependencies; uint64_t childGraph; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ } hrr_args_hipGraphAddChildGraphNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddChildGraphNode::pDependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddDependencies(hipGraph_t graph, const hipGraphNode_t* from, const hipGraphNode_t* to, size_t numDependencies) */ typedef struct { @@ -1244,7 +1287,17 @@ typedef struct { uint64_t from; uint64_t to; uint64_t numDependencies; + uint8_t from_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t from_present; /* 1 when from was non-null */ + uint32_t from_n; /* elements copied from from */ + uint8_t to_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t to_present; /* 1 when to was non-null */ + uint32_t to_n; /* elements copied from to */ } hrr_args_hipGraphAddDependencies; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddDependencies::from_bytes too small for hipGraphNode_t"); +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddDependencies::to_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddEmptyNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies) */ typedef struct { @@ -1254,7 +1307,13 @@ typedef struct { uint64_t graph; uint64_t pDependencies; uint64_t numDependencies; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ } hrr_args_hipGraphAddEmptyNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddEmptyNode::pDependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddEventRecordNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, hipEvent_t event) */ typedef struct { @@ -1265,7 +1324,13 @@ typedef struct { uint64_t pDependencies; uint64_t numDependencies; uint64_t event; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ } hrr_args_hipGraphAddEventRecordNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddEventRecordNode::pDependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddEventWaitNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, hipEvent_t event) */ typedef struct { @@ -1276,7 +1341,13 @@ typedef struct { uint64_t pDependencies; uint64_t numDependencies; uint64_t event; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ } hrr_args_hipGraphAddEventWaitNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddEventWaitNode::pDependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddHostNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const hipHostNodeParams* pNodeParams) */ typedef struct { @@ -1298,7 +1369,16 @@ typedef struct { uint64_t pDependencies; uint64_t numDependencies; uint64_t pNodeParams; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ + uint8_t pNodeParams_bytes[64]; /* hipKernelNodeParams inline copy */ + uint8_t pNodeParams_present; /* 1 when pNodeParams was non-null */ } hrr_args_hipGraphAddKernelNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddKernelNode::pDependencies_bytes too small for hipGraphNode_t"); +static_assert(sizeof(hipKernelNodeParams) <= 64, "hrr_args_hipGraphAddKernelNode::pNodeParams_bytes too small for hipKernelNodeParams"); +#endif /* hipError_t hipGraphAddMemAllocNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, hipMemAllocNodeParams* pNodeParams) */ typedef struct { @@ -1309,7 +1389,16 @@ typedef struct { uint64_t pDependencies; uint64_t numDependencies; uint64_t pNodeParams; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ + uint8_t pNodeParams_bytes[120]; /* hipMemAllocNodeParams inline copy */ + uint8_t pNodeParams_present; /* 1 when pNodeParams was non-null */ } hrr_args_hipGraphAddMemAllocNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddMemAllocNode::pDependencies_bytes too small for hipGraphNode_t"); +static_assert(sizeof(hipMemAllocNodeParams) <= 120, "hrr_args_hipGraphAddMemAllocNode::pNodeParams_bytes too small for hipMemAllocNodeParams"); +#endif /* hipError_t hipGraphAddMemFreeNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, void* dev_ptr) */ typedef struct { @@ -1320,7 +1409,13 @@ typedef struct { uint64_t pDependencies; uint64_t numDependencies; uint64_t dev_ptr; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ } hrr_args_hipGraphAddMemFreeNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddMemFreeNode::pDependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddMemcpyNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const hipMemcpy3DParms* pCopyParams) */ typedef struct { @@ -1331,7 +1426,16 @@ typedef struct { uint64_t pDependencies; uint64_t numDependencies; uint64_t pCopyParams; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ + uint8_t pCopyParams_bytes[160]; /* hipMemcpy3DParms inline copy */ + uint8_t pCopyParams_present; /* 1 when pCopyParams was non-null */ } hrr_args_hipGraphAddMemcpyNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddMemcpyNode::pDependencies_bytes too small for hipGraphNode_t"); +static_assert(sizeof(hipMemcpy3DParms) <= 160, "hrr_args_hipGraphAddMemcpyNode::pCopyParams_bytes too small for hipMemcpy3DParms"); +#endif /* hipError_t hipGraphAddMemcpyNode1D(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, hipMemcpyKind kind) */ typedef struct { @@ -1345,7 +1449,13 @@ typedef struct { uint64_t src; uint64_t count; int32_t kind; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ } hrr_args_hipGraphAddMemcpyNode1D; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddMemcpyNode1D::pDependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddMemcpyNodeFromSymbol(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* symbol, size_t count, size_t offset, hipMemcpyKind kind) */ typedef struct { @@ -1360,7 +1470,15 @@ typedef struct { uint64_t count; uint64_t offset; int32_t kind; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ + uint64_t blob_hash_lo; /* unused; the destination is written at replay */ + uint64_t blob_hash_hi; /* unused */ } hrr_args_hipGraphAddMemcpyNodeFromSymbol; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddMemcpyNodeFromSymbol::pDependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddMemcpyNodeToSymbol(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind) */ typedef struct { @@ -1375,7 +1493,15 @@ typedef struct { uint64_t count; uint64_t offset; int32_t kind; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ + uint64_t blob_hash_lo; /* host src blob hash lo, 0 if src is a device pointer */ + uint64_t blob_hash_hi; /* host src blob hash hi */ } hrr_args_hipGraphAddMemcpyNodeToSymbol; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddMemcpyNodeToSymbol::pDependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipGraphAddMemsetNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const hipMemsetParams* pMemsetParams) */ typedef struct { @@ -1386,7 +1512,16 @@ typedef struct { uint64_t pDependencies; uint64_t numDependencies; uint64_t pMemsetParams; + uint8_t pDependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t pDependencies_present; /* 1 when pDependencies was non-null */ + uint32_t pDependencies_n; /* elements copied from pDependencies */ + uint8_t pMemsetParams_bytes[48]; /* hipMemsetParams inline copy */ + uint8_t pMemsetParams_present; /* 1 when pMemsetParams was non-null */ } hrr_args_hipGraphAddMemsetNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddMemsetNode::pDependencies_bytes too small for hipGraphNode_t"); +static_assert(sizeof(hipMemsetParams) <= 48, "hrr_args_hipGraphAddMemsetNode::pMemsetParams_bytes too small for hipMemsetParams"); +#endif /* hipError_t hipGraphChildGraphNodeGetGraph(hipGraphNode_t node, hipGraph_t* pGraph) */ typedef struct { @@ -1517,7 +1652,12 @@ typedef struct { uint64_t hGraphExec; uint64_t node; uint64_t pNodeParams; + uint8_t pNodeParams_bytes[64]; /* hipKernelNodeParams inline copy */ + uint8_t pNodeParams_present; /* 1 when pNodeParams was non-null */ } hrr_args_hipGraphExecKernelNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipKernelNodeParams) <= 64, "hrr_args_hipGraphExecKernelNodeSetParams::pNodeParams_bytes too small for hipKernelNodeParams"); +#endif /* hipError_t hipGraphExecMemcpyNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t node, hipMemcpy3DParms* pNodeParams) */ typedef struct { @@ -1526,7 +1666,12 @@ typedef struct { uint64_t hGraphExec; uint64_t node; uint64_t pNodeParams; + uint8_t pNodeParams_bytes[160]; /* hipMemcpy3DParms inline copy */ + uint8_t pNodeParams_present; /* 1 when pNodeParams was non-null */ } hrr_args_hipGraphExecMemcpyNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemcpy3DParms) <= 160, "hrr_args_hipGraphExecMemcpyNodeSetParams::pNodeParams_bytes too small for hipMemcpy3DParms"); +#endif /* hipError_t hipGraphExecMemcpyNodeSetParams1D(hipGraphExec_t hGraphExec, hipGraphNode_t node, void* dst, const void* src, size_t count, hipMemcpyKind kind) */ typedef struct { @@ -1564,6 +1709,8 @@ typedef struct { uint64_t count; uint64_t offset; int32_t kind; + uint64_t blob_hash_lo; /* host src blob hash lo, 0 if src is a device pointer */ + uint64_t blob_hash_hi; /* host src blob hash hi */ } hrr_args_hipGraphExecMemcpyNodeSetParamsToSymbol; /* hipError_t hipGraphExecMemsetNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t node, const hipMemsetParams* pNodeParams) */ @@ -1573,7 +1720,12 @@ typedef struct { uint64_t hGraphExec; uint64_t node; uint64_t pNodeParams; + uint8_t pNodeParams_bytes[48]; /* hipMemsetParams inline copy */ + uint8_t pNodeParams_present; /* 1 when pNodeParams was non-null */ } hrr_args_hipGraphExecMemsetNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemsetParams) <= 48, "hrr_args_hipGraphExecMemsetNodeSetParams::pNodeParams_bytes too small for hipMemsetParams"); +#endif /* hipError_t hipGraphExecUpdate(hipGraphExec_t hGraphExec, hipGraph_t hGraph, hipGraphNode_t* hErrorNode_out, hipGraphExecUpdateResult* updateResult_out) */ typedef struct { @@ -1681,7 +1833,12 @@ typedef struct { uint64_t hNode; int32_t attr; uint64_t value; + uint8_t value_bytes[64]; /* hipKernelNodeAttrValue inline copy */ + uint8_t value_present; /* 1 when value was non-null */ } hrr_args_hipGraphKernelNodeSetAttribute; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipKernelNodeAttrValue) <= 64, "hrr_args_hipGraphKernelNodeSetAttribute::value_bytes too small for hipKernelNodeAttrValue"); +#endif /* hipError_t hipGraphKernelNodeSetParams(hipGraphNode_t node, const hipKernelNodeParams* pNodeParams) */ typedef struct { @@ -1689,7 +1846,12 @@ typedef struct { int32_t ret; uint64_t node; uint64_t pNodeParams; + uint8_t pNodeParams_bytes[64]; /* hipKernelNodeParams inline copy */ + uint8_t pNodeParams_present; /* 1 when pNodeParams was non-null */ } hrr_args_hipGraphKernelNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipKernelNodeParams) <= 64, "hrr_args_hipGraphKernelNodeSetParams::pNodeParams_bytes too small for hipKernelNodeParams"); +#endif /* hipError_t hipGraphLaunch(hipGraphExec_t graphExec, hipStream_t stream) */ typedef struct { @@ -1729,7 +1891,12 @@ typedef struct { int32_t ret; uint64_t node; uint64_t pNodeParams; + uint8_t pNodeParams_bytes[160]; /* hipMemcpy3DParms inline copy */ + uint8_t pNodeParams_present; /* 1 when pNodeParams was non-null */ } hrr_args_hipGraphMemcpyNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemcpy3DParms) <= 160, "hrr_args_hipGraphMemcpyNodeSetParams::pNodeParams_bytes too small for hipMemcpy3DParms"); +#endif /* hipError_t hipGraphMemcpyNodeSetParams1D(hipGraphNode_t node, void* dst, const void* src, size_t count, hipMemcpyKind kind) */ typedef struct { @@ -1764,6 +1931,8 @@ typedef struct { uint64_t count; uint64_t offset; int32_t kind; + uint64_t blob_hash_lo; /* host src blob hash lo, 0 if src is a device pointer */ + uint64_t blob_hash_hi; /* host src blob hash hi */ } hrr_args_hipGraphMemcpyNodeSetParamsToSymbol; /* hipError_t hipGraphMemsetNodeGetParams(hipGraphNode_t node, hipMemsetParams* pNodeParams) */ @@ -1780,7 +1949,12 @@ typedef struct { int32_t ret; uint64_t node; uint64_t pNodeParams; + uint8_t pNodeParams_bytes[48]; /* hipMemsetParams inline copy */ + uint8_t pNodeParams_present; /* 1 when pNodeParams was non-null */ } hrr_args_hipGraphMemsetNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemsetParams) <= 48, "hrr_args_hipGraphMemsetNodeSetParams::pNodeParams_bytes too small for hipMemsetParams"); +#endif /* hipError_t hipGraphNodeFindInClone(hipGraphNode_t* pNode, hipGraphNode_t originalNode, hipGraph_t clonedGraph) */ typedef struct { @@ -1877,7 +2051,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t resource; - uint64_t /* GLuint */ buffer; + uint32_t buffer; uint32_t flags; } hrr_args_hipGraphicsGLRegisterBuffer; @@ -1886,8 +2060,8 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t resource; - uint64_t /* GLuint */ image; - uint64_t /* GLenum */ target; + uint32_t image; + uint32_t target; uint32_t flags; } hrr_args_hipGraphicsGLRegisterImage; @@ -2039,24 +2213,35 @@ typedef struct { int32_t ret; uint64_t handle; uint64_t devPtr; + uint8_t handle_bytes[64]; /* hipIpcMemHandle_t inline copy */ + uint8_t handle_present; /* 1 when handle was non-null */ } hrr_args_hipIpcGetMemHandle; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipIpcMemHandle_t) <= 64, "hrr_args_hipIpcGetMemHandle::handle_bytes too small for hipIpcMemHandle_t"); +#endif /* hipError_t hipIpcOpenEventHandle(hipEvent_t* event, hipIpcEventHandle_t handle) */ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t event; - uint64_t /* hipIpcEventHandle_t */ handle; + uint8_t handle_bytes[64]; /* hipIpcEventHandle_t passed by value, inline copy */ } hrr_args_hipIpcOpenEventHandle; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipIpcEventHandle_t) <= 64, "hrr_args_hipIpcOpenEventHandle::handle_bytes too small for hipIpcEventHandle_t"); +#endif /* hipError_t hipIpcOpenMemHandle(void** devPtr, hipIpcMemHandle_t handle, unsigned int flags) */ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t devPtr; - uint64_t /* hipIpcMemHandle_t */ handle; + uint8_t handle_bytes[64]; /* hipIpcMemHandle_t passed by value, inline copy */ uint32_t flags; } hrr_args_hipIpcOpenMemHandle; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipIpcMemHandle_t) <= 64, "hrr_args_hipIpcOpenMemHandle::handle_bytes too small for hipIpcMemHandle_t"); +#endif /* const char* hipKernelNameRef(const hipFunction_t f) */ typedef struct { @@ -2143,8 +2328,11 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t pitchedDevPtr; - uint64_t /* hipExtent */ extent; + uint8_t extent_bytes[24]; /* hipExtent passed by value, inline copy */ } hrr_args_hipMalloc3D; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipExtent) <= 24, "hrr_args_hipMalloc3D::extent_bytes too small for hipExtent"); +#endif /* hipError_t hipMalloc3DArray(hipArray_t* array, const struct hipChannelFormatDesc* desc, struct hipExtent extent, unsigned int flags) */ typedef struct { @@ -2152,9 +2340,12 @@ typedef struct { int32_t ret; uint64_t array; uint64_t desc; - uint64_t /* struct hipExtent */ extent; + uint8_t extent_bytes[24]; /* hipExtent passed by value, inline copy */ uint32_t flags; } hrr_args_hipMalloc3DArray; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipExtent) <= 24, "hrr_args_hipMalloc3DArray::extent_bytes too small for hipExtent"); +#endif /* hipError_t hipMallocArray(hipArray_t* array, const hipChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) */ typedef struct { @@ -2209,10 +2400,13 @@ typedef struct { int32_t ret; uint64_t mipmappedArray; uint64_t desc; - uint64_t /* struct hipExtent */ extent; + uint8_t extent_bytes[24]; /* hipExtent passed by value, inline copy */ uint32_t numLevels; uint32_t flags; } hrr_args_hipMallocMipmappedArray; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipExtent) <= 24, "hrr_args_hipMallocMipmappedArray::extent_bytes too small for hipExtent"); +#endif /* hipError_t hipMallocPitch(void** ptr, size_t* pitch, size_t width, size_t height) */ typedef struct { @@ -2280,7 +2474,12 @@ typedef struct { uint64_t size; uint64_t prop; uint64_t flags; + uint8_t prop_bytes[32]; /* hipMemAllocationProp inline copy */ + uint8_t prop_present; /* 1 when prop was non-null */ } hrr_args_hipMemCreate; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemAllocationProp) <= 32, "hrr_args_hipMemCreate::prop_bytes too small for hipMemAllocationProp"); +#endif /* hipError_t hipMemExportToShareableHandle(void* shareableHandle, hipMemGenericAllocationHandle_t handle, hipMemAllocationHandleType handleType, unsigned long long flags) */ typedef struct { @@ -2288,7 +2487,7 @@ typedef struct { int32_t ret; uint64_t shareableHandle; uint64_t handle; - uint64_t /* hipMemAllocationHandleType */ handleType; + int32_t handleType; uint64_t flags; } hrr_args_hipMemExportToShareableHandle; @@ -2299,7 +2498,15 @@ typedef struct { uint64_t flags; uint64_t location; uint64_t ptr; + uint8_t location_bytes[8]; /* hipMemLocation inline copy */ + uint8_t location_present; /* 1 when location was non-null */ + uint8_t flags_bytes[8]; /* unsigned long long inline copy */ + uint8_t flags_present; /* 1 when flags was non-null */ } hrr_args_hipMemGetAccess; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemLocation) <= 8, "hrr_args_hipMemGetAccess::location_bytes too small for hipMemLocation"); +static_assert(sizeof(unsigned long long) <= 8, "hrr_args_hipMemGetAccess::flags_bytes too small for unsigned long long"); +#endif /* hipError_t hipMemGetAddressRange(hipDeviceptr_t* pbase, size_t* psize, hipDeviceptr_t dptr) */ typedef struct { @@ -2345,7 +2552,7 @@ typedef struct { int32_t ret; uint64_t handle; uint64_t osHandle; - uint64_t /* hipMemAllocationHandleType */ shHandleType; + int32_t shHandleType; } hrr_args_hipMemImportFromShareableHandle; /* hipError_t hipMemMap(void* ptr, size_t size, size_t offset, hipMemGenericAllocationHandle_t handle, unsigned long long flags) */ @@ -2401,7 +2608,7 @@ typedef struct { int32_t ret; uint64_t shared_handle; uint64_t mem_pool; - uint64_t /* hipMemAllocationHandleType */ handle_type; + int32_t handle_type; uint32_t flags; } hrr_args_hipMemPoolExportToShareableHandle; @@ -2430,7 +2637,7 @@ typedef struct { int32_t ret; uint64_t mem_pool; uint64_t shared_handle; - uint64_t /* hipMemAllocationHandleType */ handle_type; + int32_t handle_type; uint32_t flags; } hrr_args_hipMemPoolImportFromShareableHandle; @@ -2830,7 +3037,15 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t pCopy; + uint8_t drv2d_bytes[136]; /* hip_Memcpy2D inline copy */ + uint64_t blob_hash_lo; /* H2D blob hash lo (0 if not H2D) */ + uint64_t blob_hash_hi; /* H2D blob hash hi */ + uint64_t d2h_hash_lo; /* D2H expected-output blob hash lo (0 if not D2H) */ + uint64_t d2h_hash_hi; /* D2H expected-output blob hash hi */ } hrr_args_hipMemcpyParam2D; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hip_Memcpy2D) <= 136, "hrr_args_hipMemcpyParam2D::drv2d_bytes[136] too small for hip_Memcpy2D"); +#endif /* hipError_t hipMemcpyParam2DAsync(const hip_Memcpy2D* pCopy, hipStream_t stream) */ typedef struct { @@ -2838,7 +3053,15 @@ typedef struct { int32_t ret; uint64_t pCopy; uint64_t stream; + uint8_t drv2d_bytes[136]; /* hip_Memcpy2D inline copy */ + uint64_t blob_hash_lo; /* H2D blob hash lo (0 if not H2D) */ + uint64_t blob_hash_hi; /* H2D blob hash hi */ + uint64_t d2h_hash_lo; /* D2H expected-output blob hash lo (0 if not D2H) */ + uint64_t d2h_hash_hi; /* D2H expected-output blob hash hi */ } hrr_args_hipMemcpyParam2DAsync; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hip_Memcpy2D) <= 136, "hrr_args_hipMemcpyParam2DAsync::drv2d_bytes[136] too small for hip_Memcpy2D"); +#endif /* hipError_t hipMemcpyPeer(void* dst, int dstDeviceId, const void* src, int srcDeviceId, size_t sizeBytes) */ typedef struct { @@ -2947,20 +3170,28 @@ typedef struct { typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipPitchedPtr */ pitchedDevPtr; + uint8_t pitchedDevPtr_bytes[32]; /* hipPitchedPtr passed by value, inline copy */ int32_t value; - uint64_t /* hipExtent */ extent; + uint8_t extent_bytes[24]; /* hipExtent passed by value, inline copy */ } hrr_args_hipMemset3D; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipPitchedPtr) <= 32, "hrr_args_hipMemset3D::pitchedDevPtr_bytes too small for hipPitchedPtr"); +static_assert(sizeof(hipExtent) <= 24, "hrr_args_hipMemset3D::extent_bytes too small for hipExtent"); +#endif /* hipError_t hipMemset3DAsync(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent, hipStream_t stream) */ typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipPitchedPtr */ pitchedDevPtr; + uint8_t pitchedDevPtr_bytes[32]; /* hipPitchedPtr passed by value, inline copy */ int32_t value; - uint64_t /* hipExtent */ extent; + uint8_t extent_bytes[24]; /* hipExtent passed by value, inline copy */ uint64_t stream; } hrr_args_hipMemset3DAsync; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipPitchedPtr) <= 32, "hrr_args_hipMemset3DAsync::pitchedDevPtr_bytes too small for hipPitchedPtr"); +static_assert(sizeof(hipExtent) <= 24, "hrr_args_hipMemset3DAsync::extent_bytes too small for hipExtent"); +#endif /* hipError_t hipMemsetAsync(void* dst, int value, size_t sizeBytes, hipStream_t stream) */ typedef struct { @@ -3254,7 +3485,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t data; - uint64_t /* hipPointer_attribute */ attribute; + int32_t attribute; uint64_t ptr; } hrr_args_hipPointerGetAttribute; @@ -3271,7 +3502,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t value; - uint64_t /* hipPointer_attribute */ attribute; + int32_t attribute; uint64_t ptr; } hrr_args_hipPointerSetAttribute; @@ -3315,7 +3546,13 @@ typedef struct { uint64_t arg; uint64_t size; uint64_t offset; + uint8_t arg_bytes[256]; /* unsigned char[256] inline copy */ + uint8_t arg_present; /* 1 when arg was non-null */ + uint32_t arg_n; /* elements copied from arg */ } hrr_args_hipSetupArgument; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(unsigned char) <= 1, "hrr_args_hipSetupArgument::arg_bytes too small for unsigned char"); +#endif /* hipError_t hipSignalExternalSemaphoresAsync(const hipExternalSemaphore_t* extSemArray, const hipExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, hipStream_t stream) */ typedef struct { @@ -3713,7 +3950,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t texRef; - uint64_t /* hipArray_Format */ fmt; + int32_t fmt; int32_t NumPackedComponents; } hrr_args_hipTexRefSetFormat; @@ -3813,7 +4050,7 @@ typedef struct { /* hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f) */ typedef struct { hrr_event_header hdr; - uint64_t /* hipChannelFormatDesc */ ret; + uint8_t ret_bytes[20]; int32_t x; int32_t y; int32_t z; @@ -3980,20 +4217,28 @@ typedef struct { typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipPitchedPtr */ pitchedDevPtr; + uint8_t pitchedDevPtr_bytes[32]; /* hipPitchedPtr passed by value, inline copy */ int32_t value; - uint64_t /* hipExtent */ extent; + uint8_t extent_bytes[24]; /* hipExtent passed by value, inline copy */ uint64_t stream; } hrr_args_hipMemset3DAsync_spt; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipPitchedPtr) <= 32, "hrr_args_hipMemset3DAsync_spt::pitchedDevPtr_bytes too small for hipPitchedPtr"); +static_assert(sizeof(hipExtent) <= 24, "hrr_args_hipMemset3DAsync_spt::extent_bytes too small for hipExtent"); +#endif /* hipError_t hipMemset3D_spt(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent) */ typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipPitchedPtr */ pitchedDevPtr; + uint8_t pitchedDevPtr_bytes[32]; /* hipPitchedPtr passed by value, inline copy */ int32_t value; - uint64_t /* hipExtent */ extent; + uint8_t extent_bytes[24]; /* hipExtent passed by value, inline copy */ } hrr_args_hipMemset3D_spt; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipPitchedPtr) <= 32, "hrr_args_hipMemset3D_spt::pitchedDevPtr_bytes too small for hipPitchedPtr"); +static_assert(sizeof(hipExtent) <= 24, "hrr_args_hipMemset3D_spt::extent_bytes too small for hipExtent"); +#endif /* hipError_t hipMemcpyAsync_spt(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind, hipStream_t stream) */ typedef struct { @@ -4284,7 +4529,16 @@ typedef struct { uint64_t numDependencies; uint64_t memsetParams; uint64_t ctx; + uint8_t dependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t dependencies_present; /* 1 when dependencies was non-null */ + uint32_t dependencies_n; /* elements copied from dependencies */ + uint8_t memsetParams_bytes[48]; /* hipMemsetParams inline copy */ + uint8_t memsetParams_present; /* 1 when memsetParams was non-null */ } hrr_args_hipDrvGraphAddMemsetNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipDrvGraphAddMemsetNode::dependencies_bytes too small for hipGraphNode_t"); +static_assert(sizeof(hipMemsetParams) <= 48, "hrr_args_hipDrvGraphAddMemsetNode::memsetParams_bytes too small for hipMemsetParams"); +#endif /* hipError_t hipGraphAddExternalSemaphoresWaitNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, const hipGraphNode_t* pDependencies, size_t numDependencies, const hipExternalSemaphoreWaitNodeParams* nodeParams) */ typedef struct { @@ -4409,7 +4663,12 @@ typedef struct { int32_t hipVersion; uint64_t flags; uint64_t symbolStatus; + uint8_t symbol_bytes[256]; /* char inline copy */ + uint8_t symbol_present; /* 1 when symbol was non-null */ } hrr_args_hipGetProcAddress; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(char) <= 1, "hrr_args_hipGetProcAddress::symbol_bytes too small for char"); +#endif /* hipError_t hipStreamBeginCaptureToGraph(hipStream_t stream, hipGraph_t graph, const hipGraphNode_t* dependencies, const hipGraphEdgeData* dependencyData, size_t numDependencies, hipStreamCaptureMode mode) */ typedef struct { @@ -4421,7 +4680,17 @@ typedef struct { uint64_t dependencyData; uint64_t numDependencies; int32_t mode; + uint8_t dependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t dependencies_present; /* 1 when dependencies was non-null */ + uint32_t dependencies_n; /* elements copied from dependencies */ + uint8_t dependencyData_bytes[128]; /* hipGraphEdgeData[16] inline copy */ + uint8_t dependencyData_present; /* 1 when dependencyData was non-null */ + uint32_t dependencyData_n; /* elements copied from dependencyData */ } hrr_args_hipStreamBeginCaptureToGraph; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipStreamBeginCaptureToGraph::dependencies_bytes too small for hipGraphNode_t"); +static_assert(sizeof(hipGraphEdgeData) <= 8, "hrr_args_hipStreamBeginCaptureToGraph::dependencyData_bytes too small for hipGraphEdgeData"); +#endif /* hipError_t hipGetFuncBySymbol(hipFunction_t* functionPtr, const void* symbolPtr) */ typedef struct { @@ -4516,7 +4785,13 @@ typedef struct { uint64_t dependencies; uint64_t numDependencies; uint64_t dptr; + uint8_t dependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t dependencies_present; /* 1 when dependencies was non-null */ + uint32_t dependencies_n; /* elements copied from dependencies */ } hrr_args_hipDrvGraphAddMemFreeNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipDrvGraphAddMemFreeNode::dependencies_bytes too small for hipGraphNode_t"); +#endif /* hipError_t hipDrvGraphExecMemcpyNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t hNode, const HIP_MEMCPY3D* copyParams, hipCtx_t ctx) */ typedef struct { @@ -4526,7 +4801,12 @@ typedef struct { uint64_t hNode; uint64_t copyParams; uint64_t ctx; + uint8_t copyParams_bytes[184]; /* HIP_MEMCPY3D inline copy */ + uint8_t copyParams_present; /* 1 when copyParams was non-null */ } hrr_args_hipDrvGraphExecMemcpyNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(HIP_MEMCPY3D) <= 184, "hrr_args_hipDrvGraphExecMemcpyNodeSetParams::copyParams_bytes too small for HIP_MEMCPY3D"); +#endif /* hipError_t hipDrvGraphExecMemsetNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t hNode, const hipMemsetParams* memsetParams, hipCtx_t ctx) */ typedef struct { @@ -4536,7 +4816,12 @@ typedef struct { uint64_t hNode; uint64_t memsetParams; uint64_t ctx; + uint8_t memsetParams_bytes[48]; /* hipMemsetParams inline copy */ + uint8_t memsetParams_present; /* 1 when memsetParams was non-null */ } hrr_args_hipDrvGraphExecMemsetNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemsetParams) <= 48, "hrr_args_hipDrvGraphExecMemsetNodeSetParams::memsetParams_bytes too small for hipMemsetParams"); +#endif /* hipError_t hipGraphExecGetFlags(hipGraphExec_t graphExec, unsigned long long* flags) */ typedef struct { @@ -4586,7 +4871,12 @@ typedef struct { int32_t ret; uint64_t hNode; uint64_t nodeParams; + uint8_t nodeParams_bytes[184]; /* HIP_MEMCPY3D inline copy */ + uint8_t nodeParams_present; /* 1 when nodeParams was non-null */ } hrr_args_hipDrvGraphMemcpyNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(HIP_MEMCPY3D) <= 184, "hrr_args_hipDrvGraphMemcpyNodeSetParams::nodeParams_bytes too small for HIP_MEMCPY3D"); +#endif /* hipError_t hipExtHostAlloc(void** ptr, size_t size, unsigned int flags) */ typedef struct { @@ -4604,7 +4894,15 @@ typedef struct { uint64_t maxWidthInElements; uint64_t fmtDesc; int32_t device; + uint8_t fmtDesc_bytes[20]; /* hipChannelFormatDesc inline copy */ + uint8_t fmtDesc_present; /* 1 when fmtDesc was non-null */ + uint8_t maxWidthInElements_bytes[8]; /* size_t inline copy */ + uint8_t maxWidthInElements_present; /* 1 when maxWidthInElements was non-null */ } hrr_args_hipDeviceGetTexture1DLinearMaxWidth; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipChannelFormatDesc) <= 20, "hrr_args_hipDeviceGetTexture1DLinearMaxWidth::fmtDesc_bytes too small for hipChannelFormatDesc"); +static_assert(sizeof(size_t) <= 8, "hrr_args_hipDeviceGetTexture1DLinearMaxWidth::maxWidthInElements_bytes too small for size_t"); +#endif /* hipError_t hipStreamBatchMemOp(hipStream_t stream, unsigned int count, hipStreamBatchMemOpParams* paramArray, unsigned int flags) */ typedef struct { @@ -4614,7 +4912,13 @@ typedef struct { uint32_t count; uint64_t paramArray; uint32_t flags; + uint8_t paramArray_bytes[768]; /* hipStreamBatchMemOpParams[16] inline copy */ + uint8_t paramArray_present; /* 1 when paramArray was non-null */ + uint32_t paramArray_n; /* elements copied from paramArray */ } hrr_args_hipStreamBatchMemOp; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipStreamBatchMemOpParams) <= 48, "hrr_args_hipStreamBatchMemOp::paramArray_bytes too small for hipStreamBatchMemOpParams"); +#endif /* hipError_t hipGraphAddBatchMemOpNode(hipGraphNode_t* phGraphNode, hipGraph_t hGraph, const hipGraphNode_t* dependencies, size_t numDependencies, const hipBatchMemOpNodeParams* nodeParams) */ typedef struct { @@ -4625,7 +4929,16 @@ typedef struct { uint64_t dependencies; uint64_t numDependencies; uint64_t nodeParams; + uint8_t dependencies_bytes[128]; /* hipGraphNode_t[16] inline copy */ + uint8_t dependencies_present; /* 1 when dependencies was non-null */ + uint32_t dependencies_n; /* elements copied from dependencies */ + uint8_t nodeParams_bytes[32]; /* hipBatchMemOpNodeParams inline copy */ + uint8_t nodeParams_present; /* 1 when nodeParams was non-null */ } hrr_args_hipGraphAddBatchMemOpNode; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipGraphNode_t) <= 8, "hrr_args_hipGraphAddBatchMemOpNode::dependencies_bytes too small for hipGraphNode_t"); +static_assert(sizeof(hipBatchMemOpNodeParams) <= 32, "hrr_args_hipGraphAddBatchMemOpNode::nodeParams_bytes too small for hipBatchMemOpNodeParams"); +#endif /* hipError_t hipGraphBatchMemOpNodeGetParams(hipGraphNode_t hNode, hipBatchMemOpNodeParams* nodeParams_out) */ typedef struct { @@ -4641,7 +4954,12 @@ typedef struct { int32_t ret; uint64_t hNode; uint64_t nodeParams; + uint8_t nodeParams_bytes[32]; /* hipBatchMemOpNodeParams inline copy */ + uint8_t nodeParams_present; /* 1 when nodeParams was non-null */ } hrr_args_hipGraphBatchMemOpNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipBatchMemOpNodeParams) <= 32, "hrr_args_hipGraphBatchMemOpNodeSetParams::nodeParams_bytes too small for hipBatchMemOpNodeParams"); +#endif /* hipError_t hipGraphExecBatchMemOpNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t hNode, const hipBatchMemOpNodeParams* nodeParams) */ typedef struct { @@ -4650,39 +4968,72 @@ typedef struct { uint64_t hGraphExec; uint64_t hNode; uint64_t nodeParams; + uint8_t nodeParams_bytes[32]; /* hipBatchMemOpNodeParams inline copy */ + uint8_t nodeParams_present; /* 1 when nodeParams was non-null */ } hrr_args_hipGraphExecBatchMemOpNodeSetParams; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipBatchMemOpNodeParams) <= 32, "hrr_args_hipGraphExecBatchMemOpNodeSetParams::nodeParams_bytes too small for hipBatchMemOpNodeParams"); +#endif /* hipError_t hipLinkAddData(hipLinkState_t state, hipJitInputType type, void* data, size_t size, const char* name, unsigned int numOptions, hipJitOption* options, void** optionValues) */ typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipLinkState_t */ state; - uint64_t /* hipJitInputType */ type; + uint64_t state; + int32_t type; uint64_t data; uint64_t size; uint64_t name; uint32_t numOptions; uint64_t options; uint64_t optionValues; + uint8_t name_bytes[256]; /* char inline copy */ + uint8_t name_present; /* 1 when name was non-null */ + uint8_t options_bytes[128]; /* hipJitOption[32] inline copy */ + uint8_t options_present; /* 1 when options was non-null */ + uint32_t options_n; /* elements copied from options */ + uint8_t optionValues_bytes[256]; /* void*[32] inline copy */ + uint8_t optionValues_present; /* 1 when optionValues was non-null */ + uint32_t optionValues_n; /* elements copied from optionValues */ + uint64_t blob_hash_lo; /* linker input image blob hash lo */ + uint64_t blob_hash_hi; /* linker input image blob hash hi */ } hrr_args_hipLinkAddData; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(char) <= 1, "hrr_args_hipLinkAddData::name_bytes too small for char"); +static_assert(sizeof(hipJitOption) <= 4, "hrr_args_hipLinkAddData::options_bytes too small for hipJitOption"); +static_assert(sizeof(void*) <= 8, "hrr_args_hipLinkAddData::optionValues_bytes too small for void*"); +#endif /* hipError_t hipLinkAddFile(hipLinkState_t state, hipJitInputType type, const char* path, unsigned int numOptions, hipJitOption* options, void** optionValues) */ typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipLinkState_t */ state; - uint64_t /* hipJitInputType */ type; + uint64_t state; + int32_t type; uint64_t path; uint32_t numOptions; uint64_t options; uint64_t optionValues; + uint8_t path_bytes[256]; /* char inline copy */ + uint8_t path_present; /* 1 when path was non-null */ + uint8_t options_bytes[128]; /* hipJitOption[32] inline copy */ + uint8_t options_present; /* 1 when options was non-null */ + uint32_t options_n; /* elements copied from options */ + uint8_t optionValues_bytes[256]; /* void*[32] inline copy */ + uint8_t optionValues_present; /* 1 when optionValues was non-null */ + uint32_t optionValues_n; /* elements copied from optionValues */ } hrr_args_hipLinkAddFile; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(char) <= 1, "hrr_args_hipLinkAddFile::path_bytes too small for char"); +static_assert(sizeof(hipJitOption) <= 4, "hrr_args_hipLinkAddFile::options_bytes too small for hipJitOption"); +static_assert(sizeof(void*) <= 8, "hrr_args_hipLinkAddFile::optionValues_bytes too small for void*"); +#endif /* hipError_t hipLinkComplete(hipLinkState_t state, void** hipBinOut, size_t* sizeOut) */ typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipLinkState_t */ state; + uint64_t state; uint64_t hipBinOut; uint64_t sizeOut; } hrr_args_hipLinkComplete; @@ -4695,13 +5046,23 @@ typedef struct { uint64_t options; uint64_t optionValues; uint64_t stateOut; + uint8_t options_bytes[128]; /* hipJitOption[32] inline copy */ + uint8_t options_present; /* 1 when options was non-null */ + uint32_t options_n; /* elements copied from options */ + uint8_t optionValues_bytes[256]; /* void*[32] inline copy */ + uint8_t optionValues_present; /* 1 when optionValues was non-null */ + uint32_t optionValues_n; /* elements copied from optionValues */ } hrr_args_hipLinkCreate; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipJitOption) <= 4, "hrr_args_hipLinkCreate::options_bytes too small for hipJitOption"); +static_assert(sizeof(void*) <= 8, "hrr_args_hipLinkCreate::optionValues_bytes too small for void*"); +#endif /* hipError_t hipLinkDestroy(hipLinkState_t state) */ typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipLinkState_t */ state; + uint64_t state; } hrr_args_hipLinkDestroy; /* hipError_t hipEventRecordWithFlags(hipEvent_t event, hipStream_t stream, unsigned int flags) */ @@ -4739,7 +5100,7 @@ typedef struct { uint64_t handle; uint64_t dptr; uint64_t size; - uint64_t /* hipMemRangeHandleType */ handleType; + int32_t handleType; uint64_t flags; } hrr_args_hipMemGetHandleForAddressRange; @@ -4825,7 +5186,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t stream; - uint64_t /* hipStreamAttrID */ attr; + int32_t attr; uint64_t value_out; } hrr_args_hipStreamGetAttribute; @@ -4834,7 +5195,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t stream; - uint64_t /* hipStreamAttrID */ attr; + int32_t attr; uint64_t value; uint8_t stream_attr_bytes[64]; /* hipStreamAttrValue inline copy */ } hrr_args_hipStreamSetAttribute; @@ -4863,7 +5224,32 @@ typedef struct { uint64_t numAttrs; uint64_t failIdx; uint64_t stream; + uint8_t dsts_bytes[128]; /* void*[16] inline copy */ + uint8_t dsts_present; /* 1 when dsts was non-null */ + uint32_t dsts_n; /* elements copied from dsts */ + uint8_t srcs_bytes[128]; /* void*[16] inline copy */ + uint8_t srcs_present; /* 1 when srcs was non-null */ + uint32_t srcs_n; /* elements copied from srcs */ + uint8_t sizes_bytes[128]; /* size_t[16] inline copy */ + uint8_t sizes_present; /* 1 when sizes was non-null */ + uint32_t sizes_n; /* elements copied from sizes */ + uint8_t attrs_bytes[384]; /* hipMemcpyAttributes[16] inline copy */ + uint8_t attrs_present; /* 1 when attrs was non-null */ + uint32_t attrs_n; /* elements copied from attrs */ + uint8_t attrsIdxs_bytes[128]; /* size_t[16] inline copy */ + uint8_t attrsIdxs_present; /* 1 when attrsIdxs was non-null */ + uint32_t attrsIdxs_n; /* elements copied from attrsIdxs */ + uint8_t failIdx_bytes[8]; /* size_t inline copy */ + uint8_t failIdx_present; /* 1 when failIdx was non-null */ } hrr_args_hipMemcpyBatchAsync; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(void*) <= 8, "hrr_args_hipMemcpyBatchAsync::dsts_bytes too small for void*"); +static_assert(sizeof(void*) <= 8, "hrr_args_hipMemcpyBatchAsync::srcs_bytes too small for void*"); +static_assert(sizeof(size_t) <= 8, "hrr_args_hipMemcpyBatchAsync::sizes_bytes too small for size_t"); +static_assert(sizeof(hipMemcpyAttributes) <= 24, "hrr_args_hipMemcpyBatchAsync::attrs_bytes too small for hipMemcpyAttributes"); +static_assert(sizeof(size_t) <= 8, "hrr_args_hipMemcpyBatchAsync::attrsIdxs_bytes too small for size_t"); +static_assert(sizeof(size_t) <= 8, "hrr_args_hipMemcpyBatchAsync::failIdx_bytes too small for size_t"); +#endif /* hipError_t hipMemcpy3DBatchAsync(size_t numOps, struct hipMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, hipStream_t stream) */ typedef struct { @@ -4874,7 +5260,16 @@ typedef struct { uint64_t failIdx; uint64_t flags; uint64_t stream; + uint8_t opList_bytes[1792]; /* hipMemcpy3DBatchOp[16] inline copy */ + uint8_t opList_present; /* 1 when opList was non-null */ + uint32_t opList_n; /* elements copied from opList */ + uint8_t failIdx_bytes[8]; /* size_t inline copy */ + uint8_t failIdx_present; /* 1 when failIdx was non-null */ } hrr_args_hipMemcpy3DBatchAsync; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemcpy3DBatchOp) <= 112, "hrr_args_hipMemcpy3DBatchAsync::opList_bytes too small for hipMemcpy3DBatchOp"); +static_assert(sizeof(size_t) <= 8, "hrr_args_hipMemcpy3DBatchAsync::failIdx_bytes too small for size_t"); +#endif /* hipError_t hipMemcpy3DPeer(hipMemcpy3DPeerParms* p) */ typedef struct { @@ -4917,10 +5312,13 @@ typedef struct { int32_t ret; uint64_t dev_ptr; uint64_t count; - uint64_t /* hipMemLocation */ location; + uint8_t location_bytes[8]; /* hipMemLocation passed by value, inline copy */ uint32_t flags; uint64_t stream; } hrr_args_hipMemPrefetchAsync_v2; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemLocation) <= 8, "hrr_args_hipMemPrefetchAsync_v2::location_bytes too small for hipMemLocation"); +#endif /* hipError_t hipMemAdvise_v2(const void* dev_ptr, size_t count, hipMemoryAdvise advice, hipMemLocation device) */ typedef struct { @@ -4929,8 +5327,11 @@ typedef struct { uint64_t dev_ptr; uint64_t count; int32_t advice; - uint64_t /* hipMemLocation */ device; + uint8_t device_bytes[8]; /* hipMemLocation passed by value, inline copy */ } hrr_args_hipMemAdvise_v2; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemLocation) <= 8, "hrr_args_hipMemAdvise_v2::device_bytes too small for hipMemLocation"); +#endif /* hipError_t hipStreamGetId(hipStream_t stream, unsigned long long* streamId) */ typedef struct { @@ -5044,7 +5445,12 @@ typedef struct { int32_t hipVersion; uint64_t flags; uint64_t symbolStatus; + uint8_t symbol_bytes[256]; /* char inline copy */ + uint8_t symbol_present; /* 1 when symbol was non-null */ } hrr_args_hipGetProcAddress_spt; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(char) <= 1, "hrr_args_hipGetProcAddress_spt::symbol_bytes too small for char"); +#endif /* hipError_t hipKernelGetParamInfo(hipKernel_t kernel, size_t paramIndex, size_t* paramOffset, size_t* paramSize) */ typedef struct { @@ -5084,7 +5490,12 @@ typedef struct { uint64_t location; int32_t type; uint64_t pool; + uint8_t location_bytes[8]; /* hipMemLocation inline copy */ + uint8_t location_present; /* 1 when location was non-null */ } hrr_args_hipMemSetMemPool; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemLocation) <= 8, "hrr_args_hipMemSetMemPool::location_bytes too small for hipMemLocation"); +#endif /* hipError_t hipMemGetMemPool(hipMemPool_t* pool, hipMemLocation* location, hipMemAllocationType type) */ typedef struct { @@ -5093,7 +5504,12 @@ typedef struct { uint64_t pool; uint64_t location; int32_t type; + uint8_t location_bytes[8]; /* hipMemLocation inline copy */ + uint8_t location_present; /* 1 when location was non-null */ } hrr_args_hipMemGetMemPool; +#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H +static_assert(sizeof(hipMemLocation) <= 8, "hrr_args_hipMemGetMemPool::location_bytes too small for hipMemLocation"); +#endif /* hipError_t hipMipmappedArrayGetMemoryRequirements(hipArrayMemoryRequirements* memoryRequirements, hipMipmappedArray_t mipmap, hipDevice_t device) */ typedef struct { @@ -5109,7 +5525,7 @@ typedef struct { hrr_event_header hdr; int32_t ret; uint64_t pi; - uint64_t /* hipFunction_attribute */ attrib; + int32_t attrib; uint64_t kernel; uint64_t dev; } hrr_args_hipKernelGetAttribute; @@ -5118,7 +5534,7 @@ typedef struct { typedef struct { hrr_event_header hdr; int32_t ret; - uint64_t /* hipFunction_attribute */ attrib; + int32_t attrib; int32_t value; uint64_t kernel; uint64_t dev; @@ -5197,7 +5613,7 @@ typedef struct { int32_t ret; uint64_t device; uint64_t resource; - uint64_t /* hipDevResourceType */ type; + int32_t type; } hrr_args_hipDeviceGetDevResource; /* hipError_t hipDevSmResourceSplitByCount(hipDevResource* result, unsigned int* nbGroups, const hipDevResource* input, hipDevResource* remainder, unsigned int flags, unsigned int minCount) */ @@ -5247,7 +5663,7 @@ typedef struct { int32_t ret; uint64_t ctx; uint64_t resource; - uint64_t /* hipDevResourceType */ type; + int32_t type; } hrr_args_hipExecutionCtxGetDevResource; /* hipError_t hipExecutionCtxGetDevice(int* device, hipExecutionCtx_t ctx) */ @@ -5272,7 +5688,7 @@ typedef struct { int32_t ret; uint64_t hStream; uint64_t resource; - uint64_t /* hipDevResourceType */ type; + int32_t type; } hrr_args_hipStreamGetDevResource; /* hipError_t hipExecutionCtxRecordEvent(hipExecutionCtx_t ctx, hipEvent_t event) */ diff --git a/projects/hrr/playback/hip_playback.cpp b/projects/hrr/playback/hip_playback.cpp index 0843a0dae14f..5a5aa6633a33 100644 --- a/projects/hrr/playback/hip_playback.cpp +++ b/projects/hrr/playback/hip_playback.cpp @@ -52,6 +52,49 @@ // immediately unblock the next thread before doing timing/sync. thread_local uint64_t hrr_dispatch_seq = 0; +void hrr_note_unreplayable(PlaybackContext& ctx, const char* api, + const char* reason) { + { + std::unique_lock lk(ctx.map_mutex); + if (!ctx.unreplayable_apis.emplace(api, reason ? reason : "").second) + return; + } + fprintf(stderr, + "[HRR] %s: NOT REPLAYABLE — %s. The call is in the archive but its " + "effect cannot be reproduced here, so it is skipped; this replay is " + "not a faithful reproduction of the recording.\n", + api, reason ? reason : "(unspecified)"); +} + +void hrr_note_recorded_error(PlaybackContext& ctx, const char* api, + int recorded_ret) { + { + std::unique_lock lk(ctx.map_mutex); + if (!ctx.reproduced_errors.emplace(api, recorded_ret).second) return; + } + fprintf(stderr, + "[HRR] %s: returned %d (%s) at replay, which is what it returned at " + "capture — the recorded failure was reproduced faithfully.\n", + api, recorded_ret, + hipGetErrorString(static_cast(recorded_ret))); +} + +bool hrr_replayed_recorded_error(PlaybackContext& ctx, const char* api, + int32_t recorded_ret, hipError_t replayed) { + if (replayed == hipSuccess || recorded_ret == 0 || + static_cast(replayed) != recorded_ret) + return false; + hrr_note_recorded_error(ctx, api, recorded_ret); + return true; +} + +hipCtx_t hrr_live_ctx(uint64_t recorded) { + if (recorded == 0) return nullptr; + hipCtx_t cur = nullptr; + if (hipCtxGetCurrent(&cur) != hipSuccess) return nullptr; + return cur; +} + // --------------------------------------------------------------------------- // HIP error checking — returns the hipError_t so callers can branch on it. // Usage: HRR_HIP_CHECK(hipFoo(...)); // log only @@ -397,6 +440,13 @@ hipFunction_t PlaybackContext::resolve_replacement(const std::string& kernel_nam // hipExtModuleLaunchKernel is declared via (included at the top // of this file behind a -Wattributes diagnostic guard) so the prototype always // tracks the library ABI instead of a hand-maintained copy. +// launch_ex: the event came from hipDrvLaunchKernelEx or hipLaunchKernelExC. +// Their recorded attribute list is applied by replaying through +// hipDrvLaunchKernelEx with a rebuilt descriptor, so a cluster dimension or a +// cooperative flag the program asked for is not silently dropped. The C +// spelling replays through the driver entry point too: its host function +// address means nothing here, while the resolved hipFunction_t does. + // ---- External region annotations at the translation sites ------------------- // // Every recorded device pointer a kernel receives passes through one of the @@ -482,66 +532,15 @@ static void* hrr_region_check_ptr(PlaybackContext& ctx, RegionLaunchState* rls, return live; } -static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, - bool ext_global_worksize = false) { - // Skip the 32-byte header; kernel launch has a variable-length binary format. - const auto* hdr = reinterpret_cast(pl); - const uint8_t* p = pl + sizeof(hrr_event_header); - const uint8_t* end = pl + hdr->payload_length; - - if (p + 8 > end) return hipErrorInvalidValue; - uint64_t stream_rec; memcpy(&stream_rec, p, 8); p += 8; - - if (p + 2 > end) return hipErrorInvalidValue; - uint16_t name_len; memcpy(&name_len, p, 2); p += 2; - if (p + name_len > end) return hipErrorInvalidValue; - std::string kernel_name(reinterpret_cast(p), name_len); - p += name_len; - - // Workaround for recordings made before the capture side tagged Ext launches: - // hipBLASLt/Tensile ("Cijk_*") StreamK kernels are launched via - // hipExtModuleLaunchKernel, whose grid[] is *global work-item counts*. If such - // a launch was collapsed into the generic (workgroup-count) launch event, the - // grid is over-launched by blockDim and the persistent producer/consumer - // handshake deadlocks. Setting HIP_HRR_REPLAY_FORCE_EXT_CIJK=1 reinterprets - // these grids as global work items (replay through the Ext API). - // - // SUNSET: this is a backward-compat escape hatch only. New recordings record - // hipExtModuleLaunchKernel under HRR_API_HIPEXTMODULELAUNCHKERNEL, whose - // dedicated playback handler already passes ext_global_worksize=true (see - // playback_hipExtModuleLaunchKernel below), so they never need this path. The - // heuristic depends on the third-party Tensile/hipBLASLt "Cijk_" naming - // convention, which can change without notice. Safe to delete once no archive - // predating the capture-side Ext-tagging fix is still being replayed (i.e. - // every recording in use routes Ext launches through their own event id). - if (!ext_global_worksize && kernel_name.compare(0, 5, "Cijk_") == 0 && - std::getenv("HIP_HRR_REPLAY_FORCE_EXT_CIJK")) - ext_global_worksize = true; - - uint64_t co_hash_lo = 0, co_hash_hi = 0; - if (p + 16 <= end) { - memcpy(&co_hash_lo, p, 8); p += 8; - memcpy(&co_hash_hi, p, 8); p += 8; - } - - if (p + 32 > end) return hipErrorInvalidValue; - uint32_t grid[3], block[3], shared_mem; - memcpy(grid, p, 12); p += 12; - memcpy(block, p, 12); p += 12; - memcpy(&shared_mem, p, 4); p += 4; - - uint16_t num_args, num_snapshots; - memcpy(&num_args, p, 2); p += 2; - memcpy(&num_snapshots, p, 2); p += 2; - - // Apply kernel filter if set - if (!ctx.kernel_filter.empty() && - kernel_name.find(ctx.kernel_filter) == std::string::npos) - return hipSuccess; - - // Resolve hipFunction_t — cache hit avoids repeated hipModuleGetFunction - // searches. Locked because multiple threads can now be in kernel launch - // preparation concurrently (only the HIP call itself is serialized). +// Resolve the hipFunction_t a recorded kernel name (plus the code-object hash +// that disambiguates it) refers to in this process, or nullptr with a message +// naming the kernel. Shared by kernel launches and graph kernel nodes, which +// name their kernel the same way and for the same reason: the recorded host +// function address belongs to the capturing process. +static hipFunction_t resolve_kernel_function(PlaybackContext& ctx, + const std::string& kernel_name, + uint64_t co_hash_lo, + uint64_t co_hash_hi) { hipFunction_t func = nullptr; // Playback-time kernel override: if this kernel matches a --replace-kernel @@ -600,21 +599,82 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, if (!func) { fprintf(stderr, "[HRR] Kernel '%s' not found in any loaded module\n", kernel_name.c_str()); - return hipErrorNotFound; + return nullptr; } std::unique_lock lk(ctx.map_mutex); ctx.func_cache.emplace(cache_key, func); } + return func; +} + +// Decode the recorded argument list at `p` into `arg_storage` (owning) and +// `arg_ptrs` (what a launch or node-parameter build wants), translating every +// recorded device address on the way. `p` is advanced past the arguments. +// +// Shared by the kernel-launch path and the graph kernel-node path: a kernel +// node's arguments are the same bytes with the same pointers inside them, and +// a second decoder would be a second place for the pointer heuristics below to +// drift out of agreement with what capture recorded. +// +// `rls` is the caller's per-launch guard bookkeeping, or null on the graph +// kernel-node path where a relocation could never be undone. +static void decode_kernel_args( + PlaybackContext& ctx, const uint8_t*& p, const uint8_t* end, + uint16_t num_args, const std::string& kernel_name, + std::vector& arg_ptrs, + std::vector>& arg_storage, + RegionLaunchState* rls = nullptr, + std::vector>* dbg_ptrs_out = nullptr, + std::string* dbg_args_out = nullptr) { + (void)kernel_name; + std::vector> dbg_sink; + const bool dbg_dump_ptrs = (dbg_ptrs_out != nullptr); + auto& dbg_ptrs = dbg_ptrs_out ? *dbg_ptrs_out : dbg_sink; + + // A pointer that the kernel's own metadata does not describe as one. + // Hand-written assembly kernels carry hand-written metadata, and aiter's MLA + // decode kernel declares a 64-bit buffer address as two 4-byte by_value i32 + // fields — "out16" followed by a "pad". Neither half is eight bytes wide, so + // neither the capture-side detector nor the per-argument rescan can find a + // pointer in it, and the capture-time address reaches the GPU unchanged; + // the kernel builds a buffer descriptor from it and faults on an address + // belonging to the recording process. Join adjacent scalar halves and + // translate the pair when it resolves to a recorded allocation. + struct { bool valid = false; uint16_t idx = 0; + const uint8_t* data = nullptr; size_t store = 0; } prev_half; + static const bool no_rescan = + (std::getenv("HIP_HRR_REPLAY_NO_RESCAN") != nullptr); + + // Reject packed-integer false positives. The capture-side detector + // flags any 8-byte word that resolves to a device VA, but two adjacent + // 32-bit struct fields {uint32 lo, uint32 hi} can coincidentally form + // such a value: ATen elementwise kernels embed an OffsetCalculator + // (per-arg uint32 strides/sizes + IntDivider magic constants) in the + // functor. When `hi` happens to hold a value whose top bits match the + // device-VA prefix (0x7e../0x7f..) and `lo` holds a small integer (a + // stride/size/dim), the combined 64-bit word lands inside a real + // allocation and gets "translated" — corrupting the OffsetCalculator + // and producing an out-of-bounds VM fault (e.g. the recurring + // elementwise_kernel_manual_unroll<...MulFunctor> crash). + // + // A genuine 64-bit device pointer carries a full 48-bit address, so its + // low 32 bits are part of that address and are effectively never this + // small. The FP, by construction, needs its HIGH word to be the VA + // prefix and its LOW word to be a small scalar — so a tiny low-32 value + // is the reliable FP signature. (Set HIP_HRR_PTR_RELAX=1 to disable.) + static const bool ptr_relax = + (std::getenv("HIP_HRR_PTR_RELAX") != nullptr); + auto pointer_like = [&](uint64_t v) -> bool { + if (ptr_relax) return true; + return (v & 0xFFFFFFFFULL) >= 0x10000ULL; + }; + // Is this value shaped like an address at all? Below 0x10000 no device VA + // ever lands, and a packed integer fails pointer_like. Both paths ask this + // before deciding what an unresolvable value means. + auto va_shaped = [&](uint64_t v) -> bool { + return v >= 0x10000ULL && pointer_like(v); + }; - // Build kernelParams[] from captured args, translating GPU pointers. - std::vector arg_ptrs; - std::vector> arg_storage; - // Guarded blocks this launch relocated, if block guarding is on. Scoped to - // the launch so the relocation is undone before the next event runs. - RegionLaunchState rls; - // Optional recorded->live pointer dump for one target kernel (diff tooling). - const bool dbg_dump_ptrs = (ctx.dump_ptrs_ordinal != 0); - std::vector> dbg_ptrs; // (arg_idx, recorded, live) for (uint16_t i = 0; i < num_args; i++) { if (p + 3 > end) break; uint8_t value_kind = *p++; @@ -625,6 +685,34 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, const uint8_t* data = p; p += arg_size; + // Every argument as capture classified it, for --dump-ptrs. A pointer + // reaching the GPU untranslated is invisible in the translated-pointer + // list precisely because nothing translated it, so the raw bytes are + // what identifies it. + if (dbg_args_out) { + char line[128]; + snprintf(line, sizeof(line), " arg[%u] kind=%u size=%u", i, + value_kind, arg_size); + *dbg_args_out += line; + if (arg_size >= 8) { + *dbg_args_out += " words:"; + for (uint16_t off = 0; off + 8 <= arg_size; off += 8) { + uint64_t w; memcpy(&w, data + off, 8); + snprintf(line, sizeof(line), " 0x%llx", (unsigned long long)w); + *dbg_args_out += line; + } + } + // Raw bytes as well: a pointer split across two 4-byte arguments is + // invisible as words, and those are exactly the arguments nothing + // translates. + *dbg_args_out += " bytes:"; + for (uint16_t off = 0; off < arg_size && off < 32; off++) { + snprintf(line, sizeof(line), " %02x", data[off]); + *dbg_args_out += line; + } + *dbg_args_out += "\n"; + } + // value_kind 3 carries a trailing list of embedded-pointer byte offsets. std::vector ptr_offsets; if (value_kind == 3) { @@ -653,16 +741,22 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, if (!live && rec_ptr != 0 && ctx.regions_enabled) live = ctx.regions.materialize_for(ctx, rec_ptr); + // A pointer-typed argument does not always hold a pointer. Kernels + // pass sentinels in these slots — aiter's MLA decode kernel takes a + // literal 1 in one — and substituting null for such a value changes + // what the kernel does, silently and only on replay. + // + // The two cases are told apart by shape. A value no device VA could + // be is a sentinel and keeps its meaning. A VA-shaped value that + // resolves nowhere is a pointer HRR genuinely lost, and it goes to + // the kernel as null so it faults at first use: the driver tends to + // reproduce VA layout across runs, so passing the recorded address + // through would more likely scribble over an unrelated live buffer + // than fault. if (!live && rec_ptr != 0) { - // Counted and reported, but still handed to the kernel as null. - // Forwarding the recorded address instead would be worse than - // useless: the driver tends to reproduce a VA layout across - // runs, so a capture-time address is quite likely to be mapped - // in the replay process too, and the kernel would then scribble - // over an unrelated live buffer with nothing to show for it. A - // null dereference stops at the first kernel that uses the - // pointer, which is the failure worth having. ctx.untranslated_ptr_args.fetch_add(1, std::memory_order_relaxed); + const bool sentinel = !va_shaped(rec_ptr); + if (sentinel) live = reinterpret_cast(rec_ptr); static std::mutex mu; static std::set warned; char key[160]; @@ -672,13 +766,15 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, if (first) fprintf(stderr, "[HRR] '%s' arg[%u]: recorded 0x%llx is in no known " - "allocation — passing null\n", + "allocation — %s\n", compact_kernel_name(kernel_name).c_str(), i, - (unsigned long long)rec_ptr); + (unsigned long long)rec_ptr, + sentinel ? "not an address, passing it through unchanged" + : "passing null so it faults at first use"); } else { // Region check: an intra-segment OOB/stale pointer, or a block to be // relocated behind a guard page. No-op when no sidecar was loaded. - live = hrr_region_check_ptr(ctx, &rls, kernel_name, i, rec_ptr, live); + live = hrr_region_check_ptr(ctx, rls, kernel_name, i, rec_ptr, live); } if (dbg_dump_ptrs) dbg_ptrs.emplace_back(i, rec_ptr, live); storage.resize(sizeof(void*)); @@ -700,34 +796,11 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, // may be a genuine scalar (a large count, a double, a packed value) // that was mis-flagged — overwriting it with null would silently // corrupt it. Only rewrite the word when it actually resolves. - // Reject packed-integer false positives. The capture-side detector - // flags any 8-byte word that resolves to a device VA, but two adjacent - // 32-bit struct fields {uint32 lo, uint32 hi} can coincidentally form - // such a value: ATen elementwise kernels embed an OffsetCalculator - // (per-arg uint32 strides/sizes + IntDivider magic constants) in the - // functor. When `hi` happens to hold a value whose top bits match the - // device-VA prefix (0x7e../0x7f..) and `lo` holds a small integer (a - // stride/size/dim), the combined 64-bit word lands inside a real - // allocation and gets "translated" — corrupting the OffsetCalculator - // and producing an out-of-bounds VM fault (e.g. the recurring - // elementwise_kernel_manual_unroll<...MulFunctor> crash). - // - // A genuine 64-bit device pointer carries a full 48-bit address, so its - // low 32 bits are part of that address and are effectively never this - // small. The FP, by construction, needs its HIGH word to be the VA - // prefix and its LOW word to be a small scalar — so a tiny low-32 value - // is the reliable FP signature. (Set HIP_HRR_PTR_RELAX=1 to disable.) - static const bool ptr_relax = - (std::getenv("HIP_HRR_PTR_RELAX") != nullptr); - auto pointer_like = [&](uint64_t v) -> bool { - if (ptr_relax) return true; - return (v & 0xFFFFFFFFULL) >= 0x10000ULL; - }; auto try_translate_word = [&](size_t off, const char* src) -> bool { if (off + 8 > arg_size) return false; uint64_t rec_ptr; memcpy(&rec_ptr, data + off, 8); - if (rec_ptr < 0x10000ULL) return false; // null/small — never a VA - if (!pointer_like(rec_ptr)) return false; // packed-int false positive + // null, small, or a packed-integer false positive + if (!va_shaped(rec_ptr)) return false; void* live = ctx.translate_ptr(rec_ptr); // Same bypassed-segment materialisation as a whole-pointer // argument: a struct field pointing into memory HIP never saw @@ -740,7 +813,7 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, // Same region check as a whole-pointer argument: a device // address embedded in a by-value struct addresses a tensor // block just as much as one passed directly. - live = hrr_region_check_ptr(ctx, &rls, kernel_name, i, rec_ptr, live); + live = hrr_region_check_ptr(ctx, rls, kernel_name, i, rec_ptr, live); memcpy(storage.data() + off, &live, sizeof(void*)); if (dbg_dump_ptrs) dbg_ptrs.emplace_back(i, rec_ptr, live); if (ctx.verbose) @@ -793,22 +866,302 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, } } arg_ptrs.push_back(storage.data()); + + const size_t store_idx = arg_storage.size() - 1; + if (!no_rescan && value_kind == 0 && arg_size == 4) { + bool joined = false; + if (prev_half.valid && prev_half.idx + 1 == i) { + uint32_t lo, hi; + memcpy(&lo, prev_half.data, 4); + memcpy(&hi, data, 4); + const uint64_t rec = (static_cast(hi) << 32) | lo; + // A pointer split across two scalars is still a pointer, so it + // takes the same steps as the other two forms: the shared shape + // guard, then the bypassed-segment materialisation, then the + // region check. A pair that resolves nowhere stays two untouched + // scalars — the conservative outcome, and there is no sentinel + // to tell apart here. + void* live = nullptr; + if (va_shaped(rec)) { + live = ctx.translate_ptr(rec); + if (!live && ctx.regions_enabled) + live = ctx.regions.materialize_for(ctx, rec); + if (live) + live = hrr_region_check_ptr(ctx, rls, kernel_name, + prev_half.idx, rec, live); + } + if (live) { + const uint64_t lv = reinterpret_cast(live); + const uint32_t l32 = static_cast(lv); + const uint32_t h32 = static_cast(lv >> 32); + memcpy(arg_storage[prev_half.store].data(), &l32, 4); + memcpy(arg_storage[store_idx].data(), &h32, 4); + if (dbg_dump_ptrs) dbg_ptrs.emplace_back(prev_half.idx, rec, live); + static std::mutex mu; + static std::set warned; + char key[160]; + snprintf(key, sizeof(key), "%s#%u", kernel_name.c_str(), + prev_half.idx); + bool first; + { std::lock_guard lk(mu); first = warned.insert(key).second; } + if (first) + fprintf(stderr, + "[HRR] '%s' arg[%u]+arg[%u]: two scalar halves " + "form 0x%llx, a recorded address the metadata " + "does not call a pointer — translated to %p\n", + compact_kernel_name(kernel_name).c_str(), + prev_half.idx, i, (unsigned long long)rec, live); + joined = true; + } + } + prev_half = joined ? decltype(prev_half){} + : decltype(prev_half){true, i, data, store_idx}; + } else { + prev_half.valid = false; + } + } +} + +static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, + bool ext_global_worksize = false, + bool launch_ex = false, + bool cooperative = false) { + // Skip the 32-byte header; kernel launch has a variable-length binary format. + const auto* hdr = reinterpret_cast(pl); + const uint8_t* p = pl + sizeof(hrr_event_header); + const uint8_t* end = pl + hdr->payload_length; + + if (p + 8 > end) return hipErrorInvalidValue; + uint64_t stream_rec; memcpy(&stream_rec, p, 8); p += 8; + + if (p + 2 > end) return hipErrorInvalidValue; + uint16_t name_len; memcpy(&name_len, p, 2); p += 2; + if (p + name_len > end) return hipErrorInvalidValue; + std::string kernel_name(reinterpret_cast(p), name_len); + p += name_len; + + // Workaround for recordings made before the capture side tagged Ext launches: + // hipBLASLt/Tensile ("Cijk_*") StreamK kernels are launched via + // hipExtModuleLaunchKernel, whose grid[] is *global work-item counts*. If such + // a launch was collapsed into the generic (workgroup-count) launch event, the + // grid is over-launched by blockDim and the persistent producer/consumer + // handshake deadlocks. Setting HIP_HRR_REPLAY_FORCE_EXT_CIJK=1 reinterprets + // these grids as global work items (replay through the Ext API). + // + // SUNSET: this is a backward-compat escape hatch only. New recordings record + // hipExtModuleLaunchKernel under HRR_API_HIPEXTMODULELAUNCHKERNEL, whose + // dedicated playback handler already passes ext_global_worksize=true (see + // playback_hipExtModuleLaunchKernel below), so they never need this path. The + // heuristic depends on the third-party Tensile/hipBLASLt "Cijk_" naming + // convention, which can change without notice. Safe to delete once no archive + // predating the capture-side Ext-tagging fix is still being replayed (i.e. + // every recording in use routes Ext launches through their own event id). + if (!ext_global_worksize && kernel_name.compare(0, 5, "Cijk_") == 0 && + std::getenv("HIP_HRR_REPLAY_FORCE_EXT_CIJK")) + ext_global_worksize = true; + + uint64_t co_hash_lo = 0, co_hash_hi = 0; + if (p + 16 <= end) { + memcpy(&co_hash_lo, p, 8); p += 8; + memcpy(&co_hash_hi, p, 8); p += 8; + } + + if (p + 32 > end) return hipErrorInvalidValue; + uint32_t grid[3], block[3], shared_mem; + memcpy(grid, p, 12); p += 12; + memcpy(block, p, 12); p += 12; + memcpy(&shared_mem, p, 4); p += 4; + + uint16_t num_args, num_snapshots; + memcpy(&num_args, p, 2); p += 2; + memcpy(&num_snapshots, p, 2); p += 2; + + // Apply kernel filter if set + if (!ctx.kernel_filter.empty() && + kernel_name.find(ctx.kernel_filter) == std::string::npos) + return hipSuccess; + + hipFunction_t func = resolve_kernel_function(ctx, kernel_name, + co_hash_lo, co_hash_hi); + if (!func) return hipErrorNotFound; + + // Build kernelParams[] from captured args, translating GPU pointers. + std::vector arg_ptrs; + std::vector> arg_storage; + // Guarded blocks this launch relocated, if block guarding is on. Scoped to + // the launch so the relocation is undone before the next event runs. + RegionLaunchState rls; + // Optional recorded->live pointer dump for one target kernel (diff tooling). + const bool dbg_dump_ptrs = (ctx.dump_ptrs_ordinal != 0) || ctx.audit_host_args; + std::vector> dbg_ptrs; // (arg_idx, recorded, live) + std::string dbg_args; + decode_kernel_args(ctx, p, end, num_args, kernel_name, arg_ptrs, + arg_storage, &rls, dbg_dump_ptrs ? &dbg_ptrs : nullptr, + dbg_dump_ptrs ? &dbg_args : nullptr); + + // Launch-attribute tail: u32 count, u32 per-entry stride, then the + // entries. Every launch payload carries it (count 0 for a plain launch). + std::vector launch_attrs; + if (p + 8 <= end) { + uint32_t n_attrs = 0, stride = 0; + memcpy(&n_attrs, p, 4); p += 4; + memcpy(&stride, p, 4); p += 4; + if (n_attrs) { + const size_t have = static_cast(end - p); + if (stride != sizeof(hipLaunchAttribute) || + have < static_cast(n_attrs) * stride) { + fprintf(stderr, + "[HRR] '%s': launch attributes recorded with a %u-byte " + "entry (this build expects %zu) — launching without " + "them; the replayed launch is not the recorded one\n", + kernel_name.c_str(), stride, sizeof(hipLaunchAttribute)); + } else { + launch_attrs.resize(n_attrs); + memcpy(launch_attrs.data(), p, + static_cast(n_attrs) * stride); + p += static_cast(n_attrs) * stride; + for (auto& at : launch_attrs) { + // Two attribute values are themselves pointers. The access + // window names a device buffer, which the allocation map + // can resolve; the prefetch config is a host struct that + // was never recorded, so the attribute is dropped rather + // than passed as a capture-time address. + if (at.id == hipLaunchAttributeAccessPolicyWindow) { + at.val.accessPolicyWindow.base_ptr = ctx.translate_ptr( + reinterpret_cast( + at.val.accessPolicyWindow.base_ptr)); + } else if (at.id == hipLaunchAttributeExtDynDataPrefetch) { + fprintf(stderr, + "[HRR] '%s': dropping the dynamic-data-prefetch " + "launch attribute; its config lives in host " + "memory that the archive does not carry\n", + kernel_name.c_str()); + at.id = hipLaunchAttributeIgnore; + } + } + } + } } hipStream_t stream = ctx.translate_stream(stream_rec); const size_t kernel_ordinal = ctx.kernels_launched.load(std::memory_order_relaxed) + 1; + if (ctx.audit_host_args) { + for (auto& [idx, rec, live] : dbg_ptrs) { + void* abase = nullptr; size_t asize = 0; uint64_t arec = 0; + AllocKind akind = AllocKind::Device; + if (!live || !ctx.live_alloc_of(live, &abase, &asize, &arec, &akind)) + continue; + if (akind == AllocKind::Device) continue; + static std::mutex mu; + static std::set seen; + std::string key = kernel_name + "#" + std::to_string(idx); + bool first; + { std::lock_guard lk(mu); first = seen.insert(key).second; } + if (first) + fprintf(stderr, + "[HRR host-audit] kernel #%zu '%s' arg[%u] reads %s " + "allocation 0x%llx+%zu, which replay reallocated but " + "could not refill\n", + kernel_ordinal, compact_kernel_name(kernel_name).c_str(), + idx, PlaybackContext::alloc_kind_name(akind), + (unsigned long long)arec, asize); + } + } + if (dbg_dump_ptrs && kernel_ordinal == ctx.dump_ptrs_ordinal) { fprintf(stderr, "[HRR ptr-dump] kernel #%zu \"%s\" recorded->live pointer args:\n", kernel_ordinal, compact_kernel_name(kernel_name).c_str()); + // Name the allocation each argument lands in. Pinned host memory is + // worth calling out: the application fills it with ordinary CPU + // stores, which no HIP call reports, so replay hands the kernel a + // freshly allocated buffer that never received those writes. + for (auto& [idx, rec, live] : dbg_ptrs) { + void* abase = nullptr; size_t asize = 0; uint64_t arec = 0; + AllocKind akind = AllocKind::Device; + if (live && ctx.live_alloc_of(live, &abase, &asize, &arec, &akind)) + fprintf(stderr, "[HRR ptr-dump] arg[%u] recorded=0x%llx -> live=%p " + "(%s allocation 0x%llx+%zu, +%lld)\n", idx, + (unsigned long long)rec, live, + PlaybackContext::alloc_kind_name(akind), + (unsigned long long)arec, asize, + (long long)(reinterpret_cast(live) - + reinterpret_cast(abase))); + } for (auto& [idx, rec, live] : dbg_ptrs) fprintf(stderr, "[HRR ptr-dump] arg[%u] recorded=0x%llx -> live=%p\n", idx, (unsigned long long)rec, live); + fprintf(stderr, "[HRR ptr-dump] all recorded arguments as captured:\n%s", + dbg_args.c_str()); fflush(stderr); } + // Read back what this kernel is about to read, looking for capture-time + // addresses. Arguments are translated on the way in, so a recorded pointer + // reaching the GPU has to arrive inside a buffer instead — written by an + // earlier kernel, or by the host into memory it shares with the device. + // Either way nothing rewrote it, and the kernel dereferences an address + // that belongs to the capturing process. + if (ctx.scan_args_ordinal && kernel_ordinal == ctx.scan_args_ordinal) { + (void)hipDeviceSynchronize(); + if (const char* a = std::getenv("HIP_HRR_REPLAY_EXPLAIN_ADDR")) { + uint64_t v = strtoull(a, nullptr, 0); + fprintf(stderr, "[HRR arg-scan] 0x%llx is %s\n", + (unsigned long long)v, ctx.explain_addr(v).c_str()); + // Whether the allocation map says an address is live and whether the + // GPU can actually reach it are different questions, and a fault on + // an address the map calls live means the map is the thing that is + // wrong. Ask the runtime directly. + uint64_t probe = 0; + hipError_t pr = hipMemcpy(&probe, reinterpret_cast(v), + sizeof(probe), hipMemcpyDeviceToHost); + fprintf(stderr, "[HRR arg-scan] reading 8 bytes at 0x%llx: %s", + (unsigned long long)v, hipGetErrorString(pr)); + if (pr == hipSuccess) + fprintf(stderr, " (value 0x%llx)", (unsigned long long)probe); + fprintf(stderr, "\n"); + } + const size_t cap = ctx.scan_args_bytes; + const auto ranges = ctx.recorded_ranges(); + std::vector host(cap); + fprintf(stderr, "[HRR arg-scan] kernel #%zu \"%s\": reading %zu bytes " + "behind each pointer argument\n", kernel_ordinal, + compact_kernel_name(kernel_name).c_str(), cap); + for (auto& [idx, rec, live] : dbg_ptrs) { + if (!live) continue; + // Scan from the start of the enclosing allocation, not from the + // argument, so bytes ahead of the pointer are covered too. + void* abase = live; + size_t asize = 0; + uint64_t arec = rec; + ctx.live_alloc_of(live, &abase, &asize, &arec); + size_t n = asize ? std::min(cap, asize) : cap; + n -= n % 8; + if (!n) continue; + if (hipMemcpy(host.data(), abase, n, hipMemcpyDeviceToHost) != hipSuccess) { + fprintf(stderr, "[HRR arg-scan] arg[%u]: unreadable\n", idx); + continue; + } + size_t hits = 0; + for (size_t off = 0; off + 8 <= n; off += 8) { + uint64_t w; memcpy(&w, host.data() + off, 8); + uint64_t base = 0; size_t sz = 0; + if (!PlaybackContext::range_contains(ranges, w, &base, &sz)) continue; + if (++hits <= 8) + fprintf(stderr, "[HRR arg-scan] arg[%u] allocation 0x%llx " + "+%zu holds 0x%llx — recorded allocation 0x%llx+%zu\n", + idx, (unsigned long long)arec, off, + (unsigned long long)w, (unsigned long long)base, sz); + } + if (hits) + fprintf(stderr, "[HRR arg-scan] arg[%u]: %zu recorded addresses " + "in %zu bytes of allocation 0x%llx\n", idx, hits, n, + (unsigned long long)arec); + } + fflush(stderr); + } // Skip HIP event timing during graph capture: recording events on a // captured stream inserts them into the graph and invalidates the @@ -866,8 +1219,23 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, // HIP C++ kernels (clang-compiled) work correctly with kernelParams[]: the // runtime handles hidden args internally, so no packed buffer is needed. hipError_t r; + // The extensible descriptor is rebuilt here rather than at parse time so + // it sees the translated stream and the argument buffer chosen below. + HIP_LAUNCH_CONFIG ex_cfg{}; + if (launch_ex) { + ex_cfg.gridDimX = grid[0]; ex_cfg.gridDimY = grid[1]; ex_cfg.gridDimZ = grid[2]; + ex_cfg.blockDimX = block[0]; ex_cfg.blockDimY = block[1]; ex_cfg.blockDimZ = block[2]; + ex_cfg.sharedMemBytes = shared_mem; + ex_cfg.hStream = stream; + ex_cfg.attrs = launch_attrs.empty() ? nullptr : launch_attrs.data(); + ex_cfg.numAttrs = static_cast(launch_attrs.size()); + } { - bool is_sp3 = (kernel_name.find("Sp3") != std::string::npos || + // The cooperative entry point takes no `extra`, so the packed-kernarg + // route below is not available to it. Cooperative kernels are HIP C++ + // in every case seen here, which is the kernelParams[] path anyway. + bool is_sp3 = !cooperative && + (kernel_name.find("Sp3") != std::string::npos || kernel_name.find("sp3") != std::string::npos); if (is_sp3 && !arg_ptrs.empty()) { // Compute kernarg layout from captured arg sizes using natural alignment. @@ -896,7 +1264,9 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, HIP_LAUNCH_PARAM_BUFFER_SIZE, &extra_sz, HIP_LAUNCH_PARAM_END }; - if (ext_global_worksize) { + if (launch_ex) { + r = hipDrvLaunchKernelEx(&ex_cfg, func, nullptr, extra); + } else if (ext_global_worksize) { // grid[] = global work-item counts: replay through the Ext API. r = hipExtModuleLaunchKernel( func, @@ -915,7 +1285,18 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, } } else { // HIP C++ kernels: kernelParams[] path — runtime handles hidden args. - if (ext_global_worksize) { + if (cooperative) { + r = hipModuleLaunchCooperativeKernel( + func, + grid[0], grid[1], grid[2], + block[0], block[1], block[2], + shared_mem, stream, + arg_ptrs.empty() ? nullptr : arg_ptrs.data()); + } else if (launch_ex) { + r = hipDrvLaunchKernelEx( + &ex_cfg, func, + arg_ptrs.empty() ? nullptr : arg_ptrs.data(), nullptr); + } else if (ext_global_worksize) { // grid[] = global work-item counts: replay through the Ext API. r = hipExtModuleLaunchKernel( func, @@ -1038,6 +1419,47 @@ hipError_t playback_hipLaunchByPtr(PlaybackContext& ctx, return replay_kernel_launch(ctx, payload); } +hipError_t playback_hipLaunchKernel_spt(PlaybackContext& ctx, + const uint8_t* payload) { + // Which stream the launch goes on is recorded in the payload, so the + // stream-per-thread spelling replays through the same path as the plain + // one. Only the event id differs. + return replay_kernel_launch(ctx, payload); +} + +hipError_t playback_hipLaunchCooperativeKernel(PlaybackContext& ctx, + const uint8_t* payload) { + return replay_kernel_launch(ctx, payload, /*ext_global_worksize=*/false, + /*launch_ex=*/false, /*cooperative=*/true); +} + +hipError_t playback_hipLaunchCooperativeKernel_spt(PlaybackContext& ctx, + const uint8_t* payload) { + return replay_kernel_launch(ctx, payload, /*ext_global_worksize=*/false, + /*launch_ex=*/false, /*cooperative=*/true); +} + +hipError_t playback_hipDrvLaunchKernelEx(PlaybackContext& ctx, + const uint8_t* payload) { + return replay_kernel_launch(ctx, payload, /*ext_global_worksize=*/false, + /*launch_ex=*/true); +} + +hipError_t playback_hipLaunchKernelExC(PlaybackContext& ctx, + const uint8_t* payload) { + return replay_kernel_launch(ctx, payload, /*ext_global_worksize=*/false, + /*launch_ex=*/true); +} + +hipError_t playback_hipModuleLaunchCooperativeKernel(PlaybackContext& ctx, + const uint8_t* payload) { + // A cooperative launch has to go back through the cooperative entry point: + // it is what reserves the whole grid as co-resident, and a grid-wide + // barrier replayed through the ordinary launch hangs instead of failing. + return replay_kernel_launch(ctx, payload, /*ext_global_worksize=*/false, + /*launch_ex=*/false, /*cooperative=*/true); +} + // --------------------------------------------------------------------------- // Manual playback: __hipRegisterFatBinary // --------------------------------------------------------------------------- @@ -1046,6 +1468,17 @@ hipError_t playback_hipLaunchByPtr(PlaybackContext& ctx, // Stored in co_modules keyed by the full 32-char hex hash — collision-free // and consistent with load_module(), so kernel name scans find it automatically. +// A well-formed offload bundle starts with one of the two magics the bundler +// emits: the clang text magic, or the compressed-bundle "CCOB" magic. +static bool is_offload_bundle(const void* p, size_t n) { + if (!p) return false; + static const char kClangMagic[] = "__CLANG_OFFLOAD_BUNDLE__"; + const size_t kClangMagicLen = sizeof(kClangMagic) - 1; // 24, no NUL + if (n >= kClangMagicLen && std::memcmp(p, kClangMagic, kClangMagicLen) == 0) return true; + if (n >= 4 && std::memcmp(p, "CCOB", 4) == 0) return true; + return false; +} + hipError_t playback___hipRegisterFatBinary(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); @@ -1057,39 +1490,505 @@ hipError_t playback___hipRegisterFatBinary(PlaybackContext& ctx, std::string hex = hrr::hash_hex(blob_hash_lo, blob_hash_hi); - // Deduplicate: if already loaded (e.g. multiple __hipRegisterFatBinary events - // for the same binary), skip the load. - { - std::shared_lock lk(ctx.map_mutex); - if (ctx.co_modules.count(hex)) return hipSuccess; + // Deduplicate: if already loaded (e.g. multiple __hipRegisterFatBinary events + // for the same binary), skip the load. Binaries already known to hold no code + // for this GPU are skipped the same way. + { + std::shared_lock lk(ctx.map_mutex); + if (ctx.co_modules.count(hex)) return hipSuccess; + if (ctx.co_no_device_code.count(hex)) return hipSuccess; + } + + size_t sz = 0; + const void* blob = ctx.load_blob(blob_hash_lo, blob_hash_hi, &sz); + if (!blob || sz == 0) { + fprintf(stderr, "[HRR] __hipRegisterFatBinary: blob not found in archive\n"); + return hipSuccess; // non-fatal — kernels will fail at launch but don't abort + } + + hipModule_t mod = nullptr; + hipError_t err = hipModuleLoadData(&mod, blob); + if (err != hipSuccess) { + // A bundle with nothing built for the live GPU is expected, not a defect: + // see co_no_device_code. Launches are assumed not to reference it because + // the archive is replayed on the capture's architecture — nothing checks + // that, so on a cross-arch replay this is where missing kernels start. + // Only a well-formed bundle qualifies; anything else is a damaged blob + // and is reported below. + if ((err == hipErrorInvalidImage || err == hipErrorNoBinaryForGpu) && + is_offload_bundle(blob, sz)) { + { + std::unique_lock lk(ctx.map_mutex); + ctx.co_no_device_code.insert(hex); + } + if (ctx.verbose) + fprintf(stderr, "[HRR] Fat binary %s holds no code for this GPU — skipped\n", + hex.c_str()); + return hipSuccess; + } + fprintf(stderr, "[HRR] __hipRegisterFatBinary: hipModuleLoadData failed: %d (%s)\n", + err, hipGetErrorString(err)); + return hipSuccess; // non-fatal + } + + { + std::unique_lock lk(ctx.map_mutex); + auto [it, inserted] = ctx.co_modules.emplace(hex, mod); + if (!inserted) { + // Another thread raced us between the shared_lock check and here — discard ours. + (void)hipModuleUnload(mod); + } + } + if (ctx.verbose) + fprintf(stderr, "[HRR] Loaded fat binary blob (%zu bytes) -> hipModule_t\n", sz); + return hipSuccess; +} + +// --------------------------------------------------------------------------- +// Symbol resolution +// --------------------------------------------------------------------------- +// A __device__ global is named in the recording by the host shadow address the +// compiler emitted, which means nothing here. The code object carrying the +// global is loaded, though, so the name recorded beside that address resolves +// it — the same lazy name lookup kernel launches already use for functions. + +void* PlaybackContext::resolve_symbol_by_name(const char* name, + size_t* sz_out) const { + if (!name || !*name) return nullptr; + std::shared_lock lk(map_mutex); + for (const auto& [hex, mod] : co_modules) { + (void)hex; + hipDeviceptr_t dptr = nullptr; + size_t bytes = 0; + if (hipModuleGetGlobal(&dptr, &bytes, mod, name) == hipSuccess && dptr) { + if (sz_out) *sz_out = bytes; + return dptr; + } + } + for (const auto& [rec_mod, mod] : module_map) { + (void)rec_mod; + hipDeviceptr_t dptr = nullptr; + size_t bytes = 0; + if (hipModuleGetGlobal(&dptr, &bytes, mod, name) == hipSuccess && dptr) { + if (sz_out) *sz_out = bytes; + return dptr; + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- +// Manual playback: __hipRegisterVar +// --------------------------------------------------------------------------- +// Registrations fire at the capturing process's static-init time, before the +// capture shims are live, so what the archive holds is the post-registration +// sweep hip_capture_init() writes: one event per __device__ global, carrying +// its name, its host shadow address and its capture-time device address. +// +// Resolving the name here and recording both mappings is what makes the symbol +// family replayable. The device-address mapping matters most: hipMemcpyToSymbol +// and friends are recorded as an inner hipMemcpy against the symbol's device +// address, and without this that source translated to nothing. + +hipError_t playback___hipRegisterVar(PlaybackContext& ctx, + const uint8_t* payload) { + const auto* a = reinterpret_cast(payload); + if (!a->deviceVar_present || a->deviceVar_bytes[0] == '\0') return hipSuccess; + + const char* name = reinterpret_cast(a->deviceVar_bytes); + size_t live_size = 0; + void* live = ctx.resolve_symbol_by_name(name, &live_size); + if (!live) { + // Replay dispatches events from more than one thread, so the log-once + // flag has to be atomic even though it only guards a diagnostic. + static std::atomic warned{false}; + if (!warned.exchange(true)) { + fprintf(stderr, + "[HRR] __hipRegisterVar: symbol '%s' is not in any module " + "this replay loaded, so copies naming it will fail rather " + "than write somewhere else.\n", name); + } + return hipSuccess; + } + + if (live_size == 0) live_size = static_cast(a->size); + ctx.record_symbol(a->var, name, live, live_size); + if (a->dev_addr) + ctx.record_alloc(a->dev_addr, live, live_size); + if (ctx.verbose) + fprintf(stderr, "[HRR] symbol '%s': 0x%llx -> %p (%zu bytes)\n", name, + (unsigned long long)a->dev_addr, live, live_size); + return hipSuccess; +} + +// --------------------------------------------------------------------------- +// Manual playback: hipGetSymbolAddress / hipGetSymbolSize +// --------------------------------------------------------------------------- +// Both take the host shadow address, which the symbol sweep has already tied +// to a live global. Answering from that map also registers the recorded device +// address the capturing process got back, so a later copy against it resolves. + +hipError_t playback_hipGetSymbolAddress(PlaybackContext& ctx, + const uint8_t* payload) { + const auto* a = reinterpret_cast(payload); + size_t sz = 0; + void* live = ctx.translate_symbol(a->symbol, &sz); + if (!live) { + fprintf(stderr, + "[HRR] hipGetSymbolAddress: symbol 0x%llx was never registered " + "in this archive.\n", (unsigned long long)a->symbol); + return hipErrorInvalidSymbol; + } + if (a->devPtr) ctx.record_alloc(a->devPtr, live, sz); + return hipSuccess; +} + +hipError_t playback_hipGetSymbolSize(PlaybackContext& ctx, + const uint8_t* payload) { + const auto* a = reinterpret_cast(payload); + size_t sz = 0; + if (!ctx.translate_symbol(a->symbol, &sz)) { + fprintf(stderr, + "[HRR] hipGetSymbolSize: symbol 0x%llx was never registered in " + "this archive.\n", (unsigned long long)a->symbol); + return hipErrorInvalidSymbol; + } + return hipSuccess; +} + +// --------------------------------------------------------------------------- +// Manual playback: hipGraphAddMemcpyNodeToSymbol / FromSymbol +// --------------------------------------------------------------------------- +// The node is built against the live global the symbol registry resolved. The +// host side of the copy is the recorded blob for the to-symbol spelling and a +// context-owned landing buffer for the from-symbol one, which has to outlive +// this call: the copy happens when the graph is launched, not here. +// +// The node is added through the 1D spelling rather than the symbol one. The +// symbol entry points want the host shadow address the compiler emitted for +// the global, and a replay has no such shadow — it loaded the code object +// through hipModuleLoadData. Passing the device address instead is rejected +// with hipErrorInvalidDeviceSymbol, so the copy is expressed directly against +// the resolved device address, which is what the symbol spelling decays to +// inside the runtime anyway. + +static hipGraphNode_t* dep_array(PlaybackContext& ctx, const uint8_t* bytes, + uint8_t present, uint32_t n, + std::vector& out) { + if (!present || n == 0) return nullptr; + out.resize(n); + std::memcpy(out.data(), bytes, n * sizeof(hipGraphNode_t)); + for (auto& node : out) + node = ctx.translate_graph_node(reinterpret_cast(node)); + return out.data(); +} + +hipError_t playback_hipGraphAddMemcpyNodeToSymbol(PlaybackContext& ctx, + const uint8_t* payload) { + const auto* a = + reinterpret_cast(payload); + const auto kind = static_cast(a->kind); + void* symbol = ctx.translate_symbol(a->symbol); + if (!symbol) { + fprintf(stderr, + "[HRR] hipGraphAddMemcpyNodeToSymbol: symbol 0x%llx is not in " + "this archive's symbol registry.\n", + (unsigned long long)a->symbol); + return hipErrorInvalidSymbol; + } + + const void* src = nullptr; + if (a->blob_hash_lo || a->blob_hash_hi) { + size_t blob_sz = 0; + src = ctx.load_blob(a->blob_hash_lo, a->blob_hash_hi, &blob_sz); + if (!src || blob_sz < a->count) { + fprintf(stderr, + "[HRR] hipGraphAddMemcpyNodeToSymbol: host source blob " + "missing from the archive.\n"); + return hipErrorInvalidValue; + } + } else { + src = ctx.translate_ptr(a->src); + if (!src) { + fprintf(stderr, + "[HRR] hipGraphAddMemcpyNodeToSymbol: device source 0x%llx " + "is not mapped.\n", (unsigned long long)a->src); + return hipErrorInvalidValue; + } + } + + std::vector deps; + hipGraphNode_t* dep_ptr = dep_array(ctx, a->pDependencies_bytes, + a->pDependencies_present, + a->pDependencies_n, deps); + hipGraphNode_t node = nullptr; + hipError_t r = hipGraphAddMemcpyNode1D( + &node, ctx.translate_graph(a->graph), dep_ptr, deps.size(), + static_cast(symbol) + a->offset, src, + static_cast(a->count), kind); + if (r == hipSuccess && a->pGraphNode) + ctx.record_graph_node(a->pGraphNode, node); + return r; +} + +hipError_t playback_hipGraphAddMemcpyNodeFromSymbol(PlaybackContext& ctx, + const uint8_t* payload) { + const auto* a = + reinterpret_cast(payload); + const auto kind = static_cast(a->kind); + void* symbol = ctx.translate_symbol(a->symbol); + if (!symbol) { + fprintf(stderr, + "[HRR] hipGraphAddMemcpyNodeFromSymbol: symbol 0x%llx is not in " + "this archive's symbol registry.\n", + (unsigned long long)a->symbol); + return hipErrorInvalidSymbol; + } + + void* dst = nullptr; + if (kind == hipMemcpyDeviceToHost || kind == hipMemcpyHostToHost) { + dst = ctx.host_landing_buffer(a->dst, static_cast(a->count)); + if (!dst) { + fprintf(stderr, + "[HRR] hipGraphAddMemcpyNodeFromSymbol: the recorded host " + "destination is null, so this node is not reproduced.\n"); + return hipErrorInvalidValue; + } + } else { + dst = ctx.translate_ptr(a->dst); + if (!dst) { + fprintf(stderr, + "[HRR] hipGraphAddMemcpyNodeFromSymbol: device destination " + "0x%llx is not mapped.\n", (unsigned long long)a->dst); + return hipErrorInvalidValue; + } + } + + std::vector deps; + hipGraphNode_t* dep_ptr = dep_array(ctx, a->pDependencies_bytes, + a->pDependencies_present, + a->pDependencies_n, deps); + hipGraphNode_t node = nullptr; + hipError_t r = hipGraphAddMemcpyNode1D( + &node, ctx.translate_graph(a->graph), dep_ptr, deps.size(), dst, + static_cast(symbol) + a->offset, static_cast(a->count), + kind); + if (r == hipSuccess && a->pGraphNode) + ctx.record_graph_node(a->pGraphNode, node); + return r; +} + +// --------------------------------------------------------------------------- +// Manual playback: the four *MemcpyNodeSetParams*Symbol spellings +// --------------------------------------------------------------------------- +// Same substitution as the two construction handlers above, for the same +// reason: the symbol entry points need a host shadow address, so the mutation +// is expressed against the resolved device address through the 1D spelling. + +namespace { + +// The host source for a to-symbol mutation: the recorded blob when the copy +// came from host memory, the translated device address otherwise. +template +static const void* to_symbol_source(PlaybackContext& ctx, const A* a, + const char* api) { + if (a->blob_hash_lo || a->blob_hash_hi) { + size_t blob_sz = 0; + const void* blob = ctx.load_blob(a->blob_hash_lo, a->blob_hash_hi, + &blob_sz); + if (!blob || blob_sz < a->count) { + fprintf(stderr, "[HRR] %s: host source blob missing from the " + "archive.\n", api); + return nullptr; + } + return blob; + } + const void* src = ctx.translate_ptr(a->src); + if (!src) + fprintf(stderr, "[HRR] %s: device source 0x%llx is not mapped.\n", api, + (unsigned long long)a->src); + return src; +} + +static void* symbol_or_complain(PlaybackContext& ctx, uint64_t rec, + const char* api) { + void* symbol = ctx.translate_symbol(rec); + if (!symbol) + fprintf(stderr, "[HRR] %s: symbol 0x%llx is not in this archive's " + "symbol registry.\n", api, (unsigned long long)rec); + return symbol; +} + +} // namespace + +hipError_t playback_hipGraphMemcpyNodeSetParamsToSymbol(PlaybackContext& ctx, + const uint8_t* payload) { + const char* kApi = "hipGraphMemcpyNodeSetParamsToSymbol"; + const auto* a = + reinterpret_cast(payload); + void* symbol = symbol_or_complain(ctx, a->symbol, kApi); + if (!symbol) return hipErrorInvalidSymbol; + const void* src = to_symbol_source(ctx, a, kApi); + if (!src) return hipErrorInvalidValue; + hipGraphNode_t node = ctx.translate_graph_node(a->node); + if (!node) { + fprintf(stderr, "[HRR] %s: node 0x%llx was never built at replay.\n", + kApi, (unsigned long long)a->node); + return hipErrorInvalidValue; + } + return hipGraphMemcpyNodeSetParams1D( + node, static_cast(symbol) + a->offset, src, + static_cast(a->count), static_cast(a->kind)); +} + +hipError_t playback_hipGraphMemcpyNodeSetParamsFromSymbol(PlaybackContext& ctx, + const uint8_t* payload) { + const char* kApi = "hipGraphMemcpyNodeSetParamsFromSymbol"; + const auto* a = + reinterpret_cast(payload); + void* symbol = symbol_or_complain(ctx, a->symbol, kApi); + if (!symbol) return hipErrorInvalidSymbol; + hipGraphNode_t node = ctx.translate_graph_node(a->node); + if (!node) { + fprintf(stderr, "[HRR] %s: node 0x%llx was never built at replay.\n", + kApi, (unsigned long long)a->node); + return hipErrorInvalidValue; + } + const auto kind = static_cast(a->kind); + void* dst = (kind == hipMemcpyDeviceToHost || kind == hipMemcpyHostToHost) + ? ctx.host_landing_buffer(a->dst, static_cast(a->count)) + : ctx.translate_ptr(a->dst); + if (!dst) { + fprintf(stderr, "[HRR] %s: destination 0x%llx is not mapped.\n", kApi, + (unsigned long long)a->dst); + return hipErrorInvalidValue; + } + return hipGraphMemcpyNodeSetParams1D( + node, dst, static_cast(symbol) + a->offset, + static_cast(a->count), kind); +} + +hipError_t playback_hipGraphExecMemcpyNodeSetParamsToSymbol( + PlaybackContext& ctx, const uint8_t* payload) { + const char* kApi = "hipGraphExecMemcpyNodeSetParamsToSymbol"; + const auto* a = + reinterpret_cast(payload); + void* symbol = symbol_or_complain(ctx, a->symbol, kApi); + if (!symbol) return hipErrorInvalidSymbol; + const void* src = to_symbol_source(ctx, a, kApi); + if (!src) return hipErrorInvalidValue; + hipGraphExec_t exec = ctx.translate_graph_exec(a->hGraphExec); + hipGraphNode_t node = ctx.translate_graph_node(a->node); + if (!exec || !node) { + fprintf(stderr, "[HRR] %s: exec 0x%llx / node 0x%llx was never built " + "at replay.\n", kApi, (unsigned long long)a->hGraphExec, + (unsigned long long)a->node); + return hipErrorInvalidValue; + } + return hipGraphExecMemcpyNodeSetParams1D( + exec, node, static_cast(symbol) + a->offset, src, + static_cast(a->count), static_cast(a->kind)); +} + +hipError_t playback_hipGraphExecMemcpyNodeSetParamsFromSymbol( + PlaybackContext& ctx, const uint8_t* payload) { + const char* kApi = "hipGraphExecMemcpyNodeSetParamsFromSymbol"; + const auto* a = + reinterpret_cast(payload); + void* symbol = symbol_or_complain(ctx, a->symbol, kApi); + if (!symbol) return hipErrorInvalidSymbol; + hipGraphExec_t exec = ctx.translate_graph_exec(a->hGraphExec); + hipGraphNode_t node = ctx.translate_graph_node(a->node); + if (!exec || !node) { + fprintf(stderr, "[HRR] %s: exec 0x%llx / node 0x%llx was never built " + "at replay.\n", kApi, (unsigned long long)a->hGraphExec, + (unsigned long long)a->node); + return hipErrorInvalidValue; + } + const auto kind = static_cast(a->kind); + void* dst = (kind == hipMemcpyDeviceToHost || kind == hipMemcpyHostToHost) + ? ctx.host_landing_buffer(a->dst, static_cast(a->count)) + : ctx.translate_ptr(a->dst); + if (!dst) { + fprintf(stderr, "[HRR] %s: destination 0x%llx is not mapped.\n", kApi, + (unsigned long long)a->dst); + return hipErrorInvalidValue; } + return hipGraphExecMemcpyNodeSetParams1D( + exec, node, dst, static_cast(symbol) + a->offset, + static_cast(a->count), kind); +} - size_t sz = 0; - const void* blob = ctx.load_blob(blob_hash_lo, blob_hash_hi, &sz); - if (!blob || sz == 0) { - fprintf(stderr, "[HRR] __hipRegisterFatBinary: blob not found in archive\n"); - return hipSuccess; // non-fatal — kernels will fail at launch but don't abort +// --------------------------------------------------------------------------- +// Manual playback: hipLinkAddData +// --------------------------------------------------------------------------- +// The image comes back from its blob and the linker state from the map +// hipLinkCreate fills. Option values are replayed as recorded: an option whose +// value is a pointer into the capturing process cannot be reconstructed, and +// the link then fails here rather than producing a silently different binary. + +hipError_t playback_hipLinkAddData(PlaybackContext& ctx, + const uint8_t* payload) { + const auto* a = reinterpret_cast(payload); + hipLinkState_t state = ctx.translate_link_state(a->state); + if (!state) { + fprintf(stderr, + "[HRR] hipLinkAddData: linker state 0x%llx was never created " + "at replay.\n", (unsigned long long)a->state); + return hipErrorInvalidValue; } - hipModule_t mod = nullptr; - hipError_t err = hipModuleLoadData(&mod, blob); - if (err != hipSuccess) { - fprintf(stderr, "[HRR] __hipRegisterFatBinary: hipModuleLoadData failed: %d (%s)\n", - err, hipGetErrorString(err)); - return hipSuccess; // non-fatal + size_t blob_sz = 0; + const void* image = (a->blob_hash_lo || a->blob_hash_hi) + ? ctx.load_blob(a->blob_hash_lo, a->blob_hash_hi, + &blob_sz) + : nullptr; + if (!image) { + fprintf(stderr, + "[HRR] hipLinkAddData: the linker input image is not in this " + "archive, so the link cannot be reproduced.\n"); + return hipErrorInvalidValue; } - { - std::unique_lock lk(ctx.map_mutex); - auto [it, inserted] = ctx.co_modules.emplace(hex, mod); - if (!inserted) { - // Another thread raced us between the shared_lock check and here — discard ours. - (void)hipModuleUnload(mod); + hipJitOption options[32]{}; + void* option_values[32]{}; + uint32_t n_opts = a->options_n > 32u ? 32u : a->options_n; + if (a->options_present) + std::memcpy(options, a->options_bytes, n_opts * sizeof(hipJitOption)); + if (a->optionValues_present) + std::memcpy(option_values, a->optionValues_bytes, + n_opts * sizeof(void*)); + + // A recorded option value is whatever void* the capturing process passed. + // Some hipJitOptions take a pointer to a caller-owned buffer + // (hipJitOptionInfoLogBuffer and hipJitOptionErrorLogBuffer among them) + // and others carry a small integer cast to void*; the archive does not + // record which kind each one was. Forwarding a recorded pointer would hand + // the runtime an address belonging to a process that no longer exists, so + // a link carrying any non-null option value is refused by name rather than + // replayed into a wild dereference. Links with no options, and options + // whose values are all null, replay unchanged — which is what the captured + // workloads use. + for (uint32_t i = 0; i < n_opts; ++i) { + if (option_values[i]) { + fprintf(stderr, + "[HRR] hipLinkAddData: option %u was recorded with a " + "capture-process address (%p) that replay cannot " + "reconstruct, so the link is not reproduced.\n", + i, option_values[i]); + return hipErrorInvalidValue; } } - if (ctx.verbose) - fprintf(stderr, "[HRR] Loaded fat binary blob (%zu bytes) -> hipModule_t\n", sz); - return hipSuccess; + + hipError_t r = hipLinkAddData( + state, static_cast(a->type), const_cast(image), + blob_sz, + a->name_present ? reinterpret_cast(a->name_bytes) + : nullptr, + n_opts, n_opts ? options : nullptr, n_opts ? option_values : nullptr); + if (hrr_replayed_recorded_error(ctx, "hipLinkAddData", a->ret, r)) + return hipSuccess; + return r; } // --------------------------------------------------------------------------- @@ -1280,6 +2179,24 @@ static bool hrr_replay_zero_init() { return g_enabled; } +// Byte to fill fresh replay allocations with, normally zero. Set it to +// something no program would compute with — 0xa5, say — to find out whether a +// kernel is consuming memory that nothing in the archive ever wrote: the +// pattern then shows up in the values it derives, and a wild pointer built +// from it is recognisable on sight in a fault address. +static int hrr_replay_fill_byte() { + static std::once_flag once; + static int g_byte = 0; + std::call_once(once, [] { + if (const char* e = std::getenv("HIP_HRR_REPLAY_FILL_BYTE")) { + g_byte = static_cast(strtoul(e, nullptr, 0) & 0xff); + fprintf(stderr, "[HRR] filling fresh allocations with 0x%02x " + "(HIP_HRR_REPLAY_FILL_BYTE)\n", g_byte); + } + }); + return g_byte; +} + // Zero-initialise a host-synchronous replay allocation, ordered. // // ROCM-27985. hipMalloc is host-synchronous by contract, so the recorded @@ -1305,7 +2222,8 @@ static void hrr_zero_init_alloc(PlaybackContext& ctx, void* live, size_t sz) { if (!live || sz == 0) return; // nothing written, so nothing to order if (!hrr_zero_init_needs_drain(hrr_replay_zero_init(), ctx.in_graph_capture)) return; - if (hipMemsetAsync(live, 0, sz, nullptr) != hipSuccess) return; + if (hipMemsetAsync(live, hrr_replay_fill_byte(), sz, nullptr) != hipSuccess) + return; (void)hipStreamSynchronize(nullptr); } @@ -1878,7 +2796,8 @@ hipError_t playback_hipMallocFromPoolAsync(PlaybackContext& ctx, // --------------------------------------------------------------------------- // Manual playback: hipMemPoolSetAttribute / hipMemPoolGetAttribute -// value is void*; stored inline as value_u64 (8 bytes covers all attr sizes). +// value is void*, stored inline in the low bytes of value_u64 (an int32_t for the +// three reuse policies, uint64_t otherwise). // GetAttribute is a no-op at playback (output only; pool state matches capture). // --------------------------------------------------------------------------- @@ -2285,6 +3204,36 @@ static void* translate_or_materialize(PlaybackContext& ctx, uint64_t rec_addr) { return live; } +// A recorded H2D payload is replayed byte for byte, so any device pointer the +// application had placed in that host buffer arrives on the GPU as a +// capture-time address: nothing in the pointer-translation path ever sees it. +// Report those words rather than translate them — a payload is opaque bytes, +// and a value that merely looks like an address may be data. +static void scan_h2d_payload(PlaybackContext& ctx, const void* blob, size_t n, + uint64_t dst_rec) { + const uint8_t* p = static_cast(blob); + const auto ranges = ctx.recorded_ranges(); + size_t hits = 0; + for (size_t off = 0; off + 8 <= n; off += 8) { + uint64_t w; + memcpy(&w, p + off, 8); + uint64_t base = 0; + size_t sz = 0; + if (!PlaybackContext::range_contains(ranges, w, &base, &sz)) continue; + if (++hits <= 4) + fprintf(stderr, + "[HRR h2d-scan] payload for 0x%llx +%zu holds 0x%llx — a " + "recorded address in allocation 0x%llx+%zu\n", + (unsigned long long)dst_rec, off, (unsigned long long)w, + (unsigned long long)base, sz); + } + if (hits) + fprintf(stderr, + "[HRR h2d-scan] payload for 0x%llx: %zu recorded addresses in " + "%zu bytes\n", + (unsigned long long)dst_rec, hits, n); +} + // This mirrors the captured API exactly — hipMemcpyAsync on the default stream // (stream_rec==0, translated to nullptr) must still use the async variant. static hipError_t replay_memcpy_impl(PlaybackContext& ctx, @@ -2315,6 +3264,7 @@ static hipError_t replay_memcpy_impl(PlaybackContext& ctx, (unsigned long long)dst_rec, copy_sz, avail); copy_sz = avail; } + if (ctx.scan_h2d) scan_h2d_payload(ctx, blob, copy_sz, dst_rec); if (is_async) r = hipMemcpyAsync(dst, blob, copy_sz, hipMemcpyHostToDevice, stream); else @@ -2652,17 +3602,25 @@ hipError_t playback_hipGraphInstantiate(PlaybackContext& ctx, hipGraph_t graph = ctx.translate_graph(a->graph); if (!graph) { - // graph_map is populated ONLY by the stream-capture chain - // (hipStreamEndCapture). A miss means the graph was built through the - // explicit node API (hipGraphCreate + hipGraphAdd*Node), which HRR does - // not replay. Replaying an empty graph would silently skip every launch - // and corrupt downstream buffers, so fail loudly instead. + // graph_map holds every graph this replay built, whether by stream + // capture (hipStreamEndCapture) or by the node API (hipGraphCreate). + // A miss means the graph does not exist here at all. + fprintf(stderr, + "[HRR] hipGraphInstantiate: graph 0x%llx was never built at " + "replay, so there is nothing to instantiate. Aborting rather " + "than running an empty graph.\n", + (unsigned long long)a->graph); + return hipErrorNotSupported; + } + if (ctx.graph_is_incomplete(a->graph)) { + // The graph exists but is missing at least one node HRR could not + // reconstruct (each one said so when it was skipped). Instantiating it + // would run a graph short of work and quietly produce wrong buffers. fprintf(stderr, - "[HRR] hipGraphInstantiate: graph 0x%llx not in graph_map. HRR only " - "replays stream-capture graphs (hipStreamBeginCapture/EndCapture); " - "explicit node-API graph construction (hipGraphCreate + " - "hipGraphAdd*Node) is NOT supported. Aborting replay rather than " - "running an empty graph.\n", + "[HRR] hipGraphInstantiate: graph 0x%llx is missing nodes this " + "replay could not reconstruct (see the earlier per-node " + "messages). Refusing to instantiate a graph that would run " + "with work missing.\n", (unsigned long long)a->graph); return hipErrorNotSupported; } @@ -2690,12 +3648,18 @@ hipError_t playback_hipGraphInstantiateWithFlags(PlaybackContext& ctx, hipGraph_t graph = ctx.translate_graph(a->graph); if (!graph) { - // See playback_hipGraphInstantiate: a graph_map miss means explicit - // node-API construction, which HRR does not replay. Fail loudly. fprintf(stderr, - "[HRR] hipGraphInstantiateWithFlags: graph 0x%llx not in graph_map. " - "HRR only replays stream-capture graphs; explicit node-API graph " - "construction is NOT supported. Aborting replay.\n", + "[HRR] hipGraphInstantiateWithFlags: graph 0x%llx was never " + "built at replay. Aborting replay.\n", + (unsigned long long)a->graph); + return hipErrorNotSupported; + } + if (ctx.graph_is_incomplete(a->graph)) { + // See playback_hipGraphInstantiate. + fprintf(stderr, + "[HRR] hipGraphInstantiateWithFlags: graph 0x%llx is missing " + "nodes this replay could not reconstruct. Refusing to " + "instantiate it.\n", (unsigned long long)a->graph); return hipErrorNotSupported; } @@ -3151,20 +4115,25 @@ hipError_t playback_hipDrvMemcpy3DAsync(PlaybackContext& ctx, const uint8_t* pl) ctx.translate_stream(a->stream), /*is_async=*/true); } -hipError_t playback_hipDrvMemcpy2DUnaligned(PlaybackContext& ctx, const uint8_t* pl) { - const auto* a = reinterpret_cast(pl); - static constexpr char kApi[] = "hipDrvMemcpy2DUnaligned"; +// The three spellings of the driver 2D copy — hipDrvMemcpy2DUnaligned, +// hipMemcpyParam2D and hipMemcpyParam2DAsync — take the same hip_Memcpy2D and +// need the same treatment, so `issue` is the only thing that varies: it runs +// the rebuilt descriptor through whichever entry point was recorded. +template +static hipError_t replay_drvmemcpy2d(PlaybackContext& ctx, const T* a, + const char* api, hipStream_t stream, + bool is_async, Issue&& issue) { hip_Memcpy2D parms{}; std::memcpy(&parms, a->drv2d_bytes, sizeof(parms)); static bool array_warned = false; - if (drvmemcpy_declines_array_rect(kApi, array_warned, + if (drvmemcpy_declines_array_rect(api, array_warned, parms.srcMemoryType, parms.dstMemoryType)) return hipSuccess; if (parms.srcMemoryType == hipMemoryTypeHost) { if (parms.dstMemoryType == hipMemoryTypeHost) { - fprintf(stderr, "[HRR] %s: host-to-host copy, skipped\n", kApi); + fprintf(stderr, "[HRR] %s: host-to-host copy, skipped\n", api); return hipSuccess; } // The runtime widens hip_Memcpy2D to a HIP_MEMCPY3D with Depth == 1 and @@ -3175,13 +4144,13 @@ hipError_t playback_hipDrvMemcpy2DUnaligned(PlaybackContext& ctx, const uint8_t* size_t need = drvmemcpy_host_bytes(pitch, /*pitch_height=*/0, parms.srcXInBytes, parms.srcY, /*z=*/0, parms.WidthInBytes, parms.Height, /*depth=*/1); - const void* blob = drvmemcpy_h2d_src_blob(ctx, kApi, a->blob_hash_lo, + const void* blob = drvmemcpy_h2d_src_blob(ctx, api, a->blob_hash_lo, a->blob_hash_hi, need); if (!blob) return hipSuccess; parms.srcHost = blob; parms.dstDevice = reinterpret_cast( ctx.translate_ptr(reinterpret_cast(parms.dstDevice))); - hipError_t r = hipDrvMemcpy2DUnaligned(&parms); + hipError_t r = issue(&parms); if (r == hipSuccess) r = hrr_sync_after_replayed_h2d(ctx, "replayed driver 2D H2D memcpy"); return r; @@ -3192,20 +4161,89 @@ hipError_t playback_hipDrvMemcpy2DUnaligned(PlaybackContext& ctx, const uint8_t* if (!src_live) { fprintf(stderr, "[HRR] %s D2H validate FAIL: src 0x%llx not mapped - " "pointer translation bug\n", - kApi, (unsigned long long)src_rec); + api, (unsigned long long)src_rec); ctx.d2h_attempted++; ctx.note_d2h_fail(hrr_dispatch_seq); return hipSuccess; } size_t byte_count = parms.WidthInBytes * parms.Height; return replay_memcpy3d_d2h(ctx, src_live, byte_count, - a->d2h_hash_lo, a->d2h_hash_hi, nullptr, false, kApi); + a->d2h_hash_lo, a->d2h_hash_hi, + stream, is_async, api); } parms.srcDevice = reinterpret_cast( ctx.translate_ptr(reinterpret_cast(parms.srcDevice))); parms.dstDevice = reinterpret_cast( ctx.translate_ptr(reinterpret_cast(parms.dstDevice))); - return hipDrvMemcpy2DUnaligned(&parms); + return issue(&parms); +} + +hipError_t playback_hipDrvMemcpy2DUnaligned(PlaybackContext& ctx, const uint8_t* pl) { + const auto* a = reinterpret_cast(pl); + return replay_drvmemcpy2d(ctx, a, "hipDrvMemcpy2DUnaligned", nullptr, + /*is_async=*/false, + [](const hip_Memcpy2D* p) { + return hipDrvMemcpy2DUnaligned(p); + }); +} + +hipError_t playback_hipMemcpyParam2D(PlaybackContext& ctx, const uint8_t* pl) { + const auto* a = reinterpret_cast(pl); + return replay_drvmemcpy2d(ctx, a, "hipMemcpyParam2D", nullptr, + /*is_async=*/false, + [](const hip_Memcpy2D* p) { + return hipMemcpyParam2D(p); + }); +} + +hipError_t playback_hipMemcpyParam2DAsync(PlaybackContext& ctx, const uint8_t* pl) { + const auto* a = reinterpret_cast(pl); + hipStream_t stream = ctx.translate_stream(a->stream); + return replay_drvmemcpy2d(ctx, a, "hipMemcpyParam2DAsync", stream, + /*is_async=*/true, + [stream](const hip_Memcpy2D* p) { + return hipMemcpyParam2DAsync(p, stream); + }); +} + +// --------------------------------------------------------------------------- +// Manual playback: hipMemcpy3DBatchAsync +// +// Each operand of each op is a tagged union: a pointer with a layout and a +// location hint, or an array handle. Which member holds the recorded address +// therefore depends on the tag beside it, which is why the op list is restored +// here rather than by a ptr_members rewrite in the generator. +// --------------------------------------------------------------------------- + +static void translate_batch_operand(PlaybackContext& ctx, + hipMemcpy3DOperand& operand) { + if (operand.type == hipMemcpyOperandTypePointer) { + operand.op.ptr.ptr = ctx.translate_ptr( + reinterpret_cast(operand.op.ptr.ptr)); + } else { + operand.op.array.array = ctx.translate_array( + reinterpret_cast(operand.op.array.array)); + } +} + +hipError_t playback_hipMemcpy3DBatchAsync(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = reinterpret_cast(pl); + if (!a->opList_present || a->opList_n == 0) return hipSuccess; + + uint32_t n = a->opList_n; + if (n > 16u) n = 16u; + std::vector ops(n); + std::memcpy(ops.data(), a->opList_bytes, n * sizeof(hipMemcpy3DBatchOp)); + for (auto& op : ops) { + translate_batch_operand(ctx, op.src); + translate_batch_operand(ctx, op.dst); + } + + size_t fail_idx = 0; + return hipMemcpy3DBatchAsync(n, ops.data(), &fail_idx, + static_cast(a->flags), + ctx.translate_stream(a->stream)); } // --------------------------------------------------------------------------- @@ -3342,6 +4380,8 @@ hipError_t playback_hipArrayCreate(PlaybackContext& ctx, const uint8_t* pl) { hipArray_t arr = nullptr; hipError_t r = hipArrayCreate(&arr, &desc); if (r == hipSuccess) ctx.record_array(a->pHandle, arr); + if (hrr_replayed_recorded_error(ctx, "hipArrayCreate", a->ret, r)) + return hipSuccess; return r; } @@ -3352,6 +4392,8 @@ hipError_t playback_hipArray3DCreate(PlaybackContext& ctx, const uint8_t* pl) { hipArray_t arr = nullptr; hipError_t r = hipArray3DCreate(&arr, &desc); if (r == hipSuccess) ctx.record_array(a->array, arr); + if (hrr_replayed_recorded_error(ctx, "hipArray3DCreate", a->ret, r)) + return hipSuccess; return r; } @@ -3455,15 +4497,34 @@ hipError_t playback_hipMemAddressFree(PlaybackContext& ctx, const uint8_t* pl) { hipError_t playback_hipMemCreate(PlaybackContext& ctx, const uint8_t* pl) { const auto* a = reinterpret_cast(pl); uint64_t rec_handle = a->handle; // recorded output handle - // Reconstruct the hipMemAllocationProp — it's a regular struct, not stored inline - // Generator stores handle (u64) and prop (u64 stale ptr) and flags (u64). - // We re-query granularity with the same type/location as captured. + // The allocation property is carried inline (DEREF_FIELDS). It used to be + // hardcoded to Pinned on device 0, which builds the wrong topology without + // saying so for a heap that maps cross-node peers. hipMemAllocationProp prop{}; - prop.type = hipMemAllocationTypePinned; - prop.location.type = hipMemLocationTypeDevice; - prop.location.id = 0; // device 0; matches test workload + if (a->prop_present) { + std::memcpy(&prop, a->prop_bytes, sizeof(prop)); + } else { + prop.type = hipMemAllocationTypePinned; + prop.location.type = hipMemLocationTypeDevice; + prop.location.id = 0; + } + // A recorded device ordinal that does not exist here would otherwise be + // answered by the runtime with a bare error code, or worse, silently + // satisfied from the wrong device. + if (prop.location.type == hipMemLocationTypeDevice) { + int ndev = 0; + (void)hipGetDeviceCount(&ndev); + if (prop.location.id >= ndev) { + fprintf(stderr, + "[HRR] hipMemCreate: the recording allocated on device %d " + "and this replay has %d device(s) — refusing to allocate " + "somewhere else\n", prop.location.id, ndev); + return hipErrorInvalidDevice; + } + } hipMemGenericAllocationHandle_t live_handle{}; - hipError_t r = hipMemCreate(&live_handle, static_cast(a->size), &prop, 0); + hipError_t r = hipMemCreate(&live_handle, static_cast(a->size), &prop, + static_cast(a->flags)); if (r == hipSuccess) { std::unique_lock lk(ctx.map_mutex); ctx.vmm_handle_map[rec_handle] = live_handle; @@ -3502,3 +4563,592 @@ hipError_t playback_hipMemUnmap(PlaybackContext& ctx, const uint8_t* pl) { if (!live_va) return hipSuccess; return hipMemUnmap(live_va, static_cast(a->size)); } + +// --------------------------------------------------------------------------- +// Manual playback: hipStreamBatchMemOp +// +// The op list is carried inline (DEREF_FIELDS). Every entry's address is a +// device pointer recorded in the capturing process, and which union member +// holds it depends on the op type — so the rewrite is done here rather than by +// a generic ptr_members pass. +// --------------------------------------------------------------------------- +hipError_t playback_hipStreamBatchMemOp(PlaybackContext& ctx, const uint8_t* pl) { + const auto* a = reinterpret_cast(pl); + const uint32_t inline_max = + static_cast(sizeof(a->paramArray_bytes) / + sizeof(hipStreamBatchMemOpParams)); + uint32_t n = a->paramArray_n > inline_max ? inline_max : a->paramArray_n; + if (!a->paramArray_present || n == 0) return hipSuccess; + + std::vector ops(n); + std::memcpy(ops.data(), a->paramArray_bytes, + static_cast(n) * sizeof(hipStreamBatchMemOpParams)); + + for (auto& op : ops) { + hipDeviceptr_t* addr = nullptr; + switch (op.operation) { + case hipStreamMemOpWaitValue32: + case hipStreamMemOpWaitValue64: + addr = &op.waitValue.address; break; + case hipStreamMemOpWriteValue32: + case hipStreamMemOpWriteValue64: + addr = &op.writeValue.address; break; + default: + // Barrier and flush ops carry no address. + continue; + } + void* live = ctx.translate_ptr(reinterpret_cast(*addr)); + if (!live) { + // A batch the runtime cannot address is rejected whole, so one + // untranslatable entry would cost the rest of the archive. + // Atomic for the same reason as the __hipRegisterVar warning: + // replay is multi-threaded and this flag is shared. + static std::atomic warned{false}; + if (!warned.exchange(true)) { + fprintf(stderr, + "[HRR] hipStreamBatchMemOp: op address 0x%llx is not in " + "any recorded allocation — skipping this batch\n", + (unsigned long long)reinterpret_cast(*addr)); + } + return hipSuccess; + } + *addr = live; + // The alias field is documented as unused on AMD and holds whatever + // the capturing process left there; a stale address in it is a + // pointer the runtime must never see. + if (op.operation == hipStreamMemOpWaitValue32 || + op.operation == hipStreamMemOpWaitValue64) + op.waitValue.alias = nullptr; + else + op.writeValue.alias = nullptr; + } + + return hipStreamBatchMemOp(ctx.translate_stream(a->stream), n, ops.data(), + a->flags); +} + +// --------------------------------------------------------------------------- +// Manual playback: IPC memory handles +// +// The handle is 64 opaque bytes naming an export in the process that made it. +// Replay re-exports the live allocation and pairs the recorded bytes with the +// live ones, so an import later in the same archive has something to open. An +// import whose export is in another process (the cross-rank case) has no such +// pairing and says so rather than opening a handle from a dead process. +// --------------------------------------------------------------------------- +hipError_t playback_hipIpcGetMemHandle(PlaybackContext& ctx, const uint8_t* pl) { + const auto* a = reinterpret_cast(pl); + void* live_ptr = ctx.translate_ptr(a->devPtr); + if (!live_ptr) { + fprintf(stderr, + "[HRR] hipIpcGetMemHandle: recorded 0x%llx is not in any live " + "allocation — skipping the export\n", + (unsigned long long)a->devPtr); + return hipSuccess; + } + hipIpcMemHandle_t live{}; + hipError_t r = hipIpcGetMemHandle(&live, live_ptr); + if (r != hipSuccess) return r; + if (a->handle_present) + ctx.record_ipc_handle( + PlaybackContext::ipc_key(a->handle_bytes, sizeof(live)), + PlaybackContext::ipc_key(&live, sizeof(live))); + return hipSuccess; +} + +hipError_t playback_hipIpcOpenMemHandle(PlaybackContext& ctx, const uint8_t* pl) { + const auto* a = reinterpret_cast(pl); + hipIpcMemHandle_t handle{}; + const std::string live_bytes = ctx.translate_ipc_handle( + PlaybackContext::ipc_key(a->handle_bytes, sizeof(handle))); + if (live_bytes.size() != sizeof(handle)) { + fprintf(stderr, + "[HRR] hipIpcOpenMemHandle: no replayed export matches this " + "handle. It was exported by another process, which a " + "single-archive replay does not reproduce — skipping the " + "import; anything reading through this pointer will differ " + "from the recording.\n"); + return hipSuccess; + } + std::memcpy(&handle, live_bytes.data(), sizeof(handle)); + void* live = nullptr; + hipError_t r = hipIpcOpenMemHandle(&live, handle, a->flags); + if (r == hipSuccess && live) + ctx.record_alloc(a->devPtr, live, 0, AllocKind::DevicePtrAlias); + return r; +} + +// --------------------------------------------------------------------------- +// Manual playback: graph nodes whose parameters are more than one pointer deep +// +// The generated handlers cover every node kind whose parameter struct is flat +// enough for DEREF_FIELDS. These four are not: a kernel node names a host +// function and an argument array, a batch-memory-operation node points at an +// op array, a memory-allocation node hands back an address that only this +// replay's pool can choose, and a driver memcpy node decides between host and +// device operands by a member the archive cannot resolve for the host case. +// --------------------------------------------------------------------------- + +// Resolve a recorded dependency list into live nodes. Returns false when a +// dependency names a node this replay never built — the ordering constraint +// would be silently dropped, which is the class of bug the node map exists to +// prevent, so the caller marks the graph incomplete instead. +static bool translate_node_deps(PlaybackContext& ctx, const char* api, + const uint8_t* bytes, uint32_t n, + uint8_t present, + std::vector& out) { + out.clear(); + if (!present || n == 0) return true; + out.resize(n); + std::memcpy(out.data(), bytes, static_cast(n) * sizeof(hipGraphNode_t)); + for (auto& node : out) { + const uint64_t rec = reinterpret_cast(node); + node = ctx.translate_graph_node(rec); + if (!node && rec) { + fprintf(stderr, + "[HRR] %s: dependency 0x%llx was never built at replay, so " + "this node's ordering constraint cannot be reproduced.\n", + api, (unsigned long long)rec); + return false; + } + } + return true; +} + +// What a hand-written graph-node handler returns when the runtime refused the +// call. Unlike the generated shims, these record the event even when the call +// failed at capture, so meeting the same refusal here is fidelity. Any other +// error leaves the graph a node short, which is what the incompleteness flag +// exists to catch at instantiation. +static hipError_t node_add_failed(PlaybackContext& ctx, const char* api, + int32_t recorded_ret, hipError_t r, + uint64_t graph) { + if (hrr_replayed_recorded_error(ctx, api, recorded_ret, r)) + return hipSuccess; + ctx.mark_graph_incomplete(graph, api); + return r; +} + +// Rebuild hipKernelNodeParams from the recorded struct plus the kernel tail. +// `storage` and `arg_ptrs` own what the returned params point at and must +// outlive the call that consumes them. +static bool rebuild_kernel_node_params( + PlaybackContext& ctx, const char* api, const uint8_t* payload, + size_t fixed_size, const uint8_t* params_bytes, uint8_t params_present, + hipKernelNodeParams& out, + std::vector& arg_ptrs, + std::vector>& arg_storage) { + if (!params_present) { + fprintf(stderr, "[HRR] %s: the node parameters were not recorded.\n", api); + return false; + } + std::memcpy(&out, params_bytes, sizeof(hipKernelNodeParams)); + + const auto* hdr = reinterpret_cast(payload); + const uint8_t* p = payload + fixed_size; + const uint8_t* end = payload + hdr->payload_length; + + if (p + 2 > end) { fprintf(stderr, "[HRR] %s: truncated kernel tail.\n", api); return false; } + uint16_t name_len; memcpy(&name_len, p, 2); p += 2; + if (p + name_len > end) { fprintf(stderr, "[HRR] %s: truncated kernel name.\n", api); return false; } + std::string kernel_name(reinterpret_cast(p), name_len); + p += name_len; + if (name_len == 0) { + fprintf(stderr, + "[HRR] %s: the recording could not name this node's kernel " + "(its host function did not resolve at capture).\n", api); + return false; + } + + if (p + 16 > end) return false; + uint64_t co_lo, co_hi; + memcpy(&co_lo, p, 8); p += 8; + memcpy(&co_hi, p, 8); p += 8; + + if (p + 2 > end) return false; + uint16_t num_args; memcpy(&num_args, p, 2); p += 2; + + hipFunction_t func = resolve_kernel_function(ctx, kernel_name, co_lo, co_hi); + if (!func) return false; + + arg_storage.reserve(num_args); + decode_kernel_args(ctx, p, end, num_args, kernel_name, arg_ptrs, arg_storage); + + // CLR's GraphKernelNode::getFunc first asks the statically-registered + // code-object table what host address this is, and falls back to reading + // the field as a hipFunction_t when that lookup says "not a symbol". + // The recorded host address belongs to the capturing process, so the + // resolved function is what goes in. + out.func = reinterpret_cast(func); + out.kernelParams = arg_ptrs.empty() ? nullptr : arg_ptrs.data(); + out.extra = nullptr; + return true; +} + +hipError_t playback_hipGraphAddKernelNode(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = reinterpret_cast(pl); + hipGraph_t graph = ctx.translate_graph(a->graph); + if (!graph) { + fprintf(stderr, "[HRR] hipGraphAddKernelNode: graph 0x%llx was never " + "built at replay; skipping this node.\n", + (unsigned long long)a->graph); + return hipSuccess; + } + + hipKernelNodeParams knp{}; + std::vector arg_ptrs; + std::vector> arg_storage; + std::vector deps; + if (!rebuild_kernel_node_params(ctx, "hipGraphAddKernelNode", pl, + sizeof(*a), a->pNodeParams_bytes, + a->pNodeParams_present, knp, arg_ptrs, + arg_storage) || + !translate_node_deps(ctx, "hipGraphAddKernelNode", + a->pDependencies_bytes, a->pDependencies_n, + a->pDependencies_present, deps)) { + ctx.mark_graph_incomplete(a->graph, "hipGraphAddKernelNode"); + return hipSuccess; + } + + hipGraphNode_t node = nullptr; + hipError_t r = hipGraphAddKernelNode(&node, graph, + deps.empty() ? nullptr : deps.data(), + deps.size(), &knp); + if (r != hipSuccess) + return node_add_failed(ctx, "hipGraphAddKernelNode", a->ret, r, + a->graph); + ctx.record_graph_node(a->pGraphNode, node); + return hipSuccess; +} + +hipError_t playback_hipGraphKernelNodeSetParams(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraphNode_t node = ctx.translate_graph_node(a->node); + if (!node) return hipSuccess; // the node was never built; already reported + + hipKernelNodeParams knp{}; + std::vector arg_ptrs; + std::vector> arg_storage; + if (!rebuild_kernel_node_params(ctx, "hipGraphKernelNodeSetParams", pl, + sizeof(*a), a->pNodeParams_bytes, + a->pNodeParams_present, knp, arg_ptrs, + arg_storage)) + return hipSuccess; + hipError_t r = hipGraphKernelNodeSetParams(node, &knp); + return hrr_replayed_recorded_error(ctx, "hipGraphKernelNodeSetParams", + a->ret, r) ? hipSuccess : r; +} + +hipError_t playback_hipGraphExecKernelNodeSetParams(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraphExec_t exec = ctx.translate_graph_exec(a->hGraphExec); + hipGraphNode_t node = ctx.translate_graph_node(a->node); + if (!exec || !node) return hipSuccess; + + hipKernelNodeParams knp{}; + std::vector arg_ptrs; + std::vector> arg_storage; + if (!rebuild_kernel_node_params(ctx, "hipGraphExecKernelNodeSetParams", pl, + sizeof(*a), a->pNodeParams_bytes, + a->pNodeParams_present, knp, arg_ptrs, + arg_storage)) + return hipSuccess; + hipError_t r = hipGraphExecKernelNodeSetParams(exec, node, &knp); + return hrr_replayed_recorded_error(ctx, "hipGraphExecKernelNodeSetParams", + a->ret, r) ? hipSuccess : r; +} + +// Rebuild hipBatchMemOpNodeParams: the struct is recorded inline, the op array +// follows as a tail, and each op's address is translated the same way +// hipStreamBatchMemOp's is. +static bool rebuild_batch_memop_params( + PlaybackContext& ctx, const char* api, const uint8_t* payload, + size_t fixed_size, const uint8_t* params_bytes, uint8_t params_present, + hipBatchMemOpNodeParams& out, + std::vector& ops) { + if (!params_present) { + fprintf(stderr, "[HRR] %s: the node parameters were not recorded.\n", api); + return false; + } + std::memcpy(&out, params_bytes, sizeof(hipBatchMemOpNodeParams)); + + const auto* hdr = reinterpret_cast(payload); + const uint8_t* p = payload + fixed_size; + const uint8_t* end = payload + hdr->payload_length; + if (p + 8 > end) return false; + uint32_t n = 0, stride = 0; + memcpy(&n, p, 4); p += 4; + memcpy(&stride, p, 4); p += 4; + if (n && (stride != sizeof(hipStreamBatchMemOpParams) || + static_cast(end - p) < static_cast(n) * stride)) { + fprintf(stderr, + "[HRR] %s: the op array was recorded with a %u-byte entry " + "(this build expects %zu).\n", + api, stride, sizeof(hipStreamBatchMemOpParams)); + return false; + } + + ops.assign(n, hipStreamBatchMemOpParams{}); + if (n) memcpy(ops.data(), p, static_cast(n) * stride); + for (auto& op : ops) { + hipDeviceptr_t* addr = nullptr; + switch (op.operation) { + case hipStreamMemOpWaitValue32: + case hipStreamMemOpWaitValue64: + addr = &op.waitValue.address; break; + case hipStreamMemOpWriteValue32: + case hipStreamMemOpWriteValue64: + addr = &op.writeValue.address; break; + default: + continue; // barrier / flush carry no address + } + void* live = ctx.translate_ptr(reinterpret_cast(*addr)); + if (!live) { + fprintf(stderr, + "[HRR] %s: op address 0x%llx is not in any recorded " + "allocation.\n", api, + (unsigned long long)reinterpret_cast(*addr)); + return false; + } + *addr = live; + // The alias member is unused on AMD and still holds a capture-time + // address; the runtime must never see it. + if (op.operation == hipStreamMemOpWaitValue32 || + op.operation == hipStreamMemOpWaitValue64) + op.waitValue.alias = nullptr; + else + op.writeValue.alias = nullptr; + } + // hipGraphAddBatchMemOpNode rejects a null context (hip_graph.cpp:3736), + // so the recorded capture-time address is swapped for the live one rather + // than cleared. A recording made with a null context keeps it, and fails + // here exactly as it failed there. + out.ctx = hrr_live_ctx(reinterpret_cast(out.ctx)); + out.count = n; + out.paramArray = ops.empty() ? nullptr : ops.data(); + return true; +} + +hipError_t playback_hipGraphAddBatchMemOpNode(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraph_t graph = ctx.translate_graph(a->hGraph); + if (!graph) return hipSuccess; + + hipBatchMemOpNodeParams bnp{}; + std::vector ops; + std::vector deps; + if (!rebuild_batch_memop_params(ctx, "hipGraphAddBatchMemOpNode", pl, + sizeof(*a), a->nodeParams_bytes, + a->nodeParams_present, bnp, ops) || + !translate_node_deps(ctx, "hipGraphAddBatchMemOpNode", + a->dependencies_bytes, a->dependencies_n, + a->dependencies_present, deps)) { + ctx.mark_graph_incomplete(a->hGraph, "hipGraphAddBatchMemOpNode"); + return hipSuccess; + } + + hipGraphNode_t node = nullptr; + hipError_t r = hipGraphAddBatchMemOpNode(&node, graph, + deps.empty() ? nullptr : deps.data(), + deps.size(), &bnp); + if (r != hipSuccess) + return node_add_failed(ctx, "hipGraphAddBatchMemOpNode", a->ret, r, + a->hGraph); + ctx.record_graph_node(a->phGraphNode, node); + return hipSuccess; +} + +hipError_t playback_hipGraphBatchMemOpNodeSetParams(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraphNode_t node = ctx.translate_graph_node(a->hNode); + if (!node) return hipSuccess; + hipBatchMemOpNodeParams bnp{}; + std::vector ops; + if (!rebuild_batch_memop_params(ctx, "hipGraphBatchMemOpNodeSetParams", pl, + sizeof(*a), a->nodeParams_bytes, + a->nodeParams_present, bnp, ops)) + return hipSuccess; + hipError_t r = hipGraphBatchMemOpNodeSetParams(node, &bnp); + return hrr_replayed_recorded_error(ctx, "hipGraphBatchMemOpNodeSetParams", + a->ret, r) ? hipSuccess : r; +} + +hipError_t playback_hipGraphExecBatchMemOpNodeSetParams(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraphExec_t exec = ctx.translate_graph_exec(a->hGraphExec); + hipGraphNode_t node = ctx.translate_graph_node(a->hNode); + if (!exec || !node) return hipSuccess; + hipBatchMemOpNodeParams bnp{}; + std::vector ops; + if (!rebuild_batch_memop_params(ctx, "hipGraphExecBatchMemOpNodeSetParams", + pl, sizeof(*a), a->nodeParams_bytes, + a->nodeParams_present, bnp, ops)) + return hipSuccess; + hipError_t r = hipGraphExecBatchMemOpNodeSetParams(exec, node, &bnp); + return hrr_replayed_recorded_error( + ctx, "hipGraphExecBatchMemOpNodeSetParams", a->ret, r) + ? hipSuccess : r; +} + +hipError_t playback_hipGraphAddMemAllocNode(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraph_t graph = ctx.translate_graph(a->graph); + if (!graph) return hipSuccess; + + std::vector deps; + if (!a->pNodeParams_present || + !translate_node_deps(ctx, "hipGraphAddMemAllocNode", + a->pDependencies_bytes, a->pDependencies_n, + a->pDependencies_present, deps)) { + ctx.mark_graph_incomplete(a->graph, "hipGraphAddMemAllocNode"); + return hipSuccess; + } + + hipMemAllocNodeParams anp{}; + std::memcpy(&anp, a->pNodeParams_bytes, sizeof(anp)); + const uint64_t rec_dptr = reinterpret_cast(anp.dptr); + anp.dptr = nullptr; // written by the call; the recorded value is not ours + if (anp.accessDescs && anp.accessDescCount) { + // The descriptor array is a second pointer hop the archive does not + // carry. Dropping it would quietly give the allocation different peer + // visibility than the recording had. + fprintf(stderr, + "[HRR] hipGraphAddMemAllocNode: the node's %zu access " + "descriptors were not recorded, so its peer visibility cannot " + "be reproduced.\n", anp.accessDescCount); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddMemAllocNode"); + return hipSuccess; + } + anp.accessDescs = nullptr; + anp.accessDescCount = 0; + + hipGraphNode_t node = nullptr; + hipError_t r = hipGraphAddMemAllocNode(&node, graph, + deps.empty() ? nullptr : deps.data(), + deps.size(), &anp); + if (r != hipSuccess) + return node_add_failed(ctx, "hipGraphAddMemAllocNode", a->ret, r, + a->graph); + ctx.record_graph_node(a->pGraphNode, node); + // The pool picks the address, so it is not the recorded one; a later free + // node or memcpy naming the recorded address needs the pairing. + if (rec_dptr && anp.dptr) + ctx.record_alloc(rec_dptr, anp.dptr, + static_cast(anp.bytesize), + AllocKind::DevicePtrAlias); + return hipSuccess; +} + +// Translate a recorded HIP_MEMCPY3D in place. A host operand is a payload the +// archive never carried, so it is reported rather than passed through. +static bool translate_drv_memcpy3d(PlaybackContext& ctx, const char* api, + HIP_MEMCPY3D& c) { + struct Side { const char* what; hipMemoryType type; const void* host; + hipDeviceptr_t* dev; hipArray_t* arr; }; + Side sides[2] = { + {"source", c.srcMemoryType, c.srcHost, &c.srcDevice, &c.srcArray}, + {"destination", c.dstMemoryType, c.dstHost, &c.dstDevice, &c.dstArray}, + }; + for (const Side& s : sides) { + if (s.type == hipMemoryTypeHost || s.host) { + fprintf(stderr, + "[HRR] %s: the %s is host memory, whose contents the " + "archive does not carry.\n", api, s.what); + return false; + } + if (*s.dev) { + void* live = ctx.translate_ptr(reinterpret_cast(*s.dev)); + if (!live) { + fprintf(stderr, + "[HRR] %s: the %s address 0x%llx is not in any recorded " + "allocation.\n", api, s.what, + (unsigned long long)reinterpret_cast(*s.dev)); + return false; + } + *s.dev = live; + } + if (*s.arr) + *s.arr = ctx.translate_array(reinterpret_cast(*s.arr)); + } + return true; +} + +hipError_t playback_hipDrvGraphAddMemcpyNode(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraph_t graph = ctx.translate_graph(a->hGraph); + if (!graph) return hipSuccess; + + HIP_MEMCPY3D copy{}; + std::vector deps; + if (!a->copyParams_present) { + ctx.mark_graph_incomplete(a->hGraph, "hipDrvGraphAddMemcpyNode"); + return hipSuccess; + } + std::memcpy(©, a->copyParams_bytes, sizeof(copy)); + if (!translate_drv_memcpy3d(ctx, "hipDrvGraphAddMemcpyNode", copy) || + !translate_node_deps(ctx, "hipDrvGraphAddMemcpyNode", + a->dependencies_bytes, a->dependencies_n, + a->dependencies_present, deps)) { + ctx.mark_graph_incomplete(a->hGraph, "hipDrvGraphAddMemcpyNode"); + return hipSuccess; + } + + hipGraphNode_t node = nullptr; + hipError_t r = hipDrvGraphAddMemcpyNode(&node, graph, + deps.empty() ? nullptr : deps.data(), + deps.size(), ©, + hrr_live_ctx(a->ctx)); + if (r != hipSuccess) + return node_add_failed(ctx, "hipDrvGraphAddMemcpyNode", a->ret, r, + a->hGraph); + ctx.record_graph_node(a->phGraphNode, node); + return hipSuccess; +} + +hipError_t playback_hipDrvGraphMemcpyNodeSetParams(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraphNode_t node = ctx.translate_graph_node(a->hNode); + if (!node || !a->nodeParams_present) return hipSuccess; + HIP_MEMCPY3D copy{}; + std::memcpy(©, a->nodeParams_bytes, sizeof(copy)); + if (!translate_drv_memcpy3d(ctx, "hipDrvGraphMemcpyNodeSetParams", copy)) + return hipSuccess; + hipError_t r = hipDrvGraphMemcpyNodeSetParams(node, ©); + return hrr_replayed_recorded_error(ctx, "hipDrvGraphMemcpyNodeSetParams", + a->ret, r) ? hipSuccess : r; +} + +hipError_t playback_hipDrvGraphExecMemcpyNodeSetParams(PlaybackContext& ctx, + const uint8_t* pl) { + const auto* a = + reinterpret_cast(pl); + hipGraphExec_t exec = ctx.translate_graph_exec(a->hGraphExec); + hipGraphNode_t node = ctx.translate_graph_node(a->hNode); + if (!exec || !node || !a->copyParams_present) return hipSuccess; + HIP_MEMCPY3D copy{}; + std::memcpy(©, a->copyParams_bytes, sizeof(copy)); + if (!translate_drv_memcpy3d(ctx, "hipDrvGraphExecMemcpyNodeSetParams", copy)) + return hipSuccess; + hipError_t r = hipDrvGraphExecMemcpyNodeSetParams(exec, node, ©, + hrr_live_ctx(a->ctx)); + return hrr_replayed_recorded_error( + ctx, "hipDrvGraphExecMemcpyNodeSetParams", a->ret, r) + ? hipSuccess : r; +} diff --git a/projects/hrr/playback/hip_playback.h b/projects/hrr/playback/hip_playback.h index a2cdf688d81d..a1065429a60e 100644 --- a/projects/hrr/playback/hip_playback.h +++ b/projects/hrr/playback/hip_playback.h @@ -7,6 +7,9 @@ #pragma once #include +#include +#include +#include #include #include #include @@ -16,6 +19,7 @@ #include #include #include +#include #include "hrr/hrr_api_args.h" // for HRR_API_COUNT, hrr_api_id_t #include "hrr_region_map.h" // external region annotations (regions/*.hrrr) @@ -81,16 +85,47 @@ struct PlaybackContext { std::unordered_map array_map; std::unordered_map mipmapped_map; std::unordered_map graph_map; + std::unordered_map graph_node_map; std::unordered_map graph_exec_map; std::unordered_map surface_map; std::unordered_map texture_map; + // Driver-API contexts. Without this map the family replayed asymmetrically + // — hipCtxCreate and hipCtxPopCurrent no-ops, hipCtxPushCurrent real — so + // replay pushed a capture-time hipCtx_t and the thread was left with no + // current device, which killed the process several events later. + std::unordered_map ctx_map; + // JIT linker states. hipLinkCreate used to throw its output away, so + // hipLinkComplete and hipLinkDestroy were handed the capturing process's + // pointer and failed no matter what the linker inputs were. + std::unordered_map link_state_map; // Device allocations: recorded base address -> {live ptr, size} std::unordered_map alloc_map; + // __device__ globals, keyed by the recorded host shadow address the + // capturing process passed to hipGetSymbolAddress and the hipMemcpy*Symbol + // family. A replay has no host shadow of its own — the code object is + // loaded through hipModuleLoadData — so the name recorded beside the + // address is what resolves it. + struct SymbolEntry { + std::string name; + void* live_ptr = nullptr; + size_t size = 0; + }; + std::unordered_map symbol_map; + // Code-object modules loaded by hash (not by recorded module handle) std::unordered_map co_modules; + // Fat binaries that carry no device code for the replay GPU, by blob hash. + // A process registers every fat binary its shared libraries contain, not + // just the ones it can run: a CDNA capture of a CK-linked app still + // registers CK's RDNA-only WMMA instances. The capturing runtime never + // notices, because registration does not extract device code — only an + // eager load does. Remembering them keeps the load from being retried, and + // re-reported, on every repeat registration of the same binary. + std::unordered_set co_no_device_code; + // Kernel function cache: mangled name -> resolved hipFunction_t. // Populated on first launch of each kernel; avoids repeated hipModuleGetFunction // searches across module_map / co_modules on every launch. @@ -118,6 +153,21 @@ struct PlaybackContext { // value for the kernel launch with this 1-based ordinal. Used to diff the // captured vs replay pointer contract (e.g. a StreamK synchronizer base). size_t dump_ptrs_ordinal = 0; + // When non-zero, read back each pointer argument of the launch with this + // 1-based ordinal and report any 8-byte word that is a recorded address. + // A capture-time pointer in a buffer is one nothing translated, which is + // how a replay ends up faulting on an address from the other process. + size_t scan_args_ordinal = 0; + size_t scan_args_bytes = 4096; // per argument + // Report recorded addresses found inside replayed H2D payloads. Those + // bytes are restored verbatim, so a pointer among them reaches the GPU + // untranslated no matter how well kernel arguments are handled. + bool scan_h2d = false; + // Report every kernel that takes a pointer into host memory. Replay + // reallocates those buffers but cannot refill them: the application writes + // them with ordinary CPU stores, which no HIP call reports. A kernel + // reading one is reading data the archive never captured. + bool audit_host_args = false; bool verbose = false; bool validate_d2h = false; // perform D2H validation against captured expected data std::string kernel_filter; @@ -205,6 +255,16 @@ struct PlaybackContext { // Guarded by map_mutex. std::unordered_map host_reg_bufs; + // APIs this replay refused to reproduce (UNREPLAYABLE_PLAYBACK_APIS), and + // APIs whose recorded call had already failed at capture and failed the + // same way here. Each API is logged to stderr once, on first insertion + // (hrr_note_unreplayable / hrr_note_recorded_error); the maps de-duplicate + // those lines and are not printed in the summary. The first is what the + // replay could not do, the second is what it faithfully reproduced but a + // reader would otherwise misread as a replay error. Guarded by map_mutex. + std::map unreplayable_apis; // api -> reason + std::map reproduced_errors; // api -> recorded hipError_t + // VMM replay maps (guarded by map_mutex): // vmm_handle_map: recorded hipMemGenericAllocationHandle_t (as u64) -> live handle // vmm_va_map : recorded reserved-VA base (u64) -> {live_va, size} @@ -277,6 +337,58 @@ struct PlaybackContext { }; std::unordered_map guard_allocs; + // Graphs this replay could not build faithfully: a node the recording + // added that no handler here can reconstruct. Before the node API was + // replayed at all, a graph's mere absence from graph_map was the signal + // hipGraphInstantiate refused on. Now that most construction calls do + // build something, absence no longer distinguishes "built by the node API" + // from "built correctly", so the ones that fell short say so explicitly + // and instantiate still refuses. Keyed by the recorded hipGraph_t. + std::set incomplete_graphs; + + void mark_graph_incomplete(uint64_t rec_graph, const char* api) { + if (!rec_graph) return; + bool first; + { + std::unique_lock lk(map_mutex); + first = incomplete_graphs.insert(rec_graph).second; + } + if (first) + fprintf(stderr, + "[HRR] %s: this node was not reconstructed, so graph " + "0x%llx is incomplete; instantiating it will be refused " + "rather than replayed as a graph missing work.\n", + api, static_cast(rec_graph)); + } + bool graph_is_incomplete(uint64_t rec_graph) const { + std::shared_lock lk(map_mutex); + return incomplete_graphs.count(rec_graph) != 0; + } + + // IPC handles, recorded 64 bytes -> the 64 bytes this replay's exporter + // produced. An importer's handle names an export in the process that made + // it, so replaying hipIpcOpenMemHandle with the recorded bytes opens + // nothing; it has to be matched to the handle the replayed exporter + // returned. Keyed by the recorded bytes because that is all the importing + // event carries. Guarded by map_mutex. + std::map ipc_handle_map; + + static std::string ipc_key(const void* bytes, size_t n) { + return std::string(static_cast(bytes), n); + } + void record_ipc_handle(const std::string& recorded, const std::string& live) { + std::unique_lock lk(map_mutex); + ipc_handle_map[recorded] = live; + } + // Empty when the archive holds no export for this handle: the importer is + // then in another process, which is out of scope for a single-archive + // replay. + std::string translate_ipc_handle(const std::string& recorded) const { + std::shared_lock lk(map_mutex); + auto it = ipc_handle_map.find(recorded); + return it != ipc_handle_map.end() ? it->second : std::string(); + } + // ---- Pointer translation ---- // Translates a recorded GPU address to a live pointer. // Checks alloc_map (exact + range) then vmm_va_map (exact + range). @@ -319,6 +431,127 @@ struct PlaybackContext { return nullptr; } + // Is this value an address from the capturing process — i.e. does it fall + // inside an allocation as *recorded*, before translation? Reports the + // enclosing recorded base so a hit can be traced back to the event that + // allocated it. Answers the question a live pointer cannot: whether a word + // sitting in device memory is a capture-time pointer nothing rewrote. + bool is_recorded_va(uint64_t v, uint64_t* base_out = nullptr, + size_t* size_out = nullptr) const { + if (v < 0x10000ULL) return false; + std::shared_lock lk(map_mutex); + for (auto& [base, entry] : alloc_map) { + if (v >= base && v < base + entry.size) { + if (base_out) *base_out = base; + if (size_out) *size_out = entry.size; + return true; + } + } + for (auto& [base, va] : vmm_va_map) { + if (v >= base && v < base + va.size) { + if (base_out) *base_out = base; + if (size_out) *size_out = va.size; + return true; + } + } + return false; + } + + // Recorded allocation ranges, sorted by base, for scans that test many + // words against them. Testing one word walks every allocation; taking a + // snapshot once and binary searching turns a scan of a multi-gigabyte + // payload from hours into seconds. + std::vector> recorded_ranges() const { + std::vector> r; + { + std::shared_lock lk(map_mutex); + r.reserve(alloc_map.size() + vmm_va_map.size()); + for (auto& [base, e] : alloc_map) r.emplace_back(base, base + e.size); + for (auto& [base, va] : vmm_va_map) r.emplace_back(base, base + va.size); + } + std::sort(r.begin(), r.end()); + return r; + } + + static bool range_contains( + const std::vector>& ranges, uint64_t v, + uint64_t* base_out = nullptr, size_t* size_out = nullptr) { + if (ranges.empty() || v < ranges.front().first || v >= ranges.back().second) + return false; + auto it = std::upper_bound( + ranges.begin(), ranges.end(), v, + [](uint64_t x, const std::pair& p) { + return x < p.first; + }); + if (it == ranges.begin()) return false; + --it; + if (v >= it->second) return false; + if (base_out) *base_out = it->first; + if (size_out) *size_out = static_cast(it->second - it->first); + return true; + } + + // The live allocation containing a live pointer. A kernel argument usually + // points into the middle of a buffer, and the interesting bytes — the ones + // whose interpretation went wrong — can lie before it as easily as after. + bool live_alloc_of(const void* p, void** base_out, size_t* size_out, + uint64_t* rec_base_out, + AllocKind* kind_out = nullptr) const { + uint64_t v = reinterpret_cast(p); + std::shared_lock lk(map_mutex); + for (auto& [base, e] : alloc_map) { + uint64_t lb = reinterpret_cast(e.live_ptr); + if (!lb || v < lb || v >= lb + e.size) continue; + if (base_out) *base_out = e.live_ptr; + if (size_out) *size_out = e.size; + if (rec_base_out) *rec_base_out = base; + if (kind_out) *kind_out = e.kind; + return true; + } + return false; + } + + static const char* alloc_kind_name(AllocKind k) { + switch (k) { + case AllocKind::HostMalloc: return "pinned host"; + case AllocKind::HostRegister: return "registered host"; + case AllocKind::DevicePtrAlias: return "pinned host alias"; + default: return "device"; + } + } + + // Describe an address from both sides of the replay: as a capture-time + // address, and as one of this process's own live allocations. A fault + // address that is only the former is a recorded pointer that reached the + // GPU; one that is the latter is this replay's own memory, and the + // divergence that produced it is upstream of any pointer translation. + std::string explain_addr(uint64_t v) const { + char buf[320]; + std::string out; + uint64_t rbase = 0; size_t rsize = 0; + if (is_recorded_va(v, &rbase, &rsize)) { + void* live = translate_ptr(rbase); + snprintf(buf, sizeof(buf), + "recorded allocation 0x%llx+%zu (live base %p)", + (unsigned long long)rbase, rsize, live); + out += buf; + } + { + std::shared_lock lk(map_mutex); + for (auto& [base, entry] : alloc_map) { + uint64_t lb = reinterpret_cast(entry.live_ptr); + if (!lb || v < lb || v >= lb + entry.size) continue; + snprintf(buf, sizeof(buf), + "%slive allocation %p+%zu (recorded base 0x%llx)", + out.empty() ? "" : "; ", entry.live_ptr, entry.size, + (unsigned long long)base); + out += buf; + break; + } + } + return out.empty() ? std::string("in no allocation this replay knows") : out; + } + // Returns bytes available from a live GPU pointer to end of its backing // alloc_map entry. Returns 0 if the pointer is not within any known alloc. size_t alloc_bytes_from(void* live_ptr) const { @@ -374,6 +607,12 @@ struct PlaybackContext { std::shared_lock lk(map_mutex); auto it = graph_map.find(rec); return it != graph_map.end() ? it->second : nullptr; } + hipGraphNode_t translate_graph_node(uint64_t rec) const { + if (rec == 0) return nullptr; + std::shared_lock lk(map_mutex); + auto it = graph_node_map.find(rec); + return it != graph_node_map.end() ? it->second : nullptr; + } hipGraphExec_t translate_graph_exec(uint64_t rec) const { if (rec == 0) return nullptr; std::shared_lock lk(map_mutex); @@ -387,6 +626,17 @@ struct PlaybackContext { std::shared_lock lk(map_mutex); auto it = texture_map.find(rec); return it != texture_map.end() ? it->second : 0; } + hipCtx_t translate_ctx(uint64_t rec) const { + if (rec == 0) return nullptr; + std::shared_lock lk(map_mutex); + auto it = ctx_map.find(rec); return it != ctx_map.end() ? it->second : nullptr; + } + hipLinkState_t translate_link_state(uint64_t rec) const { + if (rec == 0) return nullptr; + std::shared_lock lk(map_mutex); + auto it = link_state_map.find(rec); + return it != link_state_map.end() ? it->second : nullptr; + } // ---- Allocation registration (exclusive lock) ---- void record_alloc(uint64_t rec, void* live, size_t sz, @@ -404,6 +654,43 @@ struct PlaybackContext { return alloc_map.find(rec) != alloc_map.end(); } + // A graph node's host operand has to stay alive until the graph is + // launched, which is long after the call that built the node returned. + // Keyed by the recorded host address so the same recorded destination + // reuses one buffer instead of growing a new one per event. + std::unordered_map> host_landing_buffers; + + // A recorded key of 0 is a null host pointer the capturing process passed, + // and the call it belonged to failed there. Handing back a real buffer + // would turn that into a success at replay, so the sentinel is preserved + // and the caller's null check reports it. + void* host_landing_buffer(uint64_t rec, size_t sz) { + if (!rec) return nullptr; + std::unique_lock lk(map_mutex); + auto& buf = host_landing_buffers[rec]; + if (buf.size() < sz) buf.resize(sz); + return buf.data(); + } + + // ---- Symbol registration and lookup ---- + void record_symbol(uint64_t rec_host_var, const std::string& name, + void* live, size_t sz) { + std::unique_lock lk(map_mutex); + symbol_map[rec_host_var] = {name, live, sz}; + } + // Device address of the global the capturing process knew by this host + // shadow address, or nullptr if the archive never registered it. + void* translate_symbol(uint64_t rec_host_var, size_t* sz_out = nullptr) const { + std::shared_lock lk(map_mutex); + auto it = symbol_map.find(rec_host_var); + if (it == symbol_map.end()) return nullptr; + if (sz_out) *sz_out = it->second.size; + return it->second.live_ptr; + } + // Resolve a symbol by name across every module the replay has loaded, + // whether it came from a fat binary or an explicit hipModuleLoad. + void* resolve_symbol_by_name(const char* name, size_t* sz_out) const; + // ---- Handle registration (exclusive lock) ---- void record_stream (uint64_t rec, hipStream_t live) { std::unique_lock lk(map_mutex); stream_map[rec] = live; } void record_event (uint64_t rec, hipEvent_t live) { std::unique_lock lk(map_mutex); event_map[rec] = live; } @@ -413,9 +700,12 @@ struct PlaybackContext { void record_array (uint64_t rec, hipArray_t live) { std::unique_lock lk(map_mutex); array_map[rec] = live; } void record_mipmapped(uint64_t rec, hipMipmappedArray_t live) { std::unique_lock lk(map_mutex); mipmapped_map[rec] = live; } void record_graph (uint64_t rec, hipGraph_t live) { std::unique_lock lk(map_mutex); graph_map[rec] = live; } + void record_graph_node(uint64_t rec, hipGraphNode_t live){ std::unique_lock lk(map_mutex); graph_node_map[rec] = live; } void record_graph_exec(uint64_t rec, hipGraphExec_t live){ std::unique_lock lk(map_mutex); graph_exec_map[rec] = live; } void record_surface (uint64_t rec, hipSurfaceObject_t live) { std::unique_lock lk(map_mutex); surface_map[rec] = live; } void record_texture (uint64_t rec, hipTextureObject_t live) { std::unique_lock lk(map_mutex); texture_map[rec] = live; } + void record_ctx (uint64_t rec, hipCtx_t live) { std::unique_lock lk(map_mutex); ctx_map[rec] = live; } + void record_link_state(uint64_t rec, hipLinkState_t live){ std::unique_lock lk(map_mutex); link_state_map[rec] = live; } // ---- Handle removal (exclusive lock) ---- void remove_stream (uint64_t rec) { std::unique_lock lk(map_mutex); stream_map.erase(rec); } @@ -426,9 +716,12 @@ struct PlaybackContext { void remove_array (uint64_t rec) { std::unique_lock lk(map_mutex); array_map.erase(rec); } void remove_mipmapped(uint64_t rec){ std::unique_lock lk(map_mutex); mipmapped_map.erase(rec); } void remove_graph (uint64_t rec) { std::unique_lock lk(map_mutex); graph_map.erase(rec); } + void remove_graph_node(uint64_t rec){ std::unique_lock lk(map_mutex); graph_node_map.erase(rec); } void remove_graph_exec(uint64_t rec){ std::unique_lock lk(map_mutex); graph_exec_map.erase(rec); } void remove_surface (uint64_t rec) { std::unique_lock lk(map_mutex); surface_map.erase(rec); } void remove_texture (uint64_t rec) { std::unique_lock lk(map_mutex); texture_map.erase(rec); } + void remove_ctx (uint64_t rec) { std::unique_lock lk(map_mutex); ctx_map.erase(rec); } + void remove_link_state(uint64_t rec){ std::unique_lock lk(map_mutex); link_state_map.erase(rec); } // ---- Blob/code-object loading ---- // Load a blob from archive_dir/blobs/<2-char-prefix>/.blob @@ -474,6 +767,56 @@ void hrr_release_region(PlaybackContext& ctx, uint64_t rec_base, void* live); // and hipFree cannot release it, so every teardown path has to go through here. void hrr_free_device_alloc(PlaybackContext& ctx, void* live); +// --------------------------------------------------------------------------- +// hrr_note_unreplayable — this API cannot be reproduced, and here is why. +// +// Called by the UNREPLAYABLE_PLAYBACK_APIS handlers before they return +// hipErrorNotSupported. Prints once per API: +// +// [HRR] hipStreamAddCallback: NOT REPLAYABLE — the callback is a host +// function pointer belonging to the capturing process. ... +// +// and records it so the replay summary can list every such API rather than +// leaving the reader to grep a long log for warnings. +void hrr_note_unreplayable(PlaybackContext& ctx, const char* api, + const char* reason); + +// --------------------------------------------------------------------------- +// hrr_note_recorded_error — replay reproduced a call that also failed at +// capture, which is fidelity rather than failure. +// +// Most capture shims only record calls that succeeded, so a handler returning +// an error normally means replay diverged. The exceptions are the hand-written +// shims that record the call before the runtime rejects it (hipArrayCreate on a +// part with no image support, say). Returning that same error to dispatch_event +// would abort the replay over a call the recording shows failing identically. +void hrr_note_recorded_error(PlaybackContext& ctx, const char* api, + int recorded_ret); + +// --------------------------------------------------------------------------- +// hrr_replayed_recorded_error — did this call fail exactly as the recording +// says it failed? +// +// The generated handlers get this from the generator; the hand-written ones +// call it. Both are needed because the hand-written capture shims are the ones +// that record a call the runtime rejected, so their handlers are the ones most +// likely to meet a recorded error. +bool hrr_replayed_recorded_error(PlaybackContext& ctx, const char* api, + int32_t recorded_ret, hipError_t replayed); + +// --------------------------------------------------------------------------- +// hrr_live_ctx — a usable hipCtx_t for a recorded one. +// +// A recorded hipCtx_t names a context object in the capturing process and +// there is no map from it to anything here. Passing it through would hand the +// runtime a stale address; passing nullptr fails outright, because the +// driver-API graph node calls reject a null context. Replay therefore +// substitutes the context the calling thread is already on, which on AMD is +// the primary context of the current device — the same device the archive was +// recorded against. A recording that used a null context keeps one, so a call +// that capture shows failing that way fails the same way here. +hipCtx_t hrr_live_ctx(uint64_t recorded); + // Thread-local sequence ID — set by dispatch_event before calling any handler. // Kernel-launch handlers read this to wait for their submission turn at the // exact point of the HIP call, allowing preparation work to run in parallel. diff --git a/projects/hrr/playback/hip_playback_generated.cpp b/projects/hrr/playback/hip_playback_generated.cpp index 0c87b6798928..ca58638644fa 100644 --- a/projects/hrr/playback/hip_playback_generated.cpp +++ b/projects/hrr/playback/hip_playback_generated.cpp @@ -92,6 +92,10 @@ static hipError_t playback_hipBindTexture(PlaybackContext& ctx, const uint8_t* p size_t _out_offset{}; const hipChannelFormatDesc* _s_desc{}; hipError_t _r = (hipError_t)hipBindTexture(&_out_offset, (const textureReference*)a->tex, ctx.translate_ptr(a->devPtr), _s_desc, (size_t)a->size); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipBindTexture", a->ret); + return hipSuccess; + } return _r; } @@ -100,6 +104,10 @@ static hipError_t playback_hipBindTexture2D(PlaybackContext& ctx, const uint8_t* size_t _out_offset{}; const hipChannelFormatDesc* _s_desc{}; hipError_t _r = (hipError_t)hipBindTexture2D(&_out_offset, (const textureReference*)a->tex, ctx.translate_ptr(a->devPtr), _s_desc, (size_t)a->width, (size_t)a->height, (size_t)a->pitch); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipBindTexture2D", a->ret); + return hipSuccess; + } return _r; } @@ -107,6 +115,10 @@ static hipError_t playback_hipBindTextureToArray(PlaybackContext& ctx, const uin const auto* a = reinterpret_cast(payload); const hipChannelFormatDesc* _s_desc{}; hipError_t _r = (hipError_t)hipBindTextureToArray((const textureReference*)a->tex, (hipArray_t)ctx.translate_array(a->array), _s_desc); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipBindTextureToArray", a->ret); + return hipSuccess; + } return _r; } @@ -114,29 +126,39 @@ static hipError_t playback_hipBindTextureToMipmappedArray(PlaybackContext& ctx, const auto* a = reinterpret_cast(payload); const hipChannelFormatDesc* _s_desc{}; hipError_t _r = (hipError_t)hipBindTextureToMipmappedArray((const textureReference*)a->tex, (hipMipmappedArray_t)ctx.translate_mipmapped(a->mipmappedArray), _s_desc); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipBindTextureToMipmappedArray", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipChooseDevice(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipChooseDevice — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + int _out_device{}; + hipDeviceProp_t _d_prop{}; + if (a->prop_present) + std::memcpy(&_d_prop, a->prop_bytes, sizeof(_d_prop)); + hipError_t _r = (hipError_t)hipChooseDevice(&_out_device, (a->prop_present ? (const hipDeviceProp_tR0600*)&_d_prop : (const hipDeviceProp_tR0600*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipChooseDevice", a->ret); + return hipSuccess; } - return hipSuccess; + return _r; } static hipError_t playback_hipChooseDeviceR0000(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipChooseDeviceR0000 — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + int _out_device{}; + hipDeviceProp_tR0000 _d_properties{}; + if (a->properties_present) + std::memcpy(&_d_properties, a->properties_bytes, sizeof(_d_properties)); + hipError_t _r = (hipError_t)hipChooseDeviceR0000(&_out_device, (a->properties_present ? (const hipDeviceProp_tR0000*)&_d_properties : (const hipDeviceProp_tR0000*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipChooseDeviceR0000", a->ret); + return hipSuccess; } - return hipSuccess; + return _r; } static hipError_t playback_hipConfigureCall(PlaybackContext& ctx, const uint8_t* payload) { @@ -144,6 +166,10 @@ static hipError_t playback_hipConfigureCall(PlaybackContext& ctx, const uint8_t* dim3 _dim_gridDim(a->gridDim_x, a->gridDim_y, a->gridDim_z); dim3 _dim_blockDim(a->blockDim_x, a->blockDim_y, a->blockDim_z); hipError_t _r = (hipError_t)hipConfigureCall(_dim_gridDim, _dim_blockDim, (size_t)a->sharedMem, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipConfigureCall", a->ret); + return hipSuccess; + } return _r; } @@ -151,6 +177,10 @@ static hipError_t playback_hipCreateSurfaceObject(PlaybackContext& ctx, const ui const auto* a = reinterpret_cast(payload); hipSurfaceObject_t _out_pSurfObject = 0; hipError_t _r = (hipError_t)hipCreateSurfaceObject(&_out_pSurfObject, (const hipResourceDesc*)a->pResDesc); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCreateSurfaceObject", a->ret); + return hipSuccess; + } if (_r == hipSuccess) { ctx.record_surface(a->pSurfObject, _out_pSurfObject); } @@ -169,19 +199,30 @@ static hipError_t playback_hipCreateTextureObject(PlaybackContext& ctx, const ui } static hipError_t playback_hipCtxCreate(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipCtxCreate — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + hipCtx_t _out_ctx = nullptr; + hipError_t _r = (hipError_t)hipCtxCreate(&_out_ctx, (unsigned int)a->flags, (hipDevice_t)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCtxCreate", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_ctx(a->ctx, _out_ctx); + } + return _r; } static hipError_t playback_hipCtxDestroy(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipCtxDestroy((hipCtx_t)a->ctx); + uint64_t _rec_hdl = a->ctx; + hipError_t _r = (hipError_t)hipCtxDestroy((hipCtx_t)ctx.translate_ctx(_rec_hdl)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCtxDestroy", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.remove_ctx(_rec_hdl); + } return _r; } @@ -230,14 +271,17 @@ static hipError_t playback_hipCtxGetCacheConfig(PlaybackContext& ctx, const uint } static hipError_t playback_hipCtxGetCurrent(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipCtxGetCurrent — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + hipCtx_t _out_ctx = nullptr; + hipError_t _r = (hipError_t)hipCtxGetCurrent(&_out_ctx); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCtxGetCurrent", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_ctx(a->ctx, _out_ctx); + } + return _r; } static hipError_t playback_hipCtxGetDevice(PlaybackContext& ctx, const uint8_t* payload) { @@ -274,25 +318,36 @@ static hipError_t playback_hipCtxGetSharedMemConfig(PlaybackContext& ctx, const } static hipError_t playback_hipCtxPopCurrent(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipCtxPopCurrent — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + hipCtx_t _out_ctx = nullptr; + hipError_t _r = (hipError_t)hipCtxPopCurrent(&_out_ctx); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCtxPopCurrent", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_ctx(a->ctx, _out_ctx); + } + return _r; } static hipError_t playback_hipCtxPushCurrent(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipCtxPushCurrent((hipCtx_t)a->ctx); + hipError_t _r = (hipError_t)hipCtxPushCurrent((hipCtx_t)ctx.translate_ctx(a->ctx)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCtxPushCurrent", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipCtxSetCacheConfig(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipCtxSetCacheConfig((hipFuncCache_t)a->cacheConfig); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCtxSetCacheConfig", a->ret); + return hipSuccess; + } return _r; } @@ -310,24 +365,40 @@ static hipError_t playback_hipCtxSetCurrent(PlaybackContext& ctx, const uint8_t* static hipError_t playback_hipCtxSetSharedMemConfig(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipCtxSetSharedMemConfig((hipSharedMemConfig)a->config); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCtxSetSharedMemConfig", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipCtxSynchronize(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipCtxSynchronize(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipCtxSynchronize", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipDestroyExternalMemory(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDestroyExternalMemory((hipExternalMemory_t)a->extMem); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDestroyExternalMemory", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipDestroyExternalSemaphore(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDestroyExternalSemaphore((hipExternalSemaphore_t)a->extSem); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDestroyExternalSemaphore", a->ret); + return hipSuccess; + } return _r; } @@ -335,6 +406,10 @@ static hipError_t playback_hipDestroySurfaceObject(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); uint64_t _rec_hdl = a->surfaceObject; hipError_t _r = (hipError_t)hipDestroySurfaceObject((hipSurfaceObject_t)ctx.translate_surface(_rec_hdl)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDestroySurfaceObject", a->ret); + return hipSuccess; + } if (_r == hipSuccess) { ctx.remove_surface(_rec_hdl); } @@ -356,6 +431,10 @@ static hipError_t playback_hipDeviceCanAccessPeer(PlaybackContext& ctx, const ui const auto* a = reinterpret_cast(payload); int _out_canAccessPeer{}; hipError_t _r = (hipError_t)hipDeviceCanAccessPeer(&_out_canAccessPeer, (int)a->deviceId, (int)a->peerDeviceId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceCanAccessPeer", a->ret); + return hipSuccess; + } return _r; } @@ -364,18 +443,30 @@ static hipError_t playback_hipDeviceComputeCapability(PlaybackContext& ctx, cons int _out_major{}; int _out_minor{}; hipError_t _r = (hipError_t)hipDeviceComputeCapability(&_out_major, &_out_minor, (hipDevice_t)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceComputeCapability", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipDeviceDisablePeerAccess(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceDisablePeerAccess((int)a->peerDeviceId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceDisablePeerAccess", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipDeviceEnablePeerAccess(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceEnablePeerAccess((int)a->peerDeviceId, (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceEnablePeerAccess", a->ret); + return hipSuccess; + } return _r; } @@ -394,6 +485,10 @@ static hipError_t playback_hipDeviceGetAttribute(PlaybackContext& ctx, const uin const auto* a = reinterpret_cast(payload); int _out_pi{}; hipError_t _r = (hipError_t)hipDeviceGetAttribute(&_out_pi, (hipDeviceAttribute_t)a->attr, (int)a->deviceId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetAttribute", a->ret); + return hipSuccess; + } return _r; } @@ -412,6 +507,10 @@ static hipError_t playback_hipDeviceGetCacheConfig(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); hipFuncCache_t _out_cacheConfig{}; hipError_t _r = (hipError_t)hipDeviceGetCacheConfig(&_out_cacheConfig); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetCacheConfig", a->ret); + return hipSuccess; + } return _r; } @@ -419,6 +518,13 @@ static hipError_t playback_hipDeviceGetDefaultMemPool(PlaybackContext& ctx, cons const auto* a = reinterpret_cast(payload); hipMemPool_t _out_mem_pool = nullptr; hipError_t _r = (hipError_t)hipDeviceGetDefaultMemPool(&_out_mem_pool, (int)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetDefaultMemPool", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_mempool(a->mem_pool, _out_mem_pool); + } return _r; } @@ -437,6 +543,10 @@ static hipError_t playback_hipDeviceGetLimit(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); size_t _out_pValue{}; hipError_t _r = (hipError_t)hipDeviceGetLimit(&_out_pValue, (enum hipLimit_t)a->limit); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetLimit", a->ret); + return hipSuccess; + } return _r; } @@ -444,6 +554,13 @@ static hipError_t playback_hipDeviceGetMemPool(PlaybackContext& ctx, const uint8 const auto* a = reinterpret_cast(payload); hipMemPool_t _out_mem_pool = nullptr; hipError_t _r = (hipError_t)hipDeviceGetMemPool(&_out_mem_pool, (int)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetMemPool", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_mempool(a->mem_pool, _out_mem_pool); + } return _r; } @@ -462,6 +579,10 @@ static hipError_t playback_hipDeviceGetP2PAttribute(PlaybackContext& ctx, const const auto* a = reinterpret_cast(payload); int _out_value{}; hipError_t _r = (hipError_t)hipDeviceGetP2PAttribute(&_out_value, (hipDeviceP2PAttr)a->attr, (int)a->srcDevice, (int)a->dstDevice); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetP2PAttribute", a->ret); + return hipSuccess; + } return _r; } @@ -480,6 +601,10 @@ static hipError_t playback_hipDeviceGetSharedMemConfig(PlaybackContext& ctx, con const auto* a = reinterpret_cast(payload); hipSharedMemConfig _out_pConfig{}; hipError_t _r = (hipError_t)hipDeviceGetSharedMemConfig(&_out_pConfig); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetSharedMemConfig", a->ret); + return hipSuccess; + } return _r; } @@ -488,6 +613,10 @@ static hipError_t playback_hipDeviceGetStreamPriorityRange(PlaybackContext& ctx, int _out_leastPriority{}; int _out_greatestPriority{}; hipError_t _r = (hipError_t)hipDeviceGetStreamPriorityRange(&_out_leastPriority, &_out_greatestPriority); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetStreamPriorityRange", a->ret); + return hipSuccess; + } return _r; } @@ -505,6 +634,10 @@ static hipError_t playback_hipDeviceGetUuid(PlaybackContext& ctx, const uint8_t* static hipError_t playback_hipDeviceGraphMemTrim(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceGraphMemTrim((int)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGraphMemTrim", a->ret); + return hipSuccess; + } return _r; } @@ -542,14 +675,17 @@ static hipError_t playback_hipDevicePrimaryCtxReset(PlaybackContext& ctx, const } static hipError_t playback_hipDevicePrimaryCtxRetain(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipDevicePrimaryCtxRetain — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + hipCtx_t _out_pctx = nullptr; + hipError_t _r = (hipError_t)hipDevicePrimaryCtxRetain(&_out_pctx, (hipDevice_t)a->dev); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDevicePrimaryCtxRetain", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_ctx(a->pctx, _out_pctx); + } + return _r; } static hipError_t playback_hipDevicePrimaryCtxSetFlags(PlaybackContext& ctx, const uint8_t* payload) { @@ -566,12 +702,20 @@ static hipError_t playback_hipDevicePrimaryCtxSetFlags(PlaybackContext& ctx, con static hipError_t playback_hipDeviceReset(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceReset(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceReset", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipDeviceSetCacheConfig(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceSetCacheConfig((hipFuncCache_t)a->cacheConfig); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceSetCacheConfig", a->ret); + return hipSuccess; + } return _r; } @@ -589,24 +733,40 @@ static hipError_t playback_hipDeviceSetGraphMemAttribute(PlaybackContext& ctx, c static hipError_t playback_hipDeviceSetLimit(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceSetLimit((enum hipLimit_t)a->limit, (size_t)a->value); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceSetLimit", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipDeviceSetMemPool(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceSetMemPool((int)a->device, (hipMemPool_t)ctx.translate_mempool(a->mem_pool)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceSetMemPool", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipDeviceSetSharedMemConfig(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceSetSharedMemConfig((hipSharedMemConfig)a->config); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceSetSharedMemConfig", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipDeviceSynchronize(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipDeviceSynchronize(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceSynchronize", a->ret); + return hipSuccess; + } return _r; } @@ -614,6 +774,10 @@ static hipError_t playback_hipDeviceTotalMem(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); size_t _out_bytes{}; hipError_t _r = (hipError_t)hipDeviceTotalMem(&_out_bytes, (hipDevice_t)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceTotalMem", a->ret); + return hipSuccess; + } return _r; } @@ -621,6 +785,10 @@ static hipError_t playback_hipDriverGetVersion(PlaybackContext& ctx, const uint8 const auto* a = reinterpret_cast(payload); int _out_driverVersion{}; hipError_t _r = (hipError_t)hipDriverGetVersion(&_out_driverVersion); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDriverGetVersion", a->ret); + return hipSuccess; + } return _r; } @@ -646,19 +814,7 @@ static hipError_t playback_hipDrvGetErrorString(PlaybackContext& ctx, const uint return hipSuccess; } -static hipError_t playback_hipDrvGraphAddMemcpyNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipDrvGraphAddMemcpyNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipDrvGraphAddMemcpyNode(PlaybackContext& ctx, const uint8_t* payload); extern hipError_t playback_hipDrvMemcpy2DUnaligned(PlaybackContext& ctx, const uint8_t* payload); @@ -687,6 +843,10 @@ static hipError_t playback_hipEventElapsedTime(PlaybackContext& ctx, const uint8 const auto* a = reinterpret_cast(payload); float _out_ms{}; hipError_t _r = (hipError_t)hipEventElapsedTime(&_out_ms, (hipEvent_t)ctx.translate_event(a->start), (hipEvent_t)ctx.translate_event(a->stop)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipEventElapsedTime", a->ret); + return hipSuccess; + } return _r; } @@ -695,12 +855,20 @@ extern hipError_t playback_hipEventQuery(PlaybackContext& ctx, const uint8_t* pa static hipError_t playback_hipEventRecord(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipEventRecord((hipEvent_t)ctx.translate_event(a->event), (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipEventRecord", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipEventSynchronize(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipEventSynchronize((hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipEventSynchronize", a->ret); + return hipSuccess; + } return _r; } @@ -727,10 +895,10 @@ static hipError_t playback_hipExtLaunchKernel(PlaybackContext& ctx, const uint8_ } static hipError_t playback_hipExtLaunchMultiKernelMultiDevice(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipLaunchParams _out_launchParamsList{}; - hipError_t _r = (hipError_t)hipExtLaunchMultiKernelMultiDevice(&_out_launchParamsList, (int)a->numDevices, (unsigned int)a->flags); - return _r; + (void)payload; + hrr_note_unreplayable(ctx, "hipExtLaunchMultiKernelMultiDevice", + "each hipLaunchParams entry names its kernel by a host function address in the capturing process, and a cooperative multi-device launch cannot be decomposed into per-device launches without breaking the grid-wide barrier it exists for"); + return hipErrorNotSupported; } extern hipError_t playback_hipExtMallocWithFlags(PlaybackContext& ctx, const uint8_t* payload); @@ -761,6 +929,10 @@ static hipError_t playback_hipExternalMemoryGetMappedBuffer(PlaybackContext& ctx const auto* a = reinterpret_cast(payload); void* _out_devPtr = nullptr; hipError_t _r = (hipError_t)hipExternalMemoryGetMappedBuffer((void**)&_out_devPtr, (hipExternalMemory_t)a->extMem, (const hipExternalMemoryBufferDesc*)a->bufferDesc); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExternalMemoryGetMappedBuffer", a->ret); + return hipSuccess; + } return _r; } @@ -873,6 +1045,10 @@ static hipError_t playback_hipGetDevice(PlaybackContext& ctx, const uint8_t* pay const auto* a = reinterpret_cast(payload); int _out_deviceId{}; hipError_t _r = (hipError_t)hipGetDevice(&_out_deviceId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetDevice", a->ret); + return hipSuccess; + } return _r; } @@ -880,6 +1056,10 @@ static hipError_t playback_hipGetDeviceCount(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); int _out_count{}; hipError_t _r = (hipError_t)hipGetDeviceCount(&_out_count); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetDeviceCount", a->ret); + return hipSuccess; + } return _r; } @@ -887,6 +1067,10 @@ static hipError_t playback_hipGetDeviceFlags(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); unsigned int _out_flags{}; hipError_t _r = (hipError_t)hipGetDeviceFlags(&_out_flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetDeviceFlags", a->ret); + return hipSuccess; + } return _r; } @@ -894,6 +1078,10 @@ static hipError_t playback_hipGetDevicePropertiesR0600(PlaybackContext& ctx, con const auto* a = reinterpret_cast(payload); hipDeviceProp_tR0600 _out_prop{}; hipError_t _r = (hipError_t)hipGetDevicePropertiesR0600(&_out_prop, (int)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetDevicePropertiesR0600", a->ret); + return hipSuccess; + } return _r; } @@ -933,6 +1121,10 @@ static hipError_t playback_hipGetErrorString(PlaybackContext& ctx, const uint8_t static hipError_t playback_hipGetLastError(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipGetLastError(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetLastError", a->ret); + return hipSuccess; + } return _r; } @@ -947,32 +1139,18 @@ static hipError_t playback_hipGetMipmappedArrayLevel(PlaybackContext& ctx, const return hipSuccess; } -static hipError_t playback_hipGetSymbolAddress(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGetSymbolAddress — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipGetSymbolAddress(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipGetSymbolSize(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGetSymbolSize — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipGetSymbolSize(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGetTextureAlignmentOffset(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); size_t _out_offset{}; hipError_t _r = (hipError_t)hipGetTextureAlignmentOffset(&_out_offset, (const textureReference*)a->texref); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetTextureAlignmentOffset", a->ret); + return hipSuccess; + } return _r; } @@ -980,6 +1158,10 @@ static hipError_t playback_hipGetTextureObjectResourceDesc(PlaybackContext& ctx, const auto* a = reinterpret_cast(payload); hipResourceDesc _out_pResDesc{}; hipError_t _r = (hipError_t)hipGetTextureObjectResourceDesc(&_out_pResDesc, (hipTextureObject_t)ctx.translate_texture(a->textureObject)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetTextureObjectResourceDesc", a->ret); + return hipSuccess; + } return _r; } @@ -987,6 +1169,10 @@ static hipError_t playback_hipGetTextureObjectResourceViewDesc(PlaybackContext& const auto* a = reinterpret_cast(payload); struct hipResourceViewDesc _out_pResViewDesc{}; hipError_t _r = (hipError_t)hipGetTextureObjectResourceViewDesc(&_out_pResViewDesc, (hipTextureObject_t)ctx.translate_texture(a->textureObject)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetTextureObjectResourceViewDesc", a->ret); + return hipSuccess; + } return _r; } @@ -994,6 +1180,10 @@ static hipError_t playback_hipGetTextureObjectTextureDesc(PlaybackContext& ctx, const auto* a = reinterpret_cast(payload); hipTextureDesc _out_pTexDesc{}; hipError_t _r = (hipError_t)hipGetTextureObjectTextureDesc(&_out_pTexDesc, (hipTextureObject_t)ctx.translate_texture(a->textureObject)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetTextureObjectTextureDesc", a->ret); + return hipSuccess; + } return _r; } @@ -1009,199 +1199,354 @@ static hipError_t playback_hipGetTextureReference(PlaybackContext& ctx, const ui } static hipError_t playback_hipGraphAddChildGraphNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddChildGraphNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_pGraphNode = nullptr; + hipGraphNode_t _d_pDependencies[16]{}; + uint32_t _d_pDependencies_n = a->pDependencies_n > 16u ? 16u : a->pDependencies_n; + (void)_d_pDependencies_n; + if (a->pDependencies_present) + std::memcpy(_d_pDependencies, a->pDependencies_bytes, static_cast(_d_pDependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_pDependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_pDependencies[_i])); + if (!_live && _d_pDependencies[_i]) { + fprintf(stderr, "[HRR] hipGraphAddChildGraphNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_pDependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddChildGraphNode"); + return hipSuccess; + } + _d_pDependencies[_i] = _live; } - return hipSuccess; + hipError_t _r = (hipError_t)hipGraphAddChildGraphNode(&_out_pGraphNode, (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_pDependencies_n ? _d_pDependencies : nullptr), (size_t)_d_pDependencies_n, (hipGraph_t)ctx.translate_graph(a->childGraph)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddChildGraphNode", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_graph_node(a->pGraphNode, _out_pGraphNode); + } + if (_r == hipSuccess && ctx.graph_is_incomplete(a->childGraph)) + ctx.mark_graph_incomplete(a->graph, "hipGraphAddChildGraphNode"); + return _r; } static hipError_t playback_hipGraphAddDependencies(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddDependencies: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _d_from[16]{}; + uint32_t _d_from_n = a->from_n > 16u ? 16u : a->from_n; + (void)_d_from_n; + if (a->from_present) + std::memcpy(_d_from, a->from_bytes, static_cast(_d_from_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_from_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_from[_i])); + if (!_live && _d_from[_i]) { + fprintf(stderr, "[HRR] hipGraphAddDependencies: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_from[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddDependencies"); + return hipSuccess; + } + _d_from[_i] = _live; + } + hipGraphNode_t _d_to[16]{}; + uint32_t _d_to_n = a->to_n > 16u ? 16u : a->to_n; + (void)_d_to_n; + if (a->to_present) + std::memcpy(_d_to, a->to_bytes, static_cast(_d_to_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_to_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_to[_i])); + if (!_live && _d_to[_i]) { + fprintf(stderr, "[HRR] hipGraphAddDependencies: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_to[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddDependencies"); + return hipSuccess; + } + _d_to[_i] = _live; } - return hipSuccess; + hipError_t _r = (hipError_t)hipGraphAddDependencies((hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_from_n ? _d_from : nullptr), (const hipGraphNode_t*)(_d_to_n ? _d_to : nullptr), (size_t)_d_from_n); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddDependencies", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphAddEmptyNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddEmptyNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_pGraphNode = nullptr; + hipGraphNode_t _d_pDependencies[16]{}; + uint32_t _d_pDependencies_n = a->pDependencies_n > 16u ? 16u : a->pDependencies_n; + (void)_d_pDependencies_n; + if (a->pDependencies_present) + std::memcpy(_d_pDependencies, a->pDependencies_bytes, static_cast(_d_pDependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_pDependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_pDependencies[_i])); + if (!_live && _d_pDependencies[_i]) { + fprintf(stderr, "[HRR] hipGraphAddEmptyNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_pDependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddEmptyNode"); + return hipSuccess; + } + _d_pDependencies[_i] = _live; } - return hipSuccess; + hipError_t _r = (hipError_t)hipGraphAddEmptyNode(&_out_pGraphNode, (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_pDependencies_n ? _d_pDependencies : nullptr), (size_t)_d_pDependencies_n); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddEmptyNode", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_graph_node(a->pGraphNode, _out_pGraphNode); + } + return _r; } static hipError_t playback_hipGraphAddEventRecordNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddEventRecordNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_pGraphNode = nullptr; + hipGraphNode_t _d_pDependencies[16]{}; + uint32_t _d_pDependencies_n = a->pDependencies_n > 16u ? 16u : a->pDependencies_n; + (void)_d_pDependencies_n; + if (a->pDependencies_present) + std::memcpy(_d_pDependencies, a->pDependencies_bytes, static_cast(_d_pDependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_pDependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_pDependencies[_i])); + if (!_live && _d_pDependencies[_i]) { + fprintf(stderr, "[HRR] hipGraphAddEventRecordNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_pDependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddEventRecordNode"); + return hipSuccess; + } + _d_pDependencies[_i] = _live; } - return hipSuccess; + hipError_t _r = (hipError_t)hipGraphAddEventRecordNode(&_out_pGraphNode, (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_pDependencies_n ? _d_pDependencies : nullptr), (size_t)_d_pDependencies_n, (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddEventRecordNode", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_graph_node(a->pGraphNode, _out_pGraphNode); + } + return _r; } static hipError_t playback_hipGraphAddEventWaitNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddEventWaitNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_pGraphNode = nullptr; + hipGraphNode_t _d_pDependencies[16]{}; + uint32_t _d_pDependencies_n = a->pDependencies_n > 16u ? 16u : a->pDependencies_n; + (void)_d_pDependencies_n; + if (a->pDependencies_present) + std::memcpy(_d_pDependencies, a->pDependencies_bytes, static_cast(_d_pDependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_pDependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_pDependencies[_i])); + if (!_live && _d_pDependencies[_i]) { + fprintf(stderr, "[HRR] hipGraphAddEventWaitNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_pDependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddEventWaitNode"); + return hipSuccess; + } + _d_pDependencies[_i] = _live; } - return hipSuccess; + hipError_t _r = (hipError_t)hipGraphAddEventWaitNode(&_out_pGraphNode, (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_pDependencies_n ? _d_pDependencies : nullptr), (size_t)_d_pDependencies_n, (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddEventWaitNode", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_graph_node(a->pGraphNode, _out_pGraphNode); + } + return _r; } static hipError_t playback_hipGraphAddHostNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddHostNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); - } - return hipSuccess; + const auto* a = reinterpret_cast(payload); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddHostNode"); + hrr_note_unreplayable(ctx, "hipGraphAddHostNode", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); + return hipErrorNotSupported; } -static hipError_t playback_hipGraphAddKernelNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddKernelNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipGraphAddKernelNode(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipGraphAddMemAllocNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddMemAllocNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipGraphAddMemAllocNode(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGraphAddMemFreeNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddMemFreeNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_pGraphNode = nullptr; + hipGraphNode_t _d_pDependencies[16]{}; + uint32_t _d_pDependencies_n = a->pDependencies_n > 16u ? 16u : a->pDependencies_n; + (void)_d_pDependencies_n; + if (a->pDependencies_present) + std::memcpy(_d_pDependencies, a->pDependencies_bytes, static_cast(_d_pDependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_pDependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_pDependencies[_i])); + if (!_live && _d_pDependencies[_i]) { + fprintf(stderr, "[HRR] hipGraphAddMemFreeNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_pDependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddMemFreeNode"); + return hipSuccess; + } + _d_pDependencies[_i] = _live; } - return hipSuccess; + hipError_t _r = (hipError_t)hipGraphAddMemFreeNode(&_out_pGraphNode, (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_pDependencies_n ? _d_pDependencies : nullptr), (size_t)_d_pDependencies_n, ctx.translate_ptr(a->dev_ptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddMemFreeNode", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_graph_node(a->pGraphNode, _out_pGraphNode); + } + return _r; } static hipError_t playback_hipGraphAddMemcpyNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddMemcpyNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_pGraphNode = nullptr; + hipGraphNode_t _d_pDependencies[16]{}; + uint32_t _d_pDependencies_n = a->pDependencies_n > 16u ? 16u : a->pDependencies_n; + (void)_d_pDependencies_n; + if (a->pDependencies_present) + std::memcpy(_d_pDependencies, a->pDependencies_bytes, static_cast(_d_pDependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_pDependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_pDependencies[_i])); + if (!_live && _d_pDependencies[_i]) { + fprintf(stderr, "[HRR] hipGraphAddMemcpyNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_pDependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddMemcpyNode"); + return hipSuccess; + } + _d_pDependencies[_i] = _live; + } + hipMemcpy3DParms _d_pCopyParams{}; + if (a->pCopyParams_present) + std::memcpy(&_d_pCopyParams, a->pCopyParams_bytes, sizeof(_d_pCopyParams)); + _d_pCopyParams.srcPtr.ptr = ctx.translate_ptr(reinterpret_cast(_d_pCopyParams.srcPtr.ptr)); + _d_pCopyParams.dstPtr.ptr = ctx.translate_ptr(reinterpret_cast(_d_pCopyParams.dstPtr.ptr)); + _d_pCopyParams.srcArray = (hipArray_t)ctx.translate_array(reinterpret_cast(_d_pCopyParams.srcArray)); + _d_pCopyParams.dstArray = (hipArray_t)ctx.translate_array(reinterpret_cast(_d_pCopyParams.dstArray)); + hipError_t _r = (hipError_t)hipGraphAddMemcpyNode(&_out_pGraphNode, (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_pDependencies_n ? _d_pDependencies : nullptr), (size_t)_d_pDependencies_n, (a->pCopyParams_present ? (const hipMemcpy3DParms*)&_d_pCopyParams : (const hipMemcpy3DParms*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddMemcpyNode", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_graph_node(a->pGraphNode, _out_pGraphNode); + } + return _r; } static hipError_t playback_hipGraphAddMemcpyNode1D(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddMemcpyNode1D: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_pGraphNode = nullptr; + hipGraphNode_t _d_pDependencies[16]{}; + uint32_t _d_pDependencies_n = a->pDependencies_n > 16u ? 16u : a->pDependencies_n; + (void)_d_pDependencies_n; + if (a->pDependencies_present) + std::memcpy(_d_pDependencies, a->pDependencies_bytes, static_cast(_d_pDependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_pDependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_pDependencies[_i])); + if (!_live && _d_pDependencies[_i]) { + fprintf(stderr, "[HRR] hipGraphAddMemcpyNode1D: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_pDependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddMemcpyNode1D"); + return hipSuccess; + } + _d_pDependencies[_i] = _live; } - return hipSuccess; -} - -static hipError_t playback_hipGraphAddMemcpyNodeFromSymbol(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddMemcpyNodeFromSymbol: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + hipError_t _r = (hipError_t)hipGraphAddMemcpyNode1D(&_out_pGraphNode, (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_pDependencies_n ? _d_pDependencies : nullptr), (size_t)_d_pDependencies_n, ctx.translate_ptr(a->dst), ctx.translate_ptr(a->src), (size_t)a->count, (hipMemcpyKind)a->kind); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddMemcpyNode1D", a->ret); + return hipSuccess; } - return hipSuccess; -} - -static hipError_t playback_hipGraphAddMemcpyNodeToSymbol(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddMemcpyNodeToSymbol: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + if (_r == hipSuccess) { + ctx.record_graph_node(a->pGraphNode, _out_pGraphNode); } - return hipSuccess; + return _r; } +extern hipError_t playback_hipGraphAddMemcpyNodeFromSymbol(PlaybackContext& ctx, const uint8_t* payload); + +extern hipError_t playback_hipGraphAddMemcpyNodeToSymbol(PlaybackContext& ctx, const uint8_t* payload); + static hipError_t playback_hipGraphAddMemsetNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddMemsetNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_pGraphNode = nullptr; + hipGraphNode_t _d_pDependencies[16]{}; + uint32_t _d_pDependencies_n = a->pDependencies_n > 16u ? 16u : a->pDependencies_n; + (void)_d_pDependencies_n; + if (a->pDependencies_present) + std::memcpy(_d_pDependencies, a->pDependencies_bytes, static_cast(_d_pDependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_pDependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_pDependencies[_i])); + if (!_live && _d_pDependencies[_i]) { + fprintf(stderr, "[HRR] hipGraphAddMemsetNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_pDependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddMemsetNode"); + return hipSuccess; + } + _d_pDependencies[_i] = _live; + } + hipMemsetParams _d_pMemsetParams{}; + if (a->pMemsetParams_present) + std::memcpy(&_d_pMemsetParams, a->pMemsetParams_bytes, sizeof(_d_pMemsetParams)); + _d_pMemsetParams.dst = ctx.translate_ptr(reinterpret_cast(_d_pMemsetParams.dst)); + hipError_t _r = (hipError_t)hipGraphAddMemsetNode(&_out_pGraphNode, (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_pDependencies_n ? _d_pDependencies : nullptr), (size_t)_d_pDependencies_n, (a->pMemsetParams_present ? (const hipMemsetParams*)&_d_pMemsetParams : (const hipMemsetParams*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphAddMemsetNode", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_graph_node(a->pGraphNode, _out_pGraphNode); + } + return _r; } static hipError_t playback_hipGraphChildGraphNodeGetGraph(PlaybackContext& ctx, const uint8_t* payload) { @@ -1216,31 +1561,33 @@ static hipError_t playback_hipGraphChildGraphNodeGetGraph(PlaybackContext& ctx, } static hipError_t playback_hipGraphClone(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphClone: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraph_t _out_pGraphClone = nullptr; + hipError_t _r = (hipError_t)hipGraphClone(&_out_pGraphClone, (hipGraph_t)ctx.translate_graph(a->originalGraph)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphClone", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_graph(a->pGraphClone, _out_pGraphClone); + } + if (_r == hipSuccess && ctx.graph_is_incomplete(a->originalGraph)) + ctx.mark_graph_incomplete(a->pGraphClone, "hipGraphClone"); + return _r; } static hipError_t playback_hipGraphCreate(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphCreate: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraph_t _out_pGraph = nullptr; + hipError_t _r = (hipError_t)hipGraphCreate(&_out_pGraph, (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphCreate", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_graph(a->pGraph, _out_pGraph); + } + return _r; } static hipError_t playback_hipGraphDebugDotPrint(PlaybackContext& ctx, const uint8_t* payload) { @@ -1267,7 +1614,25 @@ static hipError_t playback_hipGraphDestroy(PlaybackContext& ctx, const uint8_t* static hipError_t playback_hipGraphDestroyNode(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphDestroyNode((hipGraphNode_t)a->node); + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphDestroyNode: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; + } + uint64_t _rec_hdl = a->node; + hipError_t _r = (hipError_t)hipGraphDestroyNode((hipGraphNode_t)ctx.translate_graph_node(_rec_hdl)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphDestroyNode", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.remove_graph_node(_rec_hdl); + } return _r; } @@ -1283,14 +1648,23 @@ static hipError_t playback_hipGraphEventRecordNodeGetEvent(PlaybackContext& ctx, } static hipError_t playback_hipGraphEventRecordNodeSetEvent(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphEventRecordNodeSetEvent — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphEventRecordNodeSetEvent: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; } - return hipSuccess; + hipError_t _r = (hipError_t)hipGraphEventRecordNodeSetEvent((hipGraphNode_t)ctx.translate_graph_node(a->node), (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphEventRecordNodeSetEvent", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphEventWaitNodeGetEvent(PlaybackContext& ctx, const uint8_t* payload) { @@ -1305,25 +1679,53 @@ static hipError_t playback_hipGraphEventWaitNodeGetEvent(PlaybackContext& ctx, c } static hipError_t playback_hipGraphEventWaitNodeSetEvent(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphEventWaitNodeSetEvent — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphEventWaitNodeSetEvent: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; } - return hipSuccess; + hipError_t _r = (hipError_t)hipGraphEventWaitNodeSetEvent((hipGraphNode_t)ctx.translate_graph_node(a->node), (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphEventWaitNodeSetEvent", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphExecChildGraphNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecChildGraphNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecChildGraphNodeSetParams: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; } - return hipSuccess; + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecChildGraphNodeSetParams: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphExecChildGraphNodeSetParams((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->node), (hipGraph_t)ctx.translate_graph(a->childGraph)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExecChildGraphNodeSetParams", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphExecDestroy(PlaybackContext& ctx, const uint8_t* payload) { @@ -1338,92 +1740,177 @@ static hipError_t playback_hipGraphExecDestroy(PlaybackContext& ctx, const uint8 } static hipError_t playback_hipGraphExecEventRecordNodeSetEvent(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecEventRecordNodeSetEvent — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecEventRecordNodeSetEvent: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; } - return hipSuccess; + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecEventRecordNodeSetEvent: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphExecEventRecordNodeSetEvent((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->hNode), (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExecEventRecordNodeSetEvent", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphExecEventWaitNodeSetEvent(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecEventWaitNodeSetEvent — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecEventWaitNodeSetEvent: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; } - return hipSuccess; + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecEventWaitNodeSetEvent: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphExecEventWaitNodeSetEvent((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->hNode), (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExecEventWaitNodeSetEvent", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphExecHostNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecHostNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; + (void)payload; + hrr_note_unreplayable(ctx, "hipGraphExecHostNodeSetParams", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); + return hipErrorNotSupported; } -static hipError_t playback_hipGraphExecKernelNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecKernelNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipGraphExecKernelNodeSetParams(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGraphExecMemcpyNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecMemcpyNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecMemcpyNodeSetParams: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; } - return hipSuccess; + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecMemcpyNodeSetParams: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; + } + hipMemcpy3DParms _d_pNodeParams{}; + if (a->pNodeParams_present) + std::memcpy(&_d_pNodeParams, a->pNodeParams_bytes, sizeof(_d_pNodeParams)); + _d_pNodeParams.srcPtr.ptr = ctx.translate_ptr(reinterpret_cast(_d_pNodeParams.srcPtr.ptr)); + _d_pNodeParams.dstPtr.ptr = ctx.translate_ptr(reinterpret_cast(_d_pNodeParams.dstPtr.ptr)); + _d_pNodeParams.srcArray = (hipArray_t)ctx.translate_array(reinterpret_cast(_d_pNodeParams.srcArray)); + _d_pNodeParams.dstArray = (hipArray_t)ctx.translate_array(reinterpret_cast(_d_pNodeParams.dstArray)); + hipError_t _r = (hipError_t)hipGraphExecMemcpyNodeSetParams((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->node), (a->pNodeParams_present ? (hipMemcpy3DParms*)&_d_pNodeParams : (hipMemcpy3DParms*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExecMemcpyNodeSetParams", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphExecMemcpyNodeSetParams1D(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecMemcpyNodeSetParams1D — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecMemcpyNodeSetParams1D: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; + } + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecMemcpyNodeSetParams1D: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphExecMemcpyNodeSetParams1D((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->node), ctx.translate_ptr(a->dst), ctx.translate_ptr(a->src), (size_t)a->count, (hipMemcpyKind)a->kind); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExecMemcpyNodeSetParams1D", a->ret); + return hipSuccess; } - return hipSuccess; -} - -static hipError_t playback_hipGraphExecMemcpyNodeSetParamsFromSymbol(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphExecMemcpyNodeSetParamsFromSymbol((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)a->node, ctx.translate_ptr(a->dst), (const void*)a->symbol, (size_t)a->count, (size_t)a->offset, (hipMemcpyKind)a->kind); return _r; } -static hipError_t playback_hipGraphExecMemcpyNodeSetParamsToSymbol(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphExecMemcpyNodeSetParamsToSymbol((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)a->node, (const void*)a->symbol, ctx.translate_ptr(a->src), (size_t)a->count, (size_t)a->offset, (hipMemcpyKind)a->kind); - return _r; -} +extern hipError_t playback_hipGraphExecMemcpyNodeSetParamsFromSymbol(PlaybackContext& ctx, const uint8_t* payload); + +extern hipError_t playback_hipGraphExecMemcpyNodeSetParamsToSymbol(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGraphExecMemsetNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecMemsetNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecMemsetNodeSetParams: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; } - return hipSuccess; + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecMemsetNodeSetParams: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; + } + hipMemsetParams _d_pNodeParams{}; + if (a->pNodeParams_present) + std::memcpy(&_d_pNodeParams, a->pNodeParams_bytes, sizeof(_d_pNodeParams)); + _d_pNodeParams.dst = ctx.translate_ptr(reinterpret_cast(_d_pNodeParams.dst)); + hipError_t _r = (hipError_t)hipGraphExecMemsetNodeSetParams((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->node), (a->pNodeParams_present ? (const hipMemsetParams*)&_d_pNodeParams : (const hipMemsetParams*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExecMemsetNodeSetParams", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphExecUpdate(PlaybackContext& ctx, const uint8_t* payload) { @@ -1482,14 +1969,10 @@ static hipError_t playback_hipGraphHostNodeGetParams(PlaybackContext& ctx, const } static hipError_t playback_hipGraphHostNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphHostNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; + (void)payload; + hrr_note_unreplayable(ctx, "hipGraphHostNodeSetParams", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); + return hipErrorNotSupported; } extern hipError_t playback_hipGraphInstantiate(PlaybackContext& ctx, const uint8_t* payload); @@ -1497,14 +1980,33 @@ extern hipError_t playback_hipGraphInstantiate(PlaybackContext& ctx, const uint8 extern hipError_t playback_hipGraphInstantiateWithFlags(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGraphKernelNodeCopyAttributes(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphKernelNodeCopyAttributes — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hSrc != 0 && ctx.translate_graph_node(a->hSrc) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphKernelNodeCopyAttributes: hSrc 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hSrc); + } + return hipSuccess; } - return hipSuccess; + if (a->hDst != 0 && ctx.translate_graph_node(a->hDst) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphKernelNodeCopyAttributes: hDst 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hDst); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphKernelNodeCopyAttributes((hipGraphNode_t)ctx.translate_graph_node(a->hSrc), (hipGraphNode_t)ctx.translate_graph_node(a->hDst)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphKernelNodeCopyAttributes", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphKernelNodeGetAttribute(PlaybackContext& ctx, const uint8_t* payload) { @@ -1530,27 +2032,30 @@ static hipError_t playback_hipGraphKernelNodeGetParams(PlaybackContext& ctx, con } static hipError_t playback_hipGraphKernelNodeSetAttribute(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphKernelNodeSetAttribute — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphKernelNodeSetAttribute: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; } - return hipSuccess; -} - -static hipError_t playback_hipGraphKernelNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphKernelNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); + hipKernelNodeAttrValue _d_value{}; + if (a->value_present) + std::memcpy(&_d_value, a->value_bytes, sizeof(_d_value)); + hipError_t _r = (hipError_t)hipGraphKernelNodeSetAttribute((hipGraphNode_t)ctx.translate_graph_node(a->hNode), (hipKernelNodeAttrID)a->attr, (a->value_present ? (const hipKernelNodeAttrValue*)&_d_value : (const hipKernelNodeAttrValue*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphKernelNodeSetAttribute", a->ret); + return hipSuccess; } - return hipSuccess; + return _r; } +extern hipError_t playback_hipGraphKernelNodeSetParams(PlaybackContext& ctx, const uint8_t* payload); + extern hipError_t playback_hipGraphLaunch(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGraphMemAllocNodeGetParams(PlaybackContext& ctx, const uint8_t* payload) { @@ -1587,38 +2092,55 @@ static hipError_t playback_hipGraphMemcpyNodeGetParams(PlaybackContext& ctx, con } static hipError_t playback_hipGraphMemcpyNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphMemcpyNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphMemcpyNodeSetParams: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; } - return hipSuccess; + hipMemcpy3DParms _d_pNodeParams{}; + if (a->pNodeParams_present) + std::memcpy(&_d_pNodeParams, a->pNodeParams_bytes, sizeof(_d_pNodeParams)); + _d_pNodeParams.srcPtr.ptr = ctx.translate_ptr(reinterpret_cast(_d_pNodeParams.srcPtr.ptr)); + _d_pNodeParams.dstPtr.ptr = ctx.translate_ptr(reinterpret_cast(_d_pNodeParams.dstPtr.ptr)); + _d_pNodeParams.srcArray = (hipArray_t)ctx.translate_array(reinterpret_cast(_d_pNodeParams.srcArray)); + _d_pNodeParams.dstArray = (hipArray_t)ctx.translate_array(reinterpret_cast(_d_pNodeParams.dstArray)); + hipError_t _r = (hipError_t)hipGraphMemcpyNodeSetParams((hipGraphNode_t)ctx.translate_graph_node(a->node), (a->pNodeParams_present ? (const hipMemcpy3DParms*)&_d_pNodeParams : (const hipMemcpy3DParms*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphMemcpyNodeSetParams", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphMemcpyNodeSetParams1D(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphMemcpyNodeSetParams1D — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphMemcpyNodeSetParams1D: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphMemcpyNodeSetParams1D((hipGraphNode_t)ctx.translate_graph_node(a->node), ctx.translate_ptr(a->dst), ctx.translate_ptr(a->src), (size_t)a->count, (hipMemcpyKind)a->kind); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphMemcpyNodeSetParams1D", a->ret); + return hipSuccess; } - return hipSuccess; -} - -static hipError_t playback_hipGraphMemcpyNodeSetParamsFromSymbol(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphMemcpyNodeSetParamsFromSymbol((hipGraphNode_t)a->node, ctx.translate_ptr(a->dst), (const void*)a->symbol, (size_t)a->count, (size_t)a->offset, (hipMemcpyKind)a->kind); return _r; } -static hipError_t playback_hipGraphMemcpyNodeSetParamsToSymbol(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphMemcpyNodeSetParamsToSymbol((hipGraphNode_t)a->node, (const void*)a->symbol, ctx.translate_ptr(a->src), (size_t)a->count, (size_t)a->offset, (hipMemcpyKind)a->kind); - return _r; -} +extern hipError_t playback_hipGraphMemcpyNodeSetParamsFromSymbol(PlaybackContext& ctx, const uint8_t* payload); + +extern hipError_t playback_hipGraphMemcpyNodeSetParamsToSymbol(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGraphMemsetNodeGetParams(PlaybackContext& ctx, const uint8_t* payload) { (void)ctx; (void)payload; @@ -1632,25 +2154,51 @@ static hipError_t playback_hipGraphMemsetNodeGetParams(PlaybackContext& ctx, con } static hipError_t playback_hipGraphMemsetNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphMemsetNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->node != 0 && ctx.translate_graph_node(a->node) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphMemsetNodeSetParams: node 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->node); + } + return hipSuccess; } - return hipSuccess; + hipMemsetParams _d_pNodeParams{}; + if (a->pNodeParams_present) + std::memcpy(&_d_pNodeParams, a->pNodeParams_bytes, sizeof(_d_pNodeParams)); + _d_pNodeParams.dst = ctx.translate_ptr(reinterpret_cast(_d_pNodeParams.dst)); + hipError_t _r = (hipError_t)hipGraphMemsetNodeSetParams((hipGraphNode_t)ctx.translate_graph_node(a->node), (a->pNodeParams_present ? (const hipMemsetParams*)&_d_pNodeParams : (const hipMemsetParams*)nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphMemsetNodeSetParams", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphNodeFindInClone(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphNodeFindInClone — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->originalNode != 0 && ctx.translate_graph_node(a->originalNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphNodeFindInClone: originalNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->originalNode); + } + return hipSuccess; } - return hipSuccess; + hipGraphNode_t _out_pNode = nullptr; + hipError_t _r = (hipError_t)hipGraphNodeFindInClone(&_out_pNode, (hipGraphNode_t)ctx.translate_graph_node(a->originalNode), (hipGraph_t)ctx.translate_graph(a->clonedGraph)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphNodeFindInClone", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_graph_node(a->pNode, _out_pNode); + } + return _r; } static hipError_t playback_hipGraphNodeGetDependencies(PlaybackContext& ctx, const uint8_t* payload) { @@ -1698,14 +2246,33 @@ static hipError_t playback_hipGraphNodeGetType(PlaybackContext& ctx, const uint8 } static hipError_t playback_hipGraphNodeSetEnabled(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphNodeSetEnabled — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphNodeSetEnabled: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; } - return hipSuccess; + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphNodeSetEnabled: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphNodeSetEnabled((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->hNode), (unsigned int)a->isEnabled); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphNodeSetEnabled", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphReleaseUserObject(PlaybackContext& ctx, const uint8_t* payload) { @@ -1778,6 +2345,10 @@ static hipError_t playback_hipGraphicsMapResources(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); hipGraphicsResource_t _out_resources{}; hipError_t _r = (hipError_t)hipGraphicsMapResources((int)a->count, &_out_resources, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphicsMapResources", a->ret); + return hipSuccess; + } return _r; } @@ -1786,6 +2357,10 @@ static hipError_t playback_hipGraphicsResourceGetMappedPointer(PlaybackContext& void* _out_devPtr = nullptr; size_t _out_size{}; hipError_t _r = (hipError_t)hipGraphicsResourceGetMappedPointer((void**)&_out_devPtr, &_out_size, (hipGraphicsResource_t)a->resource); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphicsResourceGetMappedPointer", a->ret); + return hipSuccess; + } return _r; } @@ -1793,6 +2368,10 @@ static hipError_t playback_hipGraphicsSubResourceGetMappedArray(PlaybackContext& const auto* a = reinterpret_cast(payload); hipArray_t _out_array = nullptr; hipError_t _r = (hipError_t)hipGraphicsSubResourceGetMappedArray(&_out_array, (hipGraphicsResource_t)a->resource, (unsigned int)a->arrayIndex, (unsigned int)a->mipLevel); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphicsSubResourceGetMappedArray", a->ret); + return hipSuccess; + } return _r; } @@ -1800,12 +2379,20 @@ static hipError_t playback_hipGraphicsUnmapResources(PlaybackContext& ctx, const const auto* a = reinterpret_cast(payload); hipGraphicsResource_t _out_resources{}; hipError_t _r = (hipError_t)hipGraphicsUnmapResources((int)a->count, &_out_resources, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphicsUnmapResources", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipGraphicsUnregisterResource(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipGraphicsUnregisterResource((hipGraphicsResource_t)a->resource); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphicsUnregisterResource", a->ret); + return hipSuccess; + } return _r; } @@ -1825,6 +2412,10 @@ static hipError_t playback_hipHostFree(PlaybackContext& ctx, const uint8_t* payl uint64_t _rec_ptr = a->ptr; void* _live_ptr = ctx.translate_ptr(_rec_ptr); hipError_t _r = (hipError_t)hipHostFree(_live_ptr); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipHostFree", a->ret); + return hipSuccess; + } if (_r == hipSuccess) { ctx.remove_alloc(_rec_ptr); } @@ -1848,6 +2439,10 @@ static hipError_t playback_hipHostMalloc(PlaybackContext& ctx, const uint8_t* pa const auto* a = reinterpret_cast(payload); void* _out_ptr = nullptr; hipError_t _r = (hipError_t)hipHostMalloc((void**)&_out_ptr, (size_t)a->size, (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipHostMalloc", a->ret); + return hipSuccess; + } if (_r == hipSuccess) { ctx.record_alloc(a->ptr, _out_ptr, static_cast(a->size), AllocKind::HostMalloc); } @@ -1862,6 +2457,10 @@ static hipError_t playback_hipImportExternalMemory(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); hipExternalMemory_t _out_extMem_out{}; hipError_t _r = (hipError_t)hipImportExternalMemory(&_out_extMem_out, (const hipExternalMemoryHandleDesc*)a->memHandleDesc); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipImportExternalMemory", a->ret); + return hipSuccess; + } return _r; } @@ -1869,18 +2468,30 @@ static hipError_t playback_hipImportExternalSemaphore(PlaybackContext& ctx, cons const auto* a = reinterpret_cast(payload); hipExternalSemaphore_t _out_extSem_out{}; hipError_t _r = (hipError_t)hipImportExternalSemaphore(&_out_extSem_out, (const hipExternalSemaphoreHandleDesc*)a->semHandleDesc); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipImportExternalSemaphore", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipInit(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipInit((unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipInit", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipIpcCloseMemHandle(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipIpcCloseMemHandle(ctx.translate_ptr(a->devPtr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipIpcCloseMemHandle", a->ret); + return hipSuccess; + } return _r; } @@ -1888,31 +2499,29 @@ static hipError_t playback_hipIpcGetEventHandle(PlaybackContext& ctx, const uint const auto* a = reinterpret_cast(payload); hipIpcEventHandle_t* _s_handle{}; hipError_t _r = (hipError_t)hipIpcGetEventHandle(_s_handle, (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipIpcGetEventHandle", a->ret); + return hipSuccess; + } return _r; } -static hipError_t playback_hipIpcGetMemHandle(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipIpcMemHandle_t* _s_handle{}; - hipError_t _r = (hipError_t)hipIpcGetMemHandle(_s_handle, ctx.translate_ptr(a->devPtr)); - return _r; -} +extern hipError_t playback_hipIpcGetMemHandle(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipIpcOpenEventHandle(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipEvent_t _out_event = nullptr; - hipIpcEventHandle_t _s_handle{}; - hipError_t _r = (hipError_t)hipIpcOpenEventHandle(&_out_event, _s_handle); + hipIpcEventHandle_t _v_handle{}; + std::memcpy(&_v_handle, a->handle_bytes, sizeof(_v_handle)); + hipError_t _r = (hipError_t)hipIpcOpenEventHandle(&_out_event, _v_handle); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipIpcOpenEventHandle", a->ret); + return hipSuccess; + } return _r; } -static hipError_t playback_hipIpcOpenMemHandle(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - void* _out_devPtr = nullptr; - hipIpcMemHandle_t _s_handle{}; - hipError_t _r = (hipError_t)hipIpcOpenMemHandle((void**)&_out_devPtr, _s_handle, (unsigned int)a->flags); - return _r; -} +extern hipError_t playback_hipIpcOpenMemHandle(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipKernelNameRef(PlaybackContext& ctx, const uint8_t* payload) { (void)ctx; (void)payload; @@ -1938,28 +2547,20 @@ static hipError_t playback_hipKernelNameRefByPtr(PlaybackContext& ctx, const uin extern hipError_t playback_hipLaunchByPtr(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipLaunchCooperativeKernel(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipLaunchCooperativeKernel — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipLaunchCooperativeKernel(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipLaunchCooperativeKernelMultiDevice(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipLaunchParams _out_launchParamsList{}; - hipError_t _r = (hipError_t)hipLaunchCooperativeKernelMultiDevice(&_out_launchParamsList, (int)a->numDevices, (unsigned int)a->flags); - return _r; + (void)payload; + hrr_note_unreplayable(ctx, "hipLaunchCooperativeKernelMultiDevice", + "each hipLaunchParams entry names its kernel by a host function address in the capturing process, and a cooperative multi-device launch cannot be decomposed into per-device launches without breaking the grid-wide barrier it exists for"); + return hipErrorNotSupported; } static hipError_t playback_hipLaunchHostFunc(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipLaunchHostFunc((hipStream_t)ctx.translate_stream(a->stream), (hipHostFn_t)a->fn, (void*)a->userData); - return _r; + (void)payload; + hrr_note_unreplayable(ctx, "hipLaunchHostFunc", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); + return hipErrorNotSupported; } extern hipError_t playback_hipLaunchKernel(PlaybackContext& ctx, const uint8_t* payload); @@ -2032,6 +2633,10 @@ static hipError_t playback_hipMallocPitch(PlaybackContext& ctx, const uint8_t* p void* _out_ptr = nullptr; size_t _out_pitch{}; hipError_t _r = (hipError_t)hipMallocPitch((void**)&_out_ptr, &_out_pitch, (size_t)a->width, (size_t)a->height); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMallocPitch", a->ret); + return hipSuccess; + } if (_r == hipSuccess) { ctx.record_alloc(a->ptr, _out_ptr, static_cast(a->width), AllocKind::Device); } @@ -2052,6 +2657,10 @@ static hipError_t playback_hipMemAdvise(PlaybackContext& ctx, const uint8_t* pay #else hipError_t _r = (hipError_t)hipMemAdvise(ctx.translate_ptr(a->dev_ptr), (size_t)a->count, (hipMemoryAdvise)a->advice, (int)a->device); #endif + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemAdvise", a->ret); + return hipSuccess; + } return _r; } @@ -2081,19 +2690,27 @@ extern hipError_t playback_hipMemCreate(PlaybackContext& ctx, const uint8_t* pay static hipError_t playback_hipMemExportToShareableHandle(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipMemExportToShareableHandle((void*)a->shareableHandle, (hipMemGenericAllocationHandle_t)a->handle, (hipMemAllocationHandleType)a->handleType, (unsigned long long)a->flags); + alignas(8) unsigned char _outbuf_shareableHandle[8]{}; + hipError_t _r = (hipError_t)hipMemExportToShareableHandle((void*)_outbuf_shareableHandle, (hipMemGenericAllocationHandle_t)ctx.translate_vmm_handle(a->handle), (hipMemAllocationHandleType)a->handleType, (unsigned long long)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemExportToShareableHandle", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemGetAccess(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipMemGetAccess — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + unsigned long long _d_flags{}; + hipMemLocation _d_location{}; + if (a->location_present) + std::memcpy(&_d_location, a->location_bytes, sizeof(_d_location)); + hipError_t _r = (hipError_t)hipMemGetAccess((unsigned long long*)&_d_flags, (a->location_present ? (const hipMemLocation*)&_d_location : (const hipMemLocation*)nullptr), ctx.translate_ptr(a->ptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemGetAccess", a->ret); + return hipSuccess; } - return hipSuccess; + return _r; } static hipError_t playback_hipMemGetAddressRange(PlaybackContext& ctx, const uint8_t* payload) { @@ -2125,18 +2742,18 @@ static hipError_t playback_hipMemGetInfo(PlaybackContext& ctx, const uint8_t* pa size_t _out_free{}; size_t _out_total{}; hipError_t _r = (hipError_t)hipMemGetInfo(&_out_free, &_out_total); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemGetInfo", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemImportFromShareableHandle(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipMemImportFromShareableHandle — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; + (void)payload; + hrr_note_unreplayable(ctx, "hipMemImportFromShareableHandle", + "the recorded argument is an OS handle (a POSIX fd or a Win32 HANDLE) belonging to the process that exported it, and the same number in the replaying process names a different object or nothing at all"); + return hipErrorNotSupported; } extern hipError_t playback_hipMemMap(PlaybackContext& ctx, const uint8_t* payload); @@ -2158,6 +2775,10 @@ static hipError_t playback_hipMemPoolDestroy(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); uint64_t _rec_hdl = a->mem_pool; hipError_t _r = (hipError_t)hipMemPoolDestroy((hipMemPool_t)ctx.translate_mempool(_rec_hdl)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemPoolDestroy", a->ret); + return hipSuccess; + } if (_r == hipSuccess) { ctx.remove_mempool(_rec_hdl); } @@ -2177,7 +2798,12 @@ static hipError_t playback_hipMemPoolExportPointer(PlaybackContext& ctx, const u static hipError_t playback_hipMemPoolExportToShareableHandle(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipMemPoolExportToShareableHandle((void*)a->shared_handle, (hipMemPool_t)ctx.translate_mempool(a->mem_pool), (hipMemAllocationHandleType)a->handle_type, (unsigned int)a->flags); + alignas(8) unsigned char _outbuf_shared_handle[8]{}; + hipError_t _r = (hipError_t)hipMemPoolExportToShareableHandle((void*)_outbuf_shared_handle, (hipMemPool_t)ctx.translate_mempool(a->mem_pool), (hipMemAllocationHandleType)a->handle_type, (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemPoolExportToShareableHandle", a->ret); + return hipSuccess; + } return _r; } @@ -2195,13 +2821,10 @@ static hipError_t playback_hipMemPoolGetAccess(PlaybackContext& ctx, const uint8 extern hipError_t playback_hipMemPoolGetAttribute(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipMemPoolImportFromShareableHandle(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipMemPool_t _out_mem_pool = nullptr; - hipError_t _r = (hipError_t)hipMemPoolImportFromShareableHandle(&_out_mem_pool, (void*)a->shared_handle, (hipMemAllocationHandleType)a->handle_type, (unsigned int)a->flags); - if (_r == hipSuccess) { - ctx.record_mempool(a->mem_pool, _out_mem_pool); - } - return _r; + (void)payload; + hrr_note_unreplayable(ctx, "hipMemPoolImportFromShareableHandle", + "the recorded argument is an OS handle (a POSIX fd or a Win32 HANDLE) belonging to the process that exported it, and the same number in the replaying process names a different object or nothing at all"); + return hipErrorNotSupported; } static hipError_t playback_hipMemPoolImportPointer(PlaybackContext& ctx, const uint8_t* payload) { @@ -2222,12 +2845,20 @@ extern hipError_t playback_hipMemPoolSetAttribute(PlaybackContext& ctx, const ui static hipError_t playback_hipMemPoolTrimTo(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemPoolTrimTo((hipMemPool_t)ctx.translate_mempool(a->mem_pool), (size_t)a->min_bytes_to_hold); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemPoolTrimTo", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemPrefetchAsync(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemPrefetchAsync(ctx.translate_ptr(a->dev_ptr), (size_t)a->count, (int)a->device, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemPrefetchAsync", a->ret); + return hipSuccess; + } return _r; } @@ -2351,12 +2982,20 @@ static hipError_t playback_hipMemcpyAtoH(PlaybackContext& ctx, const uint8_t* pa static hipError_t playback_hipMemcpyDtoD(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemcpyDtoD((hipDeviceptr_t)ctx.translate_ptr(a->dst), (hipDeviceptr_t)ctx.translate_ptr(a->src), (size_t)a->sizeBytes); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemcpyDtoD", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemcpyDtoDAsync(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemcpyDtoDAsync((hipDeviceptr_t)ctx.translate_ptr(a->dst), (hipDeviceptr_t)ctx.translate_ptr(a->src), (size_t)a->sizeBytes, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemcpyDtoDAsync", a->ret); + return hipSuccess; + } return _r; } @@ -2412,37 +3051,27 @@ extern hipError_t playback_hipMemcpyHtoD(PlaybackContext& ctx, const uint8_t* pa extern hipError_t playback_hipMemcpyHtoDAsync(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipMemcpyParam2D(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipMemcpyParam2D — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipMemcpyParam2D(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipMemcpyParam2DAsync(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipMemcpyParam2DAsync — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipMemcpyParam2DAsync(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipMemcpyPeer(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemcpyPeer(ctx.translate_ptr(a->dst), (int)a->dstDeviceId, ctx.translate_ptr(a->src), (int)a->srcDeviceId, (size_t)a->sizeBytes); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemcpyPeer", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemcpyPeerAsync(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemcpyPeerAsync(ctx.translate_ptr(a->dst), (int)a->dstDeviceId, ctx.translate_ptr(a->src), (int)a->srcDevice, (size_t)a->sizeBytes, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemcpyPeerAsync", a->ret); + return hipSuccess; + } return _r; } @@ -2484,18 +3113,30 @@ extern hipError_t playback_hipMemcpyWithStream(PlaybackContext& ctx, const uint8 static hipError_t playback_hipMemset(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemset(ctx.translate_ptr(a->dst), (int)a->value, (size_t)a->sizeBytes); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemset", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemset2D(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemset2D(ctx.translate_ptr(a->dst), (size_t)a->pitch, (int)a->value, (size_t)a->width, (size_t)a->height); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemset2D", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemset2DAsync(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemset2DAsync(ctx.translate_ptr(a->dst), (size_t)a->pitch, (int)a->value, (size_t)a->width, (size_t)a->height, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemset2DAsync", a->ret); + return hipSuccess; + } return _r; } @@ -2524,42 +3165,70 @@ static hipError_t playback_hipMemset3DAsync(PlaybackContext& ctx, const uint8_t* static hipError_t playback_hipMemsetAsync(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemsetAsync(ctx.translate_ptr(a->dst), (int)a->value, (size_t)a->sizeBytes, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetAsync", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemsetD16(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemsetD16((hipDeviceptr_t)ctx.translate_ptr(a->dest), (unsigned short)a->value, (size_t)a->count); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD16", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemsetD16Async(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemsetD16Async((hipDeviceptr_t)ctx.translate_ptr(a->dest), (unsigned short)a->value, (size_t)a->count, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD16Async", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemsetD32(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemsetD32((hipDeviceptr_t)ctx.translate_ptr(a->dest), (int)a->value, (size_t)a->count); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD32", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemsetD32Async(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemsetD32Async((hipDeviceptr_t)ctx.translate_ptr(a->dst), (int)a->value, (size_t)a->count, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD32Async", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemsetD8(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemsetD8((hipDeviceptr_t)ctx.translate_ptr(a->dest), (unsigned char)a->value, (size_t)a->count); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD8", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemsetD8Async(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemsetD8Async((hipDeviceptr_t)ctx.translate_ptr(a->dest), (unsigned char)a->value, (size_t)a->count, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD8Async", a->ret); + return hipSuccess; + } return _r; } @@ -2620,17 +3289,16 @@ static hipError_t playback_hipModuleGetTexRef(PlaybackContext& ctx, const uint8_ return hipSuccess; } -static hipError_t playback_hipModuleLaunchCooperativeKernel(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - void* _out_kernelParams = nullptr; - hipError_t _r = (hipError_t)hipModuleLaunchCooperativeKernel((hipFunction_t)ctx.translate_func(a->f), (unsigned int)a->gridDimX, (unsigned int)a->gridDimY, (unsigned int)a->gridDimZ, (unsigned int)a->blockDimX, (unsigned int)a->blockDimY, (unsigned int)a->blockDimZ, (unsigned int)a->sharedMemBytes, (hipStream_t)ctx.translate_stream(a->stream), (void**)&_out_kernelParams); - return _r; -} +extern hipError_t playback_hipModuleLaunchCooperativeKernel(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipModuleLaunchCooperativeKernelMultiDevice(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipFunctionLaunchParams _out_launchParamsList{}; hipError_t _r = (hipError_t)hipModuleLaunchCooperativeKernelMultiDevice(&_out_launchParamsList, (unsigned int)a->numDevices, (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipModuleLaunchCooperativeKernelMultiDevice", a->ret); + return hipSuccess; + } return _r; } @@ -2733,6 +3401,10 @@ static hipError_t playback_hipOccupancyMaxPotentialBlockSize(PlaybackContext& ct static hipError_t playback_hipPeekAtLastError(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipPeekAtLastError(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipPeekAtLastError", a->ret); + return hipSuccess; + } return _r; } @@ -2751,6 +3423,10 @@ static hipError_t playback_hipPointerGetAttributes(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); hipPointerAttribute_t _out_attributes{}; hipError_t _r = (hipError_t)hipPointerGetAttributes(&_out_attributes, ctx.translate_ptr(a->ptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipPointerGetAttributes", a->ret); + return hipSuccess; + } return _r; } @@ -2768,12 +3444,20 @@ static hipError_t playback_hipPointerSetAttribute(PlaybackContext& ctx, const ui static hipError_t playback_hipProfilerStart(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipProfilerStart(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipProfilerStart", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipProfilerStop(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipProfilerStop(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipProfilerStop", a->ret); + return hipSuccess; + } return _r; } @@ -2781,37 +3465,63 @@ static hipError_t playback_hipRuntimeGetVersion(PlaybackContext& ctx, const uint const auto* a = reinterpret_cast(payload); int _out_runtimeVersion{}; hipError_t _r = (hipError_t)hipRuntimeGetVersion(&_out_runtimeVersion); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipRuntimeGetVersion", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipSetDevice(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipSetDevice((int)a->deviceId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipSetDevice", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipSetDeviceFlags(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipSetDeviceFlags((unsigned)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipSetDeviceFlags", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipSetupArgument(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipSetupArgument((const void*)a->arg, (size_t)a->size, (size_t)a->offset); + unsigned char _d_arg[256]{}; + uint32_t _d_arg_n = a->arg_n > 256u ? 256u : a->arg_n; + (void)_d_arg_n; + if (a->arg_present) + std::memcpy(_d_arg, a->arg_bytes, static_cast(_d_arg_n) * sizeof(unsigned char)); + hipError_t _r = (hipError_t)hipSetupArgument((const void*)(_d_arg_n ? _d_arg : nullptr), (size_t)_d_arg_n, (size_t)a->offset); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipSetupArgument", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipSignalExternalSemaphoresAsync(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipSignalExternalSemaphoresAsync((const hipExternalSemaphore_t*)a->extSemArray, (const hipExternalSemaphoreSignalParams*)a->paramsArray, (unsigned int)a->numExtSems, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipSignalExternalSemaphoresAsync", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipStreamAddCallback(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipStreamAddCallback((hipStream_t)ctx.translate_stream(a->stream), (hipStreamCallback_t)a->callback, (void*)a->userData, (unsigned int)a->flags); - return _r; + (void)payload; + hrr_note_unreplayable(ctx, "hipStreamAddCallback", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); + return hipErrorNotSupported; } static hipError_t playback_hipStreamAttachMemAsync(PlaybackContext& ctx, const uint8_t* payload) { @@ -2842,6 +3552,10 @@ static hipError_t playback_hipStreamGetCaptureInfo(PlaybackContext& ctx, const u hipStreamCaptureStatus _out_pCaptureStatus{}; unsigned long long _out_pId{}; hipError_t _r = (hipError_t)hipStreamGetCaptureInfo((hipStream_t)ctx.translate_stream(a->stream), &_out_pCaptureStatus, &_out_pId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetCaptureInfo", a->ret); + return hipSuccess; + } return _r; } @@ -2871,6 +3585,10 @@ static hipError_t playback_hipStreamGetFlags(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); unsigned int _out_flags{}; hipError_t _r = (hipError_t)hipStreamGetFlags((hipStream_t)ctx.translate_stream(a->stream), &_out_flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetFlags", a->ret); + return hipSuccess; + } return _r; } @@ -2878,6 +3596,10 @@ static hipError_t playback_hipStreamGetPriority(PlaybackContext& ctx, const uint const auto* a = reinterpret_cast(payload); int _out_priority{}; hipError_t _r = (hipError_t)hipStreamGetPriority((hipStream_t)ctx.translate_stream(a->stream), &_out_priority); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetPriority", a->ret); + return hipSuccess; + } return _r; } @@ -2885,6 +3607,10 @@ static hipError_t playback_hipStreamIsCapturing(PlaybackContext& ctx, const uint const auto* a = reinterpret_cast(payload); hipStreamCaptureStatus _out_pCaptureStatus{}; hipError_t _r = (hipError_t)hipStreamIsCapturing((hipStream_t)ctx.translate_stream(a->stream), &_out_pCaptureStatus); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamIsCapturing", a->ret); + return hipSuccess; + } return _r; } @@ -2893,6 +3619,10 @@ extern hipError_t playback_hipStreamQuery(PlaybackContext& ctx, const uint8_t* p static hipError_t playback_hipStreamSynchronize(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipStreamSynchronize((hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamSynchronize", a->ret); + return hipSuccess; + } return _r; } @@ -2910,6 +3640,10 @@ static hipError_t playback_hipStreamUpdateCaptureDependencies(PlaybackContext& c static hipError_t playback_hipStreamWaitEvent(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipStreamWaitEvent((hipStream_t)ctx.translate_stream(a->stream), (hipEvent_t)ctx.translate_event(a->event), (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamWaitEvent", a->ret); + return hipSuccess; + } return _r; } @@ -2950,6 +3684,10 @@ static hipError_t playback_hipStreamWriteValue32(PlaybackContext& ctx, const uin return hipSuccess; } hipError_t _r = (hipError_t)hipStreamWriteValue32((hipStream_t)ctx.translate_stream(a->stream), _live_dst, (uint32_t)a->value, (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamWriteValue32", a->ret); + return hipSuccess; + } return _r; } @@ -2968,6 +3706,10 @@ static hipError_t playback_hipStreamWriteValue64(PlaybackContext& ctx, const uin return hipSuccess; } hipError_t _r = (hipError_t)hipStreamWriteValue64((hipStream_t)ctx.translate_stream(a->stream), _live_dst, (uint64_t)a->value, (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamWriteValue64", a->ret); + return hipSuccess; + } return _r; } @@ -2997,6 +3739,10 @@ static hipError_t playback_hipTexObjectGetResourceDesc(PlaybackContext& ctx, con const auto* a = reinterpret_cast(payload); HIP_RESOURCE_DESC _out_pResDesc{}; hipError_t _r = (hipError_t)hipTexObjectGetResourceDesc(&_out_pResDesc, (hipTextureObject_t)ctx.translate_texture(a->texObject)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexObjectGetResourceDesc", a->ret); + return hipSuccess; + } return _r; } @@ -3004,6 +3750,10 @@ static hipError_t playback_hipTexObjectGetResourceViewDesc(PlaybackContext& ctx, const auto* a = reinterpret_cast(payload); HIP_RESOURCE_VIEW_DESC _out_pResViewDesc{}; hipError_t _r = (hipError_t)hipTexObjectGetResourceViewDesc(&_out_pResViewDesc, (hipTextureObject_t)ctx.translate_texture(a->texObject)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexObjectGetResourceViewDesc", a->ret); + return hipSuccess; + } return _r; } @@ -3011,6 +3761,10 @@ static hipError_t playback_hipTexObjectGetTextureDesc(PlaybackContext& ctx, cons const auto* a = reinterpret_cast(payload); HIP_TEXTURE_DESC _out_pTexDesc{}; hipError_t _r = (hipError_t)hipTexObjectGetTextureDesc(&_out_pTexDesc, (hipTextureObject_t)ctx.translate_texture(a->texObject)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexObjectGetTextureDesc", a->ret); + return hipSuccess; + } return _r; } @@ -3029,6 +3783,10 @@ static hipError_t playback_hipTexRefGetAddressMode(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); enum hipTextureAddressMode _out_pam{}; hipError_t _r = (hipError_t)hipTexRefGetAddressMode(&_out_pam, (const textureReference*)a->texRef, (int)a->dim); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetAddressMode", a->ret); + return hipSuccess; + } return _r; } @@ -3036,6 +3794,10 @@ static hipError_t playback_hipTexRefGetFilterMode(PlaybackContext& ctx, const ui const auto* a = reinterpret_cast(payload); enum hipTextureFilterMode _out_pfm{}; hipError_t _r = (hipError_t)hipTexRefGetFilterMode(&_out_pfm, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetFilterMode", a->ret); + return hipSuccess; + } return _r; } @@ -3043,6 +3805,10 @@ static hipError_t playback_hipTexRefGetFlags(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); unsigned int _out_pFlags{}; hipError_t _r = (hipError_t)hipTexRefGetFlags(&_out_pFlags, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetFlags", a->ret); + return hipSuccess; + } return _r; } @@ -3061,6 +3827,10 @@ static hipError_t playback_hipTexRefGetMaxAnisotropy(PlaybackContext& ctx, const const auto* a = reinterpret_cast(payload); int _out_pmaxAnsio{}; hipError_t _r = (hipError_t)hipTexRefGetMaxAnisotropy(&_out_pmaxAnsio, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetMaxAnisotropy", a->ret); + return hipSuccess; + } return _r; } @@ -3068,6 +3838,10 @@ static hipError_t playback_hipTexRefGetMipMappedArray(PlaybackContext& ctx, cons const auto* a = reinterpret_cast(payload); hipMipmappedArray_t _out_pArray = nullptr; hipError_t _r = (hipError_t)hipTexRefGetMipMappedArray(&_out_pArray, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetMipMappedArray", a->ret); + return hipSuccess; + } return _r; } @@ -3075,6 +3849,10 @@ static hipError_t playback_hipTexRefGetMipmapFilterMode(PlaybackContext& ctx, co const auto* a = reinterpret_cast(payload); enum hipTextureFilterMode _out_pfm{}; hipError_t _r = (hipError_t)hipTexRefGetMipmapFilterMode(&_out_pfm, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetMipmapFilterMode", a->ret); + return hipSuccess; + } return _r; } @@ -3082,6 +3860,10 @@ static hipError_t playback_hipTexRefGetMipmapLevelBias(PlaybackContext& ctx, con const auto* a = reinterpret_cast(payload); float _out_pbias{}; hipError_t _r = (hipError_t)hipTexRefGetMipmapLevelBias(&_out_pbias, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetMipmapLevelBias", a->ret); + return hipSuccess; + } return _r; } @@ -3090,6 +3872,10 @@ static hipError_t playback_hipTexRefGetMipmapLevelClamp(PlaybackContext& ctx, co float _out_pminMipmapLevelClamp{}; float _out_pmaxMipmapLevelClamp{}; hipError_t _r = (hipError_t)hipTexRefGetMipmapLevelClamp(&_out_pminMipmapLevelClamp, &_out_pmaxMipmapLevelClamp, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetMipmapLevelClamp", a->ret); + return hipSuccess; + } return _r; } @@ -3098,6 +3884,10 @@ static hipError_t playback_hipTexRefSetAddress(PlaybackContext& ctx, const uint8 size_t _out_ByteOffset{}; textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetAddress(&_out_ByteOffset, &_out_texRef, (hipDeviceptr_t)ctx.translate_ptr(a->dptr), (size_t)a->bytes); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetAddress", a->ret); + return hipSuccess; + } return _r; } @@ -3105,6 +3895,10 @@ static hipError_t playback_hipTexRefSetAddress2D(PlaybackContext& ctx, const uin const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetAddress2D(&_out_texRef, (const HIP_ARRAY_DESCRIPTOR*)a->desc, (hipDeviceptr_t)ctx.translate_ptr(a->dptr), (size_t)a->Pitch); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetAddress2D", a->ret); + return hipSuccess; + } return _r; } @@ -3112,6 +3906,10 @@ static hipError_t playback_hipTexRefSetAddressMode(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetAddressMode(&_out_texRef, (int)a->dim, (enum hipTextureAddressMode)a->am); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetAddressMode", a->ret); + return hipSuccess; + } return _r; } @@ -3119,6 +3917,10 @@ static hipError_t playback_hipTexRefSetArray(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); textureReference _out_tex{}; hipError_t _r = (hipError_t)hipTexRefSetArray(&_out_tex, (hipArray_t)ctx.translate_array(a->array), (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetArray", a->ret); + return hipSuccess; + } return _r; } @@ -3127,6 +3929,10 @@ static hipError_t playback_hipTexRefSetBorderColor(PlaybackContext& ctx, const u textureReference _out_texRef{}; float _out_pBorderColor{}; hipError_t _r = (hipError_t)hipTexRefSetBorderColor(&_out_texRef, &_out_pBorderColor); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetBorderColor", a->ret); + return hipSuccess; + } return _r; } @@ -3134,6 +3940,10 @@ static hipError_t playback_hipTexRefSetFilterMode(PlaybackContext& ctx, const ui const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetFilterMode(&_out_texRef, (enum hipTextureFilterMode)a->fm); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetFilterMode", a->ret); + return hipSuccess; + } return _r; } @@ -3141,6 +3951,10 @@ static hipError_t playback_hipTexRefSetFlags(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetFlags(&_out_texRef, (unsigned int)a->Flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetFlags", a->ret); + return hipSuccess; + } return _r; } @@ -3159,6 +3973,10 @@ static hipError_t playback_hipTexRefSetMaxAnisotropy(PlaybackContext& ctx, const const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetMaxAnisotropy(&_out_texRef, (unsigned int)a->maxAniso); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetMaxAnisotropy", a->ret); + return hipSuccess; + } return _r; } @@ -3166,6 +3984,10 @@ static hipError_t playback_hipTexRefSetMipmapFilterMode(PlaybackContext& ctx, co const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetMipmapFilterMode(&_out_texRef, (enum hipTextureFilterMode)a->fm); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetMipmapFilterMode", a->ret); + return hipSuccess; + } return _r; } @@ -3173,6 +3995,10 @@ static hipError_t playback_hipTexRefSetMipmapLevelBias(PlaybackContext& ctx, con const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetMipmapLevelBias(&_out_texRef, (float)a->bias); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetMipmapLevelBias", a->ret); + return hipSuccess; + } return _r; } @@ -3180,6 +4006,10 @@ static hipError_t playback_hipTexRefSetMipmapLevelClamp(PlaybackContext& ctx, co const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetMipmapLevelClamp(&_out_texRef, (float)a->minMipMapLevelClamp, (float)a->maxMipMapLevelClamp); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetMipmapLevelClamp", a->ret); + return hipSuccess; + } return _r; } @@ -3187,6 +4017,10 @@ static hipError_t playback_hipTexRefSetMipmappedArray(PlaybackContext& ctx, cons const auto* a = reinterpret_cast(payload); textureReference _out_texRef{}; hipError_t _r = (hipError_t)hipTexRefSetMipmappedArray(&_out_texRef, (hipMipmappedArray_t)ctx.translate_mipmapped(a->mipmappedArray), (unsigned int)a->Flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefSetMipmappedArray", a->ret); + return hipSuccess; + } return _r; } @@ -3201,18 +4035,18 @@ static hipError_t playback_hipThreadExchangeStreamCaptureMode(PlaybackContext& c static hipError_t playback_hipUnbindTexture(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipUnbindTexture((const textureReference*)a->tex); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipUnbindTexture", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipUserObjectCreate(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipUserObjectCreate — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; + (void)payload; + hrr_note_unreplayable(ctx, "hipUserObjectCreate", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); + return hipErrorNotSupported; } static hipError_t playback_hipUserObjectRelease(PlaybackContext& ctx, const uint8_t* payload) { @@ -3240,6 +4074,10 @@ static hipError_t playback_hipUserObjectRetain(PlaybackContext& ctx, const uint8 static hipError_t playback_hipWaitExternalSemaphoresAsync(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipWaitExternalSemaphoresAsync((const hipExternalSemaphore_t*)a->extSemArray, (const hipExternalSemaphoreWaitParams*)a->paramsArray, (unsigned int)a->numExtSems, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipWaitExternalSemaphoresAsync", a->ret); + return hipSuccess; + } return _r; } @@ -3327,24 +4165,40 @@ extern hipError_t playback_hipMemcpy3D_spt(PlaybackContext& ctx, const uint8_t* static hipError_t playback_hipMemset_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemset_spt(ctx.translate_ptr(a->dst), (int)a->value, (size_t)a->sizeBytes); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemset_spt", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemsetAsync_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemsetAsync_spt(ctx.translate_ptr(a->dst), (int)a->value, (size_t)a->sizeBytes, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetAsync_spt", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemset2D_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemset2D_spt(ctx.translate_ptr(a->dst), (size_t)a->pitch, (int)a->value, (size_t)a->width, (size_t)a->height); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemset2D_spt", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemset2DAsync_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipMemset2DAsync_spt(ctx.translate_ptr(a->dst), (size_t)a->pitch, (int)a->value, (size_t)a->width, (size_t)a->height, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemset2DAsync_spt", a->ret); + return hipSuccess; + } return _r; } @@ -3465,6 +4319,10 @@ extern hipError_t playback_hipStreamQuery_spt(PlaybackContext& ctx, const uint8_ static hipError_t playback_hipStreamSynchronize_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipStreamSynchronize_spt((hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamSynchronize_spt", a->ret); + return hipSuccess; + } return _r; } @@ -3472,12 +4330,20 @@ static hipError_t playback_hipStreamGetPriority_spt(PlaybackContext& ctx, const const auto* a = reinterpret_cast(payload); int _out_priority{}; hipError_t _r = (hipError_t)hipStreamGetPriority_spt((hipStream_t)ctx.translate_stream(a->stream), &_out_priority); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetPriority_spt", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipStreamWaitEvent_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipStreamWaitEvent_spt((hipStream_t)ctx.translate_stream(a->stream), (hipEvent_t)ctx.translate_event(a->event), (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamWaitEvent_spt", a->ret); + return hipSuccess; + } return _r; } @@ -3485,48 +4351,61 @@ static hipError_t playback_hipStreamGetFlags_spt(PlaybackContext& ctx, const uin const auto* a = reinterpret_cast(payload); unsigned int _out_flags{}; hipError_t _r = (hipError_t)hipStreamGetFlags_spt((hipStream_t)ctx.translate_stream(a->stream), &_out_flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetFlags_spt", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipStreamAddCallback_spt(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipStreamAddCallback_spt((hipStream_t)ctx.translate_stream(a->stream), (hipStreamCallback_t)a->callback, (void*)a->userData, (unsigned int)a->flags); - return _r; + (void)payload; + hrr_note_unreplayable(ctx, "hipStreamAddCallback_spt", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); + return hipErrorNotSupported; } static hipError_t playback_hipEventRecord_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipEventRecord_spt((hipEvent_t)ctx.translate_event(a->event), (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipEventRecord_spt", a->ret); + return hipSuccess; + } return _r; } -static hipError_t playback_hipLaunchCooperativeKernel_spt(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - dim3 _dim_gridDim(a->gridDim_x, a->gridDim_y, a->gridDim_z); - dim3 _dim_blockDim(a->blockDim_x, a->blockDim_y, a->blockDim_z); - void* _out_kernelParams = nullptr; - hipError_t _r = (hipError_t)hipLaunchCooperativeKernel_spt((const void*)a->f, _dim_gridDim, _dim_blockDim, (void**)&_out_kernelParams, (uint32_t)a->sharedMemBytes, (hipStream_t)ctx.translate_stream(a->hStream)); - return _r; -} +extern hipError_t playback_hipLaunchCooperativeKernel_spt(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipLaunchKernel_spt(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - dim3 _dim_numBlocks(a->numBlocks_x, a->numBlocks_y, a->numBlocks_z); - dim3 _dim_dimBlocks(a->dimBlocks_x, a->dimBlocks_y, a->dimBlocks_z); - void* _out_args = nullptr; - hipError_t _r = (hipError_t)hipLaunchKernel_spt((const void*)a->function_address, _dim_numBlocks, _dim_dimBlocks, (void**)&_out_args, (size_t)a->sharedMemBytes, (hipStream_t)ctx.translate_stream(a->stream)); - return _r; -} +extern hipError_t playback_hipLaunchKernel_spt(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGraphLaunch_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); + if (a->graphExec != 0 && ctx.translate_graph_exec(a->graphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphLaunch_spt: graphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->graphExec); + } + return hipSuccess; + } hipError_t _r = (hipError_t)hipGraphLaunch_spt((hipGraphExec_t)ctx.translate_graph_exec(a->graphExec), (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphLaunch_spt", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipStreamBeginCapture_spt(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipStreamBeginCapture_spt((hipStream_t)ctx.translate_stream(a->stream), (hipStreamCaptureMode)a->mode); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamBeginCapture_spt", a->ret); + return hipSuccess; + } return _r; } @@ -3534,6 +4413,10 @@ static hipError_t playback_hipStreamEndCapture_spt(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); hipGraph_t _out_pGraph = nullptr; hipError_t _r = (hipError_t)hipStreamEndCapture_spt((hipStream_t)ctx.translate_stream(a->stream), &_out_pGraph); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamEndCapture_spt", a->ret); + return hipSuccess; + } return _r; } @@ -3541,6 +4424,10 @@ static hipError_t playback_hipStreamIsCapturing_spt(PlaybackContext& ctx, const const auto* a = reinterpret_cast(payload); hipStreamCaptureStatus _out_pCaptureStatus{}; hipError_t _r = (hipError_t)hipStreamIsCapturing_spt((hipStream_t)ctx.translate_stream(a->stream), &_out_pCaptureStatus); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamIsCapturing_spt", a->ret); + return hipSuccess; + } return _r; } @@ -3549,6 +4436,10 @@ static hipError_t playback_hipStreamGetCaptureInfo_spt(PlaybackContext& ctx, con hipStreamCaptureStatus _out_pCaptureStatus{}; unsigned long long _out_pId{}; hipError_t _r = (hipError_t)hipStreamGetCaptureInfo_spt((hipStream_t)ctx.translate_stream(a->stream), &_out_pCaptureStatus, &_out_pId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetCaptureInfo_spt", a->ret); + return hipSuccess; + } return _r; } @@ -3564,9 +4455,10 @@ static hipError_t playback_hipStreamGetCaptureInfo_v2_spt(PlaybackContext& ctx, } static hipError_t playback_hipLaunchHostFunc_spt(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipLaunchHostFunc_spt((hipStream_t)ctx.translate_stream(a->stream), (hipHostFn_t)a->fn, (void*)a->userData); - return _r; + (void)payload; + hrr_note_unreplayable(ctx, "hipLaunchHostFunc_spt", + "the callback is a host function pointer belonging to the capturing process, so there is no function here to enqueue"); + return hipErrorNotSupported; } static hipError_t playback_hipGetStreamDeviceId(PlaybackContext& ctx, const uint8_t* payload) { @@ -3581,109 +4473,241 @@ static hipError_t playback_hipGetStreamDeviceId(PlaybackContext& ctx, const uint } static hipError_t playback_hipDrvGraphAddMemsetNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipDrvGraphAddMemsetNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_phGraphNode = nullptr; + hipGraphNode_t _d_dependencies[16]{}; + uint32_t _d_dependencies_n = a->dependencies_n > 16u ? 16u : a->dependencies_n; + (void)_d_dependencies_n; + if (a->dependencies_present) + std::memcpy(_d_dependencies, a->dependencies_bytes, static_cast(_d_dependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_dependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_dependencies[_i])); + if (!_live && _d_dependencies[_i]) { + fprintf(stderr, "[HRR] hipDrvGraphAddMemsetNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_dependencies[_i])); + ctx.mark_graph_incomplete(a->hGraph, "hipDrvGraphAddMemsetNode"); + return hipSuccess; + } + _d_dependencies[_i] = _live; + } + hipMemsetParams _d_memsetParams{}; + if (a->memsetParams_present) + std::memcpy(&_d_memsetParams, a->memsetParams_bytes, sizeof(_d_memsetParams)); + _d_memsetParams.dst = ctx.translate_ptr(reinterpret_cast(_d_memsetParams.dst)); + hipError_t _r = (hipError_t)hipDrvGraphAddMemsetNode(&_out_phGraphNode, (hipGraph_t)ctx.translate_graph(a->hGraph), (const hipGraphNode_t*)(_d_dependencies_n ? _d_dependencies : nullptr), (size_t)_d_dependencies_n, (a->memsetParams_present ? (const hipMemsetParams*)&_d_memsetParams : (const hipMemsetParams*)nullptr), hrr_live_ctx(a->ctx)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDrvGraphAddMemsetNode", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_graph_node(a->phGraphNode, _out_phGraphNode); + } + return _r; } static hipError_t playback_hipGraphAddExternalSemaphoresWaitNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; + const auto* a = reinterpret_cast(payload); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddExternalSemaphoresWaitNode"); static bool warned = false; if (!warned) { warned = true; - fprintf(stderr, "[HRR] hipGraphAddExternalSemaphoresWaitNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + fprintf(stderr, "[HRR] hipGraphAddExternalSemaphoresWaitNode: not reconstructable at " + "replay, so the call is skipped and the graph it belongs " + "to is marked incomplete; instantiating that graph fails " + "loudly rather than running a graph that is missing " + "work.\n"); } return hipSuccess; } static hipError_t playback_hipGraphAddExternalSemaphoresSignalNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; + const auto* a = reinterpret_cast(payload); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddExternalSemaphoresSignalNode"); static bool warned = false; if (!warned) { warned = true; - fprintf(stderr, "[HRR] hipGraphAddExternalSemaphoresSignalNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + fprintf(stderr, "[HRR] hipGraphAddExternalSemaphoresSignalNode: not reconstructable at " + "replay, so the call is skipped and the graph it belongs " + "to is marked incomplete; instantiating that graph fails " + "loudly rather than running a graph that is missing " + "work.\n"); } return hipSuccess; } static hipError_t playback_hipGraphExternalSemaphoresSignalNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphExternalSemaphoresSignalNodeSetParams((hipGraphNode_t)a->hNode, (const hipExternalSemaphoreSignalNodeParams*)a->nodeParams); + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExternalSemaphoresSignalNodeSetParams: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphExternalSemaphoresSignalNodeSetParams((hipGraphNode_t)ctx.translate_graph_node(a->hNode), (const hipExternalSemaphoreSignalNodeParams*)a->nodeParams); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExternalSemaphoresSignalNodeSetParams", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipGraphExternalSemaphoresWaitNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphExternalSemaphoresWaitNodeSetParams((hipGraphNode_t)a->hNode, (const hipExternalSemaphoreWaitNodeParams*)a->nodeParams); + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExternalSemaphoresWaitNodeSetParams: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphExternalSemaphoresWaitNodeSetParams((hipGraphNode_t)ctx.translate_graph_node(a->hNode), (const hipExternalSemaphoreWaitNodeParams*)a->nodeParams); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExternalSemaphoresWaitNodeSetParams", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipGraphExternalSemaphoresSignalNodeGetParams(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExternalSemaphoresSignalNodeGetParams: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } hipExternalSemaphoreSignalNodeParams _out_params_out{}; - hipError_t _r = (hipError_t)hipGraphExternalSemaphoresSignalNodeGetParams((hipGraphNode_t)a->hNode, &_out_params_out); + hipError_t _r = (hipError_t)hipGraphExternalSemaphoresSignalNodeGetParams((hipGraphNode_t)ctx.translate_graph_node(a->hNode), &_out_params_out); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExternalSemaphoresSignalNodeGetParams", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipGraphExternalSemaphoresWaitNodeGetParams(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExternalSemaphoresWaitNodeGetParams: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } hipExternalSemaphoreWaitNodeParams _out_params_out{}; - hipError_t _r = (hipError_t)hipGraphExternalSemaphoresWaitNodeGetParams((hipGraphNode_t)a->hNode, &_out_params_out); + hipError_t _r = (hipError_t)hipGraphExternalSemaphoresWaitNodeGetParams((hipGraphNode_t)ctx.translate_graph_node(a->hNode), &_out_params_out); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExternalSemaphoresWaitNodeGetParams", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipGraphExecExternalSemaphoresSignalNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphExecExternalSemaphoresSignalNodeSetParams((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)a->hNode, (const hipExternalSemaphoreSignalNodeParams*)a->nodeParams); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecExternalSemaphoresSignalNodeSetParams: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; + } + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecExternalSemaphoresSignalNodeSetParams: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphExecExternalSemaphoresSignalNodeSetParams((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->hNode), (const hipExternalSemaphoreSignalNodeParams*)a->nodeParams); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExecExternalSemaphoresSignalNodeSetParams", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipGraphExecExternalSemaphoresWaitNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipGraphExecExternalSemaphoresWaitNodeSetParams((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)a->hNode, (const hipExternalSemaphoreWaitNodeParams*)a->nodeParams); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecExternalSemaphoresWaitNodeSetParams: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; + } + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipGraphExecExternalSemaphoresWaitNodeSetParams: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } + hipError_t _r = (hipError_t)hipGraphExecExternalSemaphoresWaitNodeSetParams((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->hNode), (const hipExternalSemaphoreWaitNodeParams*)a->nodeParams); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGraphExecExternalSemaphoresWaitNodeSetParams", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipGraphAddNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; + const auto* a = reinterpret_cast(payload); + ctx.mark_graph_incomplete(a->graph, "hipGraphAddNode"); static bool warned = false; if (!warned) { warned = true; - fprintf(stderr, "[HRR] hipGraphAddNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + fprintf(stderr, "[HRR] hipGraphAddNode: not reconstructable at " + "replay, so the call is skipped and the graph it belongs " + "to is marked incomplete; instantiating that graph fails " + "loudly rather than running a graph that is missing " + "work.\n"); } return hipSuccess; } static hipError_t playback_hipGraphInstantiateWithParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; + const auto* a = reinterpret_cast(payload); + ctx.mark_graph_incomplete(a->graph, "hipGraphInstantiateWithParams"); static bool warned = false; if (!warned) { warned = true; - fprintf(stderr, "[HRR] hipGraphInstantiateWithParams: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + fprintf(stderr, "[HRR] hipGraphInstantiateWithParams: not reconstructable at " + "replay, so the call is skipped and the graph it belongs " + "to is marked incomplete; instantiating that graph fails " + "loudly rather than running a graph that is missing " + "work.\n"); } return hipSuccess; } @@ -3691,6 +4715,10 @@ static hipError_t playback_hipGraphInstantiateWithParams(PlaybackContext& ctx, c static hipError_t playback_hipExtGetLastError(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipExtGetLastError(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExtGetLastError", a->ret); + return hipSuccess; + } return _r; } @@ -3698,6 +4726,10 @@ static hipError_t playback_hipTexRefGetBorderColor(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); float _out_pBorderColor{}; hipError_t _r = (hipError_t)hipTexRefGetBorderColor(&_out_pBorderColor, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetBorderColor", a->ret); + return hipSuccess; + } return _r; } @@ -3705,29 +4737,58 @@ static hipError_t playback_hipTexRefGetArray(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); hipArray_t _out_pArray = nullptr; hipError_t _r = (hipError_t)hipTexRefGetArray(&_out_pArray, (const textureReference*)a->texRef); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipTexRefGetArray", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipGetProcAddress(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGetProcAddress — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + void* _out_pfn = nullptr; + hipDriverProcAddressQueryResult _out_symbolStatus{}; + hipError_t _r = (hipError_t)hipGetProcAddress((a->symbol_present ? (const char*)a->symbol_bytes : (const char*)nullptr), (void**)&_out_pfn, (int)a->hipVersion, (uint64_t)a->flags, &_out_symbolStatus); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetProcAddress", a->ret); + return hipSuccess; } - return hipSuccess; + return _r; } static hipError_t playback_hipStreamBeginCaptureToGraph(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipStreamBeginCaptureToGraph — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _d_dependencies[16]{}; + uint32_t _d_dependencies_n = a->dependencies_n > 16u ? 16u : a->dependencies_n; + (void)_d_dependencies_n; + if (a->dependencies_present) + std::memcpy(_d_dependencies, a->dependencies_bytes, static_cast(_d_dependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_dependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_dependencies[_i])); + if (!_live && _d_dependencies[_i]) { + fprintf(stderr, "[HRR] hipStreamBeginCaptureToGraph: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_dependencies[_i])); + ctx.mark_graph_incomplete(a->graph, "hipStreamBeginCaptureToGraph"); + return hipSuccess; + } + _d_dependencies[_i] = _live; + } + hipGraphEdgeData _d_dependencyData[16]{}; + uint32_t _d_dependencyData_n = a->dependencyData_n > 16u ? 16u : a->dependencyData_n; + (void)_d_dependencyData_n; + if (a->dependencyData_present) + std::memcpy(_d_dependencyData, a->dependencyData_bytes, static_cast(_d_dependencyData_n) * sizeof(hipGraphEdgeData)); + hipError_t _r = (hipError_t)hipStreamBeginCaptureToGraph((hipStream_t)ctx.translate_stream(a->stream), (hipGraph_t)ctx.translate_graph(a->graph), (const hipGraphNode_t*)(_d_dependencies_n ? _d_dependencies : nullptr), (const hipGraphEdgeData*)(_d_dependencyData_n ? _d_dependencyData : nullptr), (size_t)_d_dependencies_n, (hipStreamCaptureMode)a->mode); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamBeginCaptureToGraph", a->ret); + return hipSuccess; } - return hipSuccess; + return _r; } static hipError_t playback_hipGetFuncBySymbol(PlaybackContext& ctx, const uint8_t* payload) { @@ -3819,39 +4880,73 @@ static hipError_t playback_hipMemcpy2DArrayToArray(PlaybackContext& ctx, const u } static hipError_t playback_hipDrvGraphAddMemFreeNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipDrvGraphAddMemFreeNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); + const auto* a = reinterpret_cast(payload); + hipGraphNode_t _out_phGraphNode = nullptr; + hipGraphNode_t _d_dependencies[16]{}; + uint32_t _d_dependencies_n = a->dependencies_n > 16u ? 16u : a->dependencies_n; + (void)_d_dependencies_n; + if (a->dependencies_present) + std::memcpy(_d_dependencies, a->dependencies_bytes, static_cast(_d_dependencies_n) * sizeof(hipGraphNode_t)); + for (uint32_t _i = 0; _i < _d_dependencies_n; ++_i) { + hipGraphNode_t _live = (hipGraphNode_t)ctx.translate_graph_node(reinterpret_cast(_d_dependencies[_i])); + if (!_live && _d_dependencies[_i]) { + fprintf(stderr, "[HRR] hipDrvGraphAddMemFreeNode: dependency " + "0x%llx was never built at replay, so this call is " + "skipped and the graph it belongs to is marked " + "incomplete; instantiating that graph fails loudly " + "rather than running one that is missing an " + "ordering constraint.\n", + (unsigned long long)reinterpret_cast(_d_dependencies[_i])); + ctx.mark_graph_incomplete(a->hGraph, "hipDrvGraphAddMemFreeNode"); + return hipSuccess; + } + _d_dependencies[_i] = _live; } - return hipSuccess; -} - -static hipError_t playback_hipDrvGraphExecMemcpyNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipDrvGraphExecMemcpyNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); + hipError_t _r = (hipError_t)hipDrvGraphAddMemFreeNode(&_out_phGraphNode, (hipGraph_t)ctx.translate_graph(a->hGraph), (const hipGraphNode_t*)(_d_dependencies_n ? _d_dependencies : nullptr), (size_t)_d_dependencies_n, (hipDeviceptr_t)ctx.translate_ptr(a->dptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDrvGraphAddMemFreeNode", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_graph_node(a->phGraphNode, _out_phGraphNode); + } + return _r; } +extern hipError_t playback_hipDrvGraphExecMemcpyNodeSetParams(PlaybackContext& ctx, const uint8_t* payload); + static hipError_t playback_hipDrvGraphExecMemsetNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipDrvGraphExecMemsetNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + if (a->hGraphExec != 0 && ctx.translate_graph_exec(a->hGraphExec) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipDrvGraphExecMemsetNodeSetParams: hGraphExec 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hGraphExec); + } + return hipSuccess; } - return hipSuccess; + if (a->hNode != 0 && ctx.translate_graph_node(a->hNode) == nullptr) { + static bool warned = false; + if (!warned) { + warned = true; + fprintf(stderr, "[HRR] hipDrvGraphExecMemsetNodeSetParams: hNode 0x%llx was " + "never built at replay, so this call is skipped.\n", + (unsigned long long)a->hNode); + } + return hipSuccess; + } + hipMemsetParams _d_memsetParams{}; + if (a->memsetParams_present) + std::memcpy(&_d_memsetParams, a->memsetParams_bytes, sizeof(_d_memsetParams)); + _d_memsetParams.dst = ctx.translate_ptr(reinterpret_cast(_d_memsetParams.dst)); + hipError_t _r = (hipError_t)hipDrvGraphExecMemsetNodeSetParams((hipGraphExec_t)ctx.translate_graph_exec(a->hGraphExec), (hipGraphNode_t)ctx.translate_graph_node(a->hNode), (a->memsetParams_present ? (const hipMemsetParams*)&_d_memsetParams : (const hipMemsetParams*)nullptr), hrr_live_ctx(a->ctx)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDrvGraphExecMemsetNodeSetParams", a->ret); + return hipSuccess; + } + return _r; } static hipError_t playback_hipGraphExecGetFlags(PlaybackContext& ctx, const uint8_t* payload) { @@ -3891,6 +4986,10 @@ static hipError_t playback_hipExternalMemoryGetMappedMipmappedArray(PlaybackCont const auto* a = reinterpret_cast(payload); hipMipmappedArray_t _out_mipmap = nullptr; hipError_t _r = (hipError_t)hipExternalMemoryGetMappedMipmappedArray(&_out_mipmap, (hipExternalMemory_t)a->extMem, (const hipExternalMemoryMipmappedArrayDesc*)a->mipmapDesc); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExternalMemoryGetMappedMipmappedArray", a->ret); + return hipSuccess; + } return _r; } @@ -3905,16 +5004,7 @@ static hipError_t playback_hipDrvGraphMemcpyNodeGetParams(PlaybackContext& ctx, return hipSuccess; } -static hipError_t playback_hipDrvGraphMemcpyNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipDrvGraphMemcpyNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipDrvGraphMemcpyNodeSetParams(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipExtHostAlloc(PlaybackContext& ctx, const uint8_t* payload) { (void)ctx; (void)payload; @@ -3928,36 +5018,22 @@ static hipError_t playback_hipExtHostAlloc(PlaybackContext& ctx, const uint8_t* } static hipError_t playback_hipDeviceGetTexture1DLinearMaxWidth(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipDeviceGetTexture1DLinearMaxWidth — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + size_t _d_maxWidthInElements{}; + hipChannelFormatDesc _d_fmtDesc{}; + if (a->fmtDesc_present) + std::memcpy(&_d_fmtDesc, a->fmtDesc_bytes, sizeof(_d_fmtDesc)); + hipError_t _r = (hipError_t)hipDeviceGetTexture1DLinearMaxWidth((size_t*)&_d_maxWidthInElements, (a->fmtDesc_present ? (const hipChannelFormatDesc*)&_d_fmtDesc : (const hipChannelFormatDesc*)nullptr), (int)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetTexture1DLinearMaxWidth", a->ret); + return hipSuccess; } - return hipSuccess; -} - -static hipError_t playback_hipStreamBatchMemOp(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipStreamBatchMemOpParams _out_paramArray{}; - hipError_t _r = (hipError_t)hipStreamBatchMemOp((hipStream_t)ctx.translate_stream(a->stream), (unsigned int)a->count, &_out_paramArray, (unsigned int)a->flags); return _r; } -static hipError_t playback_hipGraphAddBatchMemOpNode(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] hipGraphAddBatchMemOpNode: explicit (node-API) graph " - "construction is NOT supported by HRR replay. Only " - "stream-capture graphs (hipStreamBeginCapture/EndCapture) " - "are replayable; instantiating a node-API-built graph will " - "fail loudly. This call is skipped.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipStreamBatchMemOp(PlaybackContext& ctx, const uint8_t* payload); + +extern hipError_t playback_hipGraphAddBatchMemOpNode(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipGraphBatchMemOpNodeGetParams(PlaybackContext& ctx, const uint8_t* payload) { (void)ctx; (void)payload; @@ -3970,41 +5046,29 @@ static hipError_t playback_hipGraphBatchMemOpNodeGetParams(PlaybackContext& ctx, return hipSuccess; } -static hipError_t playback_hipGraphBatchMemOpNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphBatchMemOpNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipGraphBatchMemOpNodeSetParams(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipGraphExecBatchMemOpNodeSetParams(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipGraphExecBatchMemOpNodeSetParams — " - "this API is not replayed; results may differ from capture.\n"); - } - return hipSuccess; -} +extern hipError_t playback_hipGraphExecBatchMemOpNodeSetParams(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipLinkAddData(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - hipJitOption _out_options{}; - void* _out_optionValues = nullptr; - hipError_t _r = (hipError_t)hipLinkAddData((hipLinkState_t)a->state, (hipJitInputType)a->type, (void*)a->data, (size_t)a->size, (const char*)a->name, (unsigned int)a->numOptions, &_out_options, (void**)&_out_optionValues); - return _r; -} +extern hipError_t playback_hipLinkAddData(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipLinkAddFile(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipJitOption _out_options{}; - void* _out_optionValues = nullptr; - hipError_t _r = (hipError_t)hipLinkAddFile((hipLinkState_t)a->state, (hipJitInputType)a->type, (const char*)a->path, (unsigned int)a->numOptions, &_out_options, (void**)&_out_optionValues); + hipJitOption _d_options[32]{}; + uint32_t _d_options_n = a->options_n > 32u ? 32u : a->options_n; + (void)_d_options_n; + if (a->options_present) + std::memcpy(_d_options, a->options_bytes, static_cast(_d_options_n) * sizeof(hipJitOption)); + void* _d_optionValues[32]{}; + uint32_t _d_optionValues_n = a->optionValues_n > 32u ? 32u : a->optionValues_n; + (void)_d_optionValues_n; + if (a->optionValues_present) + std::memcpy(_d_optionValues, a->optionValues_bytes, static_cast(_d_optionValues_n) * sizeof(void*)); + hipError_t _r = (hipError_t)hipLinkAddFile((hipLinkState_t)ctx.translate_link_state(a->state), (hipJitInputType)a->type, (a->path_present ? (const char*)a->path_bytes : (const char*)nullptr), (unsigned int)_d_options_n, (hipJitOption*)(_d_options_n ? _d_options : nullptr), (void**)(_d_optionValues_n ? _d_optionValues : nullptr)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipLinkAddFile", a->ret); + return hipSuccess; + } return _r; } @@ -4012,55 +5076,87 @@ static hipError_t playback_hipLinkComplete(PlaybackContext& ctx, const uint8_t* const auto* a = reinterpret_cast(payload); void* _out_hipBinOut = nullptr; size_t _out_sizeOut{}; - hipError_t _r = (hipError_t)hipLinkComplete((hipLinkState_t)a->state, (void**)&_out_hipBinOut, &_out_sizeOut); + hipError_t _r = (hipError_t)hipLinkComplete((hipLinkState_t)ctx.translate_link_state(a->state), (void**)&_out_hipBinOut, &_out_sizeOut); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipLinkComplete", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipLinkCreate(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipJitOption _out_options{}; - void* _out_optionValues = nullptr; - hipLinkState_t _out_stateOut{}; - hipError_t _r = (hipError_t)hipLinkCreate((unsigned int)a->numOptions, &_out_options, (void**)&_out_optionValues, &_out_stateOut); + hipJitOption _d_options[32]{}; + uint32_t _d_options_n = a->options_n > 32u ? 32u : a->options_n; + (void)_d_options_n; + if (a->options_present) + std::memcpy(_d_options, a->options_bytes, static_cast(_d_options_n) * sizeof(hipJitOption)); + void* _d_optionValues[32]{}; + uint32_t _d_optionValues_n = a->optionValues_n > 32u ? 32u : a->optionValues_n; + (void)_d_optionValues_n; + if (a->optionValues_present) + std::memcpy(_d_optionValues, a->optionValues_bytes, static_cast(_d_optionValues_n) * sizeof(void*)); + hipLinkState_t _out_stateOut = nullptr; + hipError_t _r = (hipError_t)hipLinkCreate((unsigned int)_d_options_n, (hipJitOption*)(_d_options_n ? _d_options : nullptr), (void**)(_d_optionValues_n ? _d_optionValues : nullptr), &_out_stateOut); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipLinkCreate", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.record_link_state(a->stateOut, _out_stateOut); + } return _r; } static hipError_t playback_hipLinkDestroy(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipError_t _r = (hipError_t)hipLinkDestroy((hipLinkState_t)a->state); + uint64_t _rec_hdl = a->state; + hipError_t _r = (hipError_t)hipLinkDestroy((hipLinkState_t)ctx.translate_link_state(_rec_hdl)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipLinkDestroy", a->ret); + return hipSuccess; + } + if (_r == hipSuccess) { + ctx.remove_link_state(_rec_hdl); + } return _r; } static hipError_t playback_hipEventRecordWithFlags(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipEventRecordWithFlags((hipEvent_t)ctx.translate_event(a->event), (hipStream_t)ctx.translate_stream(a->stream), (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipEventRecordWithFlags", a->ret); + return hipSuccess; + } return _r; } -static hipError_t playback_hipLaunchKernelExC(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - void* _out_args = nullptr; - hipError_t _r = (hipError_t)hipLaunchKernelExC((const hipLaunchConfig_t*)a->config, ctx.translate_ptr(a->fPtr), (void**)&_out_args); - return _r; -} +extern hipError_t playback_hipLaunchKernelExC(PlaybackContext& ctx, const uint8_t* payload); -static hipError_t playback_hipDrvLaunchKernelEx(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - void* _out_params = nullptr; - void* _out_extra = nullptr; - hipError_t _r = (hipError_t)hipDrvLaunchKernelEx((const HIP_LAUNCH_CONFIG*)a->config, (hipFunction_t)ctx.translate_func(a->f), (void**)&_out_params, (void**)&_out_extra); - return _r; -} +extern hipError_t playback_hipDrvLaunchKernelEx(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipMemGetHandleForAddressRange(PlaybackContext& ctx, const uint8_t* payload) { +#ifdef _WIN32 (void)ctx; (void)payload; static bool warned = false; if (!warned) { warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipMemGetHandleForAddressRange — " - "this API is not replayed; results may differ from capture.\n"); + fprintf(stderr, "[HRR] hipMemGetHandleForAddressRange is not exported by amdhip64 on " + "Windows; skipping it during replay, results may differ " + "from capture.\n"); } return hipSuccess; +#else + const auto* a = reinterpret_cast(payload); + alignas(8) unsigned char _outbuf_handle[8]{}; + hipError_t _r = (hipError_t)hipMemGetHandleForAddressRange((void*)_outbuf_handle, (hipDeviceptr_t)ctx.translate_ptr(a->dptr), (size_t)a->size, (hipMemRangeHandleType)a->handleType, (unsigned long long)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemGetHandleForAddressRange", a->ret); + return hipSuccess; + } + return _r; +#endif } static hipError_t playback_hipModuleGetFunctionCount(PlaybackContext& ctx, const uint8_t* payload) { @@ -4089,6 +5185,10 @@ static hipError_t playback_hipMemsetD2D8(PlaybackContext& ctx, const uint8_t* pa return hipSuccess; } hipError_t _r = (hipError_t)hipMemsetD2D8((hipDeviceptr_t)_live_dst, (size_t)a->dstPitch, (unsigned char)a->value, (size_t)a->width, (size_t)a->height); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD2D8", a->ret); + return hipSuccess; + } return _r; } @@ -4107,6 +5207,10 @@ static hipError_t playback_hipMemsetD2D8Async(PlaybackContext& ctx, const uint8_ return hipSuccess; } hipError_t _r = (hipError_t)hipMemsetD2D8Async((hipDeviceptr_t)_live_dst, (size_t)a->dstPitch, (unsigned char)a->value, (size_t)a->width, (size_t)a->height, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD2D8Async", a->ret); + return hipSuccess; + } return _r; } @@ -4125,6 +5229,10 @@ static hipError_t playback_hipMemsetD2D16(PlaybackContext& ctx, const uint8_t* p return hipSuccess; } hipError_t _r = (hipError_t)hipMemsetD2D16((hipDeviceptr_t)_live_dst, (size_t)a->dstPitch, (unsigned short)a->value, (size_t)a->width, (size_t)a->height); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD2D16", a->ret); + return hipSuccess; + } return _r; } @@ -4143,6 +5251,10 @@ static hipError_t playback_hipMemsetD2D16Async(PlaybackContext& ctx, const uint8 return hipSuccess; } hipError_t _r = (hipError_t)hipMemsetD2D16Async((hipDeviceptr_t)_live_dst, (size_t)a->dstPitch, (unsigned short)a->value, (size_t)a->width, (size_t)a->height, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD2D16Async", a->ret); + return hipSuccess; + } return _r; } @@ -4161,6 +5273,10 @@ static hipError_t playback_hipMemsetD2D32(PlaybackContext& ctx, const uint8_t* p return hipSuccess; } hipError_t _r = (hipError_t)hipMemsetD2D32((hipDeviceptr_t)_live_dst, (size_t)a->dstPitch, (unsigned int)a->value, (size_t)a->width, (size_t)a->height); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD2D32", a->ret); + return hipSuccess; + } return _r; } @@ -4179,6 +5295,10 @@ static hipError_t playback_hipMemsetD2D32Async(PlaybackContext& ctx, const uint8 return hipSuccess; } hipError_t _r = (hipError_t)hipMemsetD2D32Async((hipDeviceptr_t)_live_dst, (size_t)a->dstPitch, (unsigned int)a->value, (size_t)a->width, (size_t)a->height, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemsetD2D32Async", a->ret); + return hipSuccess; + } return _r; } @@ -4186,6 +5306,10 @@ static hipError_t playback_hipStreamGetAttribute(PlaybackContext& ctx, const uin const auto* a = reinterpret_cast(payload); hipStreamAttrValue _out_value_out{}; hipError_t _r = (hipError_t)hipStreamGetAttribute((hipStream_t)ctx.translate_stream(a->stream), (hipStreamAttrID)a->attr, &_out_value_out); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetAttribute", a->ret); + return hipSuccess; + } return _r; } @@ -4204,23 +5328,45 @@ static hipError_t playback_hipModuleLoadFatBinary(PlaybackContext& ctx, const ui static hipError_t playback_hipMemcpyBatchAsync(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - void* _out_dsts = nullptr; - void* _out_srcs = nullptr; - size_t _out_sizes{}; - hipMemcpyAttributes _out_attrs{}; - size_t _out_attrsIdxs{}; - size_t _out_failIdx{}; - hipError_t _r = (hipError_t)hipMemcpyBatchAsync((void**)&_out_dsts, (void**)&_out_srcs, &_out_sizes, (size_t)a->count, &_out_attrs, &_out_attrsIdxs, (size_t)a->numAttrs, &_out_failIdx, (hipStream_t)ctx.translate_stream(a->stream)); + void* _d_dsts[16]{}; + uint32_t _d_dsts_n = a->dsts_n > 16u ? 16u : a->dsts_n; + (void)_d_dsts_n; + if (a->dsts_present) + std::memcpy(_d_dsts, a->dsts_bytes, static_cast(_d_dsts_n) * sizeof(void*)); + for (uint32_t _i = 0; _i < _d_dsts_n; ++_i) + _d_dsts[_i] = ctx.translate_ptr(reinterpret_cast(_d_dsts[_i])); + void* _d_srcs[16]{}; + uint32_t _d_srcs_n = a->srcs_n > 16u ? 16u : a->srcs_n; + (void)_d_srcs_n; + if (a->srcs_present) + std::memcpy(_d_srcs, a->srcs_bytes, static_cast(_d_srcs_n) * sizeof(void*)); + for (uint32_t _i = 0; _i < _d_srcs_n; ++_i) + _d_srcs[_i] = ctx.translate_ptr(reinterpret_cast(_d_srcs[_i])); + size_t _d_sizes[16]{}; + uint32_t _d_sizes_n = a->sizes_n > 16u ? 16u : a->sizes_n; + (void)_d_sizes_n; + if (a->sizes_present) + std::memcpy(_d_sizes, a->sizes_bytes, static_cast(_d_sizes_n) * sizeof(size_t)); + hipMemcpyAttributes _d_attrs[16]{}; + uint32_t _d_attrs_n = a->attrs_n > 16u ? 16u : a->attrs_n; + (void)_d_attrs_n; + if (a->attrs_present) + std::memcpy(_d_attrs, a->attrs_bytes, static_cast(_d_attrs_n) * sizeof(hipMemcpyAttributes)); + size_t _d_attrsIdxs[16]{}; + uint32_t _d_attrsIdxs_n = a->attrsIdxs_n > 16u ? 16u : a->attrsIdxs_n; + (void)_d_attrsIdxs_n; + if (a->attrsIdxs_present) + std::memcpy(_d_attrsIdxs, a->attrsIdxs_bytes, static_cast(_d_attrsIdxs_n) * sizeof(size_t)); + size_t _d_failIdx{}; + hipError_t _r = (hipError_t)hipMemcpyBatchAsync((void**)(_d_dsts_n ? _d_dsts : nullptr), (void**)(_d_srcs_n ? _d_srcs : nullptr), (size_t*)(_d_sizes_n ? _d_sizes : nullptr), (size_t)_d_dsts_n, (hipMemcpyAttributes*)(_d_attrs_n ? _d_attrs : nullptr), (size_t*)(_d_attrsIdxs_n ? _d_attrsIdxs : nullptr), (size_t)_d_attrs_n, (size_t*)&_d_failIdx, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemcpyBatchAsync", a->ret); + return hipSuccess; + } return _r; } -static hipError_t playback_hipMemcpy3DBatchAsync(PlaybackContext& ctx, const uint8_t* payload) { - const auto* a = reinterpret_cast(payload); - struct hipMemcpy3DBatchOp _out_opList{}; - size_t _out_failIdx{}; - hipError_t _r = (hipError_t)hipMemcpy3DBatchAsync((size_t)a->numOps, &_out_opList, &_out_failIdx, (unsigned long long)a->flags, (hipStream_t)ctx.translate_stream(a->stream)); - return _r; -} +extern hipError_t playback_hipMemcpy3DBatchAsync(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback_hipMemcpy3DPeer(PlaybackContext& ctx, const uint8_t* payload) { (void)ctx; (void)payload; @@ -4268,15 +5414,25 @@ static hipError_t playback_hipGetDriverEntryPoint_spt(PlaybackContext& ctx, cons static hipError_t playback_hipMemPrefetchAsync_v2(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipMemLocation _s_location{}; - hipError_t _r = (hipError_t)hipMemPrefetchAsync_v2(ctx.translate_ptr(a->dev_ptr), (size_t)a->count, _s_location, (unsigned int)a->flags, (hipStream_t)ctx.translate_stream(a->stream)); + hipMemLocation _v_location{}; + std::memcpy(&_v_location, a->location_bytes, sizeof(_v_location)); + hipError_t _r = (hipError_t)hipMemPrefetchAsync_v2(ctx.translate_ptr(a->dev_ptr), (size_t)a->count, _v_location, (unsigned int)a->flags, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemPrefetchAsync_v2", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemAdvise_v2(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipMemLocation _s_device{}; - hipError_t _r = (hipError_t)hipMemAdvise_v2(ctx.translate_ptr(a->dev_ptr), (size_t)a->count, (hipMemoryAdvise)a->advice, _s_device); + hipMemLocation _v_device{}; + std::memcpy(&_v_device, a->device_bytes, sizeof(_v_device)); + hipError_t _r = (hipError_t)hipMemAdvise_v2(ctx.translate_ptr(a->dev_ptr), (size_t)a->count, (hipMemoryAdvise)a->advice, _v_device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemAdvise_v2", a->ret); + return hipSuccess; + } return _r; } @@ -4284,6 +5440,10 @@ static hipError_t playback_hipStreamGetId(PlaybackContext& ctx, const uint8_t* p const auto* a = reinterpret_cast(payload); unsigned long long _out_streamId{}; hipError_t _r = (hipError_t)hipStreamGetId((hipStream_t)ctx.translate_stream(a->stream), &_out_streamId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetId", a->ret); + return hipSuccess; + } return _r; } @@ -4345,6 +5505,10 @@ static hipError_t playback_hipLibraryGetKernelCount(PlaybackContext& ctx, const static hipError_t playback_hipStreamCopyAttributes(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipStreamCopyAttributes((hipStream_t)ctx.translate_stream(a->dst), (hipStream_t)ctx.translate_stream(a->src)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamCopyAttributes", a->ret); + return hipSuccess; + } return _r; } @@ -4396,7 +5560,11 @@ static hipError_t playback_hipGetProcAddress_spt(PlaybackContext& ctx, const uin const auto* a = reinterpret_cast(payload); void* _out_pfn = nullptr; hipDriverProcAddressQueryResult _out_symbolStatus{}; - hipError_t _r = (hipError_t)hipGetProcAddress_spt((const char*)a->symbol, (void**)&_out_pfn, (int)a->hipVersion, (uint64_t)a->flags, &_out_symbolStatus); + hipError_t _r = (hipError_t)hipGetProcAddress_spt((a->symbol_present ? (const char*)a->symbol_bytes : (const char*)nullptr), (void**)&_out_pfn, (int)a->hipVersion, (uint64_t)a->flags, &_out_symbolStatus); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGetProcAddress_spt", a->ret); + return hipSuccess; + } return _r; } @@ -4414,37 +5582,61 @@ static hipError_t playback_hipKernelGetParamInfo(PlaybackContext& ctx, const uin static hipError_t playback_hipExtDisableLogging(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipExtDisableLogging(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExtDisableLogging", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipExtEnableLogging(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipExtEnableLogging(); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExtEnableLogging", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipExtSetLoggingParams(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipExtSetLoggingParams((size_t)a->log_level, (size_t)a->log_size, (size_t)a->log_mask); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExtSetLoggingParams", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemSetMemPool(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); - hipMemLocation* _s_location{}; - hipError_t _r = (hipError_t)hipMemSetMemPool(_s_location, (hipMemAllocationType)a->type, (hipMemPool_t)ctx.translate_mempool(a->pool)); + hipMemLocation _d_location{}; + if (a->location_present) + std::memcpy(&_d_location, a->location_bytes, sizeof(_d_location)); + hipError_t _r = (hipError_t)hipMemSetMemPool((a->location_present ? (hipMemLocation*)&_d_location : (hipMemLocation*)nullptr), (hipMemAllocationType)a->type, (hipMemPool_t)ctx.translate_mempool(a->pool)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemSetMemPool", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipMemGetMemPool(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - static bool warned = false; - if (!warned) { - warned = true; - fprintf(stderr, "[HRR] NOOP playback handler called for hipMemGetMemPool — " - "this API is not replayed; results may differ from capture.\n"); + const auto* a = reinterpret_cast(payload); + hipMemPool_t _out_pool = nullptr; + hipMemLocation _d_location{}; + if (a->location_present) + std::memcpy(&_d_location, a->location_bytes, sizeof(_d_location)); + hipError_t _r = (hipError_t)hipMemGetMemPool(&_out_pool, (a->location_present ? (hipMemLocation*)&_d_location : (hipMemLocation*)nullptr), (hipMemAllocationType)a->type); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemGetMemPool", a->ret); + return hipSuccess; } - return hipSuccess; + if (_r == hipSuccess) { + ctx.record_mempool(a->pool, _out_pool); + } + return _r; } static hipError_t playback_hipMipmappedArrayGetMemoryRequirements(PlaybackContext& ctx, const uint8_t* payload) { @@ -4498,6 +5690,10 @@ static hipError_t playback_hipMemPrefetchBatchAsync(PlaybackContext& ctx, const hipMemLocation* _s_prefetch_locs{}; size_t _out_prefetch_loc_idxs{}; hipError_t _r = (hipError_t)hipMemPrefetchBatchAsync((void**)&_out_dev_ptrs, &_out_sizes, (size_t)a->count, _s_prefetch_locs, &_out_prefetch_loc_idxs, (size_t)a->num_prefetch_locs, (unsigned long long)a->flags, (hipStream_t)ctx.translate_stream(a->stream)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipMemPrefetchBatchAsync", a->ret); + return hipSuccess; + } return _r; } @@ -4527,12 +5723,20 @@ static hipError_t playback_hipGreenCtxCreate(PlaybackContext& ctx, const uint8_t const auto* a = reinterpret_cast(payload); hipExecutionCtx_t _out_ctx{}; hipError_t _r = (hipError_t)hipGreenCtxCreate(&_out_ctx, (hipDevResourceDesc_t)a->desc, (int)a->device, (unsigned int)a->flags); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipGreenCtxCreate", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipExecutionCtxDestroy(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipExecutionCtxDestroy((hipExecutionCtx_t)a->ctx); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExecutionCtxDestroy", a->ret); + return hipSuccess; + } return _r; } @@ -4540,6 +5744,10 @@ static hipError_t playback_hipExecutionCtxStreamCreate(PlaybackContext& ctx, con const auto* a = reinterpret_cast(payload); hipStream_t _out_stream = nullptr; hipError_t _r = (hipError_t)hipExecutionCtxStreamCreate(&_out_stream, (hipExecutionCtx_t)a->greenctx, (unsigned int)a->flags, (int)a->priority); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExecutionCtxStreamCreate", a->ret); + return hipSuccess; + } return _r; } @@ -4547,6 +5755,10 @@ static hipError_t playback_hipDeviceGetDevResource(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); hipDevResource _out_resource{}; hipError_t _r = (hipError_t)hipDeviceGetDevResource((hipDevice_t)a->device, &_out_resource, (hipDevResourceType)a->type); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetDevResource", a->ret); + return hipSuccess; + } return _r; } @@ -4556,6 +5768,10 @@ static hipError_t playback_hipDevSmResourceSplitByCount(PlaybackContext& ctx, co unsigned int _out_nbGroups{}; hipDevResource _out_remainder{}; hipError_t _r = (hipError_t)hipDevSmResourceSplitByCount(&_out_result, &_out_nbGroups, (const hipDevResource*)a->input, &_out_remainder, (unsigned int)a->flags, (unsigned int)a->minCount); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDevSmResourceSplitByCount", a->ret); + return hipSuccess; + } return _r; } @@ -4565,6 +5781,10 @@ static hipError_t playback_hipDevSmResourceSplit(PlaybackContext& ctx, const uin hipDevResource _out_remainder{}; hipDevSmResourceGroupParams _out_groupParams{}; hipError_t _r = (hipError_t)hipDevSmResourceSplit(&_out_result, (unsigned int)a->nbGroups, (const hipDevResource*)a->input, &_out_remainder, (unsigned int)a->flags, &_out_groupParams); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDevSmResourceSplit", a->ret); + return hipSuccess; + } return _r; } @@ -4573,6 +5793,10 @@ static hipError_t playback_hipDevResourceGenerateDesc(PlaybackContext& ctx, cons hipDevResourceDesc_t _out_phDesc{}; hipDevResource _out_resources{}; hipError_t _r = (hipError_t)hipDevResourceGenerateDesc(&_out_phDesc, &_out_resources, (unsigned int)a->nbResources); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDevResourceGenerateDesc", a->ret); + return hipSuccess; + } return _r; } @@ -4580,6 +5804,10 @@ static hipError_t playback_hipDeviceGetExecutionCtx(PlaybackContext& ctx, const const auto* a = reinterpret_cast(payload); hipExecutionCtx_t _out_ctx{}; hipError_t _r = (hipError_t)hipDeviceGetExecutionCtx(&_out_ctx, (int)a->device); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceGetExecutionCtx", a->ret); + return hipSuccess; + } return _r; } @@ -4587,6 +5815,10 @@ static hipError_t playback_hipExecutionCtxGetDevResource(PlaybackContext& ctx, c const auto* a = reinterpret_cast(payload); hipDevResource _out_resource{}; hipError_t _r = (hipError_t)hipExecutionCtxGetDevResource((hipExecutionCtx_t)a->ctx, &_out_resource, (hipDevResourceType)a->type); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExecutionCtxGetDevResource", a->ret); + return hipSuccess; + } return _r; } @@ -4594,6 +5826,10 @@ static hipError_t playback_hipExecutionCtxGetDevice(PlaybackContext& ctx, const const auto* a = reinterpret_cast(payload); int _out_device{}; hipError_t _r = (hipError_t)hipExecutionCtxGetDevice(&_out_device, (hipExecutionCtx_t)a->ctx); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExecutionCtxGetDevice", a->ret); + return hipSuccess; + } return _r; } @@ -4601,6 +5837,10 @@ static hipError_t playback_hipExecutionCtxGetId(PlaybackContext& ctx, const uint const auto* a = reinterpret_cast(payload); unsigned long long _out_ctxId{}; hipError_t _r = (hipError_t)hipExecutionCtxGetId((hipExecutionCtx_t)a->ctx, &_out_ctxId); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExecutionCtxGetId", a->ret); + return hipSuccess; + } return _r; } @@ -4608,24 +5848,40 @@ static hipError_t playback_hipStreamGetDevResource(PlaybackContext& ctx, const u const auto* a = reinterpret_cast(payload); hipDevResource _out_resource{}; hipError_t _r = (hipError_t)hipStreamGetDevResource((hipStream_t)ctx.translate_stream(a->hStream), &_out_resource, (hipDevResourceType)a->type); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipStreamGetDevResource", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipExecutionCtxRecordEvent(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipExecutionCtxRecordEvent((hipExecutionCtx_t)a->ctx, (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExecutionCtxRecordEvent", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipExecutionCtxSynchronize(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipExecutionCtxSynchronize((hipExecutionCtx_t)a->ctx); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExecutionCtxSynchronize", a->ret); + return hipSuccess; + } return _r; } static hipError_t playback_hipExecutionCtxWaitEvent(PlaybackContext& ctx, const uint8_t* payload) { const auto* a = reinterpret_cast(payload); hipError_t _r = (hipError_t)hipExecutionCtxWaitEvent((hipExecutionCtx_t)a->ctx, (hipEvent_t)ctx.translate_event(a->event)); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipExecutionCtxWaitEvent", a->ret); + return hipSuccess; + } return _r; } @@ -4634,6 +5890,10 @@ static hipError_t playback_hipLibraryGetGlobal(PlaybackContext& ctx, const uint8 void* _out_dptr = nullptr; size_t _out_bytes{}; hipError_t _r = (hipError_t)hipLibraryGetGlobal((void**)&_out_dptr, &_out_bytes, (hipLibrary_t)a->library, (const char*)a->name); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipLibraryGetGlobal", a->ret); + return hipSuccess; + } return _r; } @@ -4642,6 +5902,10 @@ static hipError_t playback_hipLibraryGetManaged(PlaybackContext& ctx, const uint void* _out_dptr = nullptr; size_t _out_bytes{}; hipError_t _r = (hipError_t)hipLibraryGetManaged((void**)&_out_dptr, &_out_bytes, (hipLibrary_t)a->library, (const char*)a->name); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipLibraryGetManaged", a->ret); + return hipSuccess; + } return _r; } @@ -4765,10 +6029,7 @@ static hipError_t playback___hipRegisterTexture(PlaybackContext& ctx, const uint return hipSuccess; } -static hipError_t playback___hipRegisterVar(PlaybackContext& ctx, const uint8_t* payload) { - (void)ctx; (void)payload; - return hipSuccess; -} +extern hipError_t playback___hipRegisterVar(PlaybackContext& ctx, const uint8_t* payload); static hipError_t playback___hipUnregisterFatBinary(PlaybackContext& ctx, const uint8_t* payload) { (void)ctx; (void)payload; @@ -4993,7 +6254,7 @@ const uint32_t hrr_api_min_payload_size[HRR_API_COUNT] = { static_cast(sizeof(hrr_args_hipKernelNameRef)), // [208] HRR_API_HIPKERNELNAMEREF static_cast(sizeof(hrr_args_hipKernelNameRefByPtr)), // [209] HRR_API_HIPKERNELNAMEREFBYPTR static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [210] variable-length kernel launch - static_cast(sizeof(hrr_args_hipLaunchCooperativeKernel)), // [211] HRR_API_HIPLAUNCHCOOPERATIVEKERNEL + static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [211] variable-length kernel launch static_cast(sizeof(hrr_args_hipLaunchCooperativeKernelMultiDevice)), // [212] HRR_API_HIPLAUNCHCOOPERATIVEKERNELMULTIDEVICE static_cast(sizeof(hrr_args_hipLaunchHostFunc)), // [213] HRR_API_HIPLAUNCHHOSTFUNC static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [214] variable-length kernel launch @@ -5088,7 +6349,7 @@ const uint32_t hrr_api_min_payload_size[HRR_API_COUNT] = { static_cast(sizeof(hrr_args_hipModuleGetFunction)), // [303] HRR_API_HIPMODULEGETFUNCTION static_cast(sizeof(hrr_args_hipModuleGetGlobal)), // [304] HRR_API_HIPMODULEGETGLOBAL static_cast(sizeof(hrr_args_hipModuleGetTexRef)), // [305] HRR_API_HIPMODULEGETTEXREF - static_cast(sizeof(hrr_args_hipModuleLaunchCooperativeKernel)), // [306] HRR_API_HIPMODULELAUNCHCOOPERATIVEKERNEL + static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [306] variable-length kernel launch static_cast(sizeof(hrr_args_hipModuleLaunchCooperativeKernelMultiDevice)), // [307] HRR_API_HIPMODULELAUNCHCOOPERATIVEKERNELMULTIDEVICE static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [308] variable-length kernel launch static_cast(sizeof(hrr_args_hipModuleLoad)), // [309] HRR_API_HIPMODULELOAD @@ -5200,8 +6461,8 @@ const uint32_t hrr_api_min_payload_size[HRR_API_COUNT] = { static_cast(sizeof(hrr_args_hipStreamGetFlags_spt)), // [415] HRR_API_HIPSTREAMGETFLAGS_SPT static_cast(sizeof(hrr_args_hipStreamAddCallback_spt)), // [416] HRR_API_HIPSTREAMADDCALLBACK_SPT static_cast(sizeof(hrr_args_hipEventRecord_spt)), // [417] HRR_API_HIPEVENTRECORD_SPT - static_cast(sizeof(hrr_args_hipLaunchCooperativeKernel_spt)), // [418] HRR_API_HIPLAUNCHCOOPERATIVEKERNEL_SPT - static_cast(sizeof(hrr_args_hipLaunchKernel_spt)), // [419] HRR_API_HIPLAUNCHKERNEL_SPT + static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [418] variable-length kernel launch + static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [419] variable-length kernel launch static_cast(sizeof(hrr_args_hipGraphLaunch_spt)), // [420] HRR_API_HIPGRAPHLAUNCH_SPT static_cast(sizeof(hrr_args_hipStreamBeginCapture_spt)), // [421] HRR_API_HIPSTREAMBEGINCAPTURE_SPT static_cast(sizeof(hrr_args_hipStreamEndCapture_spt)), // [422] HRR_API_HIPSTREAMENDCAPTURE_SPT @@ -5256,8 +6517,8 @@ const uint32_t hrr_api_min_payload_size[HRR_API_COUNT] = { static_cast(sizeof(hrr_args_hipLinkCreate)), // [471] HRR_API_HIPLINKCREATE static_cast(sizeof(hrr_args_hipLinkDestroy)), // [472] HRR_API_HIPLINKDESTROY static_cast(sizeof(hrr_args_hipEventRecordWithFlags)), // [473] HRR_API_HIPEVENTRECORDWITHFLAGS - static_cast(sizeof(hrr_args_hipLaunchKernelExC)), // [474] HRR_API_HIPLAUNCHKERNELEXC - static_cast(sizeof(hrr_args_hipDrvLaunchKernelEx)), // [475] HRR_API_HIPDRVLAUNCHKERNELEX + static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [474] variable-length kernel launch + static_cast(sizeof(hrr_event_header) + 8u + 2u + 16u + 12u + 12u + 4u + 2u + 2u), // [475] variable-length kernel launch static_cast(sizeof(hrr_args_hipMemGetHandleForAddressRange)), // [476] HRR_API_HIPMEMGETHANDLEFORADDRESSRANGE static_cast(sizeof(hrr_args_hipModuleGetFunctionCount)), // [477] HRR_API_HIPMODULEGETFUNCTIONCOUNT static_cast(sizeof(hrr_args_hipMemsetD2D8)), // [478] HRR_API_HIPMEMSETD2D8 diff --git a/projects/hrr/playback/hrr_playback.cpp b/projects/hrr/playback/hrr_playback.cpp index 9a7612d34851..e12aed3788bb 100644 --- a/projects/hrr/playback/hrr_playback.cpp +++ b/projects/hrr/playback/hrr_playback.cpp @@ -1383,6 +1383,17 @@ int main(int argc, char** argv) { env_size_or("HIP_HRR_REPLAY_SYNC_WATCHDOG_MS", ctx.sync_watchdog_ms)); ctx.dump_ptrs_ordinal = env_size_or("HIP_HRR_REPLAY_DUMP_PTRS_ORDINAL", ctx.dump_ptrs_ordinal); + ctx.scan_args_ordinal = + env_size_or("HIP_HRR_REPLAY_SCAN_ARGS_ORDINAL", ctx.scan_args_ordinal); + ctx.scan_args_bytes = + env_size_or("HIP_HRR_REPLAY_SCAN_ARGS_BYTES", ctx.scan_args_bytes); + ctx.scan_h2d = env_size_or("HIP_HRR_REPLAY_SCAN_H2D", 0u) != 0; + ctx.audit_host_args = + env_size_or("HIP_HRR_REPLAY_AUDIT_HOST_ARGS", 0u) != 0; + // The scan reads the pointer arguments, which the pointer dump is what + // collects, so asking for one implies the other. + if (ctx.scan_args_ordinal && !ctx.dump_ptrs_ordinal) + ctx.dump_ptrs_ordinal = ctx.scan_args_ordinal; if (archive_path.empty()) { fprintf(stderr, "[HRR] No archive path specified\n"); @@ -1633,6 +1644,9 @@ int main(int argc, char** argv) { printf("[HRR] GPU graph time : %.1f ms\n", ctx.total_graph_ms); printf("[HRR] GPU total time : %.1f ms\n", ctx.total_kernel_ms + ctx.total_graph_ms); } + if (!ctx.co_no_device_code.empty()) + printf("[HRR] Fat binaries : %zu skipped (no code for this GPU)\n", + ctx.co_no_device_code.size()); printf("[HRR] D2H checks : %zu pass (%zu exact, %zu within tol), %zu fail, %zu skipped\n", ctx.d2h_pass.load(), ctx.d2h_pass.load() - ctx.d2h_pass_tol.load(), ctx.d2h_pass_tol.load(), diff --git a/projects/hrr/skills/decode-and-triage/SKILL.md b/projects/hrr/skills/decode-and-triage/SKILL.md index 1b8834aad446..eaca21d92445 100644 --- a/projects/hrr/skills/decode-and-triage/SKILL.md +++ b/projects/hrr/skills/decode-and-triage/SKILL.md @@ -6,16 +6,20 @@ description: >- triage_archive.ps1 + ensure_playback.ps1 (PowerShell); Docker replay requires Linux or WSL2. Never edits source. Print finding summary in the chat reply. inputs: - - HRR archive path (`capture.hrr/pid-*` directory, or capture root to resolve) + - HRR archive path: one `capture.hrr/pid-*` directory. A capture root has to be resolved to a process directory first, by hand (see Workflow step 1) - Optional Docker image from capture (`HRR_DOCKER_IMAGE`) - Optional GPU ordinal (`GPU`) outputs: - Finding summary markdown in chat (outcome, fault class, kernel, fault address, event seq, archive stats) - - Optional finding file under `HRR_TRIAGE_WORKDIR` (script default) + - Optional finding file under `HRR_TRIAGE_WORKDIR`, defaulting to a temporary directory and never the archive --- # HRR Decode & Triage +**No archive yet?** This skill starts from one. To record a failing workload +into an archive first, use +[enable-recording](../enable-recording/SKILL.md). + **Platform (Linux):** AMD GPU host. Full skill workflow via `triage_archive.sh`: native GPU replay (auto-builds `hrr-playback` when missing), optional Docker replay, manifest preflight, and finding output. @@ -91,7 +95,7 @@ Environment variables on Windows: | `HIP_PATH` or `ROCM_PATH` | HIP SDK root (default: `C:\Program Files\AMD\ROCm\6.2`) | | `HRR_BUILD` | Dedicated HRR build dir for `--build` (default: `projects/hrr\build-playback`) | | `GPU` | GPU ordinal for replay (default: `0`) | -| `HRR_TRIAGE_WORKDIR` | Output dir for findings + logs | +| `HRR_TRIAGE_WORKDIR` | Output dir for findings + logs. Never the archive: a value inside it is refused. Default on Linux `$TMPDIR/hrr-triage-` created mode 0700 (`/tmp` when `TMPDIR` is inside the archive), or a `mktemp` directory when that one is not ours; on Windows the user's temp directory plus `hrr-triage` | | `HRR_CONTINUE=1` | Proceed past HIP/comgr version mismatch | | `HRR_SKIP_COMPAT=1` | Skip manifest preflight entirely | @@ -113,8 +117,8 @@ Docker GPU replay (`--device=/dev/kfd`) is **Linux-only**. On Windows: #### W4 — Print finding summary -The `.finding.md` written by `triage_archive.ps1` is identical in schema to the Linux version. -Print or paste it directly in the chat reply using the same template in Step 5 below. +The `.finding.md` written by `triage_archive.ps1` has the same schema as the Linux one, but the Windows script replays without `--sync-after-launch`, so a fault there is not attributed to a launch (see Kernel attribution below). +Print or paste it directly in the chat reply using the same template in Step 6 below. #### Copying scripts to a remote machine (no Cursor installed) @@ -151,14 +155,14 @@ Docker image used for capture: **Replay mode (`auto` default):** picks docker when `HRR_DOCKER_IMAGE` is set, else native. Use `--no-replay` for metadata-only (no GPU). -1. Resolve `capture.hrr/pid-*` (largest `events.bin` if user gives root only). +1. Resolve `capture.hrr/pid-*` yourself: `--archive` takes one process directory, and no script picks it for you. Read each `pid-*/manifest.json`: a process that died leaves `"complete": false`, and one killed outright (SIGKILL, the OOM killer, a scheduler) leaves no manifest at all. Prefer either of those over a process that shut down cleanly, and triage each of them if there are several. Size is the wrong rule for a crash, because the process that died is the one cut short. The archive still travels whole. `hrr-playback --info ` prints this as a table without touching the GPU: one row per `pid-*`, with a Complete column that reads `NO` for both. 2. **Preflight** when `manifest.json` has `metadata`: block if capture used more GPUs than replay exposes or requested `GPU` is missing. HIP/comgr mismatch prompts for confirmation (exit 2 in agents — ask user, then `HRR_CONTINUE=1`). Docker preflight uses the **image HRR stack** by default; `HRR_DOCKER_MOUNT_CLR=1` overlays host `CLR_BUILD`/`HRR_PLAYBACK` for WIP dev work. Legacy archives skip preflight. Overrides: `HRR_SKIP_COMPAT=1`, `HRR_STRICT_VERSION=1`, `HRR_STRICT_ARCH=1`. -3. **Native (Linux):** `triage_archive.sh --archive ` (builds/finds host `hrr-playback`). +3. **Native (Linux):** `triage_archive.sh --archive ` (builds/finds host `hrr-playback`). The script puts `hrr-playback`'s own `lib/` and `runtime-lib/` first on `LD_LIBRARY_PATH`, then `ROCR_LIB` (a directory holding the `libhsa-runtime64.so.1` that matches it), then your own `LD_LIBRARY_PATH`, and `$ROCM_PATH/lib` last. 4. **Native (Windows):** `triage_archive.ps1 --archive ` (finds/builds `hrr-playback.exe`). 5. **Docker (Linux only):** `export HRR_DOCKER_IMAGE=''` then `triage_archive.sh --archive --replay docker`. Image `hrr-playback` by @@ -233,7 +237,7 @@ than one kernel it stays unknown rather than guessing. Names that `--info` truncated to its column width are never used: a cut-off symbol cannot be looked up or handed to a kernel developer. -### Finding summary template (Step 5 / W4) +### Finding summary template (Step 6 / W4) Print a structured summary directly in the chat reply: @@ -294,8 +298,9 @@ Classify each finding using these labels: | Situation | Action | |-----------|--------| -| `--build` fails | Install `hrr-playback` under `$ROCM_PATH/bin` or set `HRR_PLAYBACK`; ensure a capture-enabled ROCm/HIP prefix (see README build section). | -| Native replay fails (Linux) | Docker replay with capture image (Linux host). | +| `--build` fails | Install `hrr-playback` under `$ROCM_PATH/bin` or set `HRR_PLAYBACK`; ensure a capture-enabled ROCm/HIP prefix (see `../../README.md`, "Build `hrr-playback`"). | +| `undefined symbol` from the reader itself (Linux) | Its own libraries are not in front. A packaged build ships `bin/`, `lib/` and `runtime-lib/` as siblings and the script puts `lib/` and `runtime-lib/` on the library path; a build tree may not, so point `ROCR_LIB` at the directory holding the matching `libhsa-runtime64.so.1`. This is a one-line fix and comes before trying Docker. | +| Native replay fails for any other reason (Linux) | Docker replay with capture image (Linux host). | | Native replay fails (Windows) | Check `HRR_PLAYBACK`, `HIP_PATH\bin` on `PATH`; use `--no-replay` as fallback. | | Windows host, no GPU | Metadata-only via `triage_archive.ps1 --no-replay` or `analyze_replay_finding.py --archive`. | | Docker `hrr-playback` not in image | `HRR_DOCKER_MOUNT_CLR=1` with `CLR_BUILD` / `HRR_PLAYBACK`. | @@ -310,7 +315,8 @@ Classify each finding using these labels: | `--info` | Archive summary (events, kernels, blobs, complete flag), no GPU | | `--events` | Full event log with `--info` (includes Kernel Call Counts) | | `--timing` | Kernel + graph GPU time | -| `--verbose` | Per-event trace | +| `--verbose` | Per-event trace. It prints the API name per event and not the kernel, so it cannot tell you which kernel faulted | +| `--trace-kernels` | Prints `[HRR launch] ... name="..."` before each launch. When a fault names no kernel, rerun by hand with `hrr-playback --sync-after-launch --trace-kernels`: the last launch line before the fault is the one that faulted | | `--sync-after-launch` | Surface GPU errors per kernel | | `--kernel-filter STR` | Replay one kernel only | | `--multi-thread` | One thread per captured thread | diff --git a/projects/hrr/skills/decode-and-triage/evals/evals.json b/projects/hrr/skills/decode-and-triage/evals/evals.json new file mode 100644 index 000000000000..d6cb540e2662 --- /dev/null +++ b/projects/hrr/skills/decode-and-triage/evals/evals.json @@ -0,0 +1,66 @@ +{ + "evaluations": [ + { + "id": "customer-archive-which-kernel-faulted", + "skill_should_trigger": true, + "prompt": "A customer sent us an HRR archive at /work/incoming/crash.hrr. Which kernel faulted?", + "expected_behavior": [ + "Replays the archive rather than reasoning from the archive listing alone", + "Reports the outcome and fault class alongside the kernel, rather than the kernel on its own" + ], + "unexpected_behavior": [ + "Names a kernel taken from the archive's kernel list without a replay attributing the fault to it" + ] + }, + { + "id": "multi-process-archive-which-directory", + "skill_should_trigger": true, + "prompt": "This HRR archive has three pid-* directories and one of them shows Complete: NO. Which one do I work from?", + "expected_behavior": [ + "Works from the process that shows Complete: NO, because that is the one that died", + "Says the archive travels whole even though each process is triaged separately" + ], + "unexpected_behavior": [ + "Picks the process with the largest events.bin without considering which one crashed" + ] + }, + { + "id": "replay-reports-an-aten-kernel", + "skill_should_trigger": true, + "prompt": "I replayed a customer's archive and the fault came back on at::native::vectorized_elementwise_kernel. Is that their bug?", + "expected_behavior": [ + "Asks for the customer's original failure signature before attributing the fault to their workload", + "Explains that an archive recorded before chevron-launch pointer support can fault on replay for a workload that was fine" + ] + }, + { + "id": "replay-crashed-with-no-verdict", + "skill_should_trigger": true, + "prompt": "The triage run exited 134 and the finding says outcome ABORT, fault class replay_crashed. What does that tell me about the customer's kernel?", + "expected_behavior": [ + "Says the replay process itself died and implicates no kernel" + ], + "unexpected_behavior": [ + "Treats replay_crashed as evidence about the captured workload" + ] + }, + { + "id": "no-archive-yet", + "skill_should_trigger": false, + "note": "Nothing to triage: recording the failure first is the other skill's job.", + "prompt": "Our training job crashes on an MI300 with a memory access fault. What do we send AMD so they can reproduce it?" + }, + { + "id": "performance-question", + "skill_should_trigger": false, + "note": "Replay timings are replay-side and measure nothing about the original run.", + "prompt": "Our model runs slower than we expect on MI300X. Where is the time going?" + }, + { + "id": "cuda-crash", + "skill_should_trigger": false, + "note": "No HRR archive and no AMD GPU involved.", + "prompt": "My CUDA application segfaults inside a kernel on an NVIDIA card. How do I find the cause?" + } + ] +} diff --git a/projects/hrr/skills/decode-and-triage/scripts/analyze_replay_finding.py b/projects/hrr/skills/decode-and-triage/scripts/analyze_replay_finding.py index 81c4c0433cf3..a35aa9c15b7b 100755 --- a/projects/hrr/skills/decode-and-triage/scripts/analyze_replay_finding.py +++ b/projects/hrr/skills/decode-and-triage/scripts/analyze_replay_finding.py @@ -97,6 +97,7 @@ RE_ATEN_CHEVRON = re.compile(r"_ZN2at6native|at::native::") RE_CAPTURE_MAF = RE_MAF RE_D2H_SUMMARY = re.compile(r"D2H checks\s+: (\d+) pass.*?, (\d+) fail, (\d+) skipped") +RE_KERNELS_LAUNCHED = re.compile(r"\[HRR\]\s+Kernels launched:\s*(\d+)") RE_KERNARG = re.compile(r"kernarg_address=(0x[0-9a-fA-F]+)") RE_GRID = re.compile(r"grid=\[([^\]]+)\], workgroup=\[([^\]]+)\]") RE_CIJK = re.compile(r"(Cijk_[A-Za-z0-9_]+)") @@ -343,10 +344,19 @@ def parse_text(text: str, source: str, finding: Finding) -> Finding: finding.grid = m.group(1) finding.workgroup = m.group(2) + # The replay summary carries the end-of-run totals, so it wins over the + # last progress line, which is printed at the first kernel and then only + # periodically. It gives skipped rather than attempted, and skipped is + # attempted less pass and fail. m = RE_D2H_SUMMARY.search(text) if m: finding.d2h_pass = int(m.group(1)) finding.d2h_fail = int(m.group(2)) + finding.d2h_attempted = finding.d2h_pass + finding.d2h_fail + int(m.group(3)) + + m = RE_KERNELS_LAUNCHED.search(text) + if m: + finding.kernels_launched = int(m.group(1)) new_class = _classify(text, finding) if new_class != "unknown" or finding.fault_class in (None, "unknown"): @@ -392,11 +402,18 @@ def finalize(finding: Finding) -> Finding: """ # A clean replay implicates no kernel. The archive still lists the kernels # it ran, and a GEMM matched out of that listing would sit in the report - # next to a pass as though it were a culprit. + # next to a pass as though it were a culprit. The same holds for the event + # and kernel a progress line named on the way, and the JSON report prints + # every field, so they are cleared rather than left for markdown to hide. if finding.fault_class == "replay_pass": finding.kernel_name = None finding.kernel_family = None finding.last_event_kernel = None + finding.last_progress_kernel = None + finding.failing_event_seq = None + finding.failing_call_index = None + finding.failing_thread = None + finding.failing_api = None return finding # The replay process died without reaching a verdict, so nothing here is a @@ -485,6 +502,12 @@ def run_archive_info(archive: Path, hrr_playback: str | None) -> str: def render_markdown(f: Finding) -> str: + d2h = ( + f"- **D2H**: pass={f.d2h_pass or 0} fail={f.d2h_fail or 0} " + f"attempted={f.d2h_attempted or 0}" + ) + # A replay can launch no kernels at all, and that zero is still a count. + kernels = "n/a" if f.kernels_launched is None else f.kernels_launched lines = [ "# HRR replay finding", "", @@ -494,22 +517,40 @@ def render_markdown(f: Finding) -> str: f"- **Kernel**: `{f.kernel_name or 'unknown'}`", f"- **Kernel family**: `{f.kernel_family or 'unknown'}`", "", - "## Fault details", - f"- **Fault address**: `{f.fault_address or 'n/a'}`", - f"- **Fault reason**: {f.fault_reason or 'n/a'}", - f"- **Failing event seq**: {f.failing_event_seq or 'n/a'}", - f"- **Failing call index**: {f.failing_call_index or 'n/a'}", - f"- **Failing API**: {f.failing_api or 'n/a'}", - f"- **Kernarg address**: `{f.kernarg_address or 'n/a'}`", - f"- **GPU node**: {f.gpu_node or 'n/a'}", - f"- **Grid / workgroup**: {f.grid or 'n/a'} / {f.workgroup or 'n/a'}", - "", - "## Replay progress at fault", - f"- **Kernels launched**: {f.kernels_launched or 'n/a'}", - f"- **D2H**: pass={f.d2h_pass or 0} fail={f.d2h_fail or 0} attempted={f.d2h_attempted or 0}", - f"- **Last progress kernel**: `{f.last_progress_kernel or 'n/a'}`", - f"- **Last launch before fault**: `{f.last_event_kernel or 'n/a'}`", - "", + ] + # A clean replay has no fault details and no fault to be at, and printing + # a progress counter under "at fault" read as one: a PASS came back naming + # a failing event and a kernel taken from a progress line. Keyed on the + # class, as finalize() is, so a PASS line with D2H failures, which is a + # divergence, keeps its fault details. + if f.fault_class == "replay_pass": + lines += [ + "## Replay result", + d2h, + f"- **Kernels launched**: {kernels}", + "", + ] + else: + lines += [ + "## Fault details", + f"- **Fault address**: `{f.fault_address or 'n/a'}`", + f"- **Fault reason**: {f.fault_reason or 'n/a'}", + # Both are zero-based, so 0 is the first event, not a missing one. + f"- **Failing event seq**: {'n/a' if f.failing_event_seq is None else f.failing_event_seq}", + f"- **Failing call index**: {'n/a' if f.failing_call_index is None else f.failing_call_index}", + f"- **Failing API**: {f.failing_api or 'n/a'}", + f"- **Kernarg address**: `{f.kernarg_address or 'n/a'}`", + f"- **GPU node**: {f.gpu_node or 'n/a'}", + f"- **Grid / workgroup**: {f.grid or 'n/a'} / {f.workgroup or 'n/a'}", + "", + "## Replay progress at fault", + f"- **Kernels launched**: {kernels}", + d2h, + f"- **Last progress kernel**: `{f.last_progress_kernel or 'n/a'}`", + f"- **Last launch before fault**: `{f.last_event_kernel or 'n/a'}`", + "", + ] + lines += [ "## Archive / capture", f"- **Events**: {f.archive_events or 'n/a'}", f"- **Kernels (archive)**: {f.archive_kernels or 'n/a'}", diff --git a/projects/hrr/skills/decode-and-triage/scripts/test_triage_archive.py b/projects/hrr/skills/decode-and-triage/scripts/test_triage_archive.py deleted file mode 100644 index d7524bacd920..000000000000 --- a/projects/hrr/skills/decode-and-triage/scripts/test_triage_archive.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 -"""Unit tests for triage_archive.sh (stdlib unittest, no GPU required). - -Device selection is exercised by lifting the `pick_gpu` function out of the -script and running it against recorded `rocm-smi` output, with a stub on PATH -standing in for the real tool. Picking the wrong device is not a cosmetic -mistake on a shared host: it sends the replay onto somebody else's card. -""" - -from __future__ import annotations - -import os -import re -import shutil -import subprocess -import tempfile -import unittest -from pathlib import Path - -SCRIPT_DIR = Path(__file__).resolve().parent -SCRIPT = SCRIPT_DIR / "triage_archive.sh" -FIXTURE = SCRIPT_DIR.parent / "evals" / "fixtures" / "rocm_smi_vram.txt" - - -def _shell_function(name: str) -> str: - text = SCRIPT.read_text(encoding="utf-8") - match = re.search( - rf"^{re.escape(name)}\(\)\s*\{{.*?^\}}", text, re.MULTILINE | re.DOTALL - ) - if match is None: - raise AssertionError(f"{name} not found in triage_archive.sh") - return match.group(0) - - -def _pick_gpu_function() -> str: - return _shell_function("pick_gpu") - - -@unittest.skipUnless(shutil.which("bash"), "bash is required") -class PickGpuTests(unittest.TestCase): - def _run_pick_gpu(self, smi_output: str, env_gpu: str | None = None) -> str: - with tempfile.TemporaryDirectory() as tmp: - stub_dir = Path(tmp) - data = stub_dir / "vram.txt" - data.write_text(smi_output, encoding="utf-8") - stub = stub_dir / "rocm-smi" - stub.write_text(f'#!/bin/sh\ncat "{data}"\n', encoding="utf-8") - stub.chmod(0o755) - - env = dict(os.environ) - env["PATH"] = f"{stub_dir}{os.pathsep}{env.get('PATH', '')}" - env.pop("GPU", None) - if env_gpu is not None: - env["GPU"] = env_gpu - - proc = subprocess.run( - ["bash", "-c", f"{_pick_gpu_function()}\npick_gpu"], - capture_output=True, - text=True, - env=env, - check=True, - ) - return proc.stdout.strip() - - def test_selects_the_device_with_the_most_free_memory(self) -> None: - """The freest device is neither the first nor the largest here. - - Both devices report the same total, and the first is nearly full, so a - parse that mistakes total for free selects the wrong one. - """ - self.assertEqual( - self._run_pick_gpu(FIXTURE.read_text(encoding="utf-8")), "1" - ) - - def test_used_line_is_not_read_as_the_total_line(self) -> None: - """`VRAM Total Used Memory` also contains the word `Total`.""" - smi = ( - "GPU[0]\t\t: VRAM Total Memory (B): 100\n" - "GPU[0]\t\t: VRAM Total Used Memory (B): 90\n" - "GPU[1]\t\t: VRAM Total Memory (B): 100\n" - "GPU[1]\t\t: VRAM Total Used Memory (B): 10\n" - ) - self.assertEqual(self._run_pick_gpu(smi), "1") - - def test_explicit_gpu_env_wins(self) -> None: - self.assertEqual( - self._run_pick_gpu(FIXTURE.read_text(encoding="utf-8"), env_gpu="0"), "0" - ) - - -class ReplayMaskTests(unittest.TestCase): - def test_native_replay_masks_with_hip_visible_devices(self) -> None: - """hrr-playback is a HIP program, so the HIP mask is the one that applies. - - Setting ROCR_VISIBLE_DEVICES re-indexes devices underneath a HIP mask, - which can place the replay on a device other than the one picked. The - Windows and container paths already use the HIP mask. - """ - text = SCRIPT.read_text(encoding="utf-8") - self.assertIn('HIP_VISIBLE_DEVICES="$gpu"', text) - assignments = [ - line.strip() - for line in text.splitlines() - if "ROCR_VISIBLE_DEVICES=" in line - ] - self.assertEqual(assignments, [], "ROCR_VISIBLE_DEVICES must not be set") - - -def _bash_at_least(major: int, minor: int) -> bool: - """The scripts target Linux bash. Expanding an empty array under `set -u` - is an error before bash 4.4, which the macOS system bash still is.""" - bash = shutil.which("bash") - if not bash: - return False - out = subprocess.run( - [bash, "-c", "echo ${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}"], - capture_output=True, - text=True, - ).stdout.strip() - try: - got_major, got_minor = (int(part) for part in out.split(".")[:2]) - except ValueError: - return False - return (got_major, got_minor) >= (major, minor) - - -@unittest.skipUnless(shutil.which("bash"), "bash is required") -class RecordReplayStopTests(unittest.TestCase): - """A stop the replay does not explain itself has to reach the log. - - Both cases end the log mid-stream. Without the marker the analyzer reads - that truncation as insufficient signal and then looks for a kernel to - blame, which reports a crash of the replay as a fault in the workload. - """ - - def _record(self, rc: int) -> str: - with tempfile.TemporaryDirectory() as tmp: - log = Path(tmp) / "replay.log" - log.write_text("[HRR] Replaying 5 events\n", encoding="utf-8") - script = ( - f'{_shell_function("record_replay_stop")}\n' - f'record_replay_stop {rc} "{log}" >/dev/null\n' - ) - subprocess.run(["bash", "-c", script], check=True) - return log.read_text(encoding="utf-8") - - def test_a_timeout_is_recorded_as_a_timeout(self) -> None: - self.assertIn("replay timed out after", self._record(124)) - - def test_a_signal_death_is_recorded_with_its_signal(self) -> None: - self.assertIn("replay killed by signal 11", self._record(139)) - - def test_an_ordinary_failure_adds_nothing(self) -> None: - """A replay that exits with an error has already said why in the log.""" - self.assertEqual(self._record(1), "[HRR] Replaying 5 events\n") - - -@unittest.skipUnless(shutil.which("bash"), "bash is required") -class LibraryPathTests(unittest.TestCase): - def test_packaged_playback_puts_its_own_lib_on_the_path(self) -> None: - """A playback shipped as bin/ and lib/ siblings, not a CLR build tree. - - Without its own lib dir the binary loads the system libamdhip64 and - fails on the symbols it was built against, which takes out the - metadata-only path as well as replay. - """ - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "bin").mkdir() - (root / "lib").mkdir() - play = root / "bin" / "hrr-playback" - play.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - play.chmod(0o755) - - env = dict(os.environ) - env["ROCM_PATH"] = str(root / "absent") - env.pop("LD_LIBRARY_PATH", None) - - script = ( - f'SCRIPT_DIR={root}\nROCM_PATH="${{ROCM_PATH}}"\n' - f'{_shell_function("setup_library_path")}\n' - f'setup_library_path "{play}"\necho "$LD_LIBRARY_PATH"\n' - ) - proc = subprocess.run( - ["bash", "-c", script], - capture_output=True, - text=True, - env=env, - check=True, - ) - self.assertIn(str(root / "lib"), proc.stdout.strip()) - - -@unittest.skipUnless(_bash_at_least(4, 4), "needs bash >= 4.4 (Linux target)") -class EnsurePlaybackTests(unittest.TestCase): - def test_existing_playback_is_returned_outside_a_clr_tree(self) -> None: - """A provided binary must be usable where no CLR source tree exists. - - The script runs under `set -e`, so a helper whose last statement is a - false conditional aborts it before the path is printed, and the caller - reports a build failure even though the binary was there all along. - """ - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - scripts = root / "scripts" - scripts.mkdir() - shutil.copy(SCRIPT_DIR / "ensure_playback.sh", scripts) - - play = root / "bin" / "hrr-playback" - play.parent.mkdir() - play.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - play.chmod(0o755) - - env = dict(os.environ) - env["HRR_PLAYBACK"] = str(play) - env["ROCM_PATH"] = str(root / "absent") - for stale in ("CLR_BUILD", "CLR_ROOT", "HRR_ROOT", "ROCR_LIB"): - env.pop(stale, None) - - proc = subprocess.run( - ["bash", str(scripts / "ensure_playback.sh"), "--build"], - capture_output=True, - text=True, - env=env, - ) - self.assertEqual( - proc.returncode, 0, f"ensure_playback failed: {proc.stderr}" - ) - self.assertEqual(proc.stdout.strip(), str(play)) - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/hrr/skills/decode-and-triage/scripts/triage_archive.ps1 b/projects/hrr/skills/decode-and-triage/scripts/triage_archive.ps1 index 8e0fedfc08f3..355706aad637 100644 --- a/projects/hrr/skills/decode-and-triage/scripts/triage_archive.ps1 +++ b/projects/hrr/skills/decode-and-triage/scripts/triage_archive.ps1 @@ -7,7 +7,8 @@ # [-o ] [--format markdown|json] [-h] # # Environment knobs (mirror the Linux version): -# HRR_TRIAGE_WORKDIR Output dir for findings + logs (default: current dir) +# HRR_TRIAGE_WORKDIR Output dir for findings + logs (default: a temp dir, +# never the archive: triage must not write into it) # HRR_PLAYBACK Explicit path to hrr-playback.exe # HIP_PATH / ROCM_PATH HIP SDK root (default: C:\Program Files\AMD\ROCm\6.2) # GPU Replay GPU ordinal (default: 0) @@ -117,9 +118,75 @@ if (-not (Test-Path $Archive -PathType Container)) { exit 1 } +# GetFullPath is lexical only, so a symbolic link or a junction anywhere in a +# path is followed here, as realpath does for triage_archive.sh. Components that +# do not exist yet are kept as they are. $null when the links go round in a +# loop. +function Resolve-LinkedPath([string]$Path) { + $full = [IO.Path]::GetFullPath($Path) + $hops = 0 + while ($true) { + $root = [IO.Path]::GetPathRoot($full) + $parts = @($full.Substring($root.Length).Split([char[]]'\/', [StringSplitOptions]::RemoveEmptyEntries)) + $cur = $root + $relinked = $false + for ($i = 0; $i -lt $parts.Count; $i++) { + $next = Join-Path $cur $parts[$i] + $item = Get-Item -LiteralPath $next -Force -ErrorAction SilentlyContinue + $target = if ($item -and $item.LinkType) { @($item.Target)[0] } else { $null } + if (-not $target) { $cur = $next; continue } + if (++$hops -gt 40) { return $null } + if (-not [IO.Path]::IsPathRooted($target)) { $target = Join-Path $cur $target } + # The target can hold links of its own, so the walk starts again on it. + foreach ($p in ($parts | Select-Object -Skip ($i + 1))) { $target = Join-Path $target $p } + $full = [IO.Path]::GetFullPath($target) + $relinked = $true + break + } + if (-not $relinked) { return $cur } + } +} + +# The archive's own links are followed before anything is derived from its name, +# as readlink -f does in triage_archive.sh: a link to a pid- directory has to +# protect the capture directory holding the target, not the one holding the link. +$ArchiveTarget = Resolve-LinkedPath $Archive +if (-not $ArchiveTarget) { + Write-Host "error: a symbolic link loop in the archive path: $Archive" -ForegroundColor Red + exit 1 +} +$Archive = $ArchiveTarget $Name = Split-Path $Archive -Leaf -$Ts = (Get-Date).ToUniversalTime().ToString("yyyyMMdd'T'HHmmss'Z'") -$Workdir = if ($env:HRR_TRIAGE_WORKDIR) { $env:HRR_TRIAGE_WORKDIR } else { (Get-Location).Path } +# The pid keeps two runs in the same second from sharing a log and a finding. +$Ts = (Get-Date).ToUniversalTime().ToString("yyyyMMdd'T'HHmmss'Z'") + "-$PID" +$Workdir = if ($env:HRR_TRIAGE_WORKDIR) { $env:HRR_TRIAGE_WORKDIR } else { Join-Path ([IO.Path]::GetTempPath()) 'hrr-triage' } +# The archive is not to be written to, as in triage_archive.sh: the archive here +# is the pid directory and, for a pid- one, the capture directory holding it. +# Both paths are resolved, links included, before anything is created: a work +# directory or a parent of it that is a link into the capture is inside it. +$ArchiveRoot = if ($Name -like 'pid-*') { Split-Path $Archive -Parent } else { $Archive } +$ArchiveRoot = Resolve-LinkedPath $ArchiveRoot +function Test-InArchive([string]$Path) { + $full = Resolve-LinkedPath $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Path) + if (-not $ArchiveRoot -or -not $full) { + Write-Host "error: a symbolic link loop in the archive path or in: $Path" -ForegroundColor Red + exit 1 + } + $root = $ArchiveRoot.TrimEnd('\', '/') + $full = $full.TrimEnd('\', '/') + return ($full -ieq $root -or + $full.StartsWith($root + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) +} +if (Test-InArchive $Workdir) { + Write-Host "error: the triage work directory is inside the archive: $Workdir; set HRR_TRIAGE_WORKDIR outside it" -ForegroundColor Red + exit 1 +} +# -o is checked too: a finding named into the archive writes there as surely +# as a work directory inside it does. +if ($Output -and (Test-InArchive $Output)) { + Write-Host "error: --output is inside the archive: $Output" -ForegroundColor Red + exit 1 +} New-Item -ItemType Directory -Force -Path $Workdir | Out-Null $Ext = if ($Format -eq "json") { ".finding.json" } else { ".finding.md" } $Finding = if ($Output) { $Output } else { Join-Path $Workdir "${Name}-${Ts}${Ext}" } @@ -415,5 +482,5 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# The analyzer has already printed the finding; it is the only thing on stdout. Write-Triage "finding=$Finding" -Get-Content $Finding diff --git a/projects/hrr/skills/decode-and-triage/scripts/triage_archive.sh b/projects/hrr/skills/decode-and-triage/scripts/triage_archive.sh index fef7d83460fd..088058015f13 100755 --- a/projects/hrr/skills/decode-and-triage/scripts/triage_archive.sh +++ b/projects/hrr/skills/decode-and-triage/scripts/triage_archive.sh @@ -30,6 +30,9 @@ Options: Environment (common): HRR_TRIAGE_WORKDIR Output directory for findings and replay logs + (default: $TMPDIR/hrr-triage-, mode 0700, never the + archive; /tmp if TMPDIR is inside the archive, and a + mktemp directory if that one is not ours) HRR_DOCKER_IMAGE Docker image for --replay docker / auto HRR_DOCKER_MOUNT_CLR=1 Overlay host CLR for docker replay (dev builds) GPU Replay GPU ordinal (default: auto-pick) @@ -39,17 +42,21 @@ Environment (common): EOF } +needs_value() { + (( $2 >= 2 )) || { echo "error: $1 needs a value" >&2; exit 1; } +} + while [[ $# -gt 0 ]]; do case "$1" in -h|--help) usage; exit 0 ;; - --archive) ARCHIVE="$2"; shift 2 ;; + --archive) needs_value --archive $#; ARCHIVE="$2"; shift 2 ;; --replay) if [[ $# -lt 2 || "$2" == --* ]]; then REPLAY_MODE="native"; shift else REPLAY_MODE="$2"; shift 2; fi ;; --no-replay) REPLAY_MODE="skip"; shift ;; --no-sync) NO_SYNC=1; shift ;; - -o|--output) OUTPUT="$2"; shift 2 ;; - --format) FORMAT="$2"; shift 2 ;; + -o|--output) needs_value --output $#; OUTPUT="$2"; shift 2 ;; + --format) needs_value --format $#; FORMAT="$2"; shift 2 ;; *) echo "error: unknown arg: $1" >&2; exit 1 ;; esac done @@ -59,12 +66,61 @@ ARCHIVE="$(readlink -f "$ARCHIVE" 2>/dev/null || realpath "$ARCHIVE" 2>/dev/null [[ -d "$ARCHIVE" ]] || { echo "error: archive not found: $ARCHIVE" >&2; exit 1; } name="$(basename "$ARCHIVE")" -ts="$(date -u +%Y%m%dT%H%M%SZ)" -WORKDIR="${HRR_TRIAGE_WORKDIR:-$(pwd)}" -mkdir -p "$WORKDIR" +# The pid keeps two runs in the same second from sharing a log and a finding. +ts="$(date -u +%Y%m%dT%H%M%SZ)-$$" +# Never the current directory by default: run from inside a customer's archive +# and the finding and the replay log land in it, against this skill's own rule +# that the archive is not to be written to. Per user, because a shared /tmp +# directory belongs to whoever ran first and the next user cannot write in it. +# The archive here is the pid directory and, for a pid- one, the capture +# directory holding it. Paths are resolved before anything is created, so +# TMPDIR=. from inside the archive is caught before it writes there. +ARCHIVE_ROOT="$ARCHIVE" +[[ "$name" == pid-* ]] && ARCHIVE_ROOT="$(dirname "$ARCHIVE")" +in_archive() { + local p + p="$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "$1")" + [[ "$p" == "$ARCHIVE_ROOT" || "$p" == "$ARCHIVE_ROOT"/* ]] +} +if [[ -n "${HRR_TRIAGE_WORKDIR:-}" ]]; then + WORKDIR="$HRR_TRIAGE_WORKDIR" + if in_archive "$WORKDIR"; then + echo "error: HRR_TRIAGE_WORKDIR is inside the archive: $WORKDIR" >&2 + exit 1 + fi + # Created private when we are the one creating it. An existing directory is + # the caller's business, but the finding inside it is not: see the chmod + # where it is written. + mkdir -p -m 700 "$WORKDIR" +else + tmp_base="${TMPDIR:-/tmp}" + in_archive "$tmp_base" && tmp_base=/tmp + if in_archive "$tmp_base"; then + echo "error: the archive holds /tmp; set HRR_TRIAGE_WORKDIR outside it" >&2 + exit 1 + fi + WORKDIR="$tmp_base/hrr-triage-$(id -u)" + # 0700 and ours, or somewhere else entirely. The name is predictable, so on a + # shared host another user can get there first, and a finding names a + # customer's kernels and addresses. The capture itself can carry that name + # too, and then it is ours and writable and still the archive. + mkdir -p -m 700 "$WORKDIR" 2>/dev/null || true + if [[ -L "$WORKDIR" || ! -d "$WORKDIR" || ! -O "$WORKDIR" || ! -w "$WORKDIR" ]] || + in_archive "$WORKDIR"; then + WORKDIR="$(mktemp -d "$tmp_base/hrr-triage-XXXXXX")" + else + chmod 700 "$WORKDIR" 2>/dev/null || true + fi +fi LOG="" ext=".finding.md"; [[ "$FORMAT" == "json" ]] && ext=".finding.json" FINDING="${OUTPUT:-$WORKDIR/${name}-${ts}${ext}}" +# -o is checked too: a finding named into the archive writes there as surely +# as a work directory inside it does. /dev/stdout is the caller's own stream. +if [[ -n "$OUTPUT" ]] && ! [[ "$FINDING" -ef /dev/stdout ]] && in_archive "$FINDING"; then + echo "error: --output is inside the archive: $OUTPUT" >&2 + exit 1 +fi pick_replay_mode() { if [[ "$REPLAY_MODE" != "auto" ]]; then echo "$REPLAY_MODE"; return; fi @@ -80,25 +136,49 @@ setup_library_path() { if [[ "$bin_dir" == */bin ]]; then lib_dirs+=("$(cd "$bin_dir/../lib" 2>/dev/null && pwd || true)") fi - # A packaged playback ships as bin/ and lib/ siblings rather than inside a - # build tree. Without this its libamdhip64 is never on the path, so the - # binary loads the system one and fails on the symbols it was built against. - if [[ -d "$bin_dir/../lib" ]]; then - lib_dirs+=("$(cd "$bin_dir/../lib" && pwd)") - fi + # A packaged playback ships as bin/, lib/ and runtime-lib/ siblings rather + # than inside a build tree. Without these its libamdhip64 and libhsa are + # never on the path, so the binary loads the system ones and dies on the + # symbols it was built against. runtime-lib was missing here while the + # sibling skill's inspector had it, so `verify` could launch a reader that + # `triage` could not. + for p in lib runtime-lib; do + [[ -d "$bin_dir/../$p" ]] && lib_dirs+=("$(cd "$bin_dir/../$p" && pwd)") + done repo="$(cd "$SCRIPT_DIR/../../../../.." 2>/dev/null && pwd || true)" for p in "${ROCR_LIB:-}" "${repo:+$repo/projects/rocr-runtime/build-local/rocr/lib}"; do [[ -n "$p" && -f "$p/libhsa-runtime64.so.1" ]] || continue lib_dirs+=("$p"); break done - lib_dirs+=("$ROCM_PATH/lib") - for p in "${lib_dirs[@]}"; do + for p in ${lib_dirs[@]+"${lib_dirs[@]}"}; do [[ -d "$p" ]] || continue [[ ":$seen:" == *":$p:"* ]] && continue seen="${seen:+$seen:}$p" built="${built:+$built:}$p" done - export LD_LIBRARY_PATH="${built}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH:-}}" + # The caller's own value next, and $ROCM_PATH/lib last. Putting the ROCm + # install ahead of an explicitly set LD_LIBRARY_PATH bound libhsa from the + # system release in front of the reader's own, and a newer libamdhip64 then + # fails on a symbol that release does not have. + # Assembled a component at a time. An empty component, which is what + # "${built}:..." leaves when nothing was found, means the current directory + # to the loader, and this script is run from inside customer archives: a + # shared object sitting in one would be loaded ahead of ROCm. The caller's + # value can hold empty components of its own (/mine: or a::b), so it is + # split too. + local joined="" part parts=() + for p in "$built" "${LD_LIBRARY_PATH:-}" "${ROCM_PATH:+$ROCM_PATH/lib}"; do + IFS=: read -r -a parts <<< "$p" + for part in ${parts[@]+"${parts[@]}"}; do + [[ -n "$part" ]] || continue + joined="${joined:+$joined:}$part" + done + done + if [[ -n "$joined" ]]; then + export LD_LIBRARY_PATH="$joined" + else + unset LD_LIBRARY_PATH + fi } pick_gpu() { @@ -113,6 +193,8 @@ pick_gpu() { # word Total and must be tested first. Total and used arrive on separate # lines whose order is not guaranteed, so collect both per device and # subtract at END rather than on whichever line happens to land last. + # printf, because mawk, the Ubuntu default, prints a difference that size + # as 2.73804e+11, which the bash arithmetic above cannot read. /GPU\[/ { id = $1; gsub(/[^0-9]/, "", id) if ($0 ~ /Total Used Memory/) used[id] = $NF @@ -120,7 +202,7 @@ pick_gpu() { } END { for (id in total) - if (id in used) print id, total[id] - used[id] + if (id in used) printf "%s %.0f\n", id, total[id] - used[id] }') [[ -n "$best" ]] && { echo "[triage] GPU $best (most free VRAM)" >&2; echo "$best"; return; } fi @@ -258,6 +340,11 @@ elif [[ "$mode" == "docker" && "${HRR_DOCKER_MOUNT_CLR:-0}" == "1" && -x "$ENSUR export HRR_PLAYBACK fi +# The replay log and the finding name a customer's kernels and addresses, so +# they are private from the moment they exist, not only after the chmod at the +# end. Set here rather than at the top, so a playback built above is untouched. +umask 077 + REPLAY_GPU="" if [[ "$mode" != "skip" ]]; then # Only when a device is actually going to be used: metadata-only needs no GPU @@ -277,10 +364,16 @@ elif [[ "$mode" == "native" ]]; then record_replay_stop "$replay_rc" "$LOG" fi -CMD=(python3 "$ANALYZER" --format "$FORMAT" --archive "$ARCHIVE" -o "$FINDING") +# The analyzer prints the finding as well as writing it, so it is the only +# thing on stdout, and -o naming stdout itself would print it twice. +CMD=(python3 "$ANALYZER" --format "$FORMAT" --archive "$ARCHIVE") +[[ "$FINDING" -ef /dev/stdout ]] || CMD+=(-o "$FINDING") [[ -n "${HRR_PLAYBACK:-}" ]] && CMD+=(--hrr-playback "$HRR_PLAYBACK") [[ -n "$LOG" && -f "$LOG" ]] && CMD+=(--log "$LOG") "${CMD[@]}" +# A regular file only: -o can name a device such as /dev/stdout or /dev/null, +# and -f follows a link, so /dev/stdout redirected to a file would pass it. +[[ -f "$FINDING" && ! -L "$FINDING" ]] && chmod 600 "$FINDING" 2>/dev/null || true +[[ -n "$LOG" && -f "$LOG" ]] && chmod 600 "$LOG" 2>/dev/null || true echo "[triage] finding=$FINDING" >&2 -cat "$FINDING" diff --git a/projects/hrr/skills/decode-and-triage/skill-card.md b/projects/hrr/skills/decode-and-triage/skill-card.md new file mode 100644 index 000000000000..e492f48d6be8 --- /dev/null +++ b/projects/hrr/skills/decode-and-triage/skill-card.md @@ -0,0 +1,15 @@ +# Skill Card + +## Description + +Replays an HRR capture archive on an AMD GPU and reports what failed: outcome, +fault class, faulting kernel, fault address and failing event, without the +original application, its source or its data. + +## Owner + +AMD ROCm Runtime, HRR maintainers + +## License + +MIT diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/info_crash.txt b/projects/hrr/skills/decode-and-triage/tests/fixtures/info_crash.txt similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/info_crash.txt rename to projects/hrr/skills/decode-and-triage/tests/fixtures/info_crash.txt diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/info_pass.txt b/projects/hrr/skills/decode-and-triage/tests/fixtures/info_pass.txt similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/info_pass.txt rename to projects/hrr/skills/decode-and-triage/tests/fixtures/info_pass.txt diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/replay_aten_chevron.log b/projects/hrr/skills/decode-and-triage/tests/fixtures/replay_aten_chevron.log similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/replay_aten_chevron.log rename to projects/hrr/skills/decode-and-triage/tests/fixtures/replay_aten_chevron.log diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/replay_hang_timeout.log b/projects/hrr/skills/decode-and-triage/tests/fixtures/replay_hang_timeout.log similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/replay_hang_timeout.log rename to projects/hrr/skills/decode-and-triage/tests/fixtures/replay_hang_timeout.log diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/replay_memory_fault.log b/projects/hrr/skills/decode-and-triage/tests/fixtures/replay_memory_fault.log similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/replay_memory_fault.log rename to projects/hrr/skills/decode-and-triage/tests/fixtures/replay_memory_fault.log diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/replay_pass.log b/projects/hrr/skills/decode-and-triage/tests/fixtures/replay_pass.log similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/replay_pass.log rename to projects/hrr/skills/decode-and-triage/tests/fixtures/replay_pass.log diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/replay_queue_abort_aten.log b/projects/hrr/skills/decode-and-triage/tests/fixtures/replay_queue_abort_aten.log similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/replay_queue_abort_aten.log rename to projects/hrr/skills/decode-and-triage/tests/fixtures/replay_queue_abort_aten.log diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/rocm_smi_vram.txt b/projects/hrr/skills/decode-and-triage/tests/fixtures/rocm_smi_vram.txt similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/rocm_smi_vram.txt rename to projects/hrr/skills/decode-and-triage/tests/fixtures/rocm_smi_vram.txt diff --git a/projects/hrr/skills/decode-and-triage/evals/fixtures/version_mismatch.log b/projects/hrr/skills/decode-and-triage/tests/fixtures/version_mismatch.log similarity index 100% rename from projects/hrr/skills/decode-and-triage/evals/fixtures/version_mismatch.log rename to projects/hrr/skills/decode-and-triage/tests/fixtures/version_mismatch.log diff --git a/projects/hrr/skills/decode-and-triage/scripts/test_analyze_replay_finding.py b/projects/hrr/skills/decode-and-triage/tests/test_analyze_replay_finding.py similarity index 70% rename from projects/hrr/skills/decode-and-triage/scripts/test_analyze_replay_finding.py rename to projects/hrr/skills/decode-and-triage/tests/test_analyze_replay_finding.py index eee4091b9a3d..55c0884b9719 100644 --- a/projects/hrr/skills/decode-and-triage/scripts/test_analyze_replay_finding.py +++ b/projects/hrr/skills/decode-and-triage/tests/test_analyze_replay_finding.py @@ -8,7 +8,7 @@ import unittest from pathlib import Path -SCRIPT_DIR = Path(__file__).resolve().parent +SCRIPT_DIR = Path(__file__).resolve().parent.parent / "scripts" sys.path.insert(0, str(SCRIPT_DIR)) import analyze_replay_finding as arf # noqa: E402 @@ -91,6 +91,56 @@ def test_memory_fault_outranks_the_abort_line(self) -> None: self.assertEqual(finding.fault_class, "read_only_page_fault") self.assertEqual(finding.outcome, "MAF") + def test_the_summary_outranks_a_stale_progress_line(self) -> None: + """Progress is printed at the first kernel and then periodically. + + The summary prints skipped rather than attempted, so attempted is + pass + fail + skipped, and a nonzero skipped count is what tells that + apart from pass + fail. + """ + text = ( + "[HRR progress] elapsed_s=0.0 seq=12 kernels=1 d2h_pass=0 " + 'd2h_fail=0 d2h_attempted=0 last="first_kernel"\n' + "[HRR] Kernels launched: 40\n" + "[HRR] D2H checks : 30 pass (30 exact, 0 within tol), " + "1 fail, 2 skipped\n" + "[HRR] FAIL\n" + ) + finding = arf.Finding(outcome="UNKNOWN", fault_class="unknown") + arf.parse_text(text, "replay.log", finding) + self.assertEqual(finding.kernels_launched, 40) + self.assertEqual( + (finding.d2h_pass, finding.d2h_fail, finding.d2h_attempted), (30, 1, 33) + ) + + def test_a_pass_line_with_d2h_failures_keeps_its_fault_details(self) -> None: + """It is classified as a divergence, so it is reported as one.""" + text = ( + "[HRR progress] elapsed_s=0.0 seq=12 kernels=1 d2h_pass=0 " + 'd2h_fail=2 d2h_attempted=2 last="first_kernel"\n' + "[HRR] PASS\n" + ) + finding = arf.Finding(outcome="UNKNOWN", fault_class="unknown") + arf.parse_text(text, "replay.log", finding) + report = arf.render_markdown(arf.finalize(finding)) + self.assertEqual(finding.fault_class, "nan_inf_divergence") + self.assertIn("## Fault details", report) + self.assertNotIn("## Replay result", report) + + def test_a_replay_that_launched_no_kernels_says_zero(self) -> None: + """Zero is a count the summary printed, not a missing one.""" + text = ( + "[HRR] Kernels launched: 0\n" + "[HRR] D2H checks : 1 pass (1 exact, 0 within tol), " + "0 fail, 0 skipped\n" + "[HRR] PASS\n" + ) + finding = arf.Finding(outcome="UNKNOWN", fault_class="unknown") + arf.parse_text(text, "replay.log", finding) + report = arf.render_markdown(arf.finalize(finding)) + self.assertEqual(finding.kernels_launched, 0) + self.assertIn("- **Kernels launched**: 0", report) + class RecordedCaptureTests(unittest.TestCase): """Checks against replay output recorded from a gfx950 host. @@ -100,7 +150,7 @@ class RecordedCaptureTests(unittest.TestCase): finding. """ - FIXTURES = SCRIPT_DIR.parent / "evals" / "fixtures" + FIXTURES = Path(__file__).resolve().parent / "fixtures" def _analyze(self, *names: str) -> arf.Finding: finding = arf.Finding(outcome="UNKNOWN", fault_class="unknown") @@ -123,6 +173,61 @@ def test_clean_replay_carries_archive_totals(self) -> None: self.assertEqual(finding.archive_kernels, 13233) self.assertEqual(finding.d2h_fail, 0) + def test_clean_replay_takes_its_totals_from_the_summary(self) -> None: + """This log has the summary and no progress line at all.""" + finding = self._analyze("replay_pass.log", "info_pass.txt") + self.assertEqual(finding.d2h_attempted, 32) + self.assertEqual(finding.kernels_launched, 13233) + report = arf.render_markdown(finding) + self.assertIn("## Replay result", report) + self.assertIn("pass=32 fail=0 attempted=32", report) + self.assertIn("- **Kernels launched**: 13233", report) + self.assertNotIn("## Fault details", report) + + def test_clean_replay_reports_no_failing_event_in_json(self) -> None: + """The runner asks for progress, and the first kernel always prints it. + + JSON prints every field, so the event and kernel named on the way have + to be cleared rather than hidden the way the markdown hides them. + """ + finding = arf.Finding(outcome="UNKNOWN", fault_class="unknown") + arf.parse_text( + "[HRR progress] elapsed_s=0.0 seq=12 kernels=1 d2h_pass=0 " + 'd2h_fail=0 d2h_attempted=0 last="first_kernel"\n' + "[HRR] Event 7: hipModuleLaunchKernel\n" + + (self.FIXTURES / "replay_pass.log").read_text(encoding="utf-8"), + "replay.log", + finding, + ) + payload = json.loads(json.dumps(arf.finalize(finding).to_dict())) + for key in ( + "failing_event_seq", + "failing_call_index", + "failing_api", + "last_progress_kernel", + "kernel_name", + ): + self.assertIsNone(payload[key], key) + self.assertEqual(payload["kernels_launched"], 13233) + self.assertEqual(payload["d2h_attempted"], 32) + + def test_a_fault_at_the_first_event_names_it(self) -> None: + """Event and call indices are zero-based, so 0 is a real index.""" + finding = arf.Finding( + outcome="MAF", + fault_class="illegal_memory_access", + failing_event_seq=0, + failing_call_index=0, + ) + report = arf.render_markdown(finding) + self.assertIn("- **Failing event seq**: 0", report) + self.assertIn("- **Failing call index**: 0", report) + report = arf.render_markdown( + arf.Finding(outcome="MAF", fault_class="illegal_memory_access") + ) + self.assertIn("- **Failing event seq**: n/a", report) + self.assertIn("- **Failing call index**: n/a", report) + def test_truncated_kernel_names_are_ignored(self) -> None: """Every name in this capture's table is cut off by the column width.""" finding = self._analyze("info_pass.txt") diff --git a/projects/hrr/skills/decode-and-triage/scripts/test_check_replay_compat.py b/projects/hrr/skills/decode-and-triage/tests/test_check_replay_compat.py similarity index 99% rename from projects/hrr/skills/decode-and-triage/scripts/test_check_replay_compat.py rename to projects/hrr/skills/decode-and-triage/tests/test_check_replay_compat.py index 48a62fd18af7..89a6d88452cd 100644 --- a/projects/hrr/skills/decode-and-triage/scripts/test_check_replay_compat.py +++ b/projects/hrr/skills/decode-and-triage/tests/test_check_replay_compat.py @@ -10,7 +10,7 @@ from pathlib import Path from unittest import mock -SCRIPT_DIR = Path(__file__).resolve().parent +SCRIPT_DIR = Path(__file__).resolve().parent.parent / "scripts" sys.path.insert(0, str(SCRIPT_DIR)) import check_replay_compat as crc # noqa: E402 diff --git a/projects/hrr/skills/decode-and-triage/tests/test_triage_archive.py b/projects/hrr/skills/decode-and-triage/tests/test_triage_archive.py new file mode 100644 index 000000000000..80e10ddcd6ac --- /dev/null +++ b/projects/hrr/skills/decode-and-triage/tests/test_triage_archive.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +"""Unit tests for triage_archive.sh (stdlib unittest, no GPU required). + +Device selection is exercised by lifting the `pick_gpu` function out of the +script and running it against recorded `rocm-smi` output, with a stub on PATH +standing in for the real tool. Picking the wrong device is not a cosmetic +mistake on a shared host: it sends the replay onto somebody else's card. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent.parent / "scripts" +SCRIPT = SCRIPT_DIR / "triage_archive.sh" +FIXTURE = Path(__file__).resolve().parent / "fixtures" / "rocm_smi_vram.txt" + + +def _shell_function(name: str) -> str: + text = SCRIPT.read_text(encoding="utf-8") + match = re.search( + rf"^{re.escape(name)}\(\)\s*\{{.*?^\}}", text, re.MULTILINE | re.DOTALL + ) + if match is None: + raise AssertionError(f"{name} not found in triage_archive.sh") + return match.group(0) + + +def _pick_gpu_function() -> str: + return _shell_function("pick_gpu") + + +@unittest.skipUnless(shutil.which("bash"), "bash is required") +class PickGpuTests(unittest.TestCase): + def _run_pick_gpu(self, smi_output: str, env_gpu: str | None = None) -> str: + with tempfile.TemporaryDirectory() as tmp: + stub_dir = Path(tmp) + data = stub_dir / "vram.txt" + data.write_text(smi_output, encoding="utf-8") + stub = stub_dir / "rocm-smi" + stub.write_text(f'#!/bin/sh\ncat "{data}"\n', encoding="utf-8") + stub.chmod(0o755) + + env = dict(os.environ) + env["PATH"] = f"{stub_dir}{os.pathsep}{env.get('PATH', '')}" + env.pop("GPU", None) + if env_gpu is not None: + env["GPU"] = env_gpu + + proc = subprocess.run( + ["bash", "-c", f"{_pick_gpu_function()}\npick_gpu"], + capture_output=True, + text=True, + env=env, + check=True, + ) + return proc.stdout.strip() + + def test_selects_the_device_with_the_most_free_memory(self) -> None: + """The freest device is neither the first nor the largest here. + + Both devices report the same total, and the first is nearly full, so a + parse that mistakes total for free selects the wrong one. + """ + self.assertEqual( + self._run_pick_gpu(FIXTURE.read_text(encoding="utf-8")), "1" + ) + + def test_used_line_is_not_read_as_the_total_line(self) -> None: + """`VRAM Total Used Memory` also contains the word `Total`.""" + smi = ( + "GPU[0]\t\t: VRAM Total Memory (B): 100\n" + "GPU[0]\t\t: VRAM Total Used Memory (B): 90\n" + "GPU[1]\t\t: VRAM Total Memory (B): 100\n" + "GPU[1]\t\t: VRAM Total Used Memory (B): 10\n" + ) + self.assertEqual(self._run_pick_gpu(smi), "1") + + def test_explicit_gpu_env_wins(self) -> None: + self.assertEqual( + self._run_pick_gpu(FIXTURE.read_text(encoding="utf-8"), env_gpu="0"), "0" + ) + + +class ReplayMaskTests(unittest.TestCase): + def test_native_replay_masks_with_hip_visible_devices(self) -> None: + """hrr-playback is a HIP program, so the HIP mask is the one that applies. + + Setting ROCR_VISIBLE_DEVICES re-indexes devices underneath a HIP mask, + which can place the replay on a device other than the one picked. The + Windows and container paths already use the HIP mask. + """ + text = SCRIPT.read_text(encoding="utf-8") + self.assertIn('HIP_VISIBLE_DEVICES="$gpu"', text) + assignments = [ + line.strip() + for line in text.splitlines() + if "ROCR_VISIBLE_DEVICES=" in line + ] + self.assertEqual(assignments, [], "ROCR_VISIBLE_DEVICES must not be set") + + +def _bash_at_least(major: int, minor: int) -> bool: + """The scripts target Linux bash. Expanding an empty array under `set -u` + is an error before bash 4.4, which the macOS system bash still is.""" + bash = shutil.which("bash") + if not bash: + return False + out = subprocess.run( + [bash, "-c", "echo ${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}"], + capture_output=True, + text=True, + ).stdout.strip() + try: + got_major, got_minor = (int(part) for part in out.split(".")[:2]) + except ValueError: + return False + return (got_major, got_minor) >= (major, minor) + + +@unittest.skipUnless(shutil.which("bash"), "bash is required") +class RecordReplayStopTests(unittest.TestCase): + """A stop the replay does not explain itself has to reach the log. + + Both cases end the log mid-stream. Without the marker the analyzer reads + that truncation as insufficient signal and then looks for a kernel to + blame, which reports a crash of the replay as a fault in the workload. + """ + + def _record(self, rc: int) -> str: + with tempfile.TemporaryDirectory() as tmp: + log = Path(tmp) / "replay.log" + log.write_text("[HRR] Replaying 5 events\n", encoding="utf-8") + script = ( + f'{_shell_function("record_replay_stop")}\n' + f'record_replay_stop {rc} "{log}" >/dev/null\n' + ) + subprocess.run(["bash", "-c", script], check=True) + return log.read_text(encoding="utf-8") + + def test_a_timeout_is_recorded_as_a_timeout(self) -> None: + self.assertIn("replay timed out after", self._record(124)) + + def test_a_signal_death_is_recorded_with_its_signal(self) -> None: + self.assertIn("replay killed by signal 11", self._record(139)) + + def test_an_ordinary_failure_adds_nothing(self) -> None: + """A replay that exits with an error has already said why in the log.""" + self.assertEqual(self._record(1), "[HRR] Replaying 5 events\n") + + +@unittest.skipUnless(shutil.which("bash"), "bash is required") +class LibraryPathTests(unittest.TestCase): + def test_packaged_playback_puts_its_own_lib_on_the_path(self) -> None: + """A playback shipped as bin/ and lib/ siblings, not a CLR build tree. + + Without its own lib dir the binary loads the system libamdhip64 and + fails on the symbols it was built against, which takes out the + metadata-only path as well as replay. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "bin").mkdir() + (root / "lib").mkdir() + play = root / "bin" / "hrr-playback" + play.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + play.chmod(0o755) + + env = dict(os.environ) + env["ROCM_PATH"] = str(root / "absent") + env.pop("LD_LIBRARY_PATH", None) + + script = ( + f'SCRIPT_DIR={root}\nROCM_PATH="${{ROCM_PATH}}"\n' + f'{_shell_function("setup_library_path")}\n' + f'setup_library_path "{play}"\necho "$LD_LIBRARY_PATH"\n' + ) + proc = subprocess.run( + ["bash", "-c", script], + capture_output=True, + text=True, + env=env, + check=True, + ) + self.assertIn(str(root / "lib"), proc.stdout.strip()) + + +@unittest.skipUnless(_bash_at_least(4, 4), "needs bash >= 4.4 (Linux target)") +class EnsurePlaybackTests(unittest.TestCase): + def test_existing_playback_is_returned_outside_a_clr_tree(self) -> None: + """A provided binary must be usable where no CLR source tree exists. + + The script runs under `set -e`, so a helper whose last statement is a + false conditional aborts it before the path is printed, and the caller + reports a build failure even though the binary was there all along. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + scripts = root / "scripts" + scripts.mkdir() + shutil.copy(SCRIPT_DIR / "ensure_playback.sh", scripts) + + play = root / "bin" / "hrr-playback" + play.parent.mkdir() + play.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + play.chmod(0o755) + + env = dict(os.environ) + env["HRR_PLAYBACK"] = str(play) + env["ROCM_PATH"] = str(root / "absent") + for stale in ("CLR_BUILD", "CLR_ROOT", "HRR_ROOT", "ROCR_LIB"): + env.pop(stale, None) + + proc = subprocess.run( + ["bash", str(scripts / "ensure_playback.sh"), "--build"], + capture_output=True, + text=True, + env=env, + ) + self.assertEqual( + proc.returncode, 0, f"ensure_playback failed: {proc.stderr}" + ) + self.assertEqual(proc.stdout.strip(), str(play)) + + +class WorkdirDefaultTests(unittest.TestCase): + """The findings must never land in the archive being triaged. + + Run from inside a customer's archive, the script used to default its + output directory to the working directory, which is the one place this + skill says not to write to. + """ + + def test_default_workdir_is_outside_the_archive_private_and_per_user(self): + text = SCRIPT.read_text(encoding="utf-8") + default = text.split("HRR_TRIAGE_WORKDIR:-}", 1)[1].split("\nfi\n", 1)[0] + self.assertNotIn("$(pwd)", default) + self.assertIn("id -u", default, "a shared /tmp directory locks out every other user") + self.assertIn("-m 700", default, "a finding names a customer's kernels and addresses") + self.assertIn("mktemp -d", default, "somebody else may have got to the name first") + + def test_the_archive_is_untouched_by_a_metadata_only_run(self): + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "capture.hrr" / "pid-1" + archive.mkdir(parents=True) + (archive / "events.bin").write_bytes(b"\0" * 64) + before = {p.name: p.read_bytes() for p in archive.iterdir()} + # The default is what is under test, so TMPDIR is what points + # somewhere disposable, not HRR_TRIAGE_WORKDIR. + env = {k: v for k, v in os.environ.items() if k != "HRR_TRIAGE_WORKDIR"} + env["TMPDIR"] = tmp + + proc = subprocess.run( + ["bash", str(SCRIPT), "--archive", str(archive), "--no-replay"], + cwd=archive, + capture_output=True, + text=True, + env=env, + check=False, + ) + + self.assertEqual(proc.returncode, 0, proc.stderr) + after = {p.name: p.read_bytes() for p in archive.iterdir()} + self.assertEqual(before, after) + workdir = Path(tmp) / f"hrr-triage-{os.getuid()}" + findings = list(workdir.glob("pid-1-*.finding.md")) + self.assertEqual(len(findings), 1, proc.stderr) + self.assertEqual(workdir.stat().st_mode & 0o777, 0o700) + self.assertEqual(findings[0].stat().st_mode & 0o777, 0o600) + + def _run_in_archive(self, env_extra: dict, args=(), stdout=subprocess.PIPE) -> tuple: + with tempfile.TemporaryDirectory() as tmp: + capture = Path(tmp) / "capture.hrr" + archive = capture / "pid-1" + archive.mkdir(parents=True) + (archive / "events.bin").write_bytes(b"\0" * 64) + env = {k: v for k, v in os.environ.items() if k != "HRR_TRIAGE_WORKDIR"} + env.update(env_extra) + proc = subprocess.run( + ["bash", str(SCRIPT), "--archive", str(archive), "--no-replay", *args], + cwd=archive, + stdout=stdout, + stderr=subprocess.PIPE, + text=True, + env=env, + check=False, + ) + inside = sorted(str(p.relative_to(capture)) for p in capture.rglob("*")) + return proc, inside + + def test_a_relative_tmpdir_inside_the_archive_is_not_used(self): + proc, inside = self._run_in_archive({"TMPDIR": "."}) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(inside, ["pid-1", "pid-1/events.bin"]) + self.assertIn("[triage] finding=/tmp/", proc.stderr) + finding = proc.stderr.split("[triage] finding=", 1)[1].split("\n", 1)[0] + Path(finding).unlink() + + def test_a_capture_named_like_the_default_workdir_is_not_written(self): + with tempfile.TemporaryDirectory() as tmp: + capture = Path(tmp) / f"hrr-triage-{os.getuid()}" + archive = capture / "pid-1" + archive.mkdir(parents=True) + (archive / "events.bin").write_bytes(b"\0" * 64) + env = {k: v for k, v in os.environ.items() if k != "HRR_TRIAGE_WORKDIR"} + env["TMPDIR"] = tmp + proc = subprocess.run( + ["bash", str(SCRIPT), "--archive", str(archive), "--no-replay"], + capture_output=True, + text=True, + env=env, + check=False, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + inside = sorted(str(p.relative_to(capture)) for p in capture.rglob("*")) + self.assertEqual(inside, ["pid-1", "pid-1/events.bin"]) + finding = proc.stderr.split("[triage] finding=", 1)[1].split("\n", 1)[0] + self.assertTrue(Path(finding).is_file(), proc.stderr) + + def test_an_explicit_workdir_inside_the_archive_is_refused(self): + proc, inside = self._run_in_archive({"HRR_TRIAGE_WORKDIR": "out"}) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("inside the archive", proc.stderr) + self.assertEqual(inside, ["pid-1", "pid-1/events.bin"]) + + def test_an_output_inside_the_archive_is_refused(self): + for output in ("finding.md", "../finding.md"): + with self.subTest(output=output): + proc, inside = self._run_in_archive({}, ["-o", output]) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("--output is inside the archive", proc.stderr) + self.assertEqual(inside, ["pid-1", "pid-1/events.bin"]) + + def test_stdout_redirected_to_a_file_keeps_its_mode(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "finding.md" + out.touch() + out.chmod(0o644) + with out.open("w") as fh: + proc, _ = self._run_in_archive({"TMPDIR": tmp}, ["-o", "/dev/stdout"], fh) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(out.stat().st_mode & 0o777, 0o644) + self.assertEqual(out.read_text().count("**D2H**"), 1) + + def test_the_finding_is_printed_once(self): + with tempfile.TemporaryDirectory() as tmp: + proc, _ = self._run_in_archive({"TMPDIR": tmp}) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout.count("**D2H**"), 1) + + +class LibraryPathSafetyTests(unittest.TestCase): + """An empty component in LD_LIBRARY_PATH means the current directory, and + this script is run from inside customer archives. + """ + + def _resulting_path(self, env: dict, parent: str = "bin") -> str: + body = _shell_function("setup_library_path") + # The playback has to exist: from a path that does not resolve, the + # sibling probe reads /../lib, which is /lib on any Linux host. + # ROCR_LIB is read too, so a developer's own must not leak in. + # `set -u` as in the script, where an unbound name aborts the run. + with tempfile.TemporaryDirectory() as tmp: + play = Path(tmp) / parent / "hrr-playback" + play.parent.mkdir() + play.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + play.chmod(0o755) + inherited = { + k: v + for k, v in os.environ.items() + if k not in ("LD_LIBRARY_PATH", "ROCR_LIB") + } + result = subprocess.run( + ["bash", "-c", f'set -u\nSCRIPT_DIR="{tmp}"\n{body}\n' + f'setup_library_path "{play}"\n' + 'echo "${LD_LIBRARY_PATH-unset}"'], + capture_output=True, + text=True, + env={**inherited, **env}, + check=True, + ) + return result.stdout.strip().splitlines()[-1] + + def test_no_empty_component_when_nothing_is_found(self): + self.assertEqual(self._resulting_path({"ROCM_PATH": ""}), "unset") + + def test_no_empty_component_with_only_rocm_path(self): + path = self._resulting_path({"ROCM_PATH": "/opt/rocm"}) + self.assertEqual(path, "/opt/rocm/lib") + self.assertFalse(path.startswith(":")) + + def test_the_callers_value_keeps_its_place(self): + path = self._resulting_path({"ROCM_PATH": "/opt/rocm", "LD_LIBRARY_PATH": "/mine"}) + self.assertEqual(path, "/mine:/opt/rocm/lib") + + def test_empty_components_in_the_callers_value_are_dropped(self): + for inherited in ("/mine:", ":/mine", "/mine::/theirs", ":"): + with self.subTest(inherited=inherited): + path = self._resulting_path({"ROCM_PATH": "/opt/rocm", "LD_LIBRARY_PATH": inherited}) + self.assertNotIn("::", path) + self.assertFalse(path.startswith(":") or path.endswith(":"), path) + self.assertTrue(path.endswith("/opt/rocm/lib"), path) + + def test_a_build_tree_playback_with_no_lib_dir_beside_it(self): + """`ensure_playback.sh --build` leaves the binary in build/playback/. + + Nothing is found there, and expanding the empty list under `set -u` + aborts on bash before 4.4. + """ + path = self._resulting_path({"ROCM_PATH": "/opt/rocm"}, parent="playback") + self.assertEqual(path, "/opt/rocm/lib") + + +class WorkdirSymlinkTests(unittest.TestCase): + def test_a_symlinked_default_is_not_reused(self): + """`-O` follows symlinks, so the predictable name can be a link into a + directory the caller owns, an archive among them. + """ + line = next( + ln for ln in SCRIPT.read_text(encoding="utf-8").splitlines() if '-O "$WORKDIR"' in ln + ) + self.assertIn('-L "$WORKDIR"', line) + self.assertIn('-d "$WORKDIR"', line) + + +if __name__ == "__main__": + unittest.main() diff --git a/projects/hrr/skills/enable-recording/SKILL.md b/projects/hrr/skills/enable-recording/SKILL.md new file mode 100644 index 000000000000..e4d052cd6bc3 --- /dev/null +++ b/projects/hrr/skills/enable-recording/SKILL.md @@ -0,0 +1,160 @@ +--- +name: enable-recording +description: >- + Records a failing HIP or ROCm GPU workload into an HRR archive that can be + replayed and debugged later, and checks that archive is readable before anyone + is asked to send it. Use when an AMD GPU workload crashes, hangs, aborts with a + memory access fault, or returns wrong or NaN results and the cause is not known + yet; when the question is what to send AMD to reproduce a GPU failure; or when + asked to turn on recording, capture or tracing of a workload for later replay. + Applies to any process that uses the HIP runtime, whatever it is: a training + or inference run, a simulation, a benchmark, a compiled application. Do NOT + activate for performance questions, or when an archive already exists and the + task is to replay or triage it, which is decode-and-triage. +--- + +# Record a failing workload + +Capture writes every HIP call a process makes, plus the host buffers those calls carry, into an archive directory. That archive replays on another machine without the application, its source or its data, which is what makes a failure someone else can look at. + +This skill goes from a workload that misbehaves to an archive that has been checked. Triaging the archive afterwards is a different skill, [decode-and-triage](../decode-and-triage/SKILL.md). + +Run `scripts/hrr_capture.sh` for every step below; paths are relative to this skill's own directory. It needs Linux, bash, GNU coreutils and `python3`, and runs `scripts/inspect_archive.py` itself, so invoke that directly only when a machine-readable report is wanted. `tests/` holds the inspector's unit suite and the recorded tool output it reads, run with `pytest`, and is not part of the workflow. + +## Prerequisites + +- **A ROCm build whose `libamdhip64` has HRR compiled in.** Capture is part of the runtime, not a separate package, and no ROCm version number tells you whether a given build has it. Preflight is the test: it greps each candidate runtime for `HIP_HRR_CAPTURE_OUTPUT` and marks it `capture: yes` or `capture: NO`. +- **Linux.** Capture on Windows exists but this skill's scripts do not run there. +- **CDNA data-centre parts.** Everything here was exercised on `gfx950` (MI350X). Nothing in it is architecture-specific, and nothing has been tried on RDNA, so on a consumer part treat a failure as unproven rather than as a bug. +- **Disk, in tens of gigabytes.** Archives are large and capture writes until the disk fills. Preflight refuses below 50 GiB free; `--min-free-gb` moves that for a short run. +- **A matching `hrr-playback` if you want the archive cross-checked**, which is optional. The sibling skill's `../decode-and-triage/scripts/ensure_playback.sh` finds one or builds it. + +## Environment variables that change what you get + +Each of these changes the result silently, and the first two are the ones that waste an afternoon. + +- **`HIP_HRR_CAPTURE_OUTPUT`**: switches capture on and names the archive. A runtime without HRR ignores it and reports nothing, which is why preflight exists. Let `run` set it rather than exporting it yourself, because a variable exported in the shell that starts a container does not reach the process inside it. +- **`HIP_VISIBLE_DEVICES`**: changes which devices the workload sees and therefore what gets recorded. This skill never sets it. HIP's ordering is not `rocm-smi`'s, so device 4 here is not necessarily GPU 4 there; if you pin a device, confirm which one you got. +- **`LD_PRELOAD` and `LD_LIBRARY_PATH`**: decide which runtime loads. `LD_PRELOAD` wins over everything, including a binary's own `DT_RPATH`; `LD_LIBRARY_PATH` is the one that loses to it. Winning the link is not the same as working, which is why preloading a runtime into a framework wheel breaks it. See step 1. +- **`HRR_PLAYBACK` and `ROCM_PATH`**: where `verify` looks for `hrr-playback` when `--playback` is not given, in the same order as `ensure_playback.sh`: `HRR_PLAYBACK`, then `$ROCM_PATH/bin`, then `PATH`. The first reader found is often the wrong one. +- **`HIP_HRR_DEBUG_ARGS`**: turns on capture's own argument logging, and raises the runtime's log level for the whole process, so the workload's output changes shape. +- **`PYTORCH_HIP_ALLOC_CONF`**: changes how PyTorch sub-allocates, which changes what HIP sees and therefore what the archive holds. Capture what the failing run uses, not a tidier configuration. + +## References + +- `references/workload-shapes.md`: how a workload that spawns children, has to be signalled to stop, or manages memory inside one allocation changes what comes out. **Load when** the workload is anything other than a single process that runs to completion. + +## When this is the wrong tool + +Say so rather than capturing anyway: + +- **The workload is slow, not wrong.** Capture makes a run slower and does not measure it. Use the ROCm profilers. +- **The failure is above the GPU runtime**, in the application's own logic rather than in what it asks the GPU to do. Only HIP calls are recorded. +- **The failure never reproduces.** Capture records a run that happens; it cannot record one that does not. + +## 1. Preflight + +Run this **in the same environment as the workload**, meaning inside the container and with the same user as the run itself, and pass the workload command after `--`: + +```bash +scripts/hrr_capture.sh preflight --output /data/captures/run.hrr -- \ + +``` + +The command matters. A binary's own `DT_RPATH` is searched before `LD_LIBRARY_PATH`, and no search of the environment can see it, so given the command preflight asks the loader which runtime will bind rather than guessing from what is visible. Without it the check still runs and is still worth running, but it answers a weaker question and says so. + +It answers three questions that otherwise fail silently, hours later: + +- **Which HIP runtime will the workload load, and does it have capture built in?** Capture is compiled into `libamdhip64`, so a runtime built without it ignores the environment variable, reports nothing, and creates no archive at all: `verify` afterwards reports `no archive`, which is easy to mistake for a workload that died early. The script lists every `libamdhip64` visible in load order, marks which ones can capture, and names the one the command binds when it can resolve it. It cannot always: an interpreter loads its runtime later, and a binary whose linked runtime is missing symbols makes the loader complain rather than answer, in which case the listing is what you have. An application that ships its own runtime uses that one rather than the ROCm install, which is the most common reason a capture comes back empty. An interpreter is the awkward case: `python3` links no HIP runtime itself, and the one a framework loads later cannot be resolved this way, so for a Python workload read the bundled-package rows of the listing. +- **Will the archive survive?** A path on the container's own writable layer disappears with the container. It has to be a bind mount from the host. +- **Is there room?** Archives reach tens of gigabytes for a workload of any size, and capture writes until the disk fills. + +If the runtime that will load has no capture, the fix depends on what loads it. For a compiled binary, put a capture-capable runtime in front with `LD_PRELOAD`. For a framework that carries its own ROCm in a wheel, PyTorch and vLLM being the common ones, `LD_PRELOAD` does not work: it resolves against the wheel's older HSA and the framework ends up reporting no GPU at all. There the capture-capable runtime has to be installed into the wheel's own lib directory, with the matching `libhsa-runtime64` from the same build. Both routes are in [references/workload-shapes.md](references/workload-shapes.md). Either way, preflight again rather than assuming it worked. + +## 2. Run + +```bash +scripts/hrr_capture.sh run --output /data/captures/run.hrr -- \ + +``` + +Whatever command already reproduces the failure goes after `--`, unchanged. The script preflights, sets `HIP_HRR_CAPTURE_OUTPUT`, runs the command, and reports the archive afterwards. The workload's own exit status is preserved and passed back, which also means a run that succeeds while capturing nothing exits zero: the verdict, not the exit status, says whether capture worked. + +**Run this inside the container, not around it.** The script sets the environment variable on the command it launches, and a variable set in the shell that starts a container does not cross into it. Wrapping `docker run` therefore produces exactly the empty-capture symptom this skill exists to prevent. Enter the container first, with the skill directory and the output path both visible inside it, and run the three steps there. A container running as root leaves the archive owned by root on the host, where the person who owns the directory can read it and not move it, so `chown` it back afterwards. + +Three things worth knowing before the run: + +- **A crash is the good case.** A workload that dies mid-run still produces a readable archive: the writer flushes periodically, and it finalizes from the runtime's own handler for `SIGSEGV`, `SIGABRT`, `SIGBUS`, `SIGILL` and `SIGFPE`. Nothing else is covered, so a `docker stop`, a `timeout`, or a launcher killing its ranks leaves `state: not finalized` instead. That archive is still readable and `hrr-playback --repair` gives it a clean trailer. +- **Only successful HIP calls are recorded, with a few exceptions.** An application that handles a failed call and retries will not replay down the same branch. A handful of hand-written shims record regardless of the result, among them the memory-pool and array creation calls. +- **The device mask is left alone.** This skill never picks a GPU. Capture is meant to record the run being reproduced, so run it the way it normally runs. + +Two flags change what `run` does, both off by default: `--skip-preflight` runs the command with no checks at all, and `--force` runs it even when preflight failed for any reason, including too little disk or a path that will not outlive the container. `--force` is also the only way past an output directory that already holds a capture, and it then adds this run to it, so the two become one archive nobody can separate afterwards: give `--output` a new path instead unless merging is what you want. `--min-free-gb N` moves the space threshold. + +A workload that spawns children, one that has to be stopped rather than finishing on its own, or one that manages its own memory inside a single allocation, each behave differently here. Read [references/workload-shapes.md](references/workload-shapes.md) before capturing any of those. + +## 3. Verify before sending + +```bash +scripts/hrr_capture.sh verify --output /data/captures/run.hrr \ + --playback /path/to/hrr-playback +``` + +This reads the archive's manifests, so it needs no GPU, and cross-checks against `hrr-playback --info`. Add `--json` for a machine-readable report, or `--no-playback` to report from the manifests alone, which checks what the capture writer wrote rather than that a reader can decode it. Without `--playback` it takes the first `hrr-playback` it can find, which is often the wrong one: the reader has to match the runtime that captured, or it refuses the archive with `Version mismatch in : file=N reader=M`. That means the reader is wrong, not the archive. + +A reader has libraries of its own and may not start at all, commonly inside a container that has no ROCm install to fall back on. That is a failed cross-check rather than a bad archive: the report says so and exits 3, and the archive is untouched by it. A reader that starts and then refuses the archive exits 3 for the same reason, since the archive still went unread. No reader anywhere on the machine is not a failure: that is the manifests-only path and it exits 0. + +If you are scripting a gate, run `verify` as its own step. `run` returns the workload's exit status, by design, so it cannot also report what the check found. `verify` exits 0 when the archive holds events, 1 for the three verdicts that mean it does not, 2 for a usage mistake, and 3 when a cross-check was asked for and could not run. + +A refusal is the good case. A reader from a neighbouring build can do something worse: it accepts the archive and decodes the event names against its own API table, so the counts are right, the names are wrong, and nothing reports an error. From format v5 the runtime API ids are append-only, so that is a risk for older archives and for compiler-table events rather than for every event in every archive. The report therefore names the reader it used and its version, and says so when the pairing cannot be established. If the API mix it prints does not look like the workload that ran, distrust the reader before the archive. + +Every run ends in one of four verdicts: + +- `Verdict: recorded`: events were written. The archive is worth keeping. +- `Verdict: empty`: process directories exist but hold no events. The processes opened a capture and recorded nothing. +- `Verdict: nothing captured`: the directory exists with no process directory in it, so no process ever opened a capture. The directory came from somewhere else, an earlier attempt or a `mkdir`, and the variable never reached the process that used the GPU: go back to preflight. +- `Verdict: no archive`: the directory was never created, so capture never started. Either the runtime that loaded has no capture compiled in, which is the common case, or the workload died before its first HIP call. Preflight with the command tells the two apart. + +Each process directory also reports its state: + +- `state: complete`: it shut down cleanly and wrote its trailer. +- `state: incomplete`: no trailer. Usually the process died, which is expected for a crash and still readable. It also appears when capture itself dropped an event it could not fit in the wire format, and that archive has a hole in the middle rather than a truncated tail. The workload's own stderr tells the two apart: capture prints `Archive marked INCOMPLETE` in the second case. +- `state: not finalized`: events exist but no manifest, so it was killed before it could finalize. The counts then come from `--info`. + +An archive with several `pid-*` directories is one archive. Send the whole directory, since the processes belong together, and replay them one at a time (step 5). + +## 4. Share it, knowing what is in it + +**There is no scrubbing, anonymisation or randomisation option today.** Say that plainly before anyone asks whether the archive can be sanitised, because the answer shapes what they do next and there is no flag that changes it. + +What the archive holds, per process, and it is worth reading out to whoever has to approve the transfer: + +- **`events.bin`**: every successful HIP call the process made, with its arguments: sizes, device addresses, stream handles, kernel names. +- **`blobs/`**: the host buffers those calls carried, byte for byte and in the clear. For a serving stack that means prompts and generated text; for training, the data and the weights that were copied to the device. +- **`code_objects/`**: the compiled GPU kernels the process loaded, which is the application's own binary code and a separate disclosure question from customer data. +- **`manifest.json`**: process and parent process ids, event and blob counts, the clean-shutdown flag, the HIP and comgr versions that captured, and the device's name, architecture, PCI location and UUID. +- **`regions/*.hrrr`**: the framework allocator's own layout, when a producer ran alongside capture. +- **`writer_state.json`**: the checkpoint cursor, present only while a capture is mid-flight. + +Apart from the few exceptions in step 2, failed HIP calls are not recorded, and nothing that never crossed a HIP call is. The archive is exactly as sensitive as the data the workload handled, so where it may go has to be agreed before it is uploaded. + +`verify --json` gives each process's counts and sizes in a form you can attach to the request; the device details are in each `manifest.json`. For a multi-process archive, `hrr-playback --info` against the archive root prints a table covering every process at once. The cross-check in `verify` reports on one process, which is enough to judge the capture and not enough for a disclosure inventory. + +If the content is the obstacle, three things exist and none of them is scrubbing. Re-capture the same failure driven by synthetic input, if it still reproduces. Replay the archive yourself and send only the finding, which is a few lines and carries no buffers. Or agree the handling of a production-data transfer before uploading, because that is what this is. + +## 5. Then replay it, here, now + +**Do this before sending anything anywhere.** The machine that captured has the runtime the archive was written by, so the reader that matches it is here and nowhere else in particular. It is also the fastest way to learn that the archive is worth sending: a replay that reproduces the fault is proof, where a verdict of `recorded` is only a promise. + +One thing still gets in the way. The triage skill's preflight compares the capture against the HIP version of the `hipconfig` on `PATH`, not against the reader you named, so on a host whose ROCm install differs from the build that captured it stops with a version prompt and wants `HRR_CONTINUE=1`. That is the check being coarse rather than a real mismatch. + +Offer it as soon as `verify` says `recorded`, and hand off to [decode-and-triage](../decode-and-triage/SKILL.md) on the archive just made: + +```bash +../decode-and-triage/scripts/triage_archive.sh --archive /data/captures/run.hrr/pid- +``` + +Start with the process that did not shut down cleanly, `incomplete` or `not finalized`, since that is the one that failed. A `not finalized` archive gets a clean trailer from `hrr-playback --repair` first. + +That skill replays the archive and reports what failed: fault class, faulting kernel, fault address, failing event. If no `hrr-playback` is around, its `scripts/ensure_playback.sh` finds one or builds it, which is also the answer to "where do I get the matching reader" in step 3. + +One rule for a multi-process archive, since the two skills sound contradictory and are not: **send it whole, replay one process at a time.** The processes belong together and the directory is the unit of transfer, while triage takes a single `pid-*` directory. diff --git a/projects/hrr/skills/enable-recording/evals/evals.json b/projects/hrr/skills/enable-recording/evals/evals.json new file mode 100644 index 000000000000..f563b8cce5a5 --- /dev/null +++ b/projects/hrr/skills/enable-recording/evals/evals.json @@ -0,0 +1,93 @@ +{ + "evaluations": [ + { + "id": "symptom-memory-access-fault", + "skill_should_trigger": true, + "prompt": "Our inference server on an MI300 node dies every few hours with 'Memory access fault by GPU node-4'. We cannot share the model or the code with you. What should we send you so you can debug it?", + "expected_behavior": [ + "Proposes recording the failing run into an archive that replays without the application, its source or its data", + "Checks that the HIP runtime the workload loads has capture built in, rather than only giving the environment variable", + "Says the archive has to be verified as readable before it is sent" + ], + "unexpected_behavior": [ + "Asks for the model weights, the source code or a reproducer script", + "Recommends a profiler" + ] + }, + { + "id": "wrong-results-without-a-crash", + "skill_should_trigger": true, + "prompt": "Our fine-tuning job on an AMD GPU starts producing NaN losses after a few hundred steps. Nothing crashes and we cannot hand over the dataset. How do we get this to the runtime team?", + "expected_behavior": [ + "Proposes recording the failing run into an archive rather than asking for the dataset" + ] + }, + { + "id": "empty-archive-is-the-runtime-not-the-workload", + "skill_should_trigger": true, + "prompt": "I set HIP_HRR_CAPTURE_OUTPUT and ran my PyTorch training job. It ran fine but the verify output says:\n\nArchive: /tmp/run.hrr\n\nVerdict: no archive. The directory was never created, so capture never started.\n\nWhat went wrong?", + "expected_behavior": [ + "Explains that the HIP runtime the workload loaded has no capture compiled in, so the environment variable was ignored", + "Points out that a Python workload usually loads the runtime shipped inside the torch wheel rather than the one in the ROCm install", + "Tells the user to run the skill's preflight in the workload's own environment" + ], + "unexpected_behavior": [ + "Suggests the archive is corrupt" + ] + }, + { + "id": "incomplete-archive-is-still-worth-sending", + "skill_should_trigger": true, + "prompt": "Is this capture usable?\n\nArchive: /work/captures/crash.hrr\nProcesses: 1\n\n pid-142\n events: 12,904 blobs: 3,201 state: incomplete (no clean-shutdown trailer)\n\nVerdict: recorded. 1 of 1 process directories hold events.", + "expected_behavior": [ + "Says the archive is usable and worth sending", + "Explains that the missing clean-shutdown trailer is the expected signature of a workload that crashed, not corruption" + ], + "unexpected_behavior": [ + "Tells the user to capture again because the archive is broken", + "Names a kernel or diagnoses the crash, which is a different skill's job" + ] + }, + { + "id": "what-is-inside-the-archive-before-sharing", + "skill_should_trigger": true, + "prompt": "Capture worked and we have an archive. Our legal team wants to know what is inside it before we upload it to a vendor. What do we tell them?", + "expected_behavior": [ + "States that the recorded buffers hold the real workload data in the clear, including prompts and generated text" + ], + "unexpected_behavior": [ + "Claims the archive can be anonymised, scrubbed or randomised by a flag", + "Claims the archive holds only metadata or only API calls" + ] + }, + { + "id": "multi-process-archive-is-kept-together", + "skill_should_trigger": true, + "prompt": "Our capture produced two pid directories, pid-49 and pid-579, under the same archive directory. Which one do I send?", + "expected_behavior": [ + "Says to send the whole archive directory rather than one pid directory" + ], + "unexpected_behavior": [ + "Picks a single pid directory as the one to send" + ] + }, + { + "id": "gemm-performance-question", + "skill_should_trigger": false, + "note": "Capture makes a run slower and measures nothing. This belongs to the profilers.", + "prompt": "Our GEMM kernel on MI300 reaches only 40% of peak. How do I find out which part of it is slow?" + }, + { + "id": "triage-an-archive-that-already-exists", + "skill_should_trigger": false, + "note": "An archive in hand is decode-and-triage's job, not this skill's.", + "prompt": "I already have an HRR archive at /work/captures/crash.hrr. Replay it and tell me which kernel faulted." + }, + { + "id": "failure-above-the-gpu-runtime", + "skill_should_trigger": false, + "note": "Only HIP calls are recorded, so an application-level bug is out of scope.", + "prompt": "My training script raises a KeyError in the data loader before the first epoch finishes. How do I debug it?" + } + ] +} diff --git a/projects/hrr/skills/enable-recording/references/workload-shapes.md b/projects/hrr/skills/enable-recording/references/workload-shapes.md new file mode 100644 index 000000000000..876aa6b74b36 --- /dev/null +++ b/projects/hrr/skills/enable-recording/references/workload-shapes.md @@ -0,0 +1,72 @@ +# Workload shapes that change how a capture behaves + +The main flow in [SKILL.md](../SKILL.md) wraps whatever command already reproduces the failure and does not care what that command is. A few properties of a workload do change what comes out, though, and they are worth recognising before blaming the capture. + +## The application may bring its own HIP runtime + +An application that ships a copy of `libamdhip64` alongside itself loads that copy, not the one in the ROCm install, because its own libraries are found first. So a ROCm install with capture built in does not help if the bundled copy has none. This is common wherever the runtime is delivered with the application rather than by the system package manager. + +`hrr_capture.sh preflight` lists every runtime it can see, in load order, and marks which of them can capture. Give it the workload command after `--`: a binary's own `DT_RPATH` is searched before `LD_LIBRARY_PATH` and no search of the environment can see it, so with the command preflight asks the loader instead of guessing. + +Which remedy works depends on what loads the runtime, and the difference is not cosmetic. + +**A compiled binary that links `libamdhip64` itself.** Put a capture-capable runtime in front: + +```bash +export LD_PRELOAD=/path/to/libamdhip64.so.7 +``` + +`LD_PRELOAD` wins over both a bundled copy and `LD_LIBRARY_PATH`. Preflight again afterwards rather than assuming the preload took effect. + +**A Python framework that carries its own ROCm, such as a PyTorch or vLLM wheel.** `LD_PRELOAD` does not work here, and it fails in a way that looks like broken hardware: the preloaded runtime resolves against the wheel's older HSA and dies on a missing symbol, or it initialises far enough to enumerate the GPU and then fails code-object initialisation, after which the framework reports no GPU at all. Winning the link is not enough, because the runtime also has to reach a compatible `comgr` and the device bitcode that sit beside it in its own SDK tree. + +**Find that directory by watching, not by reading.** A wheel commonly ships two SDK directories holding different builds, and the `DT_RUNPATH` on its libraries can name absolute paths that do not exist in the image it ends up running in. It is also searched after `LD_LIBRARY_PATH` rather than before it, so reading it tells you less than it looks. Ask a live process instead: + +```bash +python3 -c 'import torch; torch.cuda.init(); print(open("/proc/self/maps").read())' \ + | grep -m2 -E 'libamdhip64|libhsa-runtime64' +``` + +In-process on purpose: backgrounding the interpreter and reading `/proc/$!/maps` after a sleep races with its exit, and an empty result then reads as "nothing loaded" when the truth is "already gone". + +The directory in that output is the one to overlay. Reading `readelf -d` on the framework's libraries answers a different question and has pointed at the wrong directory of the two. + +What works is installing the capture-capable runtime into the directory the framework actually binds, together with the matching `libhsa-runtime64` from the same build, and leaving `LD_PRELOAD` unset: + +```bash +# inside the container, over the framework's own ROCm libraries. +# Move the originals aside first: this edits a shared environment, and there is +# no way back to the stock wheel afterwards short of reinstalling it. +mkdir -p ~/hrr-wheel-backup +mv /lib/libamdhip64.so.7 ~/hrr-wheel-backup/ +mv /lib/libhsa-runtime64.so.1 ~/hrr-wheel-backup/ +cp /lib/libamdhip64.so.7 /lib/ +cp /lib/libhsa-runtime64.so.1 /lib/ +unset LD_PRELOAD +``` + +Keep the backup outside that lib directory rather than beside the library: a second `libamdhip64` in there is what makes a runtime verdict unreliable in the first place. If the backup is lost, a throwaway virtual environment or `pip install --force-reinstall` gets you back to the stock wheel. + +The capture runtime and the framework have to be compatible in the other direction too, and the newest build is not automatically the right one. A HIP build that needs a newer `comgr` than the framework's own will not run it at all, so the pairing is found by trying rather than by version order. + +## One process or several + +Every process that uses HIP writes its own `pid-/` sub-archive under the same output directory, so the shape of the archive follows the shape of the run. A single process leaves one directory. A workload that forks or spawns children leaves one per child that touches the GPU, and the child doing the GPU work is usually the interesting one while the parent only sets things up. + +The whole directory is the archive. Sending one `pid-*` out of it loses the context the rest carries. + +## Stopping a long-running workload so the archive finalizes + +The clean trailer is written when the process exits normally. A process terminated with a signal it cannot handle is stopped outright, so its archive ends without a trailer. + +That archive is not lost: the writer flushes periodically, the reader keeps every complete record and stops at a torn tail, and `hrr-playback --repair` rewrites it with a clean trailer. But when the failure has already happened and the run only needs to be ended, stopping the workload from the inside, so that it returns from its own main loop, gives a complete archive with no repair step. + +## Memory the capture cannot see + +Some applications take one large allocation from HIP and then hand out pieces of it in their own code. Those sub-allocations never reach a HIP API, so the capture cannot see them, and a write that goes out of bounds while staying inside the original allocation faults neither at capture time nor at replay. A workload whose symptom is corrupted values rather than a fault may be hitting exactly this. + +## Cost + +Capture adds host CPU work and disk writes to every HIP call, and hashes the host buffers that memory copies carry. The cost is a property of the workload rather than a constant: a workload that issues many small calls pays far more than one that issues few large ones, so measure it for the workload in question rather than quoting a single figure. It is not a setting to leave on in production. + +If the archive grows faster than the disk can take it, writing to a different filesystem is the cheap experiment that separates an I/O limit from contention inside the writer. diff --git a/projects/hrr/skills/enable-recording/scripts/hrr_capture.sh b/projects/hrr/skills/enable-recording/scripts/hrr_capture.sh new file mode 100755 index 000000000000..026bbed2e148 --- /dev/null +++ b/projects/hrr/skills/enable-recording/scripts/hrr_capture.sh @@ -0,0 +1,743 @@ +#!/usr/bin/env bash +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Turn HRR capture on for a workload, then say whether it actually recorded +# anything. Sole agent entry point for this skill. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INSPECT="$SCRIPT_DIR/inspect_archive.py" +# The sibling skill, which replays an archive and says what failed. Also where +# the only script that finds or builds a matching hrr-playback lives. +TRIAGE="$SCRIPT_DIR/../../decode-and-triage/scripts/triage_archive.sh" +ENSURE_PLAYBACK="$SCRIPT_DIR/../../decode-and-triage/scripts/ensure_playback.sh" +ROCM_PATH="${ROCM_PATH:-/opt/rocm}" + +# The environment variable libamdhip64 reads to enable capture. Its presence as +# a literal in a shared object distinguishes a capture-capable runtime from one +# built without HRR. +CAPTURE_VAR="HIP_HRR_CAPTURE_OUTPUT" + +# Set by check_runtime when it could not establish which runtime will bind, so +# `run` can repeat the caveat at the moment it matters. +RUNTIME_UNRESOLVED=0 + +usage() { + cat <<'EOF' >&2 +usage: + hrr_capture.sh preflight [--output PATH] [--playback PATH] [-- [args...]] + hrr_capture.sh run --output PATH [--playback PATH] -- [args...] + hrr_capture.sh verify --output PATH [--playback PATH] [--json] [--no-playback] + + preflight Check this environment can capture: which HIP runtime the workload + will load, whether it has capture built in, and whether the output + path is big enough and outlives the container. Give it the command + after --: a binary's own RPATH beats every path a search can see, + so without the command this answers a weaker question. + run Preflight, then run the command with capture enabled, then verify. + verify Report what an existing archive contains. + +Options: + --output PATH Archive directory to write (run/verify: required) + --playback PATH hrr-playback to cross-check the archive with. It has to be + the one matching the runtime that captured, or it will + refuse the archive over its format version + --json verify: emit JSON instead of text + --no-playback verify: report from the manifests only, no cross-check + --skip-preflight run: do not preflight at all + --force run: proceed even when preflight fails, for any reason: + no capture-capable runtime, too little space, or an + output path that will not outlive the container + --min-free-gb N Minimum free space to insist on (default 50) + -h, --help Show this help + +Requires Linux, bash, GNU coreutils and python3. This script never selects a +GPU: whatever device mask the workload already uses is left alone, because +capture has to record the run being reproduced. +EOF +} + +log() { printf '[capture] %s\n' "$*" >&2; } +fail() { printf '[capture] error: %s\n' "$*" >&2; exit 1; } + +# --- runtime discovery ------------------------------------------------------ + +# Does this shared object carry HRR capture? +lib_has_capture() { + grep -a -q -- "$CAPTURE_VAR" "$1" 2>/dev/null +} + +# Only the names a linked binary can ask for. A build or backup left beside the +# real thing, `libamdhip64.so.hrr-orig` and the like, is not a runtime anything +# loads, and treating it as one has already produced a wrong verdict. +is_runtime_name() { + # Anchored: `libamdhip64.so.7.hrr-orig` and `libamdhip64.so.7.bak` match a + # `.so.[0-9]*` glob and are not runtimes anything loads. + [[ "${1##*/}" =~ ^libamdhip64\.so(\.[0-9]+)*$ ]] +} + +# Runtimes in one directory, soname first. `libamdhip64.so` is a development +# alias: nothing links it, everything links the soname `libamdhip64.so.N`, so +# the alias must never be the one a verdict is pronounced on. They are normally +# hardlinks to one file, and the order only matters when something has broken +# that, which is exactly when the verdict was wrong before. +emit_libs() { + local src="$1" dir="$2" entry + for entry in "$dir"/libamdhip64.so.[0-9]*; do + [[ -f "$entry" ]] && is_runtime_name "$entry" && echo "$src|$entry" + done + [[ -f "$dir/libamdhip64.so" ]] && echo "$src|$dir/libamdhip64.so" + return 0 +} + +# The interpreter whose packages matter. Named directly for `python train.py`, +# but the workloads this skill is aimed at are console scripts: `vllm`, +# `torchrun`, `accelerate`, `pytest`. Those are text files whose shebang names +# the interpreter of their own virtual environment, and asking the PATH's +# python3 instead reports the wrong site-packages, or none at all. +workload_interpreter() { + local bin="${WORKLOAD_BIN:-}" shebang + case "${bin##*/}" in + python|python[0-9]*) echo "$bin"; return 0 ;; + esac + if [[ -f "$bin" ]] && read -r shebang < "$bin" 2>/dev/null; then + case "$shebang" in + '#!'*python*) + shebang="${shebang#\#!}" + set -- $shebang + # `#!/usr/bin/env python3` names the interpreter in the second word, + # and `#!/usr/bin/env -S python3 -u` in the third, possibly after + # assignments. Any other option is not an interpreter. + if [[ "${1##*/}" == "env" ]]; then + shift + case "${1:-}" in + -S|--split-string) shift ;; + --split-string=*) set -- "${1#--split-string=}" "${@:2}" ;; + -S?*) set -- "${1#-S}" "${@:2}" ;; + esac + while [[ "${1:-}" == *=* ]]; do shift; done + fi + [[ -n "${1:-}" && "$1" != -* ]] && { echo "$1"; return 0; } + ;; + esac + fi + PATH="${WORKLOAD_PATH:-$PATH}" command -v python3 2>/dev/null || echo python3 +} + +# Launchers that wrap the real workload. The binary that matters is the one +# after them, otherwise preflight resolves `srun` or `env` and learns nothing. +# Prints the VAR=value assignments `env` takes, one a line, then the workload +# last: `env LD_LIBRARY_PATH=/capture/lib app` runs app with that path, so +# preflight has to resolve its runtime with it too. +strip_launchers() { + while (( $# )); do + case "${1##*/}" in + env) + # Its options change the environment in ways the assignments below + # cannot carry: -i empties it, -u removes a name, -C moves the working + # directory. Stop at env when one is given rather than resolve the + # workload in an environment it will not have. + [[ "${2:-}" == -* && "$2" != -- ]] && { printf '%s\n' "$1"; return 0; } + shift + [[ "${1:-}" == -- ]] && shift + while (( $# )) && [[ "$1" == *=* ]]; do printf '%s\n' "$1"; shift; done ;; + nohup) + shift ;; + timeout) + # Options, then the duration, which is a bare operand: skipping only + # dashed arguments left `60s` as the workload. + shift + while (( $# )) && [[ "$1" == -* ]]; do + case "$1" in + -s|--signal|-k|--kill-after) shift 2 ;; + *) shift ;; + esac + done + (( $# )) && shift ;; + nice) + shift + while (( $# )); do + case "$1" in + -n|--adjustment) shift 2 ;; + -*) shift ;; + *) break ;; + esac + done ;; + ionice) + shift + while (( $# )); do + case "$1" in + -c|-n|-p|--class|--classdata) shift 2 ;; + -*) shift ;; + *) break ;; + esac + done ;; + stdbuf) + shift + while (( $# )); do + case "$1" in + -i|-o|-e|--input|--output|--error) shift 2 ;; + -*) shift ;; + *) break ;; + esac + done ;; + mpirun|mpiexec|srun|torchrun|accelerate|deepspeed) + # Their own options and the command are not separable without knowing + # each launcher's grammar, so stop here and say so rather than guess. + printf '%s\n' "$1"; return 0 ;; + *) printf '%s\n' "$1"; return 0 ;; + esac + done + return 0 +} + +# Directories on the binary's own DT_RPATH or DT_RUNPATH, with $ORIGIN expanded, +# for whichever of the two tags is asked for. The order matters and the two are +# not interchangeable: DT_RPATH is searched before LD_LIBRARY_PATH, DT_RUNPATH +# after it, and nothing else listed here can see either. +binary_rpath_dirs() { + local bin="$1" tag="${2:-RPATH|RUNPATH}" origin list dir + [[ -n "$bin" && -f "$bin" ]] || return 0 + command -v readelf >/dev/null 2>&1 || return 0 + origin="$(dirname "$(readlink -f "$bin")")" + while read -r list; do + while IFS= read -r dir; do + [[ -n "$dir" ]] || continue + dir="${dir//\$\{ORIGIN\}/$origin}" + echo "${dir//\$ORIGIN/$origin}" + done < <(tr ':' '\n' <<<"$list") + done < <(readelf -d "$bin" 2>/dev/null | + awk -v tag="$tag" '$0 ~ "\\((" tag ")\\)" { match($0, /\[.*\]/); print substr($0, RSTART + 1, RLENGTH - 2) }') +} + +# The runtime the loader will actually bind for this binary. This is the only +# answer that accounts for DT_RPATH, and it is why the list below is a listing +# rather than a verdict. Empty when the binary links no HIP runtime of its own, +# which is the normal case for an interpreter: Python loads the runtime later, +# out of whatever package asks for it, where no static inspection can see it. +resolved_runtime() { + local bin="$1" path + [[ -n "$bin" && -f "$bin" ]] || return 1 + command -v ldd >/dev/null 2>&1 || return 1 + # Only the mapping lines: `soname => /path`, or `/path` alone for a runtime + # preloaded by absolute path, which also satisfies the binary's own entry so + # no `=>` line follows it. A binary built against a newer ROCm than the + # host's makes ldd print symbol-version errors first, and those lines also + # mention libamdhip64: taking the third field off one of them yielded the + # literal word "version", resolution then failed silently, and preflight + # blamed an interpreter for a compiled binary. + path="$(ldd "$bin" 2>/dev/null | + awk '$2 == "=>" && $1 ~ /libamdhip64\.so/ { print $3; exit } + $1 ~ /^\/.*libamdhip64\.so[^\/]*$/ && $2 ~ /^\(0x/ { print $1; exit }')" + [[ -n "$path" && "$path" == /* && -f "$path" ]] || return 1 + readlink -f "$path" +} + +# Candidate libamdhip64 paths, most-likely-to-win first. +# +# Precedence follows the dynamic loader: LD_PRELOAD beats everything, then the +# binary's own DT_RPATH, then LD_LIBRARY_PATH, then its DT_RUNPATH, then a +# runtime bundled with the application itself, then the ldconfig cache, then +# the ROCm install. Bundled +# copies are worth listing because an application that ships its own runtime +# uses that one, whatever is installed on the system. +candidate_libs() { + local entry dir pkg_dir interpreter + local IFS_SAVE="$IFS" + + # `#!/usr/bin/env python3` names an interpreter the workload finds on its own + # PATH, which under `env PATH=/venv/bin vllm` is not this script's. + interpreter="$(workload_interpreter)" + if [[ "$interpreter" != */* ]]; then + interpreter="$(PATH="${WORKLOAD_PATH:-$PATH}" command -v -- "$interpreter" 2>/dev/null || + echo "$interpreter")" + fi + + IFS=': ' + for entry in ${LD_PRELOAD:-}; do + case "$entry" in *libamdhip64*) [[ -f "$entry" ]] && echo "LD_PRELOAD|$entry" ;; esac + done + IFS="$IFS_SAVE" + + while read -r dir; do + [[ -d "$dir" ]] && emit_libs "workload RPATH" "$dir" + done < <(binary_rpath_dirs "${WORKLOAD_BIN:-}" RPATH) + + IFS=': ' + for dir in ${LD_LIBRARY_PATH:-}; do + [[ -d "$dir" ]] && emit_libs LD_LIBRARY_PATH "$dir" + done + IFS="$IFS_SAVE" + + # After LD_LIBRARY_PATH, which is where the loader looks for it. DT_RUNPATH + # is the modern linker's default, so listing it as if it were DT_RPATH put + # the wrong candidate first. + while read -r dir; do + [[ -d "$dir" ]] && emit_libs "workload RUNPATH" "$dir" + done < <(binary_rpath_dirs "${WORKLOAD_BIN:-}" RUNPATH) + + # Runtimes bundled inside installed packages. Listed generically rather than + # by package name: any package may ship one, and the site directories are + # asked of the interpreter rather than guessed. + while read -r pkg_dir; do + [[ -n "$pkg_dir" && -d "$pkg_dir" ]] || continue + while read -r dir; do + [[ -n "$dir" ]] && emit_libs "bundled package" "$dir" + done < <(find "$pkg_dir" -maxdepth 3 -name 'libamdhip64.so*' -type f -printf '%h\n' 2>/dev/null | sort -u) + # The interpreter that will run the workload, not the one on PATH. A job + # launched as /opt/venv/bin/python has different site directories from the + # shell's python3, and asking the wrong one hides the wheel that matters. + done < <("$interpreter" - <<'PY' 2>/dev/null || true +import site +seen = [] +for path in list(site.getsitepackages()) + [site.getusersitepackages()]: + if path and path not in seen: + seen.append(path) + print(path) +PY +) + + if command -v ldconfig >/dev/null 2>&1; then + while read -r entry; do + [[ -f "$entry" ]] && is_runtime_name "$entry" && echo "ldconfig|$entry" + done < <(ldconfig -p 2>/dev/null | awk '/libamdhip64\.so/ {print $NF}') + fi + + emit_libs "$ROCM_PATH/lib" "$ROCM_PATH/lib" +} + +# Identity of a file for dedup purposes. Device and inode rather than the +# resolved path, because the same library is routinely installed as several +# hardlinked names (libamdhip64.so, .so.7, .so.7.x.y) that readlink cannot +# collapse: a hardlink has no target to follow. +file_identity() { + stat -c '%d:%i' "$1" 2>/dev/null || readlink -f "$1" 2>/dev/null || echo "$1" +} + +# Print the candidate table; return 0 when the runtime that will win has capture. +check_runtime() { + local found_any=0 resolved_ok=0 effective="" effective_src="" seen="" src path resolved status ident bound + local no_capture_seen="" + log "HIP runtimes visible from here, in load order:" + while IFS='|' read -r src path; do + [[ -n "$path" ]] || continue + resolved="$(readlink -f "$path" 2>/dev/null || echo "$path")" + ident="$(file_identity "$resolved")" + case ":$seen:" in *":$ident:"*) continue ;; esac + seen="$seen:$ident" + found_any=1 + if lib_has_capture "$resolved"; then + status="capture: yes" + else + status="capture: NO" + [[ -n "$no_capture_seen" ]] || no_capture_seen="$resolved" + fi + printf '[capture] %-16s %s (%s)\n' "$src" "$resolved" "$status" >&2 + if [[ -z "$effective" ]]; then + effective="$resolved" + effective_src="$src" + fi + done < <(candidate_libs) + + # The listing above is what is visible from here. When a command was given and + # it links a HIP runtime itself, the loader's own answer replaces the guess. + if bound="$(resolved_runtime "${WORKLOAD_BIN:-}")"; then + effective="$bound" + effective_src="bound by ${WORKLOAD_BIN##*/}" + resolved_ok=1 + found_any=1 + lib_has_capture "$effective" && status="capture: yes" || status="capture: NO" + printf '[capture] %-16s %s (%s)\n' "resolved" "$effective" "$status" >&2 + elif [[ -n "${WORKLOAD_BIN:-}" ]]; then + log "note: ${WORKLOAD_BIN##*/} links no HIP runtime of its own, so the one it ends up" + log "using is loaded later, from wherever the application asks for it. For a Python" + log "workload that is the runtime inside the installed package, listed above." + else + log "note: no command given, so this is what is visible here rather than what a" + log "workload will load. Pass the command after -- to have it resolved properly." + fi + + if (( ! found_any )); then + log "no libamdhip64 found at all; this environment cannot run a HIP workload" + return 1 + fi + + if lib_has_capture "$effective"; then + if (( resolved_ok )); then + log "the runtime that will load is $effective (via $effective_src) and it has capture" + return 0 + fi + # Unresolved and the best guess looks fine. Saying "you are ready" here is + # how a Python workload gets a clean preflight and an empty capture: the + # wheel's own runtime is bound later and is not this one. + RUNTIME_UNRESOLVED=1 + if [[ -n "$no_capture_seen" ]]; then + # The PyTorch case exactly: binding cannot be established, and a runtime + # that cannot capture is sitting in the load path. Passing here is how a + # clean preflight turns into an empty archive an hour later. + log "UNRESOLVED, and refusing: nothing here was resolved, and $no_capture_seen has no" + log "capture. If the workload binds that one, which a wheel or a plugin commonly does," + log "nothing will be recorded. Install a capture-capable runtime where the workload" + log "will find it, or pass --force to capture anyway and check the verdict afterwards." + return 1 + fi + log "UNRESOLVED: nothing here was resolved, and the first candidate ($effective) has" + log "capture. That is a guess, not a verdict. Every runtime visible from here can" + log "capture, so the guess is a safe one, but the workload may still bind something" + log "none of these searches can see." + return 0 + fi + + log "the runtime that will load is $effective (via $effective_src) and it has NO capture" + log "capture is compiled into libamdhip64, so a build without it ignores $CAPTURE_VAR" + log "and no archive is created at all: verify then reports 'no archive'." + # Collected rather than piped into `grep -q`: under `pipefail` an early-closing + # reader makes the whole pipeline fail, and the advice below would then flip to + # the wrong branch precisely when a usable runtime does exist. + local alternatives + alternatives="$(candidate_libs | while IFS='|' read -r _ path; do + lib_has_capture "$path" && echo yes + done)" + if [[ -n "$alternatives" ]]; then + log "another libamdhip64 here does have capture. For a plain binary, LD_PRELOAD puts" + log "it in front. For a framework that loads its own runtime through an RPATH, such" + log "as a PyTorch or vLLM wheel, LD_PRELOAD does not work: install the capture-capable" + log "runtime into the package's own lib directory instead, with its matching" + log "libhsa-runtime64. See references/workload-shapes.md." + else + log "use a ROCm build whose libamdhip64 has HRR compiled in." + fi + return 1 +} + +# --- output path ------------------------------------------------------------ + +# Filesystem type of the mount a path sits on, from /proc/self/mountinfo, which +# knows about bind mounts where `stat -f` on a parent directory does not. Falls +# back to stat where mountinfo is unavailable. +path_fstype() { + local target="$1" resolved + resolved="$(readlink -f "$target" 2>/dev/null || echo "$target")" + if [[ -r /proc/self/mountinfo ]]; then + awk -v path="$resolved" ' + { + mount_point = $5 + for (i = 6; i <= NF; i++) if ($i == "-") { fstype = $(i + 1); break } + if (path == mount_point || index(path, mount_point == "/" ? "/" : mount_point "/") == 1) { + if (length(mount_point) >= best_len) { best_len = length(mount_point); best = fstype } + } + } + END { if (best != "") print best } + ' /proc/self/mountinfo && return 0 + fi + stat -f -c %T "$resolved" 2>/dev/null || echo unknown +} + +check_output_path() { + local out="$1" min_free_gb="$2" parent probe fstype avail_kb avail_gb + case "$out" in + /*) : ;; + *) log "output path is relative; the archive will land wherever the workload's" + log "working directory happens to be. An absolute path is safer." ;; + esac + + # Capture writes into the output directory, so when it already exists, most + # often because it is the bind mount itself, that directory is the one to + # check. Otherwise capture creates it inside its parent. + if [[ -d "$out" ]]; then + probe="$out" + elif [[ -e "$out" || -L "$out" ]]; then + # A file, or a link to nothing: capture cannot make its directory there. + fail "$out exists and is not a directory" + else + parent="$(dirname "$out")" + mkdir -p "$parent" 2>/dev/null || fail "cannot create $parent" + probe="$parent" + fi + # Creating entries in a directory takes search permission as well as write. + [[ -w "$probe" && -x "$probe" ]] || fail "not writable (needs write and search permission): $probe" + + # The filesystem of the output path itself, resolved through its own mount + # rather than its parent's. With `-v /host/captures:/data/captures` the + # parent `/data` is on the container's overlay while the archive's directory + # is the bind mount, and keying on the parent refused a perfectly good path. + fstype="$(path_fstype "$probe")" + avail_kb="$(df -Pk "$probe" 2>/dev/null | awk 'NR==2 {print $4}')" + avail_gb=$(( ${avail_kb:-0} / 1024 / 1024 )) + log "output: $out" + log " filesystem: $fstype, free: ${avail_gb} GiB" + + case "$fstype" in + overlayfs|overlay) + log " this path is on the container's own writable layer, so the archive dies with" + log " the container and there is nothing left to send. Write to a bind-mounted host" + log " directory instead, or pass --force to capture there anyway." + return 1 ;; + tmpfs) + log " this path is in RAM, so the archive competes with the workload for memory" + log " and is lost on reboot." ;; + esac + + if (( avail_gb < min_free_gb )); then + log " ${avail_gb} GiB free against the ${min_free_gb} GiB this asks for. An archive runs to" + log " tens of GiB for a workload of any size and capture writes until the disk fills, so" + log " point --output somewhere bigger, or lower the bar with --min-free-gb for a short run." + return 1 + fi + return 0 +} + +# --- the workload's process group ------------------------------------------- + +# Whether anything in process group $1 is still running. `kill -0` is true of a +# zombie, and a child that outlives the workload is reparented to PID 1: where +# PID 1 reaps nothing, a container whose entry point is `sleep infinity` or a +# Python process, that zombie stays forever and waiting on it never ended. +group_running() { + local pgid="$1" stat line fields + if [[ ! -r /proc/self/stat ]]; then + kill -0 -- "-$pgid" 2>/dev/null + return + fi + for stat in /proc/[0-9]*/stat; do + read -r line 2>/dev/null < "$stat" || continue + # The command name is in parentheses and may itself hold spaces or a `)`, + # so the fields are counted from the last one: state, ppid, pgrp. + read -r -a fields <<<"${line##*) }" + if [[ "${fields[2]:-}" == "$pgid" && "${fields[0]:-}" != Z ]]; then + return 0 + fi + done + return 1 +} + +# --- verbs ------------------------------------------------------------------ + +VERB="${1:-}" +[[ -n "$VERB" ]] || { usage; exit 1; } +shift || true +case "$VERB" in -h|--help|help) usage; exit 0 ;; esac + +OUTPUT="" +PLAYBACK="" +SKIP_PREFLIGHT=0 +FORCE=0 +MIN_FREE_GB=50 +PASSTHROUGH=() +CMD=() + +# A flag whose value is missing used to fall off the end of `shift 2` and exit 1 +# with nothing printed, and 1 is also verify's "nothing captured" code, so a +# scripted gate could not tell a typo from an empty archive. +needs_value() { + (( $2 >= 2 )) || fail "$1 needs a value" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage; exit 0 ;; + --output) needs_value --output $#; OUTPUT="$2"; shift 2 ;; + --playback) needs_value --playback $#; PLAYBACK="$2"; shift 2 ;; + --skip-preflight) SKIP_PREFLIGHT=1; shift ;; + --force) FORCE=1; shift ;; + --json) PASSTHROUGH+=(--json); shift ;; + --no-playback) PASSTHROUGH+=(--no-playback); shift ;; + --min-free-gb) + needs_value --min-free-gb $# + [[ "$2" =~ ^[0-9]+$ ]] || fail "--min-free-gb takes a whole number of GiB, not '$2'" + MIN_FREE_GB="$2"; shift 2 ;; + --) shift; CMD=("$@"); break ;; + *) fail "unexpected argument: $1. The workload command goes after --" ;; + esac +done + +# The executable the command will run, when there is one. Preflight answers a +# different and much weaker question without it. +WORKLOAD_BIN="" +# The PATH the workload is started with. Empty means this script's own. +WORKLOAD_PATH="" +LAUNCHER_ENV=() +if [[ ${#CMD[@]} -gt 0 ]]; then + WORKLOAD_NAME="" + while IFS= read -r line; do + [[ -n "$WORKLOAD_NAME" ]] && LAUNCHER_ENV+=("$WORKLOAD_NAME") + WORKLOAD_NAME="$line" + done < <(strip_launchers "${CMD[@]}") + # A command that is nothing but assignments has no workload in it. + [[ "$WORKLOAD_NAME" == *=* ]] && { LAUNCHER_ENV+=("$WORKLOAD_NAME"); WORKLOAD_NAME=""; } + # Looked up on the PATH the workload is started with: `env PATH=/x app` runs + # /x/app, and the last assignment wins as it does for env. + for assignment in ${LAUNCHER_ENV[@]+"${LAUNCHER_ENV[@]}"}; do + if [[ "$assignment" == PATH=* ]]; then WORKLOAD_PATH="${assignment#PATH=}"; fi + done + WORKLOAD_BIN="$(PATH="${WORKLOAD_PATH:-$PATH}" command -v -- "${WORKLOAD_NAME:-${CMD[0]}}" 2>/dev/null || true)" +fi + +# check_runtime in the environment the workload will have. `local -x` exports +# each assignment for the length of this call only, so the reader run after the +# capture keeps this script's own loader path. +check_runtime_as_launched() { + local assignment + for assignment in ${LAUNCHER_ENV[@]+"${LAUNCHER_ENV[@]}"}; do + # Except PATH, which is the workload's and has been used to find it and its + # interpreter. This script's own tools come from the caller's: exported, + # `env PATH=/opt/app/bin app` left grep, ldd and readlink unfound, and a + # runtime that can capture read as one that cannot. + [[ "$assignment" == PATH=* ]] && continue + [[ "${assignment%%=*}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] && local -x "$assignment" + done + check_runtime +} + +case "$VERB" in + preflight) + rc=0 + check_runtime_as_launched || rc=1 + if [[ -n "$OUTPUT" ]]; then + check_output_path "$OUTPUT" "$MIN_FREE_GB" || rc=1 + fi + if [[ -n "$PLAYBACK" ]]; then + # A regular file, as verify requires: -x alone passes a directory. + if [[ -f "$PLAYBACK" && -x "$PLAYBACK" ]]; then + log "hrr-playback given: $PLAYBACK" + else + log "the --playback path is not an executable file: $PLAYBACK" + rc=1 + fi + elif [[ -n "${HRR_PLAYBACK:-}" && -f "$HRR_PLAYBACK" && -x "$HRR_PLAYBACK" ]] || + [[ -f "$ROCM_PATH/bin/hrr-playback" && -x "$ROCM_PATH/bin/hrr-playback" ]] || + command -v hrr-playback >/dev/null 2>&1; then + log "hrr-playback is available, so the archive can be checked and replayed here" + else + log "no hrr-playback here. The archive can still be captured and checked from its" + log "manifests alone with 'verify --no-playback', and replayed elsewhere on a" + log "matching build. To get one that matches this machine: $ENSURE_PLAYBACK" + fi + exit "$rc" + ;; + + run) + [[ -n "$OUTPUT" ]] || fail "--output is required" + [[ ${#CMD[@]} -gt 0 ]] || fail "no command given" + if (( ! SKIP_PREFLIGHT )); then + if ! check_runtime_as_launched; then + (( FORCE )) || fail "preflight failed: no capture-capable runtime, so nothing would be recorded. Pass --force to run anyway." + fi + if ! check_output_path "$OUTPUT" "$MIN_FREE_GB"; then + (( FORCE )) || fail "preflight failed on the output path. Pass --force to run anyway." + fi + fi + + # Capture resumes into an archive that is already there and leaves the + # other pid-* directories where they are, so a second run against the same + # --output silently merges two attempts into what reads afterwards as one + # multi-process capture. + if compgen -G "$OUTPUT/pid-*" >/dev/null 2>&1; then + if (( FORCE )); then + log "$OUTPUT already holds a capture and --force was given: this run will be added" + log "to it, and the result will read as one multi-process archive." + else + fail "$OUTPUT already holds a capture. Capture would add this run to it and the two would be indistinguishable afterwards. Move it aside, or give --output a new path." + fi + fi + + # The archive holds blobs, prompts and weights in clear, and a writer that + # creates its files 0644 leaves them to every user on a shared host under + # the usual umask. A directory created 0700 here closes that without + # changing the umask the workload's own files are written with. + if [[ ! -e "$OUTPUT" ]]; then + mkdir -p -- "$(dirname -- "$OUTPUT")" && ( umask 077 && mkdir -- "$OUTPUT" ) || + fail "cannot create $OUTPUT" + elif [[ -n "$(find "$OUTPUT" -maxdepth 0 -perm /077 2>/dev/null)" ]]; then + log "note: $OUTPUT can be read by other users, and so can the archive inside it." + log "chmod 700 it, or give --output a new path, if this host is shared." + fi + + if (( RUNTIME_UNRESOLVED )); then + log "note: preflight could not resolve which runtime this workload binds, so this run" + log "may record nothing. The verdict afterwards is what tells you, not the exit status." + fi + log "capturing to $OUTPUT" + log "running: ${CMD[*]}" + # Backgrounded and waited on, with the signals forwarded, so that a stop + # aimed at this script reaches the workload. Left in the foreground, a + # SIGTERM from timeout, systemd, a pod stop or a cancelled CI job killed + # the wrapper only: the workload carried on capturing and nothing verified + # the archive. The workload gets a process group of its own and the signal + # goes to the whole group: a launcher that exits on it would otherwise + # leave its GPU workers capturing behind it. + workload_pid=0 + forward_signal() { + local sig="$1" + (( workload_pid )) && kill -"$sig" -- "-$workload_pid" 2>/dev/null || true + } + trap 'forward_signal TERM' TERM + trap 'forward_signal INT' INT + trap 'forward_signal HUP' HUP + + set +e + # Job control on for the launch only, which is what gives the job its own + # group. It also stops stdin defaulting to /dev/null for a background job, + # so a workload fed from a pipe or a file still reads it. Only a terminal + # is replaced: a workload reading it from outside the foreground group + # would be stopped, and the wait below with it. + # A background job starts with SIGINT and SIGQUIT ignored, and the workload + # would inherit that, so neither a Ctrl-C nor the INT forwarded above would + # ever reach it. Put them back to what this script was started with. + set -m + ( + trap - INT QUIT + export HIP_HRR_CAPTURE_OUTPUT="$OUTPUT" + if [[ -t 0 ]]; then exec "${CMD[@]}" /dev/null; do + wait "$workload_pid" + done + wait "$workload_pid" + workload_rc=$? + # Then the rest of its group, which is still writing the archive. + if group_running "$workload_pid"; then + log "the workload has exited and processes it started are still running; waiting" + log "for them, so the archive is checked once it is complete" + while group_running "$workload_pid"; do sleep 1; done + fi + set -e + trap - TERM INT HUP + log "workload exited $workload_rc" + if (( workload_rc != 0 )); then + log "a non-zero exit is fine here: a workload that crashed is the one worth capturing" + fi + + inspect_rc=0 + python3 "$INSPECT" --archive "$OUTPUT" ${PLAYBACK:+--playback "$PLAYBACK"} \ + "${PASSTHROUGH[@]+"${PASSTHROUGH[@]}"}" || inspect_rc=$? + + # Offer the replay here, while the machine that captured is still the + # machine in front of you. Its runtime is by definition the one the reader + # has to match, so the pairing problem that dogs every later attempt does + # not exist at this moment. + if (( inspect_rc == 0 )); then + log "the archive holds events. To find out what went wrong, replay it here and now:" + log " $TRIAGE --archive $OUTPUT/pid-" + log "this host captured it, so its runtime is the one the reader has to match." + fi + exit "$workload_rc" + ;; + + verify) + [[ -n "$OUTPUT" ]] || fail "--output is required" + exec python3 "$INSPECT" --archive "$OUTPUT" ${PLAYBACK:+--playback "$PLAYBACK"} \ + "${PASSTHROUGH[@]+"${PASSTHROUGH[@]}"}" + ;; + + *) + usage + exit 1 + ;; +esac diff --git a/projects/hrr/skills/enable-recording/scripts/inspect_archive.py b/projects/hrr/skills/enable-recording/scripts/inspect_archive.py new file mode 100755 index 000000000000..6036149b456b --- /dev/null +++ b/projects/hrr/skills/enable-recording/scripts/inspect_archive.py @@ -0,0 +1,870 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Report what an HRR capture archive actually contains. + +Answers the only question that matters right after a capture run: did anything +get recorded, and is the archive worth sending on. It works without a GPU and +without `hrr-playback`, reading the manifests the capture writer leaves behind, +and uses `hrr-playback --info` as a cross-check when a compatible one is around. + +Two properties of the on-disk layout drive the logic here: + +* The **root manifest is written best-effort by whichever process last flushed + it**, so its per-process counts can lag the truth. The per-process + `pid-/manifest.json` is authoritative and is what this reports. +* A process killed hard may leave `events.bin` with **no manifest at all**. + That is a capture with unknown counts, not an empty one, so it is reported as + recorded-but-unfinalized rather than as a failure. + +`complete: false` is likewise not an error. It means the clean-shutdown trailer +is missing, which is exactly what a workload that crashed is supposed to look +like. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import struct +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +EXIT_OK = 0 +EXIT_EMPTY = 1 +EXIT_USAGE = 2 +# A cross-check the caller asked for and that did not happen. Distinct from a +# bad archive: the archive may be perfect and the reader unable to start. It +# gets its own code because exiting 0 here lets a pre-send gate pass on a check +# that never ran. +EXIT_NO_CROSS_CHECK = 3 + +# Emitted by hrr_reader.cpp when the archive format and the reader disagree. +VERSION_MISMATCH = "Version mismatch" + +# `events.bin` is hrr_file_header(8) + events + optional hrr_eof_record(44). +# The header is `{ uint32 magic, uint16 version, uint16 reserved }`, little +# endian, and its version is the number the reader compares against its own +# when it says `Version mismatch in : file=N reader=M`. The root +# manifest's `version` is a different quantity and comparing that one raises a +# false alarm on a perfectly matched reader. +# See projects/hrr/include/hrr/hrr_api_args.h and projects/hrr/DESIGN.md. +HEADER_BYTES = 8 +HEADER_MAGIC = 0x52524845 # "HRRE" +# `hrr_event_header.event_type` sentinel for the clean-shutdown trailer. A file +# whose very first record is the trailer recorded nothing, however many bytes +# it holds: header plus trailer is 52 bytes of no events at all. +EOF_MARKER = 0xFFFF +# `hrr_eof_record`: the event header, uint64 total_events, uint32 eof_magic. +# The sentinel alone is not the trailer: the reader takes 0xFFFF with any other +# length or magic as an unknown event and keeps it. +EOF_RECORD_BYTES = 44 +EOF_MAGIC = 0x464F4548 # "HEOF" +EOF_MAGIC_OFFSET = 40 +# `hrr_event_header`: uint16 event_type, uint64 sequence_id, uint64 timestamp_ns, +# uint64 thread_id, uint32 payload_length, 2 bytes of padding. Packed, 32 bytes. +EVENT_HEADER_BYTES = 32 +# The reader takes a record claiming more than this as torn (`kMaxRecordBytes` +# in playback/hrr_reader.cpp), however large the file is. +MAX_RECORD_BYTES = 64 * 1024 * 1024 + + +@dataclass +class ProcessArchive: + path: Path + pid: int | None = None + parent_pid: int | None = None + complete: bool | None = None + event_count: int | None = None + blob_count: int | None = None + events_bytes: int = 0 + blob_files: int = 0 + code_object_files: int = 0 + total_bytes: int = 0 + manifest_error: str | None = None + # Manifest fields present with the wrong JSON type, read as unknown. + manifest_bad_fields: list[str] = field(default_factory=list) + hip_runtime_version: str | None = None + format_version: int | None = None + holds_an_event: bool = False + + @property + def recorded(self) -> bool: + """Something is in this process directory worth keeping. + + Not "the file is non-empty": `events.bin` opens with an 8-byte header + and a clean shutdown appends a 44-byte trailer, so a capture that + recorded nothing at all still leaves bytes behind. The manifest's count + is authoritative where it exists; a process killed before it could write + one is decided by whether the first record in the file is a real event + rather than the clean-shutdown trailer. + """ + if self.event_count is not None: + return self.event_count > 0 + return self.holds_an_event + + @property + def finalized(self) -> bool: + return self.complete is not None + + @property + def blob_count_reconciles(self) -> bool | None: + """Does the manifest's blob total agree with what is on disk? + + The manifest counts code objects among its blobs while `blobs/` and + `hrr-playback --info` do not, so `blob_count` legitimately equals either + the file count or the file count plus the code objects. Anything else is + a real discrepancy. Unknown when there is no manifest. + """ + if self.blob_count is None: + return None + return self.blob_count in (self.blob_files, self.blob_files + self.code_object_files) + + +@dataclass +class ArchiveReport: + root: Path + processes: list[ProcessArchive] = field(default_factory=list) + root_manifest: dict | None = None + root_manifest_error: str | None = None + total_bytes: int = 0 + playback_info: str | None = None + playback_error: str | None = None + playback_binary: str | None = None + playback_version: str | None = None + playback_format_version: int | None = None + capture_runtime: str | None = None + # True only when a reader actually decoded the archive. A reader that + # refused it explained itself, which is useful, but the archive still went + # unread and a pre-send gate should not pass on that. + cross_check_ran: bool = False + warnings: list[str] = field(default_factory=list) + + @property + def recorded_processes(self) -> list[ProcessArchive]: + return [p for p in self.processes if p.recorded] + + +def _dir_stats(path: Path) -> tuple[int, int]: + """(file count, total bytes) for a directory tree, cheaply and safely.""" + files = 0 + total = 0 + if not path.is_dir(): + return files, total + for entry in path.rglob("*"): + try: + if entry.is_file(): + files += 1 + total += entry.stat().st_size + except OSError: + continue + return files, total + + +def _archive_format_version(events: Path) -> int | None: + """The format version written in the first 8 bytes of `events.bin`.""" + try: + with events.open("rb") as handle: + head = handle.read(HEADER_BYTES) + except OSError: + return None + if len(head) < HEADER_BYTES: + return None + magic, version, _ = struct.unpack(" bool: + """Is there a real event in this file, rather than framing alone? + + Only asked when a process died before writing its manifest, so the count + has to come off the file. Bytes are not the answer: a clean trailer with no + events leaves 52 of them, and a process killed mid-write can leave a header + torn in half. So the first record has to be a whole event header, not the + clean-shutdown trailer, and its declared length has to make sense against the + file: a plausible record, rather than any two bytes that are not 0xFFFF. + """ + try: + size = events.stat().st_size + with events.open("rb") as handle: + head = handle.read(HEADER_BYTES + EOF_RECORD_BYTES) + except OSError: + return False + if len(head) < HEADER_BYTES + EVENT_HEADER_BYTES: + return False + # The reader rejects a file without the HRR header, whatever follows it. + if struct.unpack_from(" size - HEADER_BYTES + or payload_length > MAX_RECORD_BYTES + ): + return False + return True + + +def _read_manifest(path: Path) -> tuple[dict | None, str | None]: + if not path.is_file(): + return None, None + try: + manifest = json.loads(path.read_text(encoding="utf-8", errors="replace")) + except (OSError, ValueError) as exc: + return None, str(exc) + # Valid JSON is not yet a manifest: `[]` or `1` crashed the first `.get` + # on it, in the tool meant to report on damaged captures. + if not isinstance(manifest, dict): + return None, f"not a JSON object: {type(manifest).__name__}" + return manifest, None + + +def _load_process(pid_dir: Path) -> ProcessArchive: + proc = ProcessArchive(path=pid_dir) + + events = pid_dir / "events.bin" + try: + proc.events_bytes = events.stat().st_size if events.is_file() else 0 + except OSError: + proc.events_bytes = 0 + + proc.format_version = _archive_format_version(events) + proc.holds_an_event = _holds_an_event(events) + proc.blob_files, _ = _dir_stats(pid_dir / "blobs") + proc.code_object_files, _ = _dir_stats(pid_dir / "code_objects") + + manifest, error = _read_manifest(pid_dir / "manifest.json") + proc.manifest_error = error + # `{}` is a manifest too, and a damaged one, not a missing one. + if manifest is not None: + # A damaged manifest is reported, not raised on: any field can be any + # JSON value, and `"event_count": "100"` crashed the first comparison. + # bool is an int to Python, so it is excluded from the counts. The + # writer never writes null, so an explicit null is damage as well. Every + # integer it writes is a count or a pid, so a negative one is damage too: + # `"event_count": -1` read as zero would call a recorded stream empty. + def typed(source: dict, key: str, kind: type): + if key not in source: + return None + value = source[key] + if isinstance(value, kind) and (kind is bool or not isinstance(value, bool)): + if kind is int and value < 0: + proc.manifest_bad_fields.append(f"{key}: {value}") + return None + return value + proc.manifest_bad_fields.append(f"{key}: {type(value).__name__}") + return None + + proc.pid = typed(manifest, "pid", int) + proc.parent_pid = typed(manifest, "parent_pid", int) + proc.complete = typed(manifest, "complete", bool) + proc.event_count = typed(manifest, "event_count", int) + proc.blob_count = typed(manifest, "blob_count", int) + if "complete" not in manifest: + proc.manifest_bad_fields.append("complete: missing") + metadata = manifest.get("metadata") + runtime = metadata.get("runtime") if isinstance(metadata, dict) else None + if isinstance(runtime, dict): + proc.hip_runtime_version = typed(runtime, "hip_runtime_version", str) + + if proc.pid is None and pid_dir.name.startswith("pid-"): + suffix = pid_dir.name[len("pid-") :] + if suffix.isdigit(): + proc.pid = int(suffix) + + # The whole directory is what gets sent: the manifests and any `regions/` + # sidecars as well as the events, blobs and code objects. + _, proc.total_bytes = _dir_stats(pid_dir) + return proc + + +def _resolve(archive: Path) -> tuple[Path, list[Path]]: + """Return (root, pid directories). Accepts a root or a single pid dir. + + A single pid dir is its own root: it is what gets sent, and the parent's + manifest lists processes that were not passed. + """ + # A pid-* directory with no events.bin is a leftover the reader skips + # (playback/hrr_playback.cpp), not a process that recorded nothing. + pid_dirs = sorted( + d for d in archive.glob("pid-*") if d.is_dir() and (d / "events.bin").is_file() + ) + if pid_dirs: + return archive, pid_dirs + if (archive / "events.bin").is_file(): + return archive, [archive] + return archive, [] + + +def _find_playback(explicit: str | None) -> str | None: + """The reader to cross-check with, in the same order the sibling skill uses. + + `HRR_PLAYBACK`, then `$ROCM_PATH/bin`, then `PATH`, taking only a file that + can run, which is what `decode-and-triage/scripts/ensure_playback.sh` does. + Searching in a different order meant the same host could verify an archive + with one reader and replay it with another, on a skill whose whole point is + that the reader has to match the runtime that captured. + """ + if explicit: + return explicit if Path(explicit).is_file() else None + from_env = os.environ.get("HRR_PLAYBACK") + if from_env and Path(from_env).is_file() and os.access(from_env, os.X_OK): + return from_env + rocm = Path(os.environ.get("ROCM_PATH", "/opt/rocm")) + candidate = rocm / "bin" / "hrr-playback" + if candidate.is_file() and os.access(candidate, os.X_OK): + return str(candidate) + return shutil.which("hrr-playback") + + +def _playback_env(playback: str) -> dict[str, str]: + """Environment for running a playback binary out of a packaged build. + + Such a build ships `bin/`, `lib/` and `runtime-lib/` as siblings, and the + binary needs the HIP and HSA libraries from its own `lib/` and + `runtime-lib/`. Without them it picks up whichever libamdhip64 the host has + and dies on a missing symbol version, which reads like a broken archive. + A build-tree reader needs the matching libhsa-runtime64 the same way, and + `ROCR_LIB` names it, as it does for triage_archive.sh. + + `$ROCM_PATH/lib` goes last, after the caller's own `LD_LIBRARY_PATH`, in + the order triage_archive.sh uses: a build-tree reader still needs the rest + of ROCm from it, and ahead of the caller it would bind the system release's + libraries in front of the reader's own. + """ + env = dict(os.environ) + root = Path(playback).resolve().parent.parent + dirs = [str(root / name) for name in ("lib", "runtime-lib") if (root / name).is_dir()] + rocr = env.get("ROCR_LIB") + if rocr and (Path(rocr) / "libhsa-runtime64.so.1").is_file() and rocr not in dirs: + dirs.append(rocr) + existing = [part for part in env.get("LD_LIBRARY_PATH", "").split(":") if part] + rocm_lib = str(Path(env.get("ROCM_PATH", "/opt/rocm")) / "lib") + tail = [rocm_lib] if Path(rocm_lib).is_dir() and rocm_lib not in dirs + existing else [] + if dirs or tail: + env["LD_LIBRARY_PATH"] = ":".join(dirs + existing + tail) + return env + + +def _playback_version(playback: str, timeout: int = 20) -> tuple[str | None, int | None]: + """What the reader says it is: (identity, archive format version it reads). + + `--version` prints a banner, then `archive format version`, the source + revision and branch, and the HIP runtime version as an integer. The format + version is the number worth comparing, because it is the same quantity the + archive's own manifest carries; the HIP integer is not comparable with the + dotted version capture records. + + Older builds have no `--version` at all, so `(None, None)` is common and is + not an error. It does mean the pairing cannot be established, which + matters: a reader built from a different API table accepts the archive and + decodes the event names shifted, without complaining. + """ + try: + completed = subprocess.run( + [playback, "--version"], + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=_playback_env(playback), + ) + except (OSError, subprocess.SubprocessError): + return None, None + if completed.returncode != 0: + return None, None + output = ((completed.stdout or "") + (completed.stderr or "")).strip() + if not output: + return None, None + + fields = {} + for line in output.splitlines(): + if ":" in line: + key, _, value = line.partition(":") + fields[key.strip()] = value.strip() + + format_version = None + raw = fields.get("archive format version") + if raw and raw.isdigit(): + format_version = int(raw) + + identity = ", ".join( + part + for part in ( + f"reads archive format v{format_version}" if format_version is not None else None, + f"rev {fields['source revision']}" if fields.get("source revision") else None, + ) + if part + ) + return (identity or output.splitlines()[0]), format_version + + +def _info_field(info: str, label: str) -> int | None: + """An integer field out of `hrr-playback --info`, such as `Events:`.""" + for line in info.splitlines(): + stripped = line.strip() + if stripped.startswith(f"{label}:"): + value = stripped.split(":", 1)[1].strip().split()[0] + if value.isdigit(): + return int(value) + return None + + +def _run_playback_info(playback: str, target: Path, timeout: int) -> tuple[str | None, str | None]: + try: + completed = subprocess.run( + [playback, str(target), "--info"], + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=_playback_env(playback), + ) + except (OSError, subprocess.SubprocessError) as exc: + return None, str(exc) + output = ((completed.stdout or "") + (completed.stderr or "")).strip() + if completed.returncode != 0: + # A reader that refused the archive explains itself, and that explanation + # is worth keeping. A reader that never started prints a loader error, + # and printing that under an `--info` heading passes a crash off as + # archive information: it is a failed cross-check and has to read as one. + if VERSION_MISMATCH in output: + return output, None + detail = output.splitlines()[0] if output else "no output" + return None, f"hrr-playback --info exited {completed.returncode}: {detail}" + # Success with nothing said is not a cross-check either, and without an + # error here the report gave no reason for its failed exit status. + if not output: + return None, "hrr-playback --info exited 0 and printed nothing" + return output, None + + +def inspect( + archive: Path, + playback: str | None = None, + use_playback: bool = True, + timeout: int = 120, +) -> ArchiveReport: + root, pid_dirs = _resolve(archive) + report = ArchiveReport(root=root) + + # In a single pid dir, manifest.json is the process's own, read below. + manifest, error = None, None + if pid_dirs != [root]: + manifest, error = _read_manifest(root / "manifest.json") + report.root_manifest = manifest + report.root_manifest_error = error + if error: + # A warning, so --json carries it as the text report does. + report.warnings.append( + f"root manifest.json is unreadable ({error}); each process's own manifest " + "is what this reports" + ) + + for pid_dir in pid_dirs: + report.processes.append(_load_process(pid_dir)) + # What is sent is the root whole, its own manifest included. + _, report.total_bytes = _dir_stats(root) + + if manifest and isinstance(manifest.get("processes"), list): + by_pid = {p.pid: p for p in report.processes if p.pid is not None} + for entry in manifest["processes"]: + if not isinstance(entry, dict): + report.warnings.append( + f"root manifest has a processes entry that is not an object: " + f"{type(entry).__name__}" + ) + continue + pid = entry.get("pid") + # A list or an object here is unhashable, and the lookup raised. + proc = by_pid.get(pid) if isinstance(pid, int) else None + if proc is None: + report.warnings.append( + f"root manifest lists pid {pid} but no pid-{pid}/ directory exists" + ) + continue + if ( + proc.event_count is not None + and entry.get("event_count") is not None + and entry["event_count"] != proc.event_count + ): + report.warnings.append( + f"root manifest says pid {pid} has {entry['event_count']} events, " + f"its own manifest says {proc.event_count}; the per-process count is " + "authoritative because the root is rewritten best-effort" + ) + + for proc in report.processes: + if proc.manifest_bad_fields: + report.warnings.append( + f"{proc.path.name}: manifest.json has fields missing, of the wrong type or out of range " + f"({', '.join(proc.manifest_bad_fields)}), read as unknown" + ) + if proc.manifest_error: + report.warnings.append( + f"{proc.path.name}: manifest.json is unreadable ({proc.manifest_error}), so " + "its counts and completion state are unknown; whether it recorded anything " + "was read off events.bin instead" + ) + elif proc.recorded and not proc.finalized and not proc.manifest_bad_fields: + report.warnings.append( + f"{proc.path.name}: events.bin holds {proc.events_bytes} bytes but there is " + "no manifest, so the process died before finalizing; counts are unknown " + "until hrr-playback --info reads it" + ) + if proc.blob_count_reconciles is False: + report.warnings.append( + f"{proc.path.name}: the manifest counts {proc.blob_count} blobs but the " + f"directory holds {proc.blob_files} blob files and {proc.code_object_files} " + "code objects, which is neither total; something is missing from disk" + ) + if proc.hip_runtime_version and not report.capture_runtime: + report.capture_runtime = proc.hip_runtime_version + + if use_playback and report.recorded_processes: + binary = _find_playback(playback) + if binary: + report.playback_binary = binary + report.playback_version, report.playback_format_version = _playback_version(binary) + # The substantive process, not the alphabetically first one: pid-406 + # sorts before pid-45, and in a server capture the small process is + # usually the one that only registered fat binaries. + target = max( + report.recorded_processes, + key=lambda p: (p.event_count or 0, p.events_bytes), + ).path + report.playback_info, report.playback_error = _run_playback_info( + binary, target, timeout + ) + if report.playback_info and VERSION_MISMATCH in report.playback_info: + report.warnings.append( + "this hrr-playback cannot read this archive: the capture wrote one " + "format version and this reader expects another. The archive is fine; " + "the reader has to match the runtime that captured it" + ) + elif report.playback_info: + report.cross_check_ran = True + _cross_check(report, target) + else: + report.playback_error = ( + "no hrr-playback found in $HRR_PLAYBACK, under $ROCM_PATH/bin or on PATH, " + "so this reports " + "from the manifests alone. The sibling skill's " + "decode-and-triage/scripts/ensure_playback.sh finds or builds one that " + "matches this machine" + ) + + return report + + +def _cross_check(report: ArchiveReport, target: Path) -> None: + """Compare what the reader says with what the archive says. + + A reader that refuses an archive says so. A reader from a neighbouring + build does something worse: it accepts the archive and decodes the event + names against its own API table, so the counts are right and the names are + wrong, with no error anywhere. Nothing here can detect that from the + output, so the reader is identified instead and an unestablished pairing is + reported as exactly that. + """ + info = report.playback_info or "" + proc = next((p for p in report.processes if p.path == target), None) + + if proc is not None and proc.event_count is not None: + reported = _info_field(info, "Events") + if reported is not None and reported != proc.event_count: + report.warnings.append( + f"{proc.path.name}: the manifest records {proc.event_count:,} events and this " + f"reader counts {reported:,}. Trust neither until the reader is the one that " + "matches the capturing runtime" + ) + + kernels = _info_field(info, "Kernels") + if kernels == 0: + report.warnings.append( + "this archive contains no kernel launches at all. The process made HIP calls " + "and never reached the GPU, which is what a run that died during setup looks " + "like: check it is the run that reproduces your failure before sending it" + ) + + if proc is not None: + reported_blobs = _info_field(info, "Blobs") + if reported_blobs is not None and reported_blobs != proc.blob_files: + report.warnings.append( + f"{proc.path.name}: this reader counts {reported_blobs:,} blobs against " + f"{proc.blob_files:,} files in blobs/" + ) + + archive_format = proc.format_version if proc is not None else None + if ( + isinstance(archive_format, int) + and report.playback_format_version is not None + and archive_format != report.playback_format_version + ): + report.warnings.append( + f"the archive is format v{archive_format} and this reader reads " + f"v{report.playback_format_version}. It has not refused the archive here, but " + "the two do not agree and nothing it reports can be relied on" + ) + elif report.playback_format_version is None: + # Keyed on the format version rather than on the identity string: a + # reader whose `--version` succeeds without printing that field still + # establishes nothing, and testing the identity instead let it pass as + # though it had. + report.warnings.append( + "this hrr-playback does not report the archive format it reads, so the pairing " + "with the runtime that captured is unverified. A reader from a different build " + "reads the archive without complaining and decodes the event names against its " + "own API table: if " + "the API mix below does not look like the workload, the reader is wrong rather " + "than the archive" + ) + + +def _human(size: int) -> str: + value = float(size) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if value < 1024 or unit == "TiB": + return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} TiB" + + +def verdict(report: ArchiveReport) -> str: + """One of the four documented verdicts, for the text and JSON paths alike. + + `no archive` is decided before a report exists, in `missing_archive`. + """ + if report.recorded_processes: + return "recorded" + if not report.processes: + return "nothing captured" + return "empty" + + +def render(report: ArchiveReport) -> str: + lines = [f"Archive: {report.root}"] + + if not report.processes: + lines.extend(f"Warning: {w}" for w in report.warnings) + lines.append("") + lines.append( + "The usual cause is that the runtime the workload loaded has no capture " + "support, or that HIP_HRR_CAPTURE_OUTPUT never reached the process that " + "used the GPU. Run 'hrr_capture.sh preflight' in the same environment." + ) + lines.append("") + lines.append( + "Verdict: nothing captured. The directory exists but holds no pid-* " + "process directory, so no process ever opened a capture here." + ) + return "\n".join(lines) + + lines.append(f"Processes: {len(report.processes)} Size on disk: {_human(report.total_bytes)}") + lines.append("") + for proc in report.processes: + events = "unknown" if proc.event_count is None else f"{proc.event_count:,}" + # Files on disk rather than the manifest total, which counts code + # objects among its blobs and so disagrees with both `blobs/` and + # `hrr-playback --info` for the same archive. + blobs = f"{proc.blob_files:,}" + if proc.manifest_error: + state = "unknown (manifest unreadable)" + elif proc.complete is None and proc.manifest_bad_fields: + state = "unknown (manifest damaged)" + elif proc.complete is None: + state = "not finalized (no manifest)" + elif proc.complete: + state = "complete" + else: + state = "incomplete (no clean-shutdown trailer)" + lines.append(f" {proc.path.name}") + lines.append(f" events: {events} blobs: {blobs} state: {state}") + lines.append( + f" events.bin: {_human(proc.events_bytes)} " + f"blob files: {proc.blob_files} code objects: {proc.code_object_files}" + ) + if proc.parent_pid: + lines.append(f" parent pid: {proc.parent_pid}") + + incomplete = [p for p in report.processes if p.complete is False] + if incomplete: + lines.append("") + lines.append( + "At least one process has no clean-shutdown trailer. That is what a workload " + "that crashed looks like, and such an archive is still readable and still " + "worth sending: the events before the crash are all there. The exception is " + "a capture that dropped an event it could not fit in the wire format, which " + "leaves a hole instead; the workload's stderr then says 'Archive marked " + "INCOMPLETE'." + ) + + if report.warnings: + lines.append("") + lines.append("Warnings:") + lines.extend(f" - {w}" for w in report.warnings) + + if report.playback_info: + lines.append("") + reader = report.playback_binary or "hrr-playback" + version = report.playback_version or "version not reported" + lines.append(f"Reader: {reader} ({version})") + lines.append("hrr-playback --info:") + lines.extend(f" {line}" for line in report.playback_info.splitlines()) + elif report.playback_error: + lines.append("") + lines.append(f"Cross-check with hrr-playback skipped: {report.playback_error}") + + lines.append("") + if report.recorded_processes: + lines.append( + f"Verdict: recorded. {len(report.recorded_processes)} of {len(report.processes)} " + "process directories hold events." + ) + if len(report.processes) > 1: + lines.append( + "Send the whole archive directory, not one pid-* directory out of it; the " + "processes belong together. Replay them one at a time, starting with one " + "that did not shut down cleanly." + ) + else: + lines.append("Verdict: empty. Process directories exist but no events were written.") + + return "\n".join(lines) + + +def to_dict(report: ArchiveReport) -> dict: + return { + "archive": str(report.root), + "total_bytes": report.total_bytes, + "recorded": bool(report.recorded_processes), + "processes": [ + { + "dir": p.path.name, + "pid": p.pid, + "parent_pid": p.parent_pid, + "complete": p.complete, + "event_count": p.event_count, + "blob_count": p.blob_count, + "events_bytes": p.events_bytes, + "blob_files": p.blob_files, + "code_object_files": p.code_object_files, + "finalized": p.finalized, + } + for p in report.processes + ], + "root_manifest_error": report.root_manifest_error, + "warnings": report.warnings, + "playback_info": report.playback_info, + "playback_error": report.playback_error, + "playback_binary": report.playback_binary, + "playback_version": report.playback_version, + "playback_format_version": report.playback_format_version, + "capture_runtime": report.capture_runtime, + "verdict": verdict(report), + } + + +def missing_archive(archive: Path) -> tuple[str, dict]: + """The report for a path capture never created, in both output forms. + + Distinct from an empty archive: nothing was created at all, so capture + never ran. Usual causes are a runtime with no capture compiled in, or a + workload that died before its first HIP call. + """ + text = ( + f"Archive: {archive}\n\n" + "The runtime wrote nothing here. Either the runtime it loaded has no capture " + "support, which is the common case, or the workload failed before its first HIP " + "call. Run 'hrr_capture.sh preflight' with the workload command in the same " + "environment to tell those apart." + "\n\nVerdict: no archive. The directory was never created, so capture never started." + ) + payload = { + "archive": str(archive), + "total_bytes": 0, + "recorded": False, + "processes": [], + "root_manifest_error": None, + "warnings": [], + "playback_info": None, + "playback_error": None, + "playback_binary": None, + "playback_version": None, + "playback_format_version": None, + "capture_runtime": None, + "verdict": "no archive", + } + return text, payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--archive", required=True, help="capture root or a pid-* directory") + parser.add_argument("--playback", help="hrr-playback binary to cross-check with") + parser.add_argument( + "--no-playback", action="store_true", help="skip the hrr-playback cross-check" + ) + # --info walks the whole event stream, so a multi-million-event archive on a + # slow filesystem takes tens of seconds. Two minutes is generous for that and + # still bounds a reader that has wedged. + parser.add_argument("--timeout", type=int, default=120, help="seconds for hrr-playback --info") + parser.add_argument("--json", action="store_true", help="emit JSON instead of text") + args = parser.parse_args(argv) + + archive = Path(args.archive).expanduser() + + # A reader the caller named and that cannot run is a mistake to report, not + # one to fall back from: a silently skipped cross-check exiting 0 passes a + # scripted gate that was written to depend on it. + if args.playback and not (Path(args.playback).is_file() and os.access(args.playback, os.X_OK)): + message = f"--playback is not an executable file: {args.playback}" + if args.json: + print(json.dumps({"error": message, "verdict": "usage error"}, indent=2)) + else: + print(f"error: {message}", file=sys.stderr) + return EXIT_USAGE + + if not archive.exists(): + text, payload = missing_archive(archive) + print(json.dumps(payload, indent=2) if args.json else text) + return EXIT_EMPTY + if not archive.is_dir(): + print(f"error: not a directory: {archive}", file=sys.stderr) + return EXIT_USAGE + + report = inspect( + archive.resolve(), + playback=args.playback, + use_playback=not args.no_playback, + timeout=args.timeout, + ) + + print(json.dumps(to_dict(report), indent=2) if args.json else render(report)) + if not report.recorded_processes: + return EXIT_EMPTY + # A reader that was found and then failed is a check that did not happen, + # whoever asked for it. No reader on the machine at all is the documented + # manifests-only path and not a failure, or every host without one would + # report red. + if not report.cross_check_ran and (args.playback or report.playback_binary): + return EXIT_NO_CROSS_CHECK + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/hrr/skills/enable-recording/skill-card.md b/projects/hrr/skills/enable-recording/skill-card.md new file mode 100644 index 000000000000..5b4aa1f62ea2 --- /dev/null +++ b/projects/hrr/skills/enable-recording/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Records a failing HIP workload into an HRR capture archive and verifies that the archive is readable, so a GPU failure can be reproduced elsewhere without the original application, its source or its data. + +## Owner + +AMD ROCm Runtime, HRR maintainers + +## License + +MIT diff --git a/projects/hrr/skills/enable-recording/tests/fixtures/playback_info_pass.txt b/projects/hrr/skills/enable-recording/tests/fixtures/playback_info_pass.txt new file mode 100644 index 000000000000..1f709b9d386a --- /dev/null +++ b/projects/hrr/skills/enable-recording/tests/fixtures/playback_info_pass.txt @@ -0,0 +1,134 @@ +HRR Archive: /work/captures-develop/qwen.hrr/pid-49 +======================================== +Complete: yes (clean shutdown) +Recovered: 158361 events +Events: 158361 +Kernels: 10281 +Blobs: 4100 +Code Objects: 33 +Threads: 2 + +Event Type Breakdown: + Type Count + ---- ----- + __hipPopCallConfiguration 10001 + __hipPushCallConfiguration 10001 + __hipRegisterFatBinary 3720 + hipDeviceGetAttribute 9741 + hipDeviceGetStreamPriorityRange 1 + hipDevicePrimaryCtxGetState 92 + hipDeviceSynchronize 1 + hipDriverGetVersion 1 + hipEventCreateWithFlags 50 + hipEventDestroy 48 + hipEventQuery 23 + hipEventRecord 99 + hipEventSynchronize 50 + hipFree 11 + hipFuncGetAttribute 12 + hipGetDevice 66010 + hipGetDeviceCount 4 + hipGetDevicePropertiesR0600 4167 + hipGetLastError 1161 + hipHostMalloc 29 + hipMalloc 107 + hipMemGetInfo 5 + hipMemcpyAsync 267 + hipMemcpyWithStream 509 + hipMemset 1 + hipModuleLaunchKernel 10281 + hipModuleLoad 2 + hipModuleLoadDataEx 4 + hipModuleUnload 2 + hipPointerGetAttribute 416 + hipRuntimeGetVersion 3247 + hipSetDevice 37574 + hipStreamCreateWithPriority 1 + hipStreamGetCaptureInfo 509 + hipStreamIsCapturing 166 + hipStreamWaitEvent 24 + hipGetProcAddress 24 + +Kernel Summary (first 20): + ID Kernel Grid Block SharedMem + -- ------ ---- ----- --------- + 0 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 1 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 2 _ZN2at6native29vectorized_elementwise_kernelILi... [32,1,1] [256,1,1] 0 + 3 _ZN2at6native29vectorized_elementwise_kernelILi... [16,1,1] [256,1,1] 0 + 4 _ZN2at6native29vectorized_elementwise_kernelILi... [16,1,1] [256,1,1] 0 + 5 _ZN2at6native29vectorized_elementwise_kernelILi... [16,1,1] [256,1,1] 0 + 6 _ZN2at6native29vectorized_elementwise_kernelILi... [2,1,1] [256,1,1] 0 + 7 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 8 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 9 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 10 _ZN2at6native29vectorized_elementwise_kernelILi... [16,1,1] [256,1,1] 0 + 11 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 12 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 13 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 14 _ZN2at6native29vectorized_elementwise_kernelILi... [8192,1,1] [256,1,1] 0 + 15 _ZN2at6native29vectorized_elementwise_kernelILi... [4,1,1] [256,1,1] 0 + 16 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 17 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 18 _ZN2at6native29vectorized_elementwise_kernelILi... [1,1,1] [256,1,1] 0 + 19 _ZN2at6native29vectorized_elementwise_kernelILi... [16,1,1] [256,1,1] 0 + ... and 10261 more + +Kernel Call Counts: + Kernel Calls + ------ ----- + Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT128x128x12... 28 + Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT192x128x12... 28 + Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT256x256x64... 114 + Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x64x256_... 56 + _Z11wvSplitKrc_I14__hip_bfloat16Li64ELi16ELi4ELi8ELi1ELi1... 112 + _Z16wvSplitK_hf_sml_I14__hip_bfloat16Li64ELi1ELi16ELi8ELi... 1288 + _Z16wvSplitK_hf_sml_I14__hip_bfloat16Li64ELi2ELi16ELi8ELi... 1288 + _Z16wvSplitK_hf_sml_I14__hip_bfloat16Li64ELi4ELi16ELi8ELi... 24 + _Z35paged_attention_ll4mi_reduce_kernelI14__hip_bfloat16S... 672 + _Z38paged_attention_ll4mi_QKV_mfma4_kernelI14__hip_bfloat... 672 + _ZN12_GLOBAL__N_129elementwise_kernel_with_indexIiZZZN2at... 2 + _ZN2at6native12_GLOBAL__N_121indexSelectSmallIndexIN3c108... 24 + _ZN2at6native12_GLOBAL__N_123cunn_SoftMaxForwardGmemILi4E... 26 + _ZN2at6native12_GLOBAL__N_126CatArrayBatchedCopy_contigIN... 1 + _ZN2at6native12_GLOBAL__N_143distribution_elementwise_gri... 2 + _ZN2at6native12_GLOBAL__N_143distribution_elementwise_gri... 26 + _ZN2at6native13reduce_kernelILi512ELi1ENS0_8ReduceOpIfNS0... 28 + _ZN2at6native24index_elementwise_kernelILi128ELi4EZNS0_16... 24 + _ZN2at6native24vectorized_gather_kernelILi16ElEEvPcS2_PT0... 28 + _ZN2at6native27unrolled_elementwise_kernelINS0_21CUDAFunc... 24 + _ZN2at6native29vectorized_elementwise_kernelILi16ENS0_11F... 56 + _ZN2at6native29vectorized_elementwise_kernelILi16ENS0_11F... 3 + _ZN2at6native29vectorized_elementwise_kernelILi2ENS0_11Fi... 24 + _ZN2at6native29vectorized_elementwise_kernelILi4ENS0_11Fi... 125 + _ZN2at6native29vectorized_elementwise_kernelILi4ENS0_11Fi... 35 + _ZN2at6native29vectorized_elementwise_kernelILi4ENS0_11Fi... 9 + _ZN2at6native29vectorized_elementwise_kernelILi4ENS0_13AU... 1 + _ZN2at6native29vectorized_elementwise_kernelILi4ENS0_13BU... 1 + _ZN2at6native29vectorized_elementwise_kernelILi4ENS0_13Bi... 26 + _ZN2at6native29vectorized_elementwise_kernelILi4ENS0_15CU... 24 + _ZN2at6native29vectorized_elementwise_kernelILi4ENS0_15CU... 24 + _ZN2at6native29vectorized_elementwise_kernelILi4EZNS0_21c... 4 + _ZN2at6native29vectorized_elementwise_kernelILi4EZNS0_25b... 1 + _ZN2at6native29vectorized_elementwise_kernelILi4EZNS0_34b... 26 + _ZN2at6native29vectorized_elementwise_kernelILi4EZZZNS0_1... 2 + _ZN2at6native29vectorized_elementwise_kernelILi4EZZZNS0_1... 1 + _ZN2at6native29vectorized_elementwise_kernelILi4EZZZNS0_1... 1 + _ZN2at6native29vectorized_elementwise_kernelILi4EZZZNS0_2... 1 + _ZN2at6native29vectorized_elementwise_kernelILi8ENS0_11Fi... 170 + _ZN2at6native32elementwise_kernel_manual_unrollILi128ELi4... 1 + _ZN2at6native32elementwise_kernel_manual_unrollILi128ELi4... 28 + _ZN2at6native32elementwise_kernel_manual_unrollILi128ELi4... 74 + _ZN2at6native32elementwise_kernel_manual_unrollILi128ELi4... 26 + _ZN2at6native32elementwise_kernel_manual_unrollILi128ELi4... 1 + _ZN2at6native32elementwise_kernel_manual_unrollILi128ELi4... 2 + _ZN4vllm15rms_norm_kernelIN3c108BFloat16ELi8ELi2EEEvPT_PK... 26 + _ZN4vllm15rms_norm_kernelIN3c108BFloat16ELi8ELi3EEEvPT_PK... 1456 + _ZN4vllm18act_and_mul_kernelIN3c108BFloat16E15__hip_bfloa... 728 + _ZN4vllm23rotary_embedding_kernelIN3c108BFloat16ELb1EEEvP... 728 + _ZN4vllm24reshape_and_cache_kernelI14__hip_bfloat16S1_LNS... 700 + _ZN4vllm25fused_add_rms_norm_kernelIN3c108BFloat16ELi8EEE... 1456 + _compute_slot_mapping_kernel 24 + _fwd_kernel 28 + _topk_topp_kernel 2 + diff --git a/projects/hrr/skills/enable-recording/tests/fixtures/playback_info_version_mismatch.txt b/projects/hrr/skills/enable-recording/tests/fixtures/playback_info_version_mismatch.txt new file mode 100644 index 000000000000..e8f9e0a58d77 --- /dev/null +++ b/projects/hrr/skills/enable-recording/tests/fixtures/playback_info_version_mismatch.txt @@ -0,0 +1 @@ +[HRR] Version mismatch in /work/captures/qwen.hrr/pid-49/events.bin: file=4 reader=5 diff --git a/projects/hrr/skills/enable-recording/tests/test_hrr_capture.py b/projects/hrr/skills/enable-recording/tests/test_hrr_capture.py new file mode 100644 index 000000000000..01fe8c75613e --- /dev/null +++ b/projects/hrr/skills/enable-recording/tests/test_hrr_capture.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Tests for hrr_capture.sh, driven the way the agent drives it. + +Run: python3 -m pytest tests/test_hrr_capture.py + +Preflight decides from the loader's point of view, so these tests build it a +world it cannot tell from a real one: libamdhip64 files that do or do not carry +the capture variable, a workload, and stand-ins for ldd, readelf, ldconfig and +the interpreter's site scan first on PATH. The verdict then depends on the +test, not on what the host has installed. `run` gets workloads that write an +archive of the real shape, or nothing, or exit with a status of their own. + +The launcher parsing, signal forwarding, umask and output-permission cases +are in test_inspect_archive.py. +""" + +from __future__ import annotations + +import json +import os +import re +import signal +import struct +import subprocess +import sys +from pathlib import Path + +import pytest + +SKILL = Path(__file__).resolve().parent.parent +SCRIPT = SKILL / "scripts" / "hrr_capture.sh" + +sys.path.insert(0, str(SKILL / "scripts")) + +import inspect_archive # noqa: E402 + +# The script requires GNU find, stat and readlink, and reads /proc. The +# argument and output-path cases below run anywhere; the ones that let it +# search for runtimes are Linux only, which is what CI runs. +linux_only = pytest.mark.skipif( + sys.platform != "linux", reason="the script's runtime search needs GNU tools and /proc" +) + +CAPTURE = b"\x7fELF runtime built with HRR\0HIP_HRR_CAPTURE_OUTPUT\0" +NO_CAPTURE = b"\x7fELF runtime built without HRR\0" + +LISTING = re.compile(r"^\[capture\] (.+?) +(/\S+) \(capture: (yes|NO)\)$") + + +def _function(name: str) -> str: + function = re.search( + rf"^{name}\(\).*?^\}}", SCRIPT.read_text(encoding="utf-8"), re.M | re.S + ) + assert function, f"{name} is gone from the script" + return function.group(0) + + +def _executable(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + path.chmod(0o755) + return path + + +def _archive_events() -> bytes: + """An events.bin the inspector counts as a recording: the file header, one + whole event header and its payload.""" + header = struct.pack("/dev/null\n' + ' [ -n "$FAKE_SITE" ] && printf "%s\\n" "$FAKE_SITE"\n' + ' exit 0\n' + 'fi\n' + f'exec "{sys.executable}" "$@"\n', + ) + self.env = { + key: value + for key, value in os.environ.items() + if key not in ("LD_PRELOAD", "LD_LIBRARY_PATH", "HRR_PLAYBACK", "HIP_HRR_CAPTURE_OUTPUT") + } + self.env.update(PATH=f"{self.stubs}:/usr/bin:/bin", ROCM_PATH=str(self.rocm)) + + def lib(self, directory: str | Path, name: str = "libamdhip64.so.7", *, capture: bool) -> Path: + path = self.root / directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(CAPTURE if capture else NO_CAPTURE) + return path + + def workload( + self, + name: str = "app", + body: str = "exit 0\n", + *, + links: Path | None = None, + rpath: str | None = None, + runpath: str | None = None, + bindir: str = "app/bin", + ) -> Path: + path = _executable(self.root / bindir / name, f"#!/bin/sh\n{body}") + if links: + Path(f"{path}.ldd").write_text( + "\tlinux-vdso.so.1 (0x00007fff00000000)\n" + f"\tlibamdhip64.so.7 => {links} (0x00007f0000000000)\n" + ) + dynamic = "" + if rpath: + dynamic += f" 0x000000000000000f (RPATH) Library rpath: [{rpath}]\n" + if runpath: + dynamic += f" 0x000000000000001d (RUNPATH) Library runpath: [{runpath}]\n" + if dynamic: + Path(f"{path}.readelf").write_text(dynamic) + return path + + def script(self, *args: str, input: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + ["bash", str(SCRIPT), *args], + capture_output=True, + text=True, + env=self.env, + input=input, + stdin=None if input is not None else subprocess.DEVNULL, + timeout=120, + ) + + +@pytest.fixture +def world(tmp_path): + return World(tmp_path) + + +def listing(stderr: str) -> list[tuple[str, str, str]]: + """The candidate table preflight prints: (source, path, yes|NO).""" + return [m.groups() for m in map(LISTING.match, stderr.splitlines()) if m] + + +# --- arguments -------------------------------------------------------------- + + +def test_no_verb_is_a_usage_error_and_help_is_not(world): + bare = world.script() + assert bare.returncode == 1 + assert "usage:" in bare.stderr + + for flag in ("-h", "--help", "help"): + result = world.script(flag) + assert result.returncode == 0, flag + assert "usage:" in result.stderr + + unknown = world.script("capture") + assert unknown.returncode == 1 + assert "usage:" in unknown.stderr + + +@pytest.mark.parametrize("flag", ["--output", "--playback", "--min-free-gb"]) +def test_a_flag_without_its_value_says_so(world, flag): + """It used to fall off the end of `shift 2` and exit 1 in silence, which + is also verify's "nothing captured" status.""" + result = world.script("preflight", flag) + + assert result.returncode == 1 + assert f"error: {flag} needs a value" in result.stderr + + +def test_min_free_gb_takes_a_whole_number(world): + result = world.script("preflight", "--min-free-gb", "ten") + + assert result.returncode == 1 + assert "--min-free-gb takes a whole number of GiB, not 'ten'" in result.stderr + + +def test_the_workload_has_to_come_after_the_separator(world): + marker = world.root / "ran" + app = world.workload(body=f'touch "{marker}"\n') + result = world.script("run", "--skip-preflight", "--output", str(world.root / "out"), str(app)) + + assert result.returncode == 1 + assert f"unexpected argument: {app}. The workload command goes after --" in result.stderr + assert not marker.exists() + + +def test_run_and_verify_need_their_required_arguments(world): + no_output = world.script("run", "--", "true") + assert no_output.returncode == 1 + assert "--output is required" in no_output.stderr + + no_command = world.script("run", "--output", str(world.root / "out")) + assert no_command.returncode == 1 + assert "no command given" in no_command.stderr + + verify = world.script("verify") + assert verify.returncode == 1 + assert "--output is required" in verify.stderr + + +# --- which runtime binds ---------------------------------------------------- + + +@pytest.mark.parametrize( + "name, loadable", + [ + ("libamdhip64.so", True), + ("libamdhip64.so.7", True), + ("libamdhip64.so.7.1.70100", True), + ("/opt/rocm/lib/libamdhip64.so.7", True), + # Backups left beside the real runtime match a `.so.[0-9]*` glob and + # were judged as if the loader would pick them. + ("libamdhip64.so.7.hrr-orig", False), + ("libamdhip64.so.7.bak", False), + ("libamdhip64.so.hrr-orig", False), + ("libamdhip64.so.7x", False), + ("libamdhip64_static.so", False), + ], +) +def test_only_names_the_loader_can_ask_for_are_runtimes(name, loadable): + result = subprocess.run( + ["bash", "-c", f'{_function("is_runtime_name")}\nis_runtime_name "$1"', "-", name], + capture_output=True, + ) + assert (result.returncode == 0) == loadable + + +def test_the_soname_is_listed_before_the_development_alias(tmp_path): + """Nothing links `libamdhip64.so`; a verdict pronounced on it is about a + file no workload loads.""" + for name in ("libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.hrr-orig"): + (tmp_path / name).write_bytes(NO_CAPTURE) + result = subprocess.run( + ["bash", "-c", + f'{_function("is_runtime_name")}\n{_function("emit_libs")}\nemit_libs src "$1"', + "-", str(tmp_path)], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.splitlines() == [ + f"src|{tmp_path}/libamdhip64.so.7", + f"src|{tmp_path}/libamdhip64.so", + ] + + +@linux_only +def test_rpath_and_runpath_are_told_apart_and_origin_expanded(world): + """DT_RPATH is searched before LD_LIBRARY_PATH and DT_RUNPATH after it; + listing both as one put the wrong candidate first.""" + app = world.workload(rpath="$ORIGIN/../rpath:/opt/r2", runpath="${ORIGIN}/../runpath") + origin = app.parent + + def dirs(tag: str) -> list[str]: + result = subprocess.run( + ["bash", "-c", f'{_function("binary_rpath_dirs")}\nbinary_rpath_dirs "$1" "$2"', + "-", str(app), tag], + capture_output=True, + text=True, + env=world.env, + check=True, + ) + return result.stdout.splitlines() + + assert dirs("RPATH") == [f"{origin}/../rpath", "/opt/r2"] + assert dirs("RUNPATH") == [f"{origin}/../runpath"] + + +@linux_only +def test_candidates_are_listed_in_the_loaders_order(world): + preload = world.lib("preload", capture=True) + world.lib("app/rpath", capture=True) + world.lib("llp", capture=True) + world.lib("app/runpath", capture=True) + world.lib("site/torch/lib", "libamdhip64.so", capture=True) + ldconfig_lib = world.lib("ldcache", capture=True) + world.lib("rocm/lib", capture=True) + (world.root / "ldconfig.out").write_text( + f"\tlibamdhip64.so.7 (libc6,x86-64) => {ldconfig_lib}\n" + ) + world.env.update( + LD_PRELOAD=str(preload), + LD_LIBRARY_PATH=str(world.root / "llp"), + FAKE_SITE=str(world.root / "site"), + FAKE_LDCONFIG=str(world.root / "ldconfig.out"), + ) + app = world.workload(rpath="$ORIGIN/../rpath", runpath="$ORIGIN/../runpath", bindir="app/bin") + + result = world.script("preflight", "--", str(app)) + + assert [source for source, _, _ in listing(result.stderr)] == [ + "LD_PRELOAD", + "workload RPATH", + "LD_LIBRARY_PATH", + "workload RUNPATH", + "bundled package", + "ldconfig", + f"{world.rocm}/lib", + ] + + +@linux_only +def test_the_loaders_answer_replaces_the_guess(world): + """The first runtime on LD_LIBRARY_PATH cannot capture, but the binary + binds another one, and only that one decides.""" + world.lib("llp", capture=False) + bound = world.lib("rocm/lib", capture=True) + world.env["LD_LIBRARY_PATH"] = str(world.root / "llp") + app = world.workload(links=bound) + + result = world.script("preflight", "--", str(app)) + + assert result.returncode == 0, result.stderr + assert ("resolved", str(bound), "yes") in listing(result.stderr) + assert f"the runtime that will load is {bound} (via bound by app) and it has capture" in ( + result.stderr + ) + + +@linux_only +def test_a_bound_runtime_without_capture_is_refused_with_the_remedy(world): + bound = world.lib("rocm/lib", capture=False) + app = world.workload(links=bound) + + alone = world.script("preflight", "--", str(app)) + assert alone.returncode == 1 + assert f"the runtime that will load is {bound} (via bound by app) and it has NO capture" in ( + alone.stderr + ) + assert "use a ROCm build whose libamdhip64 has HRR compiled in" in alone.stderr + + # A capture-capable runtime elsewhere changes the advice, and the advice + # has to say where LD_PRELOAD stops working. + world.lib("capture/lib", capture=True) + world.env["LD_LIBRARY_PATH"] = str(world.root / "capture/lib") + beside = world.script("preflight", "--", str(app)) + assert beside.returncode == 1 + assert "another libamdhip64 here does have capture" in beside.stderr + assert "LD_PRELOAD does not work" in beside.stderr + + +@linux_only +def test_the_binarys_own_rpath_beats_ld_library_path(world): + """With nothing resolved, the first candidate decides, and a DT_RPATH + runtime is ahead of everything LD_LIBRARY_PATH names.""" + stale = world.lib("app/rpath", capture=False) + world.lib("llp", capture=True) + world.env["LD_LIBRARY_PATH"] = str(world.root / "llp") + app = world.workload(rpath="$ORIGIN/../rpath") + + result = world.script("preflight", "--", str(app)) + + assert result.returncode == 1 + assert f"the runtime that will load is {stale} (via workload RPATH) and it has NO capture" in ( + result.stderr + ) + + +@linux_only +def test_unresolved_with_a_runtime_that_cannot_capture_in_reach_is_refused(world): + """The PyTorch case: the interpreter links no runtime, the wheel's own + cannot capture, and a clean preflight became an empty archive.""" + world.lib("llp", capture=True) + wheel = world.lib("site/torch/lib", "libamdhip64.so", capture=False) + world.env.update(LD_LIBRARY_PATH=str(world.root / "llp"), FAKE_SITE=str(world.root / "site")) + app = world.workload() + + result = world.script("preflight", "--", str(app)) + + assert result.returncode == 1 + assert "app links no HIP runtime of its own" in result.stderr + assert f"UNRESOLVED, and refusing: nothing here was resolved, and {wheel} has no" in result.stderr + + +@linux_only +def test_unresolved_with_every_runtime_capable_passes_as_a_guess(world): + world.lib("rocm/lib", capture=True) + app = world.workload() + + result = world.script("preflight", "--", str(app)) + + assert result.returncode == 0, result.stderr + assert "UNRESOLVED: nothing here was resolved" in result.stderr + assert "That is a guess, not a verdict." in result.stderr + + +@linux_only +def test_no_runtime_anywhere_is_refused(world): + result = world.script("preflight", "--", str(world.workload())) + + assert result.returncode == 1 + assert "no libamdhip64 found at all" in result.stderr + + +@linux_only +def test_without_a_command_preflight_says_what_it_could_not_check(world): + world.lib("rocm/lib", capture=True) + + result = world.script("preflight") + + assert result.returncode == 0, result.stderr + assert "no command given, so this is what is visible here" in result.stderr + + +@linux_only +def test_one_runtime_under_several_names_is_listed_once(world): + """`libamdhip64.so`, `.so.7` and `.so.7.x.y` are hardlinks to one file, + which readlink cannot collapse, and it is reachable from two places.""" + soname = world.lib("rocm/lib", capture=True) + os.link(soname, soname.with_name("libamdhip64.so")) + os.link(soname, soname.with_name("libamdhip64.so.7.1.70100")) + world.env["LD_LIBRARY_PATH"] = str(soname.parent) + + result = world.script("preflight", "--", str(world.workload())) + + assert listing(result.stderr) == [("LD_LIBRARY_PATH", str(soname), "yes")] + + +@linux_only +def test_a_backup_beside_the_runtime_is_not_a_candidate(world): + """`libamdhip64.so.7.hrr-orig`, left by the overlay recipe, cannot capture + and is loaded by nothing; listing it refused a capture that would work.""" + runtime = world.lib("llp", capture=True) + world.lib("llp", "libamdhip64.so.7.hrr-orig", capture=False) + world.lib("llp", "libamdhip64.so.7.bak", capture=False) + world.env["LD_LIBRARY_PATH"] = str(runtime.parent) + + result = world.script("preflight", "--", str(world.workload())) + + assert result.returncode == 0, result.stderr + assert listing(result.stderr) == [("LD_LIBRARY_PATH", str(runtime), "yes")] + + +@linux_only +def test_the_workload_is_looked_up_on_the_path_its_launcher_sets(world): + """`env PATH=/x app` runs /x/app, whatever the caller's PATH finds.""" + bound = world.lib("rocm/lib", capture=True) + world.workload("job", links=bound, bindir="elsewhere") + + launcher_path = f"{world.root / 'elsewhere'}:{world.env['PATH']}" + result = world.script("preflight", "--", "env", f"PATH={launcher_path}", "job") + + assert result.returncode == 0, result.stderr + assert "(via bound by job) and it has capture" in result.stderr + + +@linux_only +def test_a_launcher_path_without_the_system_directories_does_not_blind_preflight(world): + """`env PATH=/opt/app/bin app` is the workload's PATH, not preflight's. + Applied to the whole check, it left grep, ldd and readlink unfound, and a + runtime that can capture was reported as one that cannot.""" + bound = world.lib("rocm/lib", capture=True) + world.workload("job", links=bound, bindir="opt/app/bin") + + result = world.script("preflight", "--", "env", f"PATH={world.root / 'opt/app/bin'}", "job") + + assert result.returncode == 0, result.stderr + assert ("resolved", str(bound), "yes") in listing(result.stderr) + + +@linux_only +def test_a_console_script_finds_its_interpreter_on_the_launchers_path(world): + """`#!/usr/bin/env python3` is resolved on the PATH the workload runs + with, so under `env PATH=/venv/bin serve` it is the venv's Python whose + packages count.""" + wheel = world.lib("venv/site/torch/lib", "libamdhip64.so", capture=False) + _executable( + world.root / "venv/bin/python3", + f'#!/bin/sh\ncat >/dev/null\necho "{world.root / "venv/site"}"\n', + ) + world.lib("rocm/lib", capture=True) + _executable(world.root / "venv/bin/serve", "#!/usr/bin/env python3\nimport sys\n") + + result = world.script("preflight", "--", "env", f"PATH={world.root / 'venv/bin'}", "serve") + + assert ("bundled package", str(wheel), "NO") in listing(result.stderr) + assert result.returncode == 1 + + +@linux_only +def test_a_console_script_is_scanned_with_its_own_interpreter(world): + """`vllm` and `torchrun` are text files naming their virtualenv's Python. + The PATH's python3 knows nothing of that environment's wheels.""" + wheel = world.lib("venv/site/torch/lib", "libamdhip64.so", capture=False) + _executable( + world.root / "venv/bin/python3", + f'#!/bin/sh\ncat >/dev/null\necho "{world.root / "venv/site"}"\n', + ) + world.lib("rocm/lib", capture=True) + console = _executable( + world.root / "venv/bin/serve", f"#!{world.root / 'venv/bin/python3'}\nimport sys\n" + ) + + result = world.script("preflight", "--", str(console)) + + assert ("bundled package", str(wheel), "NO") in listing(result.stderr) + assert result.returncode == 1 + + +@linux_only +@pytest.mark.parametrize("where", ["nowhere", "HRR_PLAYBACK", "ROCM_PATH"]) +def test_preflight_says_whether_a_reader_is_here(world, where): + world.lib("rocm/lib", capture=True) + if where == "HRR_PLAYBACK": + world.env["HRR_PLAYBACK"] = str(_executable(world.root / "reader/hrr-playback", "#!/bin/sh\n")) + elif where == "ROCM_PATH": + _executable(world.rocm / "bin/hrr-playback", "#!/bin/sh\n") + + result = world.script("preflight") + + # A missing reader is not a preflight failure: the manifests-only verify + # and replay elsewhere are both documented. + assert result.returncode == 0, result.stderr + if where == "nowhere": + assert "no hrr-playback here" in result.stderr + assert "decode-and-triage/scripts/ensure_playback.sh" in result.stderr + else: + assert "hrr-playback is available" in result.stderr + + +# --- the output path -------------------------------------------------------- + + +def _check_output( + out: str, *, fstype: str = "ext4", free_gib: int = 100, min_gib: int = 0, cwd: Path | None = None +) -> subprocess.CompletedProcess: + """The script's output-path check, with the filesystem and free space the + test names.""" + stubs = ( + "log() { printf '%s\\n' \"$*\" >&2; }\n" + "fail() { printf 'error: %s\\n' \"$*\" >&2; exit 1; }\n" + f"path_fstype() {{ echo {fstype}; }}\n" + "df() { printf 'Filesystem 1024-blocks Used Available Capacity Mounted\\n" + f"stub 1 0 {free_gib * 1024 * 1024} 0%% /\\n'; }}\n" + ) + return subprocess.run( + ["bash", "-c", f'{stubs}{_function("check_output_path")}\ncheck_output_path "$1" "$2"', + "-", out, str(min_gib)], + capture_output=True, + text=True, + cwd=cwd, + ) + + +@pytest.mark.parametrize("fstype", ["overlay", "overlayfs"]) +def test_an_output_path_on_the_container_layer_is_refused(tmp_path, fstype): + result = _check_output(str(tmp_path / "run.hrr"), fstype=fstype) + + assert result.returncode == 1 + assert "the archive dies with" in result.stderr + + +def test_an_output_path_in_ram_is_warned_about_not_refused(tmp_path): + result = _check_output(str(tmp_path / "run.hrr"), fstype="tmpfs") + + assert result.returncode == 0 + assert "this path is in RAM" in result.stderr + + +def test_too_little_free_space_is_refused(tmp_path): + result = _check_output(str(tmp_path / "run.hrr"), free_gib=10, min_gib=50) + assert result.returncode == 1 + assert "10 GiB free against the 50 GiB this asks for" in result.stderr + + assert _check_output(str(tmp_path / "run.hrr"), free_gib=10, min_gib=10).returncode == 0 + + +def test_a_relative_output_path_is_warned_about(tmp_path): + result = _check_output("run.hrr", cwd=tmp_path) + + assert result.returncode == 0 + assert "output path is relative" in result.stderr + + +@linux_only +@pytest.mark.parametrize( + "path, expected", + [ + # `-v /host/captures:/data/captures` under an overlay root. + ("/data/captures", "ext4"), + ("/data/captures/run.hrr", "ext4"), + ("/data", "overlay"), + # A prefix of the mount point's name is not inside it. + ("/data/captures-old", "overlay"), + ("/data/scratch/run.hrr", "tmpfs"), + ], +) +def test_the_filesystem_is_the_innermost_mount_holding_the_path(tmp_path, path, expected): + mountinfo = tmp_path / "mountinfo" + mountinfo.write_text( + "22 1 0:21 / / rw,relatime - overlay overlay rw,lowerdir=/l\n" + "30 22 8:1 /host/captures /data/captures rw,noatime master:1 - ext4 /dev/sda1 rw\n" + "31 22 0:30 / /data/scratch rw shared:5 - tmpfs tmpfs rw\n" + ) + function = _function("path_fstype").replace("/proc/self/mountinfo", str(mountinfo)) + result = subprocess.run( + ["bash", "-c", f'{function}\npath_fstype "$1"', "-", path], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.strip() == expected + + +# --- run and verify --------------------------------------------------------- + + +def _capturing_workload(world: World, *, exit_status: int = 0, record: bool = True) -> Path: + """A workload that does what libamdhip64 does with capture on: writes + pid-/ under HIP_HRR_CAPTURE_OUTPUT. It also notes the variable.""" + events = world.root / "events.bin" + events.write_bytes(_archive_events()) + body = f'printf "%s" "$HIP_HRR_CAPTURE_OUTPUT" > "{world.root / "seen-output"}"\n' + if record: + body += ( + 'dir="$HIP_HRR_CAPTURE_OUTPUT/pid-$$"\n' + 'mkdir -p "$dir/blobs"\n' + f'cp "{events}" "$dir/events.bin"\n' + "printf '{\"pid\": %s, \"parent_pid\": 1, \"complete\": true, " + "\"event_count\": 1, \"blob_count\": 0}' $$ > \"$dir/manifest.json\"\n" + ) + body += f"exit {exit_status}\n" + return world.workload("capturing", body) + + +def test_run_points_capture_at_the_output_and_offers_the_replay(world): + out = world.root / "capture" + app = _capturing_workload(world) + + result = world.script("run", "--skip-preflight", "--no-playback", "--output", str(out), "--", str(app)) + + assert result.returncode == 0, result.stderr + assert (world.root / "seen-output").read_text() == str(out) + assert "Verdict: recorded" in result.stdout + assert f"triage_archive.sh --archive {out}/pid-" in result.stderr + + +def test_run_with_nothing_captured_reports_it_and_offers_no_replay(world): + """`run` returns the workload's status by design; the verdict is what + says nothing was recorded.""" + out = world.root / "capture" + app = _capturing_workload(world, record=False) + + result = world.script("run", "--skip-preflight", "--no-playback", "--output", str(out), "--", str(app)) + + assert result.returncode == 0 + assert "Verdict: nothing captured" in result.stdout + assert "triage_archive.sh" not in result.stderr + + +def test_run_returns_the_workloads_own_status_after_verifying(world): + """A workload that crashed is the one worth capturing, and a caller + scripting around it still needs its status.""" + out = world.root / "capture" + app = _capturing_workload(world, exit_status=7) + + result = world.script("run", "--skip-preflight", "--no-playback", "--output", str(out), "--", str(app)) + + assert result.returncode == 7 + assert "workload exited 7" in result.stderr + assert "a non-zero exit is fine here" in result.stderr + assert "Verdict: recorded" in result.stdout + + +def test_run_refuses_to_merge_into_an_existing_capture(world): + """Capture resumes into what is there, and two attempts then read as one + multi-process archive.""" + out = world.root / "capture" + (out / "pid-1").mkdir(parents=True) + out.chmod(0o700) + app = _capturing_workload(world) + + refused = world.script("run", "--skip-preflight", "--no-playback", "--output", str(out), "--", str(app)) + assert refused.returncode == 1 + assert f"{out} already holds a capture" in refused.stderr + assert not (world.root / "seen-output").exists(), "the workload ran anyway" + + forced = world.script( + "run", "--skip-preflight", "--force", "--no-playback", "--output", str(out), "--", str(app) + ) + assert forced.returncode == 0, forced.stderr + assert "this run will be added" in forced.stderr + assert (world.root / "seen-output").exists() + + +@linux_only +def test_run_warns_when_an_existing_output_is_readable_by_others(world): + out = world.root / "capture" + out.mkdir() + app = _capturing_workload(world) + + out.chmod(0o755) + shared = world.script("run", "--skip-preflight", "--no-playback", "--output", str(out), "--", str(app)) + assert "can be read by other users" in shared.stderr + + for entry in out.iterdir(): + subprocess.run(["rm", "-rf", str(entry)], check=True) + out.chmod(0o700) + private = world.script("run", "--skip-preflight", "--no-playback", "--output", str(out), "--", str(app)) + assert "can be read by other users" not in private.stderr + + +def test_run_passes_piped_input_to_the_workload(world): + got = world.root / "stdin" + app = world.workload(body=f'cat > "{got}"\n') + + result = world.script( + "run", "--skip-preflight", "--no-playback", "--output", str(world.root / "capture"), "--", str(app), + input="prompt from a pipe\n", + ) + + assert result.returncode == 0, result.stderr + assert got.read_text() == "prompt from a pipe\n" + + +def test_run_waits_for_what_the_workload_left_running(world): + """A launcher that returns before its workers leaves them writing the + archive. Verifying then reads it half-written.""" + done = world.root / "done" + app = world.workload(body=f'( sleep 1; touch "{done}" ) /dev/null 2>&1 &\nexit 0\n') + + result = world.script("run", "--skip-preflight", "--no-playback", "--output", str(world.root / "capture"), "--", str(app)) + + assert result.returncode == 0, result.stderr + assert "processes it started are still running; waiting" in result.stderr + assert done.exists(), "the wrapper returned while the workload's children were running" + + +# Runs a command as a child subreaper that waits for that command only: a +# process orphaned below it is reparented here and never reaped, as under a +# container whose PID 1 is `sleep infinity` or a Python entry point. +NON_REAPING_PARENT = ( + "import ctypes, subprocess, sys\n" + "PR_SET_CHILD_SUBREAPER = 36\n" + "assert ctypes.CDLL(None).prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) == 0\n" + "sys.exit(subprocess.call(sys.argv[1:]))\n" +) + + +@linux_only +def test_run_does_not_wait_on_a_child_that_has_already_exited(world): + """`kill -0` is true of a zombie. A child that outlived the workload and + then exited stayed one under a parent that never reaps, and the wait for + the workload's group never ended.""" + done = world.root / "done" + app = world.workload(body=f'( sleep 1; touch "{done}" ) /dev/null 2>&1 &\nexit 0\n') + + harness = subprocess.Popen( + [sys.executable, "-c", NON_REAPING_PARENT, + "bash", str(SCRIPT), "run", "--skip-preflight", "--no-playback", + "--output", str(world.root / "capture"), "--", str(app)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + env=world.env, + start_new_session=True, + ) + try: + _, stderr = harness.communicate(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(harness.pid, signal.SIGKILL) + harness.communicate() + pytest.fail("the wrapper was still waiting 30 s after the workload's child exited") + + assert harness.returncode == 0, stderr + assert done.exists() + assert "workload exited 0" in stderr + + +@linux_only +def test_run_stops_before_the_workload_when_nothing_would_be_recorded(world): + bound = world.lib("rocm/lib", capture=False) + marker = world.root / "ran" + app = world.workload(body=f'touch "{marker}"\nexit 5\n', links=bound) + out = world.root / "capture" + + refused = world.script("run", "--min-free-gb", "0", "--output", str(out), "--", str(app)) + assert refused.returncode == 1 + assert "preflight failed: no capture-capable runtime" in refused.stderr + assert not marker.exists() + + forced = world.script("run", "--force", "--min-free-gb", "0", "--no-playback", "--output", str(out), "--", str(app)) + assert forced.returncode == 5 + assert marker.exists() + + +@linux_only +def test_run_repeats_that_the_runtime_was_not_resolved(world): + """A guessed preflight pass is not a promise, and the run says so where + the user is looking.""" + world.lib("rocm/lib", capture=True) + app = world.workload() + + # --force only so the host's own filesystem cannot fail the output check. + result = world.script( + "run", "--force", "--min-free-gb", "0", "--no-playback", + "--output", str(world.root / "capture"), "--", str(app), + ) + + assert result.returncode == 0, result.stderr + assert "preflight could not resolve which runtime this workload binds" in result.stderr + + +def test_verify_reports_an_existing_archive_and_passes_json_through(world): + out = world.root / "capture" + pid_dir = out / "pid-42" + (pid_dir / "blobs").mkdir(parents=True) + (pid_dir / "events.bin").write_bytes(_archive_events()) + (pid_dir / "manifest.json").write_text( + json.dumps({"pid": 42, "parent_pid": 1, "complete": True, "event_count": 1, "blob_count": 0}) + ) + + result = world.script("verify", "--output", str(out), "--json", "--no-playback") + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["verdict"] == "recorded" + + missing = world.script("verify", "--output", str(world.root / "never"), "--no-playback") + assert missing.returncode == inspect_archive.EXIT_EMPTY diff --git a/projects/hrr/skills/enable-recording/tests/test_inspect_archive.py b/projects/hrr/skills/enable-recording/tests/test_inspect_archive.py new file mode 100644 index 000000000000..f5e908149602 --- /dev/null +++ b/projects/hrr/skills/enable-recording/tests/test_inspect_archive.py @@ -0,0 +1,1422 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Unit tests for inspect_archive.py. + +Run: python3 -m pytest tests/test_inspect_archive.py + +The archive layouts here are built to match what the capture writer produces: +a root manifest listing processes, and one manifest per pid directory carrying +pid, parent_pid, complete, event_count and blob_count. +""" + +from __future__ import annotations + +import json +import os +import re +import signal +import struct +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import inspect_archive # noqa: E402 + +FIXTURES = Path(__file__).resolve().parent / "fixtures" + + +def make_process( + root: Path, + pid: int, + *, + events: int | None = 100, + blobs: int | None = 5, + complete: bool | None = True, + events_bytes: int = 4096, + write_manifest: bool = True, + format_version: int = 5, +) -> Path: + pid_dir = root / f"pid-{pid}" + (pid_dir / "blobs").mkdir(parents=True, exist_ok=True) + # Real shape: hrr_file_header { magic, version, reserved }, then a whole + # hrr_event_header whose payload_length counts itself, then the payload. + header = struct.pack("= inspect_archive.EVENT_HEADER_BYTES: + payload_length = remaining + body = struct.pack(" None: + (root / "manifest.json").write_text( + json.dumps( + { + "version": 4, + "capture_mode": "in-tree", + "owner_pid": processes[0]["pid"] if processes else 0, + "processes": processes, + } + ) + ) + + +def test_single_process_archive_is_recorded(tmp_path): + make_process(tmp_path, 42) + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert len(report.processes) == 1 + assert report.recorded_processes + proc = report.processes[0] + assert proc.pid == 42 + assert proc.event_count == 100 + assert proc.complete is True + assert "Verdict: recorded" in inspect_archive.render(report) + + +def test_a_pid_directory_can_be_passed_directly(tmp_path): + pid_dir = make_process(tmp_path, 7) + report = inspect_archive.inspect(pid_dir, use_playback=False) + + assert [p.pid for p in report.processes] == [7] + + +def test_a_pid_directory_passed_alone_is_its_own_archive(tmp_path): + """The parent's manifest lists the other processes, which were not passed, + so reading it warned that their real directories were missing. + """ + pid_dir = make_process(tmp_path, 49) + make_process(tmp_path, 50) + make_root(tmp_path, [{"pid": 49, "event_count": 100}, {"pid": 50, "event_count": 100}]) + + report = inspect_archive.inspect(pid_dir, use_playback=False) + + assert report.root == pid_dir + assert report.root_manifest is None and report.root_manifest_error is None + assert report.warnings == [] + assert inspect_archive.render(report).startswith(f"Archive: {pid_dir}") + + +def test_archive_with_no_process_directories_is_empty(tmp_path): + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert not report.processes + assert not report.recorded_processes + rendered = inspect_archive.render(report) + assert "Verdict: nothing captured" in rendered + assert "preflight" in rendered + + +def test_process_directory_without_events_is_not_recorded(tmp_path): + make_process(tmp_path, 9, events=0, blobs=0, events_bytes=0) + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert report.processes + assert not report.recorded_processes + assert "Verdict: empty" in inspect_archive.render(report) + + +def test_incomplete_process_is_reported_but_still_recorded(tmp_path): + """A crashed capture is the case worth keeping, not a failure.""" + make_process(tmp_path, 11, complete=False) + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert report.recorded_processes + rendered = inspect_archive.render(report) + assert "incomplete" in rendered + assert "still worth sending" in rendered + # A lossy capture has no trailer either, and only the always-on line of + # capture's stderr tells it apart: the louder one needs AMD_LOG_LEVEL. + assert "Archive marked INCOMPLETE" in rendered + + +def test_events_without_a_manifest_are_reported_as_unfinalized(tmp_path): + """Killed before finalizing: counts unknown, but the events are there.""" + make_process(tmp_path, 13, write_manifest=False) + report = inspect_archive.inspect(tmp_path, use_playback=False) + + proc = report.processes[0] + assert proc.recorded + assert not proc.finalized + assert proc.pid == 13, "pid must fall back to the directory name" + assert any("died before finalizing" in w for w in report.warnings) + assert "not finalized (no manifest)" in inspect_archive.render(report) + + +def test_stale_root_manifest_is_flagged_and_not_believed(tmp_path): + """The root manifest is rewritten best-effort, so it can lag the truth.""" + make_process(tmp_path, 21, events=9_000_000) + make_root(tmp_path, [{"pid": 21, "event_count": 4096, "blob_count": 5}]) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert report.processes[0].event_count == 9_000_000 + assert any("authoritative" in w for w in report.warnings) + + +def test_root_manifest_listing_a_missing_process_is_flagged(tmp_path): + make_process(tmp_path, 31) + make_root( + tmp_path, + [ + {"pid": 31, "event_count": 100, "blob_count": 5}, + {"pid": 32, "event_count": 50, "blob_count": 1}, + ], + ) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert any("no pid-32/ directory" in w for w in report.warnings) + + +def test_size_on_disk_counts_the_whole_process_directory(tmp_path): + """The manifests and the allocator's regions/ sidecars travel with the + archive too, and leaving them out understated what is being sent. + """ + pid_dir = make_process(tmp_path, 57) + (pid_dir / "regions").mkdir() + (pid_dir / "regions" / "torch-0.hrrr").write_bytes(b"r" * 1000) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + on_disk = sum(f.stat().st_size for f in pid_dir.rglob("*") if f.is_file()) + assert report.processes[0].total_bytes == on_disk + assert report.total_bytes == on_disk + + +def test_multi_process_archive_says_to_keep_it_together(tmp_path): + make_process(tmp_path, 100) + make_process(tmp_path, 101) + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert len(report.recorded_processes) == 2 + rendered = inspect_archive.render(report) + assert "whole archive directory" in rendered + # hrr-playback refuses a root holding several processes, and triage takes one. + assert "one at a time" in rendered + + +def test_unreadable_root_manifest_does_not_hide_the_processes(tmp_path): + make_process(tmp_path, 55) + (tmp_path / "manifest.json").write_text("{not json") + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert report.root_manifest_error + assert report.recorded_processes + + +@pytest.mark.parametrize("content", ["[]", "1", '"text"', "[1]"]) +def test_a_manifest_that_is_not_an_object_is_unreadable_not_fatal(tmp_path, content): + """Valid JSON that is not an object crashed the inspector on its first + `.get`, and this is the tool that reports on damaged captures. + """ + pid_dir = make_process(tmp_path, 56) + (pid_dir / "manifest.json").write_text(content) + (tmp_path / "manifest.json").write_text(content) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert report.root_manifest_error + assert report.processes[0].manifest_error + assert report.recorded_processes, "the events are still there" + # Said, rather than shown as an ordinary process that never finalized. + assert any("pid-56: manifest.json is unreadable" in w for w in report.warnings) + assert not any("died before finalizing" in w for w in report.warnings) + assert "state: unknown (manifest unreadable)" in inspect_archive.render(report) + + +def test_manifest_fields_of_the_wrong_type_are_unknown_not_fatal(tmp_path): + """`"event_count": "100"` reached `event_count > 0` and raised, so verify + produced no report at all for a damaged archive. + """ + pid_dir = make_process(tmp_path, 57) + (pid_dir / "manifest.json").write_text(json.dumps({ + "pid": "57", "parent_pid": [1], "complete": "yes", + "event_count": "100", "blob_count": True, + "metadata": {"runtime": {"hip_runtime_version": 7}}, + })) + make_root(tmp_path, [{"pid": [57], "event_count": 1}]) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + rendered = inspect_archive.render(report) + + proc = report.processes[0] + assert (proc.pid, proc.event_count, proc.blob_count, proc.complete) == (57, None, None, None) + assert report.recorded_processes, "the events are still there" + warning = next(w for w in report.warnings if "wrong type" in w) + assert "event_count: str" in warning and "blob_count: bool" in warning + assert not any("died before finalizing" in w for w in report.warnings) + assert "state: unknown (manifest damaged)" in rendered + + +def test_a_negative_count_is_damaged_not_zero(tmp_path): + """`"event_count": -1` passed the type check, `recorded` came out false and + the verdict read empty although the events file holds a stream. + """ + pid_dir = make_process(tmp_path, 60) + (pid_dir / "manifest.json").write_text(json.dumps({ + "pid": 60, "complete": True, "event_count": -1, "blob_count": -3, + })) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + proc = report.processes[0] + assert (proc.event_count, proc.blob_count) == (None, None) + assert report.recorded_processes, "the events are still there" + warning = next(w for w in report.warnings if "out of range" in w) + assert "event_count: -1" in warning and "blob_count: -3" in warning + + +@pytest.mark.parametrize("content", ["{}", '{"pid": 58}']) +def test_an_empty_or_partial_manifest_is_damaged_not_missing(tmp_path, content): + """`{}` was skipped as if the file did not exist, and the process read as + one killed before it could finalize. + """ + pid_dir = make_process(tmp_path, 58) + (pid_dir / "manifest.json").write_text(content) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert any("complete: missing" in w for w in report.warnings) + assert not any("died before finalizing" in w for w in report.warnings) + assert "state: unknown (manifest damaged)" in inspect_archive.render(report) + + +def test_a_null_manifest_field_is_damaged_not_missing(tmp_path): + """`{"complete": null}` read as no manifest at all, and the process as one + that died before finalizing, although the file was there. + """ + pid_dir = make_process(tmp_path, 59) + (pid_dir / "manifest.json").write_text( + json.dumps({"pid": 59, "complete": None, "event_count": None}) + ) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + warning = next(w for w in report.warnings if "wrong type" in w) + assert "complete: NoneType" in warning and "event_count: NoneType" in warning + assert not any("died before finalizing" in w for w in report.warnings) + assert "state: unknown (manifest damaged)" in inspect_archive.render(report) + + +def test_a_pid_directory_without_events_is_a_leftover_not_a_capture(tmp_path): + """The reader skips a pid-* directory with no events.bin, so the verdict + has to as well: an output holding only one captured nothing. + """ + stale = tmp_path / "pid-123" + stale.mkdir() + + report = inspect_archive.inspect(tmp_path, use_playback=False) + assert report.processes == [] + assert inspect_archive.verdict(report) == "nothing captured" + + direct = inspect_archive.inspect(stale, use_playback=False) + assert direct.processes == [] + assert inspect_archive.verdict(direct) == "nothing captured" + + +def test_the_json_report_carries_an_unreadable_root_manifest(tmp_path, capsys): + make_process(tmp_path, 59) + (tmp_path / "manifest.json").write_text("{not json") + + inspect_archive.main(["--archive", str(tmp_path), "--no-playback", "--json"]) + payload = json.loads(capsys.readouterr().out) + + assert payload["root_manifest_error"] + assert any("root manifest.json is unreadable" in w for w in payload["warnings"]) + + +def test_size_on_disk_counts_the_root_manifest_but_not_the_parent(tmp_path): + """The whole archive directory is what gets sent; a single pid-* directory + passed on its own is sent alone. + """ + pid_dir = make_process(tmp_path, 58) + make_root(tmp_path, [{"pid": 58, "event_count": 100, "blob_count": 5}]) + process_bytes = sum(f.stat().st_size for f in pid_dir.rglob("*") if f.is_file()) + root_bytes = (tmp_path / "manifest.json").stat().st_size + + assert inspect_archive.inspect(tmp_path, use_playback=False).total_bytes == ( + process_bytes + root_bytes + ) + assert inspect_archive.inspect(pid_dir, use_playback=False).total_bytes == process_bytes + + +@pytest.mark.parametrize( + "metadata", [[], None, "text", {"runtime": None}, {"runtime": []}, {"runtime": "7.16"}] +) +def test_metadata_of_the_wrong_type_is_not_fatal(tmp_path, metadata): + pid_dir = make_process(tmp_path, 57) + (pid_dir / "manifest.json").write_text( + json.dumps({"pid": 57, "complete": True, "metadata": metadata}) + ) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert report.processes[0].hip_runtime_version is None + assert report.recorded_processes + + +def test_version_mismatch_from_playback_is_explained(tmp_path, monkeypatch): + """A reader that cannot read the archive is a tooling problem, not a bad capture. + + The fixture is real output from a reader built on a different release line + than the runtime that captured. Format versions do not track the tool, so + this happens whenever the two come from different lines. + """ + make_process(tmp_path, 77) + fixture = (FIXTURES / "playback_info_version_mismatch.txt").read_text() + + monkeypatch.setattr(inspect_archive, "_find_playback", lambda explicit: "/usr/bin/hrr-playback") + monkeypatch.setattr( + inspect_archive, "_run_playback_info", lambda binary, target, timeout: (fixture, None) + ) + + report = inspect_archive.inspect(tmp_path, use_playback=True) + + assert any("cannot read this archive" in w for w in report.warnings) + assert report.recorded_processes, "the archive itself is still fine" + + +def test_playback_info_is_included_verbatim(tmp_path, monkeypatch): + make_process(tmp_path, 78) + fixture = (FIXTURES / "playback_info_pass.txt").read_text() + + monkeypatch.setattr(inspect_archive, "_find_playback", lambda explicit: "/usr/bin/hrr-playback") + monkeypatch.setattr( + inspect_archive, "_run_playback_info", lambda binary, target, timeout: (fixture, None) + ) + + report = inspect_archive.inspect(tmp_path, use_playback=True) + rendered = inspect_archive.render(report) + + assert "hrr-playback --info:" in rendered + for line in fixture.splitlines(): + if line.strip(): + assert line.strip() in rendered + + +def test_playback_cross_check_uses_the_substantive_process(tmp_path, monkeypatch): + """Not the alphabetically first one. + + Real capture: pid-406 held 10,512 registration events and pid-45 held + 258,042 including every kernel. Sorted as text, pid-406 comes first, so the + cross-check reported on the process with nothing in it. + """ + make_process(tmp_path, 406, events=10_512, blobs=4_082) + make_process(tmp_path, 45, events=258_042, blobs=4_515) + seen = {} + + monkeypatch.setattr(inspect_archive, "_find_playback", lambda explicit: "/usr/bin/hrr-playback") + monkeypatch.setattr( + inspect_archive, + "_run_playback_info", + lambda binary, target, timeout: (seen.update(target=target), ("info", None))[1], + ) + + inspect_archive.inspect(tmp_path, use_playback=True) + + assert seen["target"].name == "pid-45" + + +def test_packaged_playback_gets_its_own_libraries_on_the_path(tmp_path): + """A packaged build ships bin/, lib/ and runtime-lib/ as siblings. + + Run without those on the library path, the binary loads whatever + libamdhip64 the host has and dies on a missing symbol version, which looks + like a broken archive rather than a broken invocation. + """ + root = tmp_path / "playback-build" + (root / "bin").mkdir(parents=True) + (root / "lib").mkdir() + (root / "runtime-lib").mkdir() + binary = root / "bin" / "hrr-playback" + binary.write_text("#!/bin/sh\n") + + env = inspect_archive._playback_env(str(binary)) + + assert env["LD_LIBRARY_PATH"].split(":")[:2] == [str(root / "lib"), str(root / "runtime-lib")] + + +def test_build_tree_playback_gets_rocr_lib_on_the_path(tmp_path, monkeypatch): + """As triage_archive.sh does: an in-tree reader may need the matching + libhsa-runtime64, and ROCR_LIB is where it is.""" + binary = tmp_path / "build" / "playback" / "hrr-playback" + binary.parent.mkdir(parents=True) + binary.write_text("#!/bin/sh\n") + rocr = tmp_path / "rocr" / "lib" + rocr.mkdir(parents=True) + (rocr / "libhsa-runtime64.so.1").write_text("") + monkeypatch.setenv("ROCR_LIB", str(rocr)) + monkeypatch.setenv("LD_LIBRARY_PATH", "/mine") + monkeypatch.setenv("ROCM_PATH", str(tmp_path / "no-rocm")) + + env = inspect_archive._playback_env(str(binary)) + + assert env["LD_LIBRARY_PATH"] == f"{rocr}:/mine" + + # A ROCR_LIB without the library in it is not put on the path. + (rocr / "libhsa-runtime64.so.1").unlink() + assert inspect_archive._playback_env(str(binary))["LD_LIBRARY_PATH"] == "/mine" + + +def test_build_tree_playback_gets_rocm_lib_last(tmp_path, monkeypatch): + """As ensure_playback.sh and triage_archive.sh do: the rest of ROCm comes + from $ROCM_PATH/lib, after the caller's own path so the system release + does not bind ahead of it.""" + binary = tmp_path / "build" / "playback" / "hrr-playback" + binary.parent.mkdir(parents=True) + binary.write_text("#!/bin/sh\n") + rocm = tmp_path / "rocm" + (rocm / "lib").mkdir(parents=True) + monkeypatch.delenv("ROCR_LIB", raising=False) + monkeypatch.setenv("ROCM_PATH", str(rocm)) + monkeypatch.setenv("LD_LIBRARY_PATH", "/mine") + + assert inspect_archive._playback_env(str(binary))["LD_LIBRARY_PATH"] == f"/mine:{rocm / 'lib'}" + + monkeypatch.delenv("LD_LIBRARY_PATH") + assert inspect_archive._playback_env(str(binary))["LD_LIBRARY_PATH"] == str(rocm / "lib") + + +def test_missing_playback_is_reported_not_fatal(tmp_path, monkeypatch): + make_process(tmp_path, 79) + monkeypatch.setattr(inspect_archive, "_find_playback", lambda explicit: None) + + report = inspect_archive.inspect(tmp_path, use_playback=True) + + assert report.playback_error + assert report.recorded_processes + + +def test_missing_archive_says_capture_never_started(tmp_path, capsys): + """A workload that died before its first HIP call leaves no directory. + + Reporting that as a usage error hides the one useful fact: capture never + ran, which is a different problem from an archive that came out empty. + """ + rc = inspect_archive.main(["--archive", str(tmp_path / "never-created.hrr"), "--no-playback"]) + out = capsys.readouterr().out + + assert rc == inspect_archive.EXIT_EMPTY + assert "Verdict: no archive" in out + assert "capture never started" in out + assert "preflight" in out + + +def test_exit_code_is_one_when_nothing_was_captured(tmp_path, capsys): + rc = inspect_archive.main(["--archive", str(tmp_path), "--no-playback"]) + capsys.readouterr() + assert rc == inspect_archive.EXIT_EMPTY + + +def test_exit_code_is_zero_when_something_was_captured(tmp_path, capsys): + make_process(tmp_path, 88) + rc = inspect_archive.main(["--archive", str(tmp_path), "--no-playback"]) + capsys.readouterr() + assert rc == inspect_archive.EXIT_OK + + +def test_json_output_is_machine_readable(tmp_path, capsys): + make_process(tmp_path, 99, complete=False) + inspect_archive.main(["--archive", str(tmp_path), "--no-playback", "--json"]) + + payload = json.loads(capsys.readouterr().out) + assert payload["recorded"] is True + assert payload["processes"][0]["complete"] is False + assert payload["processes"][0]["pid"] == 99 + + +def test_the_json_verdict_is_one_of_the_four_documented_ones(tmp_path, capsys): + """`--json` used to know only `recorded` and `empty`, so a consumer could + not tell an archive with no events from a directory no process ever opened. + """ + (tmp_path / "manifest.json").write_text("{}") + + inspect_archive.main(["--archive", str(tmp_path), "--no-playback", "--json"]) + assert json.loads(capsys.readouterr().out)["verdict"] == "nothing captured" + + make_process(tmp_path, 75, events=0, blobs=0, events_bytes=52) + inspect_archive.main(["--archive", str(tmp_path), "--no-playback", "--json"]) + assert json.loads(capsys.readouterr().out)["verdict"] == "empty" + + make_process(tmp_path, 76) + inspect_archive.main(["--archive", str(tmp_path), "--no-playback", "--json"]) + assert json.loads(capsys.readouterr().out)["verdict"] == "recorded" + + +def test_the_reader_comes_from_hrr_playback_first(tmp_path, monkeypatch): + """Same order as the sibling skill's ensure_playback.sh, so one host does + not verify with one reader and replay with another. + """ + named = tmp_path / "from-env" / "hrr-playback" + named.parent.mkdir() + named.write_text("#!/bin/sh\n") + named.chmod(0o755) + rocm = tmp_path / "rocm" + (rocm / "bin").mkdir(parents=True) + (rocm / "bin" / "hrr-playback").write_text("#!/bin/sh\n") + (rocm / "bin" / "hrr-playback").chmod(0o755) + + monkeypatch.setenv("HRR_PLAYBACK", str(named)) + monkeypatch.setenv("ROCM_PATH", str(rocm)) + monkeypatch.setattr(inspect_archive.shutil, "which", lambda name: "/usr/bin/hrr-playback") + + assert inspect_archive._find_playback(None) == str(named) + + # Only a file that can run, as its `-x` does: one that cannot is passed over. + named.chmod(0o644) + assert inspect_archive._find_playback(None) == str(rocm / "bin" / "hrr-playback") + + monkeypatch.delenv("HRR_PLAYBACK") + assert inspect_archive._find_playback(None) == str(rocm / "bin" / "hrr-playback") + + # PATH last, and only when ROCm has none. + (rocm / "bin" / "hrr-playback").unlink() + assert inspect_archive._find_playback(None) == "/usr/bin/hrr-playback" + + +def test_preflight_sees_the_reader_hrr_playback_names(tmp_path): + """Preflight said there was no reader while verify went on to use the one + HRR_PLAYBACK named. + """ + reader = tmp_path / "hrr-playback" + reader.write_text("#!/bin/sh\n") + reader.chmod(0o755) + env = {**os.environ, "HRR_PLAYBACK": str(reader), "ROCM_PATH": str(tmp_path / "no-rocm")} + + result = subprocess.run( + ["bash", str(SCRIPT), "preflight"], capture_output=True, text=True, env=env, timeout=120 + ) + + assert "hrr-playback is available" in result.stderr + + +def test_preflight_refuses_a_directory_as_the_reader(tmp_path): + """`-x` is true of a directory, which verify then refuses as a reader.""" + env = {**os.environ, "ROCM_PATH": str(tmp_path / "no-rocm")} + + result = subprocess.run( + ["bash", str(SCRIPT), "preflight", "--playback", str(tmp_path)], + capture_output=True, + text=True, + env=env, + timeout=120, + ) + + assert result.returncode != 0 + assert "not an executable file" in result.stderr + + +def test_json_is_still_json_when_there_is_no_archive(tmp_path, capsys): + """The case a script most needs to recognise, and the one that printed prose. + + A pre-send gate parses this output. Emitting a paragraph on the one path + that says capture never ran defeats the gate exactly when it matters. + """ + rc = inspect_archive.main( + ["--archive", str(tmp_path / "never-created.hrr"), "--no-playback", "--json"] + ) + payload = json.loads(capsys.readouterr().out) + + assert rc == inspect_archive.EXIT_EMPTY + assert payload["verdict"] == "no archive" + assert payload["recorded"] is False + + +def test_an_unusable_playback_path_is_an_error_not_a_skip(tmp_path, capsys): + """A typo in --playback used to skip the cross-check and exit 0.""" + make_process(tmp_path, 55) + rc = inspect_archive.main( + ["--archive", str(tmp_path), "--playback", str(tmp_path / "no-such-binary")] + ) + + assert rc == inspect_archive.EXIT_USAGE + assert "not an executable file" in capsys.readouterr().err + + # A gate parsing --json gets JSON for this too, not prose on stderr. + rc = inspect_archive.main( + ["--archive", str(tmp_path), "--playback", str(tmp_path / "no-such-binary"), "--json"] + ) + assert rc == inspect_archive.EXIT_USAGE + assert json.loads(capsys.readouterr().out)["verdict"] == "usage error" + + +def test_an_archive_path_that_is_a_file_is_a_usage_error(tmp_path, capsys): + """Not an empty capture: exit 1 would read as "nothing recorded".""" + events = tmp_path / "events.bin" + events.write_bytes(b"") + + rc = inspect_archive.main(["--archive", str(events), "--no-playback"]) + + assert rc == inspect_archive.EXIT_USAGE + assert f"not a directory: {events}" in capsys.readouterr().err + + +def test_blob_count_including_code_objects_is_not_a_discrepancy(tmp_path): + """The manifest counts code objects among its blobs; blobs/ and --info do not. + + Real archive: manifest 8, blob files 7, code objects 1. Three numbers, one + truth, and the report used to show them side by side with no explanation. + """ + pid_dir = make_process(tmp_path, 61, blobs=7) + (pid_dir / "code_objects").mkdir() + (pid_dir / "code_objects" / "co-0").write_bytes(b"x") + (pid_dir / "manifest.json").write_text( + json.dumps({"pid": 61, "complete": True, "event_count": 89, "blob_count": 8}) + ) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + rendered = inspect_archive.render(report) + + assert report.warnings == [] + assert "blobs: 7" in rendered, "report the files on disk, which --info agrees with" + + +def test_a_blob_count_matching_nothing_on_disk_is_flagged(tmp_path): + pid_dir = make_process(tmp_path, 62, blobs=3) + (pid_dir / "manifest.json").write_text( + json.dumps({"pid": 62, "complete": True, "event_count": 10, "blob_count": 99}) + ) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert any("neither total" in w for w in report.warnings) + + +def test_a_reader_that_cannot_name_itself_is_reported_as_unverified(tmp_path, monkeypatch): + """The silent failure: a reader from another build accepts the archive and + decodes the event names against its own API table, with no error at all.""" + pid_dir = make_process(tmp_path, 63) + (pid_dir / "manifest.json").write_text( + json.dumps( + { + "pid": 63, + "complete": True, + "event_count": 100, + "blob_count": 5, + "metadata": {"runtime": {"hip_runtime_version": "7.16.26320"}}, + } + ) + ) + + monkeypatch.setattr(inspect_archive, "_find_playback", lambda explicit: "/usr/bin/hrr-playback") + monkeypatch.setattr( + inspect_archive, "_playback_version", lambda binary, timeout=20: (None, None) + ) + monkeypatch.setattr( + inspect_archive, + "_run_playback_info", + lambda binary, target, timeout: ("Events: 100\nBlobs: 5", None), + ) + + report = inspect_archive.inspect(tmp_path, use_playback=True) + + assert any("unverified" in w for w in report.warnings) + + +def test_a_reader_that_never_started_is_a_failed_cross_check(tmp_path, monkeypatch): + """Observed on a real box: the reader could not load its own libraries, and + the loader error was printed under the `hrr-playback --info:` heading with a + `Verdict: recorded` under it and exit 0. A crash is not archive information. + """ + make_process(tmp_path, 65) + + class Completed: + returncode = 127 + stdout = "" + stderr = ( + "/reader/bin/hrr-playback: error while loading shared libraries: " + "librocprofiler-register.so.0: cannot open shared object file" + ) + + monkeypatch.setattr(inspect_archive.subprocess, "run", lambda *a, **k: Completed()) + + info, error = inspect_archive._run_playback_info("/reader/bin/hrr-playback", tmp_path, 10) + + assert info is None + assert "exited 127" in error + assert "librocprofiler-register" in error + + +def test_a_reader_that_prints_nothing_is_a_failed_cross_check(tmp_path, monkeypatch): + """Exit 0 and no output returned neither text nor an error, so the report + failed its exit status with nothing to say why. + """ + make_process(tmp_path, 66) + + class Completed: + returncode = 0 + stdout = "" + stderr = " \n" + + monkeypatch.setattr(inspect_archive.subprocess, "run", lambda *a, **k: Completed()) + + info, error = inspect_archive._run_playback_info("/reader/bin/hrr-playback", tmp_path, 10) + + assert info is None + assert error == "hrr-playback --info exited 0 and printed nothing" + + +def test_a_reader_that_refuses_the_archive_still_explains_itself(tmp_path, monkeypatch): + """Non-zero, but the output is the reader's own verdict rather than a crash.""" + + class Completed: + returncode = 1 + stdout = "[HRR] Version mismatch: file=4 reader=5" + stderr = "" + + monkeypatch.setattr(inspect_archive.subprocess, "run", lambda *a, **k: Completed()) + + info, error = inspect_archive._run_playback_info("/usr/bin/hrr-playback", tmp_path, 10) + + assert error is None + assert inspect_archive.VERSION_MISMATCH in info + + +def test_a_header_only_events_file_is_not_a_recording(tmp_path): + """`events.bin` opens with an 8-byte header and a clean shutdown appends a + 44-byte trailer, so bytes on disk are not evidence that anything was + recorded. The manifest's count decides where there is one. + """ + make_process(tmp_path, 67, events=0, blobs=0, events_bytes=52) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert report.recorded_processes == [] + assert "Verdict: empty" in inspect_archive.render(report) + + +def test_a_process_killed_before_its_manifest_is_still_a_recording(tmp_path): + make_process(tmp_path, 68, events_bytes=4096, write_manifest=False) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert len(report.recorded_processes) == 1 + + +def test_a_damaged_root_entry_does_not_hide_the_processes(tmp_path): + """The root manifest is rewritten best-effort, so it can be valid JSON with + a `processes` entry that is not an object. The pid directories still count. + """ + make_process(tmp_path, 79) + (tmp_path / "manifest.json").write_text(json.dumps({"version": 4, "processes": [None]})) + + report = inspect_archive.inspect(tmp_path, use_playback=False) + + assert len(report.recorded_processes) == 1 + assert any("not an object: NoneType" in w for w in report.warnings) + + +def test_a_torn_first_event_header_is_not_a_recording(tmp_path): + """A process killed mid-write can leave a header cut in half, and half a + header is not an event. Nor is one whose declared length cannot fit. + """ + pid_dir = make_process(tmp_path, 77, write_manifest=False) + header = struct.pack(" str: + function = re.search( + r"^strip_launchers\(\).*?^\}", SCRIPT.read_text(encoding="utf-8"), re.M | re.S + ) + assert function, "strip_launchers is gone from the script" + result = subprocess.run( + ["bash", "-c", f"{function.group(0)}\nstrip_launchers {command}"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +@pytest.mark.parametrize( + "command, expected", + [ + ("python train.py", "python"), + # env's assignments come first, one a line, so preflight can apply them. + ("env FOO=1 python train.py", "FOO=1\npython"), + ("env A=1 nice -n 5 env B=2 app", "A=1\nB=2\napp"), + # The operand cases: skipping only dashed arguments used to leave the + # duration or the niceness as the workload. + ("timeout 60s python train.py", "python"), + ("timeout -s KILL 30 python train.py", "python"), + ("nice -n 10 vllm serve", "vllm"), + ("ionice -c 2 -n 4 python train.py", "python"), + ("stdbuf -oL python train.py", "python"), + # Launchers whose grammar we do not model stop the search at themselves + # rather than guessing. + ("srun -N2 python train.py", "srun"), + # env's own options change the environment in ways an assignment + # cannot carry, and used to come back as the workload themselves. + ("env -u LD_PRELOAD app", "env"), + ("env --ignore-environment app", "env"), + ("/usr/bin/env -i A=1 app", "/usr/bin/env"), + ("env -- A=1 app", "A=1\napp"), + ], +) +def test_the_workload_is_found_behind_its_launcher(command, expected): + assert _strip_launchers(command) == expected + + +@pytest.mark.parametrize( + "shebang, expected", + [ + ("#!/venv/bin/python3", "/venv/bin/python3"), + ("#!/usr/bin/env python3", "python3"), + # -S used to come back as the interpreter, and the package scan ran it. + ("#!/usr/bin/env -S python3 -u", "python3"), + ("#!/usr/bin/env -Spython3", "python3"), + ("#!/usr/bin/env --split-string=python3 -u", "python3"), + ("#!/usr/bin/env -S PYTHONUNBUFFERED=1 python3", "python3"), + # An option we do not model is not a binary: fall back to the PATH's. + ("#!/usr/bin/env -i python3", "FALLBACK"), + ], +) +def test_the_interpreter_is_read_off_the_shebang(tmp_path, shebang, expected): + function = re.search( + r"^workload_interpreter\(\).*?^\}", SCRIPT.read_text(encoding="utf-8"), re.M | re.S + ) + assert function, "workload_interpreter is gone from the script" + script = tmp_path / "console-script" + script.write_text(f"{shebang}\nimport sys\n") + # The PATH's python3 is what the fallback answers, so make it recognisable. + bindir = tmp_path / "bin" + bindir.mkdir() + fallback = bindir / "python3" + fallback.write_text("#!/bin/sh\n") + fallback.chmod(0o755) + result = subprocess.run( + ["/bin/bash", "-c", f"{function.group(0)}\nworkload_interpreter"], + capture_output=True, + text=True, + env={**os.environ, "WORKLOAD_BIN": str(script), "PATH": str(bindir)}, + check=True, + ) + assert result.stdout.strip() == (str(fallback) if expected == "FALLBACK" else expected) + + +def test_preflight_resolves_with_the_launchers_environment(): + """`env LD_LIBRARY_PATH=/capture/lib app` runs app with that path, and + preflight resolved its runtime with the parent's path instead. The + assignments apply to that check only, not to the rest of the script. + """ + function = re.search( + r"^check_runtime_as_launched\(\).*?^\}", SCRIPT.read_text(encoding="utf-8"), re.M | re.S + ) + assert function, "check_runtime_as_launched is gone from the script" + result = subprocess.run( + ["bash", "-c", + "check_runtime() { sh -c 'echo \"in=$LD_LIBRARY_PATH|$LD_PRELOAD\"'; }\n" + f"{function.group(0)}\n" + "LD_LIBRARY_PATH=/parent; export LD_LIBRARY_PATH\n" + "LAUNCHER_ENV=(LD_LIBRARY_PATH=/capture/lib LD_PRELOAD=/capture/libamdhip64.so.7 'not a name=x')\n" + "check_runtime_as_launched\n" + 'echo "after=$LD_LIBRARY_PATH|${LD_PRELOAD-unset}"'], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.split() == [ + "in=/capture/lib|/capture/libamdhip64.so.7", + "after=/parent|unset", + ] + + +def _resolved_runtime(ldd_output: str, binary: Path) -> str: + """Run the script's resolver with a stubbed ldd.""" + function = re.search( + r"^resolved_runtime\(\).*?^\}", SCRIPT.read_text(encoding="utf-8"), re.M | re.S + ) + assert function, "resolved_runtime is gone from the script" + stub = f'ldd() {{ cat <<"LDD"\n{ldd_output}\nLDD\n}}\n' + result = subprocess.run( + ["bash", "-c", f'{stub}{function.group(0)}\nresolved_runtime "{binary}" || true'], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def test_ldd_version_errors_do_not_defeat_resolution(tmp_path): + """A binary built against a newer ROCm than the host's makes ldd print + symbol-version errors first, and those lines mention libamdhip64 too. The + third field of one of them is the word `version`, which resolved to + nothing and made preflight blame an interpreter for a compiled binary. + """ + runtime = tmp_path / "libamdhip64.so.7" + runtime.write_bytes(b"\0") + binary = tmp_path / "workload" + binary.write_bytes(b"\0") + + noisy = ( + f"{binary}: {runtime}: version `hip_7.17' not found (required by {binary})\n" + "\tlinux-vdso.so.1 (0x00007fff)\n" + f"\tlibamdhip64.so.7 => {runtime} (0x00007f00)" + ) + assert _resolved_runtime(noisy, binary) == str(runtime) + + # An interpreter links no HIP runtime of its own, and that has to stay + # unresolved rather than resolving to something arbitrary. + assert _resolved_runtime("\tlibc.so.6 => /usr/lib/libc.so.6 (0x00007f00)", binary) == "" + + +def test_a_runtime_preloaded_by_path_is_the_one_resolved(tmp_path): + """ldd prints a library preloaded by absolute path as `/path (0x...)` and + prints no `=>` line for the binary's own entry, which the preload + satisfies. Reading `=>` lines only refused the LD_PRELOAD remedy the skill + recommends for a compiled binary. + """ + runtime = tmp_path / "libamdhip64.so.7" + runtime.write_bytes(b"\0") + binary = tmp_path / "workload" + binary.write_bytes(b"\0") + + preloaded = ( + f"\t{runtime} (0x00007fffff75e000)\n" + "\tlinux-vdso.so.1 (0x00007fff)\n" + "\tlibc.so.6 => /usr/lib/libc.so.6 (0x00007f00)" + ) + assert _resolved_runtime(preloaded, binary) == str(runtime) + + +def _bash_version() -> tuple[int, int]: + result = subprocess.run( + ["bash", "-c", "echo ${BASH_VERSINFO[0]} ${BASH_VERSINFO[1]}"], + capture_output=True, + text=True, + check=True, + ) + major, minor = result.stdout.split() + return int(major), int(minor) + + +def _run(tmp_path: Path, *workload: str, **popen) -> subprocess.Popen: + """Start `run` on a Python workload, with no preflight and no reader.""" + return subprocess.Popen( + ["bash", str(SCRIPT), "run", "--skip-preflight", "--output", str(tmp_path / "capture"), + "--no-playback", "--", sys.executable, *workload], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + **popen, + ) + + +def test_the_wrapper_waits_for_the_workload_it_signalled(tmp_path): + """A trapped signal ends `wait` early. The wrapper took that for the + workload's exit, verified an archive still being written, and returned + while the workload was still shutting down. + """ + ready, done = tmp_path / "ready", tmp_path / "done" + workload = tmp_path / "workload.py" + workload.write_text( + "import signal, sys, time\n" + "def stop(*_):\n" + " time.sleep(1)\n" + f" open({str(done)!r}, 'w').close()\n" + " sys.exit(0)\n" + "signal.signal(signal.SIGTERM, stop)\n" + f"open({str(ready)!r}, 'w').close()\n" + "time.sleep(60)\n" + ) + wrapper = _run(tmp_path, str(workload)) + try: + deadline = time.monotonic() + 30 + while not ready.exists(): + assert wrapper.poll() is None and time.monotonic() < deadline, "workload never started" + time.sleep(0.05) + wrapper.send_signal(signal.SIGTERM) + _, stderr = wrapper.communicate(timeout=30) + finally: + if wrapper.poll() is None: + wrapper.kill() + + assert done.exists(), "the wrapper returned before the workload had finished" + assert "workload exited 0" in stderr + assert wrapper.returncode == 0 + + +def test_a_signal_reaches_the_workers_a_launcher_started(tmp_path): + """The signal went to the wrapper's direct child only. A launcher that + exits on it left its workers capturing, and the wrapper verified the + archive they were still writing. + """ + ready, done = tmp_path / "ready", tmp_path / "done" + worker = tmp_path / "worker.py" + worker.write_text( + "import signal, sys, time\n" + "def stop(*_):\n" + " time.sleep(1)\n" + f" open({str(done)!r}, 'w').close()\n" + " sys.exit(0)\n" + "signal.signal(signal.SIGTERM, stop)\n" + f"open({str(ready)!r}, 'w').close()\n" + "time.sleep(60)\n" + ) + launcher = tmp_path / "launcher.py" + launcher.write_text( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, {str(worker)!r}])\n" + "time.sleep(60)\n" + ) + wrapper = _run(tmp_path, str(launcher)) + try: + deadline = time.monotonic() + 30 + while not ready.exists(): + assert wrapper.poll() is None and time.monotonic() < deadline, "worker never started" + time.sleep(0.05) + wrapper.send_signal(signal.SIGTERM) + _, stderr = wrapper.communicate(timeout=30) + finally: + if wrapper.poll() is None: + wrapper.kill() + + assert done.exists(), "the worker was not stopped, or the wrapper did not wait for it" + assert "workload exited" in stderr + + +@pytest.mark.skipif(_bash_version() < (4, 3), reason="needs bash >= 4.3 (Linux target)") +def test_the_workload_can_be_interrupted(tmp_path): + """A background job starts with SIGINT ignored and the workload inherited + it, so a Ctrl-C, or the INT the wrapper forwards, left it running. + """ + seen = tmp_path / "sigint" + probe = ( + "import signal, sys\n" + "open(sys.argv[1], 'w').write(str(signal.getsignal(signal.SIGINT) is signal.default_int_handler))" + ) + wrapper = _run( + tmp_path, "-c", probe, str(seen), + preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_DFL), + ) + wrapper.communicate(timeout=60) + + assert seen.read_text() == "True" + + +def test_the_archive_directory_is_private_and_the_workload_umask_is_not(tmp_path): + """The writer creates its files 0644, so under a 022 umask the archive was + readable by every user on the host. The directory is created 0700 instead, + and the workload keeps the umask it was started with. + """ + seen = tmp_path / "umask" + probe = "import os, sys\nopen(sys.argv[1], 'w').write(oct(os.umask(0)))" + wrapper = _run(tmp_path, "-c", probe, str(seen), preexec_fn=lambda: os.umask(0o022)) + wrapper.communicate(timeout=60) + + assert (tmp_path / "capture").stat().st_mode & 0o777 == 0o700 + assert seen.read_text() == "0o22" + + +def _output_path_log(out: Path) -> str: + """Run the script's output-path check with the filesystem lookups stubbed.""" + function = re.search( + r"^check_output_path\(\).*?^\}", SCRIPT.read_text(encoding="utf-8"), re.M | re.S + ) + assert function, "check_output_path is gone from the script" + stubs = ( + "log() { printf '%s\\n' \"$*\" >&2; }\n" + "fail() { printf 'error: %s\\n' \"$*\" >&2; exit 1; }\n" + 'path_fstype() { echo "fs-of:$1"; }\n' + "df() { printf 'Filesystem 1024-blocks Used Available Capacity Mounted\\n" + "stub 1 0 104857600 0%% /\\n'; }\n" + ) + result = subprocess.run( + ["bash", "-c", f'{stubs}{function.group(0)}\ncheck_output_path "{out}" 0'], + capture_output=True, + text=True, + ) + return result.stderr + + +def test_an_output_directory_that_exists_is_checked_itself(tmp_path): + """`-v /host/captures:/data/captures --output /data/captures`: the parent + `/data` is root-owned and on the container's overlay, and checking it + refused a writable bind mount, as not writable or as dying with the + container. + """ + out = tmp_path / "captures" + out.mkdir() + assert f"filesystem: fs-of:{out}," in _output_path_log(out) + + # Not there yet: capture creates it inside its parent, so the parent counts. + assert f"filesystem: fs-of:{tmp_path}," in _output_path_log(tmp_path / "run.hrr") + + +@pytest.mark.skipif(os.geteuid() == 0, reason="root passes every permission check") +def test_an_output_directory_without_search_permission_is_refused(tmp_path): + """Creating `pid-*/` takes search permission as well as write, so a + directory with only `-w` passed preflight and the run recorded nothing. + """ + out = tmp_path / "captures" + out.mkdir() + out.chmod(0o600) + try: + assert f"error: not writable (needs write and search permission): {out}" in ( + _output_path_log(out) + ) + finally: + out.chmod(0o700) + + +def test_an_output_path_that_is_not_a_directory_is_refused(tmp_path): + """Capture cannot create its archive directory on top of a file, and a + preflight that passed left the run to end with no archive at all. + """ + regular = tmp_path / "run.hrr" + regular.write_text("") + assert f"error: {regular} exists and is not a directory" in _output_path_log(regular) + + dangling = tmp_path / "gone.hrr" + dangling.symlink_to(tmp_path / "nowhere") + assert f"error: {dangling} exists and is not a directory" in _output_path_log(dangling) diff --git a/projects/hrr/tests/hrr_test_common.hh b/projects/hrr/tests/hrr_test_common.hh index 4b7845a1a633..131b5a7243c6 100644 --- a/projects/hrr/tests/hrr_test_common.hh +++ b/projects/hrr/tests/hrr_test_common.hh @@ -343,6 +343,9 @@ enum class HrrReplayClass { kErrorStub, // ERROR_STUB_PLAYBACK_APIS — named graph-construction warning. kHandlerError, // A real handler ran and returned a HIP error. kCrash, // A real handler took the replay process down with a signal. + kUnreplayable, // UNREPLAYABLE_PLAYBACK_APIS — refused by name, with the + // reason, rather than attempted with recorded arguments that + // cannot mean anything here. }; inline const char* hrr_replay_class_name(HrrReplayClass c) { @@ -352,6 +355,7 @@ inline const char* hrr_replay_class_name(HrrReplayClass c) { case HrrReplayClass::kErrorStub: return "ERROR_STUB"; case HrrReplayClass::kHandlerError: return "HANDLER_ERROR"; case HrrReplayClass::kCrash: return "CRASH"; + case HrrReplayClass::kUnreplayable: return "UNREPLAYABLE"; } return "?"; } @@ -413,7 +417,14 @@ inline std::string hrr_noop_marker(const std::string& api) { } inline std::string hrr_error_stub_marker(const std::string& api) { - return "[HRR] " + api + ": explicit (node-API) graph"; + return "[HRR] " + api + ": not reconstructable at replay"; +} + +// An unreplayable API also returns hipErrorNotSupported, so its event appears +// in the failed-API list too. This marker is what tells the two apart: a +// declared refusal with a reason, rather than a handler that tried and failed. +inline std::string hrr_unreplayable_marker(const std::string& api) { + return "[HRR] " + api + ": NOT REPLAYABLE"; } // --------------------------------------------------------------------------- @@ -515,6 +526,10 @@ inline bool hrr_replay_aborted(const std::string& merged_output) { // hrr_info_api_counts() and the second with hrr_replay_aborted(). inline HrrReplayClass hrr_observed_replay_class(const std::string& merged_output, const std::string& api) { + // Checked before the failed-API list: an unreplayable handler reports the + // event as failed as well, and the refusal is the more specific fact. + if (merged_output.find(hrr_unreplayable_marker(api)) != std::string::npos) + return HrrReplayClass::kUnreplayable; if (hrr_replay_failed_apis(merged_output).count(api)) return HrrReplayClass::kHandlerError; if (merged_output.find(hrr_error_stub_marker(api)) != std::string::npos) diff --git a/projects/hrr/tests/hrr_test_process.hh b/projects/hrr/tests/hrr_test_process.hh index 34cc1c08ed9e..6984ca287542 100644 --- a/projects/hrr/tests/hrr_test_process.hh +++ b/projects/hrr/tests/hrr_test_process.hh @@ -15,6 +15,7 @@ #include #if defined(_WIN32) +#include #include #else #include @@ -65,6 +66,11 @@ class SpawnProc { env_.push_back({key, value}); } + // Remove a variable from the child's environment (after fork, before exec). + // Prefer this over setEnv(key, ""): CLR's flag parser turns an empty + // HIP_HRR_CAPTURE_OUTPUT into a single space, which still enables capture. + void unsetEnv(const std::string& key) { unset_env_.push_back(key); } + // Exit code reported when the deadline fired and the child had to be killed. // 128 + SIGKILL matches the encoding run() already uses for a signalled // child, so callers that only distinguish "crashed" from "clean" need no @@ -140,6 +146,9 @@ class SpawnProc { if (pid == 0) { // Child: apply environment overrides (affects this process only), wire up // stdout capture if requested, then exec. On failure exit with 127. + for (const auto& key : unset_env_) { + ::unsetenv(key.c_str()); + } for (const auto& kv : env_) { ::setenv(kv.first.c_str(), kv.second.c_str(), 1); } @@ -314,6 +323,11 @@ class SpawnProc { auto ieq = [](const std::string& a, const std::string& b) { return a.size() == b.size() && _stricmp(a.c_str(), b.c_str()) == 0; }; + for (const auto& key : unset_env_) { + vars.erase(std::remove_if(vars.begin(), vars.end(), + [&](const auto& v) { return ieq(v.first, key); }), + vars.end()); + } for (const auto& kv : env_) { bool replaced = false; for (auto& v : vars) { @@ -343,6 +357,7 @@ class SpawnProc { bool capture_stderr_ = false; std::string output_; std::vector> env_; + std::vector unset_env_; }; } // namespace hrr::test diff --git a/projects/hrr/tests/integration/CMakeLists.txt b/projects/hrr/tests/integration/CMakeLists.txt index 0634b38509fc..dec0d04bcb5e 100644 --- a/projects/hrr/tests/integration/CMakeLists.txt +++ b/projects/hrr/tests/integration/CMakeLists.txt @@ -25,19 +25,32 @@ # api_matrix.yaml. Regenerate after an upstream classification change with: # aim-labs/scenarios/prep/hrr-api-matrix/check_matrix.py --emit-cxx \ # /hrr_api_matrix_expectations.h +# +# hrr_kernel_args_test.cc covers what replay does with a pointer-typed kernel +# argument no allocation explains: a sentinel keeps its value, an +# address-shaped lost pointer becomes null. Contains its own hidden +# [.][hrr-direct] workload. +# +# hrr_disk_space_test.cc covers how capture stops before the file system +# holding the archive fills. The binary defines statvfs, statvfs64 and fsync to +# fake a small file system under HIP_HRR_CAPTURE_OUTPUT when the test sets +# HRR_TEST_FAKE_FS_TOTAL; every other call goes on to libc. Contains its own +# hidden [.][hrr-direct] workloads. POSIX only. # The workload translation units carry __global__ kernels, device built-ins # (blockIdx/blockDim/threadIdx), hipLaunchKernelGGL, and HIP_SYMBOL, so they must # be compiled with the HIP device toolchain rather than a plain C++ compiler. -# hrr_roundtrip_test.cc and hrr_api_matrix_test.cc are host-only drivers -# (subprocess orchestration + hipRTC host calls) and stay plain C++ translation -# units. +# hrr_roundtrip_test.cc, hrr_api_matrix_test.cc and hrr_secure_exec_test.cc are +# host-only drivers (subprocess orchestration + hipRTC host calls) and stay plain +# C++ translation units. set(HRR_GPU_WORKLOAD_SOURCES hrr_workload_test.cc hrr_repro_workload_test.cc hrr_spt_workload_test.cc hrr_regions_test.cc hrr_api_matrix_workload_test.cc + hrr_kernel_args_test.cc + hrr_disk_space_test.cc ) if(NOT DEFINED HIP_PLATFORM) @@ -53,6 +66,7 @@ add_executable(hrr-integration-tests ${HRR_GPU_WORKLOAD_SOURCES} hrr_roundtrip_test.cc hrr_api_matrix_test.cc + hrr_secure_exec_test.cc ) set_target_properties(hrr-integration-tests PROPERTIES CXX_STANDARD 17 @@ -96,6 +110,15 @@ target_link_libraries(hrr-integration-tests PRIVATE if(NOT WIN32) find_package(hsa-runtime64 CONFIG REQUIRED) target_link_libraries(hrr-integration-tests PRIVATE hsa-runtime64::hsa-runtime64) + + # hrr_disk_space_test.cc defines these for the HIP runtime to bind to. An + # executable exports only what its link-time libraries reference, and the + # runtime may name either statvfs or statvfs64, so export all three. + set(_hrr_disk_space_exports "${CMAKE_CURRENT_BINARY_DIR}/hrr_disk_space_exports.list") + file(WRITE "${_hrr_disk_space_exports}" "{ statvfs; statvfs64; fsync; };\n") + target_link_options(hrr-integration-tests PRIVATE + "LINKER:--dynamic-list=${_hrr_disk_space_exports}") + target_link_libraries(hrr-integration-tests PRIVATE ${CMAKE_DL_LIBS}) endif() # Optional: hip_raw_trace, a genuine multi-threaded capture workload used only by diff --git a/projects/hrr/tests/integration/hrr_api_matrix_expectations.h b/projects/hrr/tests/integration/hrr_api_matrix_expectations.h index 315ee4c15e21..ceb0efa69278 100644 --- a/projects/hrr/tests/integration/hrr_api_matrix_expectations.h +++ b/projects/hrr/tests/integration/hrr_api_matrix_expectations.h @@ -32,6 +32,8 @@ enum HrrExpectCode { kHrrExpectErrorStub = 2, // ERROR_STUB_PLAYBACK_APIS kHrrExpectHandlerError = 3, // a real handler runs and returns a HIP error kHrrExpectCrash = 4, // a real handler runs and kills the replay process + kHrrExpectUnreplayable = 5, // UNREPLAYABLE_PLAYBACK_APIS: refused by name, + // with the reason, instead of being attempted }; struct HrrApiExpectation { @@ -71,7 +73,7 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"__hipRegisterFunction", "T0", 0, false, false, false, false}, {"__hipUnregisterFatBinary", "T0", 0, false, false, false, false}, {"hipDeviceSynchronize", "T0", 0, false, true, false, false}, - {"hipDrvLaunchKernelEx", "T0", 3, true, true, false, false}, + {"hipDrvLaunchKernelEx", "T0", 0, false, true, false, false}, {"hipEventCreate", "T0", 0, false, true, false, false}, {"hipEventCreateWithFlags", "T0", 0, false, true, false, false}, {"hipEventDestroy", "T0", 0, false, true, false, false}, @@ -81,9 +83,9 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipEventSynchronize", "T0", 0, false, true, false, false}, {"hipExtModuleLaunchKernel", "T0", 0, false, false, false, false}, {"hipFree", "T0", 0, false, true, false, false}, - {"hipFuncGetAttribute", "T0", 1, true, true, false, false}, + {"hipFuncGetAttribute", "T0", 1, false, true, false, false}, {"hipGetFuncBySymbol", "T0", 1, false, true, false, false}, - {"hipGetProcAddress", "T0", 1, false, true, false, false}, + {"hipGetProcAddress", "T0", 0, false, true, false, false}, {"hipGraphDestroy", "T0", 1, false, true, false, false}, {"hipGraphExecDestroy", "T0", 1, false, true, false, false}, {"hipGraphInstantiate", "T0", 0, false, true, false, false}, @@ -94,9 +96,9 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipHostRegister", "T0", 0, false, true, false, false}, {"hipHostUnregister", "T0", 0, false, true, false, false}, {"hipLaunchByPtr", "T0", 0, false, false, false, false}, - {"hipLaunchHostFunc", "T0", 3, true, true, false, false}, + {"hipLaunchHostFunc", "T0", 5, false, true, false, false}, {"hipLaunchKernel", "T0", 0, false, false, false, false}, - {"hipLaunchKernelExC", "T0", 3, true, true, false, false}, + {"hipLaunchKernelExC", "T0", 0, false, true, false, false}, {"hipMalloc", "T0", 0, false, true, false, false}, {"hipMemGetInfo", "T0", 0, false, true, false, false}, {"hipMemcpy", "T0", 0, false, true, false, false}, @@ -139,35 +141,35 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipImportExternalSemaphore", "T1", 0, true, false, false, false}, {"hipIpcCloseMemHandle", "T1", 0, false, false, false, false}, {"hipIpcGetEventHandle", "T1", 0, false, true, false, true}, - {"hipIpcGetMemHandle", "T1", 3, true, true, false, false}, - {"hipIpcOpenEventHandle", "T1", 0, true, false, false, false}, - {"hipIpcOpenMemHandle", "T1", 0, true, false, false, false}, - {"hipLaunchCooperativeKernelMultiDevice", "T1", 3, false, true, false, false}, - {"hipMemCreate", "T1", 0, true, true, false, false}, + {"hipIpcGetMemHandle", "T1", 0, false, true, false, false}, + {"hipIpcOpenEventHandle", "T1", 0, false, false, false, false}, + {"hipIpcOpenMemHandle", "T1", 0, false, false, false, false}, + {"hipLaunchCooperativeKernelMultiDevice", "T1", 5, false, true, false, false}, + {"hipMemCreate", "T1", 0, false, true, false, false}, {"hipSignalExternalSemaphoresAsync", "T1", 0, true, false, false, false}, - {"hipStreamBatchMemOp", "T1", 0, true, true, false, false}, + {"hipStreamBatchMemOp", "T1", 0, false, true, false, false}, {"hipWaitExternalSemaphoresAsync", "T1", 0, true, false, false, false}, - {"hipDrvGraphExecMemcpyNodeSetParams", "T2", 1, true, true, false, false}, - {"hipDrvGraphExecMemsetNodeSetParams", "T2", 1, true, true, false, false}, - {"hipDrvGraphMemcpyNodeSetParams", "T2", 1, true, true, false, false}, + {"hipDrvGraphExecMemcpyNodeSetParams", "T2", 0, false, true, false, false}, + {"hipDrvGraphExecMemsetNodeSetParams", "T2", 0, false, true, false, false}, + {"hipDrvGraphMemcpyNodeSetParams", "T2", 0, false, true, false, false}, {"hipFreeHost", "T2", 1, false, true, false, false}, - {"hipGetSymbolAddress", "T2", 1, false, true, false, false}, - {"hipGetSymbolSize", "T2", 1, false, true, false, false}, - {"hipGraphBatchMemOpNodeSetParams", "T2", 1, false, true, false, false}, - {"hipGraphExecBatchMemOpNodeSetParams", "T2", 1, true, true, false, false}, - {"hipGraphExecChildGraphNodeSetParams", "T2", 1, false, true, false, false}, - {"hipGraphExecHostNodeSetParams", "T2", 1, true, true, false, false}, - {"hipGraphExecKernelNodeSetParams", "T2", 1, true, true, false, false}, - {"hipGraphExecMemcpyNodeSetParams", "T2", 1, false, true, false, false}, - {"hipGraphExecMemcpyNodeSetParams1D", "T2", 1, false, true, false, false}, - {"hipGraphExecMemsetNodeSetParams", "T2", 1, true, true, false, false}, + {"hipGetSymbolAddress", "T2", 0, false, true, false, false}, + {"hipGetSymbolSize", "T2", 0, false, true, false, false}, + {"hipGraphBatchMemOpNodeSetParams", "T2", 0, false, true, false, false}, + {"hipGraphExecBatchMemOpNodeSetParams", "T2", 0, false, true, false, false}, + {"hipGraphExecChildGraphNodeSetParams", "T2", 0, false, true, false, false}, + {"hipGraphExecHostNodeSetParams", "T2", 5, true, true, false, false}, + {"hipGraphExecKernelNodeSetParams", "T2", 0, false, true, false, false}, + {"hipGraphExecMemcpyNodeSetParams", "T2", 0, false, true, false, false}, + {"hipGraphExecMemcpyNodeSetParams1D", "T2", 0, false, true, false, false}, + {"hipGraphExecMemsetNodeSetParams", "T2", 0, false, true, false, false}, {"hipGraphExecNodeSetParams", "T2", 1, false, true, false, false}, {"hipGraphExecUpdate", "T2", 1, false, true, false, false}, - {"hipGraphHostNodeSetParams", "T2", 1, true, true, false, false}, - {"hipGraphKernelNodeSetParams", "T2", 1, true, true, false, false}, - {"hipGraphMemcpyNodeSetParams", "T2", 1, true, true, false, false}, - {"hipGraphMemcpyNodeSetParams1D", "T2", 1, false, true, false, false}, - {"hipGraphMemsetNodeSetParams", "T2", 1, true, true, false, false}, + {"hipGraphHostNodeSetParams", "T2", 5, true, true, false, false}, + {"hipGraphKernelNodeSetParams", "T2", 0, false, true, false, false}, + {"hipGraphMemcpyNodeSetParams", "T2", 0, false, true, false, false}, + {"hipGraphMemcpyNodeSetParams1D", "T2", 0, false, true, false, false}, + {"hipGraphMemsetNodeSetParams", "T2", 0, false, true, false, false}, {"hipGraphNodeSetParams", "T2", 1, false, true, false, false}, {"hipHostAlloc", "T2", 1, false, true, false, false}, {"hipMallocHost", "T2", 1, false, true, false, false}, @@ -177,7 +179,7 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipMemcpyToSymbol", "T2", 1, false, true, false, false}, {"hipMemcpyToSymbolAsync", "T2", 1, false, true, false, false}, {"hipModuleGetGlobal", "T2", 1, false, true, false, false}, - {"hipModuleLaunchCooperativeKernel", "T2", 3, false, true, false, false}, + {"hipModuleLaunchCooperativeKernel", "T2", 0, false, true, false, false}, {"hipStreamWaitValue32", "T2", 1, false, true, false, false}, {"hipStreamWaitValue64", "T2", 1, false, true, false, false}, {"hipStreamWriteValue32", "T2", 0, false, true, false, false}, @@ -189,12 +191,12 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipExtGetLinkTypeAndHopCount", "T3", 1, false, true, false, false}, {"hipMemAddressFree", "T3", 0, false, true, false, false}, {"hipMemAddressReserve", "T3", 0, false, true, false, false}, - {"hipMemExportToShareableHandle", "T3", 4, true, true, false, false}, - {"hipMemGetAccess", "T3", 1, true, true, false, false}, + {"hipMemExportToShareableHandle", "T3", 0, false, true, false, false}, + {"hipMemGetAccess", "T3", 0, false, true, false, false}, {"hipMemGetAllocationGranularity", "T3", 0, false, true, false, false}, {"hipMemGetAllocationPropertiesFromHandle", "T3", 1, false, true, false, false}, - {"hipMemGetHandleForAddressRange", "T3", 1, true, true, false, false}, - {"hipMemImportFromShareableHandle", "T3", 1, true, true, false, false}, + {"hipMemGetHandleForAddressRange", "T3", 0, false, true, false, false}, + {"hipMemImportFromShareableHandle", "T3", 5, false, true, false, false}, {"hipMemMap", "T3", 0, false, true, false, false}, {"hipMemRelease", "T3", 0, false, true, false, false}, {"hipMemRetainAllocationHandle", "T3", 1, false, true, false, false}, @@ -207,21 +209,21 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"__hipRegisterManagedVar", "T4", 0, false, true, false, false}, {"__hipRegisterVar", "T4", 0, false, true, false, false}, {"hipApiName", "T4", 1, false, true, false, false}, - {"hipChooseDevice", "T4", 1, true, true, false, false}, - {"hipChooseDeviceR0000", "T4", 1, true, true, false, false}, + {"hipChooseDevice", "T4", 0, false, true, false, false}, + {"hipChooseDeviceR0000", "T4", 0, false, true, false, false}, {"hipConfigureCall", "T4", 0, false, true, false, false}, - {"hipCtxCreate", "T4", 1, false, true, false, false}, + {"hipCtxCreate", "T4", 0, false, true, false, false}, {"hipCtxDestroy", "T4", 0, false, true, false, false}, {"hipCtxDisablePeerAccess", "T4", 1, false, true, false, false}, {"hipCtxEnablePeerAccess", "T4", 1, false, true, false, false}, {"hipCtxGetApiVersion", "T4", 1, false, true, false, false}, {"hipCtxGetCacheConfig", "T4", 1, false, true, false, false}, - {"hipCtxGetCurrent", "T4", 1, false, true, false, false}, + {"hipCtxGetCurrent", "T4", 0, false, true, false, false}, {"hipCtxGetDevice", "T4", 1, false, true, false, false}, {"hipCtxGetFlags", "T4", 1, false, true, false, false}, {"hipCtxGetSharedMemConfig", "T4", 1, false, true, false, false}, - {"hipCtxPopCurrent", "T4", 1, false, true, false, false}, - {"hipCtxPushCurrent", "T4", 4, false, true, false, false}, + {"hipCtxPopCurrent", "T4", 0, false, true, false, false}, + {"hipCtxPushCurrent", "T4", 0, false, true, false, false}, {"hipCtxSetCacheConfig", "T4", 0, false, true, false, false}, {"hipCtxSetCurrent", "T4", 1, false, true, false, false}, {"hipCtxSetSharedMemConfig", "T4", 0, false, true, false, false}, @@ -235,26 +237,27 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipDeviceGetByPCIBusId", "T4", 1, false, true, false, false}, {"hipDeviceGetCacheConfig", "T4", 0, false, true, false, false}, {"hipDeviceGetDefaultMemPool", "T4", 0, false, true, false, false}, - {"hipDeviceGetDevResource", "T4", 0, true, true, false, false}, + {"hipDeviceGetDevResource", "T4", 0, false, true, false, false}, {"hipDeviceGetExecutionCtx", "T4", 0, false, true, false, false}, - {"hipDeviceGetGraphMemAttribute", "T4", 1, true, true, false, false}, + {"hipDeviceGetGraphMemAttribute", "T4", 1, false, true, false, false}, {"hipDeviceGetLimit", "T4", 0, false, true, false, false}, + {"hipDeviceGetLuid", "T4", 0, false, true, false, false}, {"hipDeviceGetMemPool", "T4", 0, false, true, false, false}, {"hipDeviceGetName", "T4", 1, false, true, false, false}, {"hipDeviceGetPCIBusId", "T4", 1, false, true, false, false}, {"hipDeviceGetSharedMemConfig", "T4", 0, false, true, false, false}, {"hipDeviceGetStreamPriorityRange", "T4", 0, false, true, false, false}, - {"hipDeviceGetTexture1DLinearMaxWidth", "T4", 1, true, true, false, false}, + {"hipDeviceGetTexture1DLinearMaxWidth", "T4", 0, false, true, false, false}, {"hipDeviceGetUuid", "T4", 1, false, true, false, false}, {"hipDeviceGraphMemTrim", "T4", 0, false, true, false, false}, {"hipDevicePrimaryCtxGetState", "T4", 1, false, true, false, false}, {"hipDevicePrimaryCtxRelease", "T4", 1, false, true, false, false}, {"hipDevicePrimaryCtxReset", "T4", 1, false, true, false, false}, - {"hipDevicePrimaryCtxRetain", "T4", 1, false, true, false, false}, + {"hipDevicePrimaryCtxRetain", "T4", 0, false, true, false, false}, {"hipDevicePrimaryCtxSetFlags", "T4", 1, false, true, false, false}, {"hipDeviceReset", "T4", 0, false, true, false, false}, {"hipDeviceSetCacheConfig", "T4", 0, false, true, false, false}, - {"hipDeviceSetGraphMemAttribute", "T4", 1, true, true, false, false}, + {"hipDeviceSetGraphMemAttribute", "T4", 1, false, true, false, false}, {"hipDeviceSetLimit", "T4", 0, false, true, false, false}, {"hipDeviceSetMemPool", "T4", 0, false, true, false, false}, {"hipDeviceSetSharedMemConfig", "T4", 0, false, true, false, false}, @@ -262,9 +265,9 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipDriverGetVersion", "T4", 0, false, true, false, false}, {"hipDrvGetErrorName", "T4", 1, false, true, false, false}, {"hipDrvGetErrorString", "T4", 1, false, true, false, false}, - {"hipDrvGraphAddMemFreeNode", "T4", 2, false, true, false, false}, - {"hipDrvGraphAddMemcpyNode", "T4", 2, true, true, false, false}, - {"hipDrvGraphAddMemsetNode", "T4", 2, true, true, false, false}, + {"hipDrvGraphAddMemFreeNode", "T4", 0, false, true, false, false}, + {"hipDrvGraphAddMemcpyNode", "T4", 0, false, true, false, false}, + {"hipDrvGraphAddMemsetNode", "T4", 0, false, true, false, false}, {"hipDrvGraphMemcpyNodeGetParams", "T4", 1, false, true, false, false}, {"hipDrvMemDiscardAndPrefetchBatchAsync", "T4", 1, false, true, false, false}, {"hipDrvMemDiscardBatchAsync", "T4", 1, false, true, false, false}, @@ -275,7 +278,7 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipEventRecordWithFlags", "T4", 0, false, true, false, false}, {"hipEventRecord_spt", "T4", 0, false, true, false, false}, {"hipExecutionCtxDestroy", "T4", 0, false, true, false, false}, - {"hipExecutionCtxGetDevResource", "T4", 0, true, true, false, false}, + {"hipExecutionCtxGetDevResource", "T4", 0, false, true, false, false}, {"hipExecutionCtxGetDevice", "T4", 0, false, true, false, false}, {"hipExecutionCtxGetId", "T4", 0, false, true, false, false}, {"hipExecutionCtxRecordEvent", "T4", 0, false, true, false, false}, @@ -286,7 +289,7 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipExtEnableLogging", "T4", 0, false, true, false, false}, {"hipExtGetLastError", "T4", 0, false, true, false, false}, {"hipExtHostAlloc", "T4", 1, false, true, false, false}, - {"hipExtLaunchMultiKernelMultiDevice", "T4", 3, false, true, false, false}, + {"hipExtLaunchMultiKernelMultiDevice", "T4", 5, false, true, false, false}, {"hipExtMallocWithFlags", "T4", 0, false, true, false, false}, {"hipExtSetLoggingParams", "T4", 0, false, true, false, false}, {"hipExtStreamCreateWithCUMask", "T4", 1, false, true, false, false}, @@ -306,41 +309,41 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipGetErrorName", "T4", 1, false, true, false, false}, {"hipGetErrorString", "T4", 1, false, true, false, false}, {"hipGetLastError", "T4", 0, false, true, false, false}, - {"hipGetProcAddress_spt", "T4", 4, false, true, false, false}, + {"hipGetProcAddress_spt", "T4", 0, false, true, false, false}, {"hipGetStreamDeviceId", "T4", 1, false, true, false, false}, {"hipGetTextureObjectResourceDesc", "T4", 0, false, true, false, false}, {"hipGetTextureObjectResourceViewDesc", "T4", 0, false, true, false, false}, {"hipGetTextureObjectTextureDesc", "T4", 0, false, true, false, false}, - {"hipGraphAddBatchMemOpNode", "T4", 2, true, true, false, false}, - {"hipGraphAddChildGraphNode", "T4", 2, false, true, false, false}, - {"hipGraphAddDependencies", "T4", 2, false, true, false, false}, - {"hipGraphAddEmptyNode", "T4", 2, false, true, false, false}, - {"hipGraphAddEventRecordNode", "T4", 2, false, true, false, false}, - {"hipGraphAddEventWaitNode", "T4", 2, false, true, false, false}, + {"hipGraphAddBatchMemOpNode", "T4", 0, false, true, false, false}, + {"hipGraphAddChildGraphNode", "T4", 0, false, true, false, false}, + {"hipGraphAddDependencies", "T4", 0, false, true, false, false}, + {"hipGraphAddEmptyNode", "T4", 0, false, true, false, false}, + {"hipGraphAddEventRecordNode", "T4", 0, false, true, false, false}, + {"hipGraphAddEventWaitNode", "T4", 0, false, true, false, false}, {"hipGraphAddExternalSemaphoresSignalNode", "T4", 2, true, true, false, false}, {"hipGraphAddExternalSemaphoresWaitNode", "T4", 2, true, true, false, false}, - {"hipGraphAddHostNode", "T4", 2, true, true, false, false}, - {"hipGraphAddKernelNode", "T4", 2, true, true, false, false}, - {"hipGraphAddMemAllocNode", "T4", 2, false, true, false, false}, - {"hipGraphAddMemFreeNode", "T4", 2, false, true, false, false}, - {"hipGraphAddMemcpyNode", "T4", 2, true, true, false, false}, - {"hipGraphAddMemcpyNode1D", "T4", 2, false, true, false, false}, - {"hipGraphAddMemcpyNodeFromSymbol", "T4", 2, false, true, false, false}, - {"hipGraphAddMemcpyNodeToSymbol", "T4", 2, false, true, false, false}, - {"hipGraphAddMemsetNode", "T4", 2, true, true, false, false}, + {"hipGraphAddHostNode", "T4", 5, true, true, false, false}, + {"hipGraphAddKernelNode", "T4", 0, false, true, false, false}, + {"hipGraphAddMemAllocNode", "T4", 0, false, true, false, false}, + {"hipGraphAddMemFreeNode", "T4", 0, false, true, false, false}, + {"hipGraphAddMemcpyNode", "T4", 0, false, true, false, false}, + {"hipGraphAddMemcpyNode1D", "T4", 0, false, true, false, false}, + {"hipGraphAddMemcpyNodeFromSymbol", "T4", 0, false, true, false, false}, + {"hipGraphAddMemcpyNodeToSymbol", "T4", 0, false, true, false, false}, + {"hipGraphAddMemsetNode", "T4", 0, false, true, false, false}, {"hipGraphAddNode", "T4", 2, false, true, false, false}, {"hipGraphBatchMemOpNodeGetParams", "T4", 1, false, true, false, false}, {"hipGraphChildGraphNodeGetGraph", "T4", 1, false, true, false, false}, - {"hipGraphClone", "T4", 2, false, true, false, false}, - {"hipGraphCreate", "T4", 2, false, true, false, false}, + {"hipGraphClone", "T4", 0, false, true, false, false}, + {"hipGraphCreate", "T4", 0, false, true, false, false}, {"hipGraphDebugDotPrint", "T4", 1, false, true, false, false}, - {"hipGraphDestroyNode", "T4", 0, false, true, true, false}, + {"hipGraphDestroyNode", "T4", 0, false, true, false, false}, {"hipGraphEventRecordNodeGetEvent", "T4", 1, false, true, false, false}, - {"hipGraphEventRecordNodeSetEvent", "T4", 1, false, true, false, false}, + {"hipGraphEventRecordNodeSetEvent", "T4", 0, false, true, false, false}, {"hipGraphEventWaitNodeGetEvent", "T4", 1, false, true, false, false}, - {"hipGraphEventWaitNodeSetEvent", "T4", 1, false, true, false, false}, - {"hipGraphExecEventRecordNodeSetEvent", "T4", 1, false, true, false, false}, - {"hipGraphExecEventWaitNodeSetEvent", "T4", 1, false, true, false, false}, + {"hipGraphEventWaitNodeSetEvent", "T4", 0, false, true, false, false}, + {"hipGraphExecEventRecordNodeSetEvent", "T4", 0, false, true, false, false}, + {"hipGraphExecEventWaitNodeSetEvent", "T4", 0, false, true, false, false}, {"hipGraphExecExternalSemaphoresSignalNodeSetParams", "T4", 0, true, true, false, false}, {"hipGraphExecExternalSemaphoresWaitNodeSetParams", "T4", 0, true, true, false, false}, {"hipGraphExecGetFlags", "T4", 1, false, true, false, false}, @@ -356,23 +359,23 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipGraphHostNodeGetParams", "T4", 1, false, true, false, false}, {"hipGraphInstantiateWithFlags", "T4", 0, false, true, false, false}, {"hipGraphInstantiateWithParams", "T4", 2, false, true, false, false}, - {"hipGraphKernelNodeCopyAttributes", "T4", 1, false, true, false, false}, + {"hipGraphKernelNodeCopyAttributes", "T4", 0, false, true, false, false}, {"hipGraphKernelNodeGetAttribute", "T4", 1, false, true, false, false}, {"hipGraphKernelNodeGetParams", "T4", 1, false, true, false, false}, - {"hipGraphKernelNodeSetAttribute", "T4", 1, true, true, false, false}, - {"hipGraphLaunch_spt", "T4", 0, false, true, true, false}, + {"hipGraphKernelNodeSetAttribute", "T4", 0, false, true, false, false}, + {"hipGraphLaunch_spt", "T4", 0, false, true, false, false}, {"hipGraphMemAllocNodeGetParams", "T4", 1, false, true, false, false}, {"hipGraphMemFreeNodeGetParams", "T4", 1, false, true, false, false}, {"hipGraphMemcpyNodeGetParams", "T4", 1, false, true, false, false}, - {"hipGraphMemcpyNodeSetParamsFromSymbol", "T4", 0, false, true, true, false}, - {"hipGraphMemcpyNodeSetParamsToSymbol", "T4", 0, false, true, true, false}, + {"hipGraphMemcpyNodeSetParamsFromSymbol", "T4", 0, false, true, false, false}, + {"hipGraphMemcpyNodeSetParamsToSymbol", "T4", 0, false, true, false, false}, {"hipGraphMemsetNodeGetParams", "T4", 1, false, true, false, false}, - {"hipGraphNodeFindInClone", "T4", 1, false, true, false, false}, + {"hipGraphNodeFindInClone", "T4", 0, false, true, false, false}, {"hipGraphNodeGetDependencies", "T4", 1, false, true, false, false}, {"hipGraphNodeGetDependentNodes", "T4", 1, false, true, false, false}, {"hipGraphNodeGetEnabled", "T4", 1, false, true, false, false}, {"hipGraphNodeGetType", "T4", 1, false, true, false, false}, - {"hipGraphNodeSetEnabled", "T4", 1, false, true, false, false}, + {"hipGraphNodeSetEnabled", "T4", 0, false, true, false, false}, {"hipGraphReleaseUserObject", "T4", 1, false, true, false, false}, {"hipGraphRemoveDependencies", "T4", 1, false, true, false, false}, {"hipGraphRetainUserObject", "T4", 1, false, true, false, false}, @@ -382,17 +385,17 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipHostGetFlags", "T4", 1, false, true, false, false}, {"hipInit", "T4", 0, false, true, false, false}, {"hipInitDevice", "T4", 0, false, true, false, false}, - {"hipKernelGetAttribute", "T4", 1, true, true, false, false}, + {"hipKernelGetAttribute", "T4", 1, false, true, false, false}, {"hipKernelGetFunction", "T4", 1, false, true, false, false}, {"hipKernelGetLibrary", "T4", 1, false, true, false, false}, {"hipKernelGetName", "T4", 1, false, true, false, false}, {"hipKernelGetParamInfo", "T4", 1, false, true, false, false}, {"hipKernelNameRef", "T4", 1, false, true, false, false}, {"hipKernelNameRefByPtr", "T4", 1, false, true, false, false}, - {"hipKernelSetAttribute", "T4", 1, true, true, false, false}, - {"hipLaunchCooperativeKernel_spt", "T4", 3, false, true, false, false}, - {"hipLaunchHostFunc_spt", "T4", 3, true, true, false, false}, - {"hipLaunchKernel_spt", "T4", 4, false, true, false, false}, + {"hipKernelSetAttribute", "T4", 1, false, true, false, false}, + {"hipLaunchCooperativeKernel_spt", "T4", 0, false, true, false, false}, + {"hipLaunchHostFunc_spt", "T4", 5, false, true, false, false}, + {"hipLaunchKernel_spt", "T4", 0, false, true, false, false}, {"hipLibraryEnumerateKernels", "T4", 1, false, true, false, false}, {"hipLibraryGetGlobal", "T4", 0, false, true, false, false}, {"hipLibraryGetKernel", "T4", 1, false, true, false, false}, @@ -401,12 +404,12 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipLibraryLoadData", "T4", 1, false, true, false, false}, {"hipLibraryLoadFromFile", "T4", 1, false, true, false, false}, {"hipLibraryUnload", "T4", 1, false, true, false, false}, - {"hipLinkAddData", "T4", 0, true, true, false, false}, - {"hipLinkAddFile", "T4", 0, true, true, false, false}, - {"hipLinkComplete", "T4", 3, true, true, false, false}, + {"hipLinkAddData", "T4", 0, false, true, false, false}, + {"hipLinkAddFile", "T4", 0, false, true, false, false}, + {"hipLinkComplete", "T4", 0, false, true, false, false}, {"hipLinkCreate", "T4", 0, false, true, false, false}, - {"hipLinkDestroy", "T4", 3, true, true, false, false}, - {"hipMalloc3D", "T4", 1, true, true, false, false}, + {"hipLinkDestroy", "T4", 0, false, true, false, false}, + {"hipMalloc3D", "T4", 1, false, true, false, false}, {"hipMallocAsync", "T4", 0, false, true, false, false}, {"hipMallocFromPoolAsync", "T4", 0, false, true, false, false}, {"hipMallocPitch", "T4", 0, false, true, false, false}, @@ -415,21 +418,21 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipMemDiscardBatchAsync", "T4", 1, false, true, false, false}, {"hipMemGetAddressRange", "T4", 1, false, true, false, false}, {"hipMemGetDefaultMemPool", "T4", 0, false, true, false, false}, - {"hipMemGetMemPool", "T4", 1, false, true, false, false}, + {"hipMemGetMemPool", "T4", 0, false, true, false, false}, {"hipMemPoolCreate", "T4", 0, false, true, false, false}, {"hipMemPoolDestroy", "T4", 0, false, true, false, false}, {"hipMemPoolExportPointer", "T4", 1, false, true, false, false}, - {"hipMemPoolExportToShareableHandle", "T4", 4, true, true, false, false}, + {"hipMemPoolExportToShareableHandle", "T4", 0, false, true, false, false}, {"hipMemPoolGetAccess", "T4", 1, false, true, false, false}, {"hipMemPoolGetAttribute", "T4", 0, false, true, false, false}, - {"hipMemPoolImportFromShareableHandle", "T4", 0, true, true, false, false}, + {"hipMemPoolImportFromShareableHandle", "T4", 5, false, true, false, false}, {"hipMemPoolImportPointer", "T4", 1, false, true, false, false}, {"hipMemPoolSetAccess", "T4", 0, false, true, false, false}, {"hipMemPoolSetAttribute", "T4", 0, false, true, false, false}, {"hipMemPoolTrimTo", "T4", 0, false, true, false, false}, {"hipMemPrefetchBatchAsync", "T4", 0, false, true, false, false}, {"hipMemPtrGetInfo", "T4", 1, false, true, false, false}, - {"hipMemSetMemPool", "T4", 0, false, true, true, false}, + {"hipMemSetMemPool", "T4", 0, false, true, false, false}, {"hipMemcpy2D", "T4", 0, false, true, false, false}, {"hipMemcpy2DAsync", "T4", 0, false, true, false, false}, {"hipMemcpy2DAsync_spt", "T4", 1, false, true, false, false}, @@ -437,22 +440,22 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipMemcpy3D", "T4", 0, false, true, false, false}, {"hipMemcpy3DAsync", "T4", 0, false, true, false, false}, {"hipMemcpy3DAsync_spt", "T4", 0, false, true, false, false}, - {"hipMemcpy3DBatchAsync", "T4", 3, false, true, false, false}, + {"hipMemcpy3DBatchAsync", "T4", 0, false, true, false, false}, {"hipMemcpy3D_spt", "T4", 0, false, true, false, false}, {"hipMemcpyAsync_spt", "T4", 1, false, true, false, false}, {"hipMemcpyAtoA", "T4", 1, false, true, false, false}, {"hipMemcpyAtoD", "T4", 1, false, true, false, false}, {"hipMemcpyAtoH", "T4", 1, false, true, false, false}, {"hipMemcpyAtoHAsync", "T4", 1, false, true, false, false}, - {"hipMemcpyBatchAsync", "T4", 3, false, true, false, false}, + {"hipMemcpyBatchAsync", "T4", 0, false, true, false, false}, {"hipMemcpyDtoA", "T4", 1, false, true, false, false}, {"hipMemcpyDtoDAsync", "T4", 0, false, true, false, false}, {"hipMemcpyFromSymbolAsync_spt", "T4", 1, false, true, false, false}, {"hipMemcpyFromSymbol_spt", "T4", 1, false, true, false, false}, {"hipMemcpyHtoA", "T4", 1, false, true, false, false}, {"hipMemcpyHtoAAsync", "T4", 1, false, true, false, false}, - {"hipMemcpyParam2D", "T4", 1, true, true, false, false}, - {"hipMemcpyParam2DAsync", "T4", 1, true, true, false, false}, + {"hipMemcpyParam2D", "T4", 0, false, true, false, false}, + {"hipMemcpyParam2DAsync", "T4", 0, false, true, false, false}, {"hipMemcpyToSymbolAsync_spt", "T4", 1, false, true, false, false}, {"hipMemcpyToSymbol_spt", "T4", 1, false, true, false, false}, {"hipMemcpy_spt", "T4", 1, false, true, false, false}, @@ -460,10 +463,10 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipMemset2DAsync", "T4", 0, false, true, false, false}, {"hipMemset2DAsync_spt", "T4", 0, false, true, false, false}, {"hipMemset2D_spt", "T4", 0, false, true, false, false}, - {"hipMemset3D", "T4", 1, true, true, false, false}, - {"hipMemset3DAsync", "T4", 1, true, true, false, false}, - {"hipMemset3DAsync_spt", "T4", 1, true, true, false, false}, - {"hipMemset3D_spt", "T4", 1, true, true, false, false}, + {"hipMemset3D", "T4", 1, false, true, false, false}, + {"hipMemset3DAsync", "T4", 1, false, true, false, false}, + {"hipMemset3DAsync_spt", "T4", 1, false, true, false, false}, + {"hipMemset3D_spt", "T4", 1, false, true, false, false}, {"hipMemsetAsync_spt", "T4", 0, false, true, false, false}, {"hipMemsetD16", "T4", 0, false, true, false, false}, {"hipMemsetD16Async", "T4", 0, false, true, false, false}, @@ -491,27 +494,27 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipOccupancyMaxPotentialBlockSize", "T4", 1, false, true, false, false}, {"hipOccupancyMaxPotentialClusterSize", "T4", 1, true, true, false, false}, {"hipPeekAtLastError", "T4", 0, false, true, false, false}, - {"hipPointerGetAttribute", "T4", 1, true, true, false, false}, - {"hipPointerSetAttribute", "T4", 1, true, true, false, false}, + {"hipPointerGetAttribute", "T4", 1, false, true, false, false}, + {"hipPointerSetAttribute", "T4", 1, false, true, false, false}, {"hipProfilerStart", "T4", 0, false, true, false, false}, {"hipProfilerStop", "T4", 0, false, true, false, false}, {"hipRuntimeGetVersion", "T4", 0, false, true, false, false}, {"hipSetDevice", "T4", 0, false, true, false, false}, {"hipSetDeviceFlags", "T4", 0, false, true, false, false}, {"hipSetValidDevices", "T4", 1, false, true, false, false}, - {"hipSetupArgument", "T4", 4, false, true, false, false}, - {"hipStreamAddCallback", "T4", 3, true, true, false, false}, - {"hipStreamAddCallback_spt", "T4", 3, true, true, false, false}, + {"hipSetupArgument", "T4", 0, false, true, false, false}, + {"hipStreamAddCallback", "T4", 5, false, true, false, false}, + {"hipStreamAddCallback_spt", "T4", 5, false, true, false, false}, {"hipStreamAttachMemAsync", "T4", 1, false, true, false, false}, - {"hipStreamBeginCaptureToGraph", "T4", 1, true, true, false, false}, + {"hipStreamBeginCaptureToGraph", "T4", 0, false, true, false, false}, {"hipStreamBeginCapture_spt", "T4", 0, false, true, false, false}, {"hipStreamCopyAttributes", "T4", 0, false, true, false, false}, {"hipStreamEndCapture_spt", "T4", 0, false, true, false, false}, - {"hipStreamGetAttribute", "T4", 0, true, true, false, false}, + {"hipStreamGetAttribute", "T4", 0, false, true, false, false}, {"hipStreamGetCaptureInfo_spt", "T4", 0, false, true, false, false}, {"hipStreamGetCaptureInfo_v2", "T4", 1, false, true, false, false}, {"hipStreamGetCaptureInfo_v2_spt", "T4", 1, false, true, false, false}, - {"hipStreamGetDevResource", "T4", 0, true, true, false, false}, + {"hipStreamGetDevResource", "T4", 0, false, true, false, false}, {"hipStreamGetDevice", "T4", 1, false, true, false, false}, {"hipStreamGetFlags_spt", "T4", 0, false, true, false, false}, {"hipStreamGetId", "T4", 0, false, true, false, false}, @@ -522,14 +525,14 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipStreamSynchronize_spt", "T4", 0, false, true, false, false}, {"hipStreamUpdateCaptureDependencies", "T4", 1, false, true, false, false}, {"hipStreamWaitEvent_spt", "T4", 0, false, true, false, false}, - {"hipUserObjectCreate", "T4", 1, true, true, false, false}, + {"hipUserObjectCreate", "T4", 5, true, true, false, false}, {"hipUserObjectRelease", "T4", 1, false, true, false, false}, {"hipUserObjectRetain", "T4", 1, false, true, false, false}, {"__hipRegisterSurface", "T5", 0, false, true, false, false}, {"__hipRegisterTexture", "T5", 0, false, true, false, false}, - {"hipArray3DCreate", "T5", 3, false, true, false, false}, + {"hipArray3DCreate", "T5", 0, false, true, false, false}, {"hipArray3DGetDescriptor", "T5", 1, false, true, false, false}, - {"hipArrayCreate", "T5", 3, false, true, false, false}, + {"hipArrayCreate", "T5", 0, false, true, false, false}, {"hipArrayDestroy", "T5", 1, false, true, false, false}, {"hipArrayGetDescriptor", "T5", 1, false, true, false, false}, {"hipArrayGetInfo", "T5", 1, false, true, false, false}, @@ -546,28 +549,28 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipExternalMemoryGetMappedMipmappedArray", "T5", 0, true, true, false, false}, {"hipFreeArray", "T5", 0, false, true, false, false}, {"hipFreeMipmappedArray", "T5", 1, false, true, false, false}, - {"hipGLGetDevices", "T5", 1, true, true, false, false}, + {"hipGLGetDevices", "T5", 1, false, true, false, false}, {"hipGetChannelDesc", "T5", 1, false, true, false, false}, {"hipGetMipmappedArrayLevel", "T5", 1, false, true, false, false}, {"hipGetTextureAlignmentOffset", "T5", 0, true, true, false, false}, {"hipGetTextureReference", "T5", 1, false, true, false, false}, - {"hipGraphicsGLRegisterBuffer", "T5", 1, true, true, false, false}, - {"hipGraphicsGLRegisterImage", "T5", 1, true, true, false, false}, + {"hipGraphicsGLRegisterBuffer", "T5", 1, false, true, false, false}, + {"hipGraphicsGLRegisterImage", "T5", 1, false, true, false, false}, {"hipGraphicsMapResources", "T5", 0, false, true, false, false}, {"hipGraphicsResourceGetMappedPointer", "T5", 0, true, true, false, false}, {"hipGraphicsSubResourceGetMappedArray", "T5", 0, true, true, false, false}, {"hipGraphicsUnmapResources", "T5", 0, false, true, false, false}, {"hipGraphicsUnregisterResource", "T5", 0, true, true, false, false}, - {"hipLaunchCooperativeKernel", "T5", 1, false, true, false, false}, + {"hipLaunchCooperativeKernel", "T5", 0, false, true, false, false}, {"hipMalloc3DArray", "T5", 1, true, true, false, false}, {"hipMallocArray", "T5", 1, true, true, false, false}, {"hipMallocManaged", "T5", 0, false, true, false, false}, {"hipMallocMipmappedArray", "T5", 1, true, true, false, false}, {"hipMemAdvise", "T5", 0, false, true, false, false}, - {"hipMemAdvise_v2", "T5", 3, true, true, false, false}, + {"hipMemAdvise_v2", "T5", 0, false, true, false, false}, {"hipMemMapArrayAsync", "T5", 1, false, true, false, false}, {"hipMemPrefetchAsync", "T5", 0, false, true, false, false}, - {"hipMemPrefetchAsync_v2", "T5", 0, true, true, false, false}, + {"hipMemPrefetchAsync_v2", "T5", 0, false, true, false, false}, {"hipMemRangeGetAttribute", "T5", 1, false, true, false, false}, {"hipMemRangeGetAttributes", "T5", 1, false, true, false, false}, {"hipMemcpy2DArrayToArray", "T5", 1, false, true, false, false}, @@ -610,7 +613,7 @@ inline constexpr HrrApiExpectation kHrrApiMatrix[] = { {"hipTexRefSetBorderColor", "T5", 0, false, true, false, false}, {"hipTexRefSetFilterMode", "T5", 0, false, true, false, false}, {"hipTexRefSetFlags", "T5", 0, false, true, false, false}, - {"hipTexRefSetFormat", "T5", 1, true, true, false, false}, + {"hipTexRefSetFormat", "T5", 1, false, true, false, false}, {"hipTexRefSetMaxAnisotropy", "T5", 0, false, true, false, false}, {"hipTexRefSetMipmapFilterMode", "T5", 0, false, true, false, false}, {"hipTexRefSetMipmapLevelBias", "T5", 0, false, true, false, false}, @@ -693,9 +696,9 @@ inline constexpr HrrTierFloor kHrrTierFloors[] = { {"T0", 65, 1, false, kHrrWorkloadsT0}, {"T1", 14, 1, false, kHrrWorkloadsT1}, {"T2", 34, 1, false, kHrrWorkloadsT2}, - {"T3", 19, 2, false, kHrrWorkloadsT3}, + {"T3", 20, 2, false, kHrrWorkloadsT3}, {"T4", 254, 1, false, kHrrWorkloadsT4}, - {"T5", 14, 1, true, kHrrWorkloadsT5}, + {"T5", 12, 1, true, kHrrWorkloadsT5}, }; inline constexpr size_t kHrrTierFloorCount = diff --git a/projects/hrr/tests/integration/hrr_api_matrix_test.cc b/projects/hrr/tests/integration/hrr_api_matrix_test.cc index 5e3788aa4a20..605a52fb7e3a 100644 --- a/projects/hrr/tests/integration/hrr_api_matrix_test.cc +++ b/projects/hrr/tests/integration/hrr_api_matrix_test.cc @@ -77,6 +77,7 @@ HrrReplayClass expect_code_to_class(int code) { case kHrrExpectErrorStub: return HrrReplayClass::kErrorStub; case kHrrExpectHandlerError: return HrrReplayClass::kHandlerError; case kHrrExpectCrash: return HrrReplayClass::kCrash; + case kHrrExpectUnreplayable: return HrrReplayClass::kUnreplayable; default: return HrrReplayClass::kReal; } } @@ -563,7 +564,7 @@ TEST_CASE("Unit_HRR_ApiMatrix_ManifestWellFormed", "[.][hrr][api-matrix][cpu]") REQUIRE(e.api != nullptr); REQUIRE(e.tier != nullptr); CHECK(e.expect >= kHrrExpectReal); - CHECK(e.expect <= kHrrExpectCrash); + CHECK(e.expect <= kHrrExpectUnreplayable); CHECK(tiers.count(e.tier) == 1); CHECK(seen.insert(e.api).second); } @@ -584,13 +585,15 @@ TEST_CASE("Unit_HRR_ApiMatrix_ReplayClassMarkers", "[.][hrr][api-matrix][cpu]") "[HRR] NOOP playback handler called for hipHostAlloc \xE2\x80\x94 this " "API is not replayed; results may differ from capture.\n"; const std::string stub_line = - "[HRR] hipGraphCreate: explicit (node-API) graph construction is NOT " - "supported by HRR replay.\n"; + "[HRR] hipGraphAddNode: not reconstructable at replay, so the call is " + "skipped and the graph it belongs to is marked incomplete; " + "instantiating that graph fails loudly rather than running a graph " + "that is missing work.\n"; const std::string clean = "[HRR] D2H checks : 4 pass, 0 fail\n"; CHECK(hrr_observed_replay_class(noop_line, "hipHostAlloc") == HrrReplayClass::kNoop); - CHECK(hrr_observed_replay_class(stub_line, "hipGraphCreate") == + CHECK(hrr_observed_replay_class(stub_line, "hipGraphAddNode") == HrrReplayClass::kErrorStub); CHECK(hrr_observed_replay_class(clean, "hipMalloc") == HrrReplayClass::kReal); @@ -598,7 +601,7 @@ TEST_CASE("Unit_HRR_ApiMatrix_ReplayClassMarkers", "[.][hrr][api-matrix][cpu]") // the API name for exactly this reason. CHECK(hrr_observed_replay_class(noop_line, "hipHostMalloc") == HrrReplayClass::kReal); - CHECK(hrr_observed_replay_class(stub_line, "hipGraphClone") == + CHECK(hrr_observed_replay_class(stub_line, "hipGraphAddMemsetNode") == HrrReplayClass::kReal); // HIP API names prefix one another, so a marker that is not bounded at the @@ -619,7 +622,7 @@ TEST_CASE("Unit_HRR_ApiMatrix_ReplayClassMarkers", "[.][hrr][api-matrix][cpu]") const std::string both = noop_line + stub_line; CHECK(hrr_observed_replay_class(both, "hipHostAlloc") == HrrReplayClass::kNoop); - CHECK(hrr_observed_replay_class(both, "hipGraphCreate") == + CHECK(hrr_observed_replay_class(both, "hipGraphAddNode") == HrrReplayClass::kErrorStub); // A handler that returns a HIP error reports itself in one of two forms @@ -652,6 +655,27 @@ TEST_CASE("Unit_HRR_ApiMatrix_ReplayClassMarkers", "[.][hrr][api-matrix][cpu]") CHECK(hrr_observed_replay_class(many, "hipHostAlloc") == HrrReplayClass::kNoop); + // An unreplayable API returns hipErrorNotSupported, so it appears in the + // failed-API list as well. The refusal is the more specific fact and must + // win, or a declared scope exclusion would read as a handler that broke. + const std::string unreplayable_line = + "[HRR] hipStreamAddCallback: NOT REPLAYABLE \xE2\x80\x94 the callback is " + "a host function pointer belonging to the capturing process. The call is " + "in the archive but its effect cannot be reproduced here.\n"; + const std::string unreplayable_pair = + unreplayable_line + + "[HRR] Error: T19 Event 71 (hipStreamAddCallback) returned 801 " + "(operation not supported) \xE2\x80\x94 continuing\n"; + CHECK(hrr_observed_replay_class(unreplayable_pair, "hipStreamAddCallback") == + HrrReplayClass::kUnreplayable); + CHECK(hrr_observed_replay_class(unreplayable_pair, "hipMalloc") == + HrrReplayClass::kReal); + // A handler that genuinely failed is still HANDLER_ERROR when some other API + // was the unreplayable one. + CHECK(hrr_observed_replay_class(unreplayable_line + continued_line, + "hipDrvLaunchKernelEx") == + HrrReplayClass::kHandlerError); + // Only the fatal form means the replay stopped; the continuing form must not // be read as a truncated run, or every tier would fail on its own design. CHECK(hrr_replay_aborted(fatal_line + diff --git a/projects/hrr/tests/integration/hrr_api_matrix_workload_test.cc b/projects/hrr/tests/integration/hrr_api_matrix_workload_test.cc index 2f42e8b9ccb8..1331bd057c40 100644 --- a/projects/hrr/tests/integration/hrr_api_matrix_workload_test.cc +++ b/projects/hrr/tests/integration/hrr_api_matrix_workload_test.cc @@ -1250,10 +1250,10 @@ TEST_CASE("Unit_HRR_ApiMatrix_Breadth_Direct", "[.][hrr-direct]") { // // The other graph coverage in this suite builds graphs by stream capture, // which is how PyTorch, vLLM and llama.cpp build theirs. This workload builds -// one the other way, node by node, because that is the half of the graph -// surface HRR answers with ERROR_STUB: "explicit (node-API) graph construction -// is NOT supported". Around 55 APIs sit behind that sentence, and a matrix -// that never calls them cannot notice if one quietly starts doing something. +// one the other way, node by node. Around 55 APIs sit on that path; most now +// replay for real, and the handful that do not answer with ERROR_STUB and +// poison the graph so instantiating it fails loudly. A matrix that never +// calls them cannot notice if one quietly starts doing something else. // // The external-semaphore node family is absent on purpose: those nodes need an // imported semaphore, which needs a real fd from another process, and a call @@ -2602,11 +2602,10 @@ TEST_CASE("Unit_HRR_ApiMatrix_Deprioritised_Direct", "[.][hrr-direct]") { } // ---- Driver-flavoured array creation ------------------------------------ - // Deliberately outside the image-support guard below. gfx950 reports no - // image support and both of these return hipErrorNotSupported, but unlike - // their runtime counterparts the capture shim records the call before the - // runtime rejects it, so they are the only two members of the array family - // that reach an archive at all. + // Deliberately outside the image-support guard below, so they run on every + // device. Capture records them only when they succeed; gfx950 reports no + // image support and returns hipErrorNotSupported for both, so there they + // reach no archive and the T5 floor does not count them. { HIP_ARRAY_DESCRIPTOR desc{}; desc.Width = 64; diff --git a/projects/hrr/tests/integration/hrr_disk_space_test.cc b/projects/hrr/tests/integration/hrr_disk_space_test.cc new file mode 100644 index 000000000000..47588c2dcc94 --- /dev/null +++ b/projects/hrr/tests/integration/hrr_disk_space_test.cc @@ -0,0 +1,849 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +/** + * @addtogroup HRR HRR capture free-space reserve + * @{ + * @ingroup HRRTest + * Tests for how capture stops before the file system holding the archive + * fills. + * + * The writer keeps free the smaller of 15% of that file system and 4 GiB. It + * refuses to open below that, and stops, leaving an archive marked incomplete, + * when a write would eat into it. CI runners have far more free space than any + * workload can fill, so this file fakes the file system instead: the test + * binary defines statvfs, statvfs64 and fsync itself. The dynamic linker binds + * the HIP runtime's calls to these definitions before libc's, and they forward + * to libc for every path outside HIP_HRR_CAPTURE_OUTPUT. + * + * Under that directory they report a file system of HRR_TEST_FAKE_FS_TOTAL + * bytes. Free space starts at HRR_TEST_FAKE_FS_AVAIL (1 TiB when unset), and a + * workload can then pin it relative to the reserve: either fixed, or shrinking + * as the archive grows on disk. Two one-shot holds park the next statvfs or + * fsync, so a test can line up the races the writer has to survive. + * + * POSIX only: Windows reads free space through GetDiskFreeSpaceExA, which this + * file does not fake. + */ + +#include "hrr_test_common.hh" +#include "hrr_test_process.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) && defined(HRR_TEST_EXE) + +#include +#include +#include +#include +#include +#include + +// One struct serves both statvfs and statvfs64: their layouts only differ on +// 32-bit targets. +static_assert(sizeof(void*) == 8, "the statvfs fake assumes an LP64 target"); + +namespace { + +constexpr uint64_t kBlock = 4096; +constexpr uint64_t kDefaultAvail = 1ull << 40; +// Free space one block short of the reserve: every check fails, whatever is +// pending. +constexpr int64_t kBelowReserve = -static_cast(kBlock); + +enum FakeMode : int { kOff = 0, kFixed = 1, kShrinking = 2 }; + +std::atomic g_mode{kOff}; +std::atomic g_headroom{0}; +std::atomic g_used_base{0}; +std::atomic g_fake_calls{0}; + +std::atomic g_statvfs_hold{false}; +std::atomic g_statvfs_entered{false}; +std::atomic g_statvfs_release{false}; +std::atomic g_fsync_hold{false}; +std::atomic g_fsync_entered{false}; + +std::atomic g_quick_exit_pid{0}; + +const char* fake_root() { + const char* root = getenv("HIP_HRR_CAPTURE_OUTPUT"); + return root != nullptr && root[0] != '\0' ? root : nullptr; +} + +uint64_t fake_total() { + const char* total = getenv("HRR_TEST_FAKE_FS_TOTAL"); + return total != nullptr ? strtoull(total, nullptr, 10) : 0; +} + +bool is_faked(const char* path) { + const char* root = fake_root(); + if (root == nullptr || path == nullptr || fake_total() == 0) return false; + const size_t n = strlen(root); + return strncmp(path, root, n) == 0 && (path[n] == '\0' || path[n] == '/'); +} + +uint64_t round_up(uint64_t v) { return (v + kBlock - 1) / kBlock * kBlock; } + +// The writer's reserve, computed the same way from the same total. +uint64_t keep_free(uint64_t total) { + return std::min(total / 100 * 15, 4ull << 30); +} + +uint64_t check_interval(uint64_t total) { + return std::max(std::min(64ull << 20, keep_free(total) / 4), 1); +} + +// Bytes the capture directory takes on disk, each file in whole blocks. +uint64_t used_bytes() { + const char* root = fake_root(); + if (root == nullptr) return 0; + uint64_t sum = 0; + std::error_code ec; + fs::recursive_directory_iterator it(root, ec), end; + for (; !ec && it != end; it.increment(ec)) { + std::error_code fe; + if (!it->is_regular_file(fe) || fe) continue; + const uint64_t size = it->file_size(fe); + if (!fe) sum += round_up(size); + } + return sum; +} + +uint64_t fake_avail() { + const int mode = g_mode.load(); + if (mode == kOff) { + const char* avail = getenv("HRR_TEST_FAKE_FS_AVAIL"); + return avail != nullptr ? strtoull(avail, nullptr, 10) : kDefaultAvail; + } + int64_t avail = static_cast(round_up(keep_free(fake_total()))) + g_headroom.load(); + if (mode == kShrinking) + avail -= static_cast(used_bytes()) - static_cast(g_used_base.load()); + return avail > 0 ? static_cast(avail) : 0; +} + +bool wait_for(const std::atomic& flag, int timeout_ms) { + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + while (!flag.load()) { + if (std::chrono::steady_clock::now() >= deadline) return false; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return true; +} + +int fake_statvfs(const char* name, const char* path, struct statvfs* buf) { + if (!is_faked(path)) { + using Fn = int (*)(const char*, struct statvfs*); + static Fn real[2] = {reinterpret_cast(dlsym(RTLD_NEXT, "statvfs")), + reinterpret_cast(dlsym(RTLD_NEXT, "statvfs64"))}; + Fn fn = real[strcmp(name, "statvfs") == 0 ? 0 : 1]; + if (fn == nullptr) { + errno = ENOSYS; + return -1; + } + return fn(path, buf); + } + g_fake_calls.fetch_add(1); + if (g_statvfs_hold.exchange(false)) { + g_statvfs_entered.store(true); + (void)wait_for(g_statvfs_release, 5000); + } + memset(buf, 0, sizeof(*buf)); + buf->f_bsize = kBlock; + buf->f_frsize = kBlock; + buf->f_blocks = fake_total() / kBlock; + buf->f_bfree = fake_avail() / kBlock; + buf->f_bavail = buf->f_bfree; + buf->f_files = 1u << 20; + buf->f_ffree = 1u << 20; + buf->f_favail = 1u << 20; + buf->f_namemax = 255; + return 0; +} + +} // namespace + +// Exported through the dynamic list in CMakeLists.txt, so the HIP runtime binds +// to them. The asm labels keep the C++ names apart from the libc declarations. +extern "C" int hrr_disk_space_statvfs(const char* path, struct statvfs* buf) + __asm__("statvfs"); +extern "C" int hrr_disk_space_statvfs64(const char* path, struct statvfs* buf) + __asm__("statvfs64"); +extern "C" int hrr_disk_space_fsync(int fd) __asm__("fsync"); + +extern "C" int hrr_disk_space_statvfs(const char* path, struct statvfs* buf) { + return fake_statvfs("statvfs", path, buf); +} + +extern "C" int hrr_disk_space_statvfs64(const char* path, struct statvfs* buf) { + return fake_statvfs("statvfs64", path, buf); +} + +extern "C" int hrr_disk_space_fsync(int fd) { + using Fn = int (*)(int); + static Fn real = reinterpret_cast(dlsym(RTLD_NEXT, "fsync")); + if (g_fsync_hold.exchange(false)) { + g_fsync_entered.store(true); + std::this_thread::sleep_for(std::chrono::seconds(3)); + } + if (real == nullptr) { + errno = ENOSYS; + return -1; + } + return real(fd); +} + +namespace { + +void arm_fixed(int64_t headroom) { + g_headroom.store(headroom); + g_mode.store(kFixed); +} + +// Free space starts `headroom` above the reserve and shrinks by whatever the +// capture writes from here on. +void arm_shrinking(int64_t headroom) { + g_used_base.store(used_bytes()); + g_headroom.store(headroom); + g_mode.store(kShrinking); +} + +void disarm() { g_mode.store(kOff); } + +// Leave through _exit right after hip_capture_shutdown when getpid() matches: +// registered before HIP initializes, so it runs after the capture's own atexit +// handler, and skips HIP teardown in a forked child or under a parked thread. +void quick_exit_hook() { + if (g_quick_exit_pid.load() != getpid()) return; + fflush(stdout); + fflush(stderr); + _exit(0); +} + +// Every workload starts here, before its first HIP call. +void begin_workload() { + // Capture opens at HIP initialization and reads free space at once, so no + // call yet means HIP has not initialized and the hook goes in first. + REQUIRE(g_fake_calls.load() == 0); + REQUIRE(std::atexit(quick_exit_hook) == 0); + HRR_HIP_CHECK(hipSetDevice(0)); + // The HIP runtime reached the fake. Without this a missing export would turn + // every test below into a capture on the real disk. + REQUIRE(g_fake_calls.load() > 0); +} + +// Distinct bytes per seed, so content-addressed blobs never deduplicate. +std::vector pattern(size_t len, uint32_t seed) { + std::vector v(len); + uint32_t x = seed * 2654435761u + 0x9e3779b9u; + for (size_t i = 0; i < len; ++i) { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + v[i] = static_cast(x); + } + return v; +} + +void copy_to_device(void* dev, uint32_t seed, size_t len) { + const std::vector host = pattern(len, seed); + HRR_HIP_CHECK(hipMemcpy(dev, host.data(), len, hipMemcpyHostToDevice)); +} + +// Wait for a forked child for at most 60 s; -1 when it had to be killed. +int wait_child(pid_t child) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60); + int status = 0; + while (std::chrono::steady_clock::now() < deadline) { + const pid_t r = waitpid(child, &status, WNOHANG); + if (r == child) return status; + if (r < 0 && errno != EINTR) return -1; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + kill(child, SIGKILL); + waitpid(child, &status, 0); + return -1; +} + +constexpr uint64_t kTotal64K = 64ull << 10; +constexpr uint64_t kTotal16M = 16ull << 20; +constexpr uint64_t kTotal64M = 64ull << 20; +// At least the check interval for a 64 MiB file system, so one copy is checked. +constexpr size_t kBigCopy = 4u << 20; +constexpr size_t kStepCopy = 256u << 10; +constexpr int kStepCount = 16; +constexpr size_t kTinyCopy = 64; +constexpr int kTinyCount = 1500; +constexpr int kMemsetCount = 200; +// Events still buffered when the capture stops are flushed into events.bin. +constexpr uint64_t kEventSlack = 64u << 10; + +struct HrrDiskSpaceBig { + unsigned char bytes[3000]; +}; + +// One launch record carrying it stays under the check interval of a 64 KiB file +// system, and two go over it. +struct HrrDiskSpaceMid { + unsigned char bytes[1400]; +}; + +} // namespace + +__global__ void hrr_disk_space_big_arg(HrrDiskSpaceBig big, int* out) { + if (threadIdx.x == 0) out[0] = big.bytes[0] + big.bytes[sizeof(big.bytes) - 1]; +} + +__global__ void hrr_disk_space_mid_arg(HrrDiskSpaceMid mid, int* out) { + if (threadIdx.x == 0) out[0] = mid.bytes[0] + mid.bytes[sizeof(mid.bytes) - 1]; +} + +// =========================================================================== +// Workloads. Each runs in its own process under the fake file system. +// =========================================================================== + +TEST_CASE("Unit_HRR_DiskSpace_RefusedAtStart_Direct", "[.][hrr-direct]") { + begin_workload(); + void* dev = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev, kStepCopy)); + const std::vector in = pattern(kStepCopy, 1); + std::vector out(kStepCopy); + HRR_HIP_CHECK(hipMemcpy(dev, in.data(), kStepCopy, hipMemcpyHostToDevice)); + HRR_HIP_CHECK(hipMemcpy(out.data(), dev, kStepCopy, hipMemcpyDeviceToHost)); + REQUIRE(in == out); + HRR_HIP_CHECK(hipFree(dev)); +} + +TEST_CASE("Unit_HRR_DiskSpace_StopsBeforeReserve_Direct", "[.][hrr-direct]") { + begin_workload(); + const uint64_t interval = check_interval(fake_total()); + void* dev = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev, 1u << 20)); + // Larger than the interval, so it is checked and the count starts from zero. + copy_to_device(dev, 100, 1u << 20); + + const int64_t headroom = 2 << 20; + arm_shrinking(headroom); + for (int i = 0; i < kStepCount; ++i) copy_to_device(dev, 200 + i, kStepCopy); + const uint64_t used = used_bytes() - g_used_base.load(); + disarm(); + + // The capture went no further into the reserve than one check interval. + INFO("used " << used << " headroom " << headroom << " interval " << interval); + CHECK(used <= static_cast(headroom) + interval + kEventSlack); + + std::vector out(kStepCopy); + HRR_HIP_CHECK(hipMemcpy(out.data(), dev, kStepCopy, hipMemcpyDeviceToHost)); + REQUIRE(out == pattern(kStepCopy, 200 + kStepCount - 1)); + HRR_HIP_CHECK(hipFree(dev)); +} + +TEST_CASE("Unit_HRR_DiskSpace_SmallBlobsCountedInBlocks_Direct", "[.][hrr-direct]") { + begin_workload(); + const uint64_t interval = check_interval(fake_total()); + void* dev = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev, kBigCopy)); + copy_to_device(dev, 300, kBigCopy); + + // Each blob is 64 bytes and one block on disk. Counted in bytes, the whole + // loop stays under one check interval and nothing is ever checked. + const int64_t headroom = 1 << 20; + arm_shrinking(headroom); + for (int i = 0; i < kTinyCount; ++i) copy_to_device(dev, 1000 + i, kTinyCopy); + const uint64_t used = used_bytes() - g_used_base.load(); + disarm(); + + INFO("used " << used << " headroom " << headroom << " interval " << interval); + CHECK(used <= static_cast(headroom) + interval + kEventSlack); + HRR_HIP_CHECK(hipFree(dev)); +} + +TEST_CASE("Unit_HRR_DiskSpace_ForkAfterStop_Direct", "[.][hrr-direct]") { + begin_workload(); + const fs::path root = fake_root(); + void* dev = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev, kBigCopy)); + arm_fixed(kBelowReserve); + copy_to_device(dev, 400, kBigCopy); + // Plenty free again: a child that ignored the stop would open an archive. + disarm(); + + const pid_t parent = getpid(); + fflush(stdout); + fflush(stderr); + const pid_t child = fork(); + if (child == 0) { + g_quick_exit_pid.store(getpid()); + std::exit(0); + } + REQUIRE(child > 0); + const int status = wait_child(child); + REQUIRE(status != -1); + REQUIRE(WIFEXITED(status)); + REQUIRE(WEXITSTATUS(status) == 0); + + // The child opened nothing and finalized nothing, its own or the parent's. + CHECK_FALSE(fs::exists(root / ("pid-" + std::to_string(child)))); + CHECK_FALSE(fs::exists(root / "manifest.json")); + CHECK_FALSE(fs::exists(root / ("pid-" + std::to_string(parent)) / "manifest.json")); + HRR_HIP_CHECK(hipFree(dev)); +} + +TEST_CASE("Unit_HRR_DiskSpace_RefusedForkChild_Direct", "[.][hrr-direct]") { + begin_workload(); + const fs::path root = fake_root(); + void* dev = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev, kStepCopy)); + copy_to_device(dev, 500, kStepCopy); + + arm_fixed(kBelowReserve); + fflush(stdout); + fflush(stderr); + const pid_t child = fork(); + if (child == 0) { + g_quick_exit_pid.store(getpid()); + std::exit(0); + } + REQUIRE(child > 0); + const int status = wait_child(child); + disarm(); + REQUIRE(status != -1); + REQUIRE(WIFEXITED(status)); + REQUIRE(WEXITSTATUS(status) == 0); + + CHECK_FALSE(fs::exists(root / ("pid-" + std::to_string(child)))); + CHECK_FALSE(fs::exists(root / "manifest.json")); + // The parent's capture is untouched. + copy_to_device(dev, 501, kStepCopy); + HRR_HIP_CHECK(hipFree(dev)); +} + +TEST_CASE("Unit_HRR_DiskSpace_CrashAfterStop_Direct", "[.][hrr-direct]") { + begin_workload(); + void* dev = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev, kBigCopy)); + arm_fixed(kBelowReserve); + copy_to_device(dev, 600, kBigCopy); + + const struct rlimit no_core = {0, 0}; + (void)setrlimit(RLIMIT_CORE, &no_core); + fflush(stdout); + fflush(stderr); + abort(); +} + +TEST_CASE("Unit_HRR_DiskSpace_StopWhileExiting_Direct", "[.][hrr-direct]") { + begin_workload(); + void* dev = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev, kBigCopy)); + HRR_HIP_CHECK(hipMemset(dev, 0x5a, kBigCopy)); + static std::vector host(kBigCopy); + + // The stop parks in its fsync, between closing events.bin and marking the + // archive incomplete, while this thread exits and flush() runs. + arm_fixed(kBelowReserve); + g_fsync_hold.store(true); + std::thread([dev] { + (void)hipMemcpy(host.data(), dev, kBigCopy, hipMemcpyDeviceToHost); + for (;;) std::this_thread::sleep_for(std::chrono::seconds(1)); + }).detach(); + + REQUIRE(wait_for(g_fsync_entered, 10000)); + g_quick_exit_pid.store(getpid()); + std::exit(0); +} + +TEST_CASE("Unit_HRR_DiskSpace_DuplicateRacingStop_Direct", "[.][hrr-direct]") { + begin_workload(); + void* dev_a = nullptr; + void* dev_b = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev_a, kBigCopy)); + HRR_HIP_CHECK(hipMalloc(&dev_b, kBigCopy)); + const std::vector a = pattern(kBigCopy, 800); + const std::vector b = a; + + // The first copy parks in its free-space check. The second copy of the same + // bytes must not take the hash as written while that blob may never be. + arm_fixed(kBelowReserve); + g_statvfs_hold.store(true); + hipError_t first = hipErrorUnknown; + std::thread worker([&] { + first = hipMemcpy(dev_a, a.data(), kBigCopy, hipMemcpyHostToDevice); + }); + const bool parked = wait_for(g_statvfs_entered, 10000); + hipError_t second = hipErrorUnknown; + if (parked) second = hipMemcpy(dev_b, b.data(), kBigCopy, hipMemcpyHostToDevice); + g_statvfs_release.store(true); + worker.join(); + disarm(); + + REQUIRE(parked); + HRR_HIP_CHECK(first); + HRR_HIP_CHECK(second); + HRR_HIP_CHECK(hipFree(dev_a)); + HRR_HIP_CHECK(hipFree(dev_b)); +} + +TEST_CASE("Unit_HRR_DiskSpace_LargeEventCheckedBeforeWrite_Direct", "[.][hrr-direct]") { + begin_workload(); + int* out = nullptr; + HRR_HIP_CHECK(hipMalloc(&out, sizeof(int))); + HrrDiskSpaceBig big{}; + big.bytes[0] = 3; + big.bytes[sizeof(big.bytes) - 1] = 4; + + // The launch record carries the 3000-byte argument, above the 2456-byte check + // interval of a 64 KiB file system. + hipLaunchKernelGGL(hrr_disk_space_big_arg, dim3(1), dim3(64), 0, nullptr, big, out); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipDeviceSynchronize()); + void* scratch = nullptr; + HRR_HIP_CHECK(hipMalloc(&scratch, kBlock)); + copy_to_device(scratch, 900, kBlock); + + arm_fixed(kBelowReserve); + hipLaunchKernelGGL(hrr_disk_space_big_arg, dim3(1), dim3(64), 0, nullptr, big, out); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipDeviceSynchronize()); + disarm(); + + int host = 0; + HRR_HIP_CHECK(hipMemcpy(&host, out, sizeof(int), hipMemcpyDeviceToHost)); + REQUIRE(host == 7); + HRR_HIP_CHECK(hipFree(scratch)); + HRR_HIP_CHECK(hipFree(out)); +} + +TEST_CASE("Unit_HRR_DiskSpace_EventsCheckedAcrossInterval_Direct", "[.][hrr-direct]") { + begin_workload(); + int* out = nullptr; + HRR_HIP_CHECK(hipMalloc(&out, sizeof(int))); + HrrDiskSpaceMid mid{}; + mid.bytes[0] = 3; + mid.bytes[sizeof(mid.bytes) - 1] = 4; + + // The first launch writes the code object while free space is plenty. + hipLaunchKernelGGL(hrr_disk_space_mid_arg, dim3(1), dim3(64), 0, nullptr, mid, out); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipDeviceSynchronize()); + // A one-block blob reaches the interval, so the count starts again from here. + void* scratch = nullptr; + HRR_HIP_CHECK(hipMalloc(&scratch, kBlock)); + copy_to_device(scratch, 970, kBlock); + + // Neither record reaches the interval alone. The second one makes the check + // due, and that check has to run before it is written. + arm_fixed(kBelowReserve); + hipLaunchKernelGGL(hrr_disk_space_mid_arg, dim3(1), dim3(64), 0, nullptr, mid, out); + hipLaunchKernelGGL(hrr_disk_space_mid_arg, dim3(1), dim3(64), 0, nullptr, mid, out); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipDeviceSynchronize()); + disarm(); + + int host = 0; + HRR_HIP_CHECK(hipMemcpy(&host, out, sizeof(int), hipMemcpyDeviceToHost)); + REQUIRE(host == 7); + HRR_HIP_CHECK(hipFree(scratch)); + HRR_HIP_CHECK(hipFree(out)); +} + +TEST_CASE("Unit_HRR_DiskSpace_BufferedEventsCounted_Direct", "[.][hrr-direct]") { + begin_workload(); + int* dev = nullptr; + HRR_HIP_CHECK(hipMalloc(&dev, sizeof(int))); + void* scratch = nullptr; + HRR_HIP_CHECK(hipMalloc(&scratch, kBlock)); + copy_to_device(scratch, 950, kBlock); + + // No blobs from here on, only small buffered records. + arm_fixed(kBelowReserve); + for (int i = 0; i < kMemsetCount; ++i) + HRR_HIP_CHECK(hipMemsetAsync(dev, i & 0xff, sizeof(int), nullptr)); + HRR_HIP_CHECK(hipStreamSynchronize(nullptr)); + disarm(); + + HRR_HIP_CHECK(hipFree(scratch)); + HRR_HIP_CHECK(hipFree(dev)); +} + +// =========================================================================== +// Drivers. +// =========================================================================== + +namespace { + +struct WorkloadRun { + int rc = -1; + std::string out; +}; + +WorkloadRun run_workload(const std::string& name, const fs::path& cap, uint64_t total, + uint64_t avail = 0) { + hrr::test::SpawnProc proc(HRR_TEST_EXE, /*capture_stdout=*/true, /*capture_stderr=*/true); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.string()); + proc.setEnv("HRR_TEST_FAKE_FS_TOTAL", std::to_string(total)); + if (avail != 0) proc.setEnv("HRR_TEST_FAKE_FS_AVAIL", std::to_string(avail)); + set_proc_search_path(proc); + WorkloadRun run; + run.rc = proc.runWithTimeout("\"" + name + "\"", 300); + run.out = proc.getOutput(); + return run; +} + +size_t count_of(const std::string& text, const std::string& needle) { + size_t n = 0; + for (size_t at = text.find(needle); at != std::string::npos; + at = text.find(needle, at + needle.size())) + ++n; + return n; +} + +constexpr const char* kStopped = "[HRR capture] Capture stopped"; +constexpr const char* kDisabled = "[HRR capture] Capture disabled"; +constexpr const char* kIncomplete = "\"complete\": false"; + +// The archive a stopped capture leaves: stopped once, not torn, marked +// incomplete in both manifests, and every recorded copy's blob on disk. +void load_stopped_archive(const fs::path& cap, const WorkloadRun& run, hrr::Archive& arc) { + CHECK(count_of(run.out, kStopped) == 1); + const fs::path proc_dir = hrr_single_process_archive(cap); + REQUIRE(hrr::load_archive(proc_dir.string(), arc)); + CHECK_FALSE(arc.complete); + CHECK_FALSE(arc.truncated); + CHECK(read_text_file(proc_dir / "manifest.json").find(kIncomplete) != std::string::npos); + CHECK(read_text_file(cap / "manifest.json").find(kIncomplete) != std::string::npos); + + size_t missing = 0; + for (const auto& ev : arc.events) { + if (ev.header().event_type != HRR_API_HIPMEMCPY) continue; + if (ev.memcpy_ev.hash_lo == 0 && ev.memcpy_ev.hash_hi == 0) continue; + std::vector data; + if (!hrr::read_blob(arc, ev.memcpy_ev.hash_lo, ev.memcpy_ev.hash_hi, data)) ++missing; + } + CHECK(missing == 0); +} + +size_t count_h2d(const hrr::Archive& arc, uint64_t size) { + size_t n = 0; + for (const auto& ev : arc.events) + if (ev.header().event_type == HRR_API_HIPMEMCPY && + ev.memcpy_ev.kind == static_cast(hipMemcpyHostToDevice) && + ev.memcpy_ev.size == size) + ++n; + return n; +} + +size_t count_processes(const fs::path& cap) { + return count_of(read_text_file(cap / "manifest.json"), "{ \"pid\": "); +} + +fs::path cap_dir(const char* name) { + return fs::temp_directory_path() / (std::string("hrr_disk_space_") + name + ".hrr"); +} + +} // namespace + +// Below the reserve at start, capture stays off, leaves nothing behind, and the +// program runs as if it were never asked to capture. +HRR_TEST_CASE(Unit_HRR_DiskSpace_RefusedAtStart) { + ScopedDir cap(cap_dir("refused")); + const WorkloadRun run = run_workload("Unit_HRR_DiskSpace_RefusedAtStart_Direct", cap.path, + kTotal64M, /*avail=*/1u << 20); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + CHECK(count_of(run.out, kDisabled) == 1); + CHECK_FALSE(fs::exists(cap.path / "manifest.json")); + std::error_code ec; + for (const auto& ent : fs::directory_iterator(cap.path, ec)) + CHECK(ent.path().filename().string().rfind("pid-", 0) != 0); +} + +// Free space shrinks with every blob written. The capture stops within one +// check interval of the reserve and leaves an archive that is incomplete but +// whole. +HRR_TEST_CASE(Unit_HRR_DiskSpace_StopsBeforeReserve) { + ScopedDir cap(cap_dir("stops")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_StopsBeforeReserve_Direct", cap.path, kTotal16M); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + hrr::Archive arc; + load_stopped_archive(cap.path, run, arc); + const size_t steps = count_h2d(arc, kStepCopy); + CHECK(steps > 0); + CHECK(steps < static_cast(kStepCount)); +} + +// Small blobs are counted in file system blocks, so a stream of them reaches a +// check, and stops, long before their byte count would. +HRR_TEST_CASE(Unit_HRR_DiskSpace_SmallBlobsCountedInBlocks) { + ScopedDir cap(cap_dir("blocks")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_SmallBlobsCountedInBlocks_Direct", cap.path, kTotal64M); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + hrr::Archive arc; + load_stopped_archive(cap.path, run, arc); + CHECK(count_h2d(arc, kTinyCopy) < static_cast(kTinyCount)); +} + +// A child forked after the stop opens no archive of its own and does not +// finalize the parent's when it exits. +HRR_TEST_CASE(Unit_HRR_DiskSpace_ForkAfterStop) { + ScopedDir cap(cap_dir("fork_stop")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_ForkAfterStop_Direct", cap.path, kTotal64M); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + hrr::Archive arc; + load_stopped_archive(cap.path, run, arc); + CHECK(count_processes(cap.path) == 1); +} + +// A child whose own open is refused leaves no trace, and the parent's archive +// stays complete. +HRR_TEST_CASE(Unit_HRR_DiskSpace_RefusedForkChild) { + ScopedDir cap(cap_dir("fork_refused")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_RefusedForkChild_Direct", cap.path, kTotal64M); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + CHECK(count_of(run.out, kDisabled) == 1); + CHECK(count_of(run.out, kStopped) == 0); + CHECK(count_processes(cap.path) == 1); + const std::string root = read_text_file(cap.path / "manifest.json"); + CHECK(root.find("\"complete\": true") != std::string::npos); + CHECK(root.find(kIncomplete) == std::string::npos); +} + +// A crash after the stop still leaves a manifest that says the archive is +// incomplete, although events.bin is already closed. +HRR_TEST_CASE(Unit_HRR_DiskSpace_CrashAfterStop) { + ScopedDir cap(cap_dir("crash")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_CrashAfterStop_Direct", cap.path, kTotal64M); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 128 + SIGABRT); + CHECK(count_of(run.out, kStopped) == 1); + const fs::path proc_dir = hrr_single_process_archive(cap.path); + REQUIRE(fs::exists(proc_dir / "manifest.json")); + CHECK(read_text_file(proc_dir / "manifest.json").find(kIncomplete) != std::string::npos); +} + +// The process exits while another thread is stopping the capture. The final +// manifests still say incomplete. +HRR_TEST_CASE(Unit_HRR_DiskSpace_StopWhileExiting) { + ScopedDir cap(cap_dir("exiting")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_StopWhileExiting_Direct", cap.path, kTotal64M); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + hrr::Archive arc; + load_stopped_archive(cap.path, run, arc); +} + +// Two threads copy the same bytes while the capture stops. Neither copy may be +// recorded against a blob that was never written. +HRR_TEST_CASE(Unit_HRR_DiskSpace_DuplicateRacingStop) { + ScopedDir cap(cap_dir("duplicate")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_DuplicateRacingStop_Direct", cap.path, kTotal64M); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + hrr::Archive arc; + load_stopped_archive(cap.path, run, arc); +} + +// A record of at least the check interval is checked before it is written, so +// the launch that would cross into the reserve is not recorded. +HRR_TEST_CASE(Unit_HRR_DiskSpace_LargeEventCheckedBeforeWrite) { + ScopedDir cap(cap_dir("large_event")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_LargeEventCheckedBeforeWrite_Direct", cap.path, kTotal64K); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + hrr::Archive arc; + load_stopped_archive(cap.path, run, arc); + size_t launches = 0; + for (const auto& ev : arc.events) + if (ev.kernel_launch != nullptr && + ev.kernel_launch->kernel_name.find("hrr_disk_space_big_arg") != std::string::npos) + ++launches; + CHECK(launches == 1); +} + +// Records under the check interval are checked before the one that makes the +// check due is written, so less than one interval goes in unchecked, however +// the bytes add up. +HRR_TEST_CASE(Unit_HRR_DiskSpace_EventsCheckedAcrossInterval) { + ScopedDir cap(cap_dir("event_interval")); + const WorkloadRun run = run_workload("Unit_HRR_DiskSpace_EventsCheckedAcrossInterval_Direct", + cap.path, kTotal64K); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + hrr::Archive arc; + load_stopped_archive(cap.path, run, arc); + + // Everything recorded from the scratch copy on, the last check that passed. + const uint64_t interval = check_interval(kTotal64K); + size_t from = arc.events.size(); + for (size_t i = 0; i < arc.events.size(); ++i) + if (arc.events[i].header().event_type == HRR_API_HIPMEMCPY && + arc.events[i].memcpy_ev.kind == static_cast(hipMemcpyHostToDevice) && + arc.events[i].memcpy_ev.size == kBlock) + from = i; + REQUIRE(from < arc.events.size()); + uint64_t unchecked = 0; + size_t launches = 0; + for (size_t i = from; i < arc.events.size(); ++i) { + const auto& ev = arc.events[i]; + unchecked += ev.header().payload_length; + if (ev.kernel_launch != nullptr && + ev.kernel_launch->kernel_name.find("hrr_disk_space_mid_arg") != std::string::npos) { + // Each launch alone stays under the interval, so it takes the counted path. + CHECK(ev.header().payload_length < interval); + ++launches; + } + } + INFO("unchecked " << unchecked << " interval " << interval << " launches " << launches); + CHECK(unchecked < interval); + CHECK(launches < 2); +} + +// Event records count toward the next check like blobs do, so a run of +// buffered events with no blobs still stops. +HRR_TEST_CASE(Unit_HRR_DiskSpace_BufferedEventsCounted) { + ScopedDir cap(cap_dir("events")); + const WorkloadRun run = + run_workload("Unit_HRR_DiskSpace_BufferedEventsCounted_Direct", cap.path, kTotal64K); + INFO("Capture output:\n" << run.out); + REQUIRE(run.rc == 0); + hrr::Archive arc; + load_stopped_archive(cap.path, run, arc); + size_t memsets = 0; + for (const auto& ev : arc.events) + if (ev.header().event_type == HRR_API_HIPMEMSETASYNC) ++memsets; + CHECK(memsets > 0); + CHECK(memsets < static_cast(kMemsetCount)); +} + +#endif // !_WIN32 && HRR_TEST_EXE + +/** + * @} + */ diff --git a/projects/hrr/tests/integration/hrr_kernel_args_test.cc b/projects/hrr/tests/integration/hrr_kernel_args_test.cc new file mode 100644 index 000000000000..8f3b4fa6d293 --- /dev/null +++ b/projects/hrr/tests/integration/hrr_kernel_args_test.cc @@ -0,0 +1,183 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +/** + * @addtogroup HRR HRR kernel-argument translation + * @{ + * @ingroup HRRTest + * Tests for what replay does with a pointer-typed kernel argument the + * allocation map cannot explain. + * + * Two things reach a kernel through the same kind of slot and mean opposite + * things. A sentinel is a small literal a kernel branches on — aiter's MLA + * decode kernel takes a 1 in one of its pointer slots — and substituting null + * for it changes what the kernel does, silently and only on replay. A pointer + * HRR genuinely lost is address-shaped, and handing it to the GPU unchanged is + * worse than null: the driver tends to reproduce VA layout across runs, so the + * recorded address is likely to land in an unrelated live buffer and be + * scribbled over instead of faulting. + * + * Replay tells them apart by shape, and this asserts both halves of that. + */ + +#include "hrr_test_common.hh" +#include "hrr_test_process.hh" + +#include +#include +#include +#include +#include + +#if defined(HRR_PLAYBACK_EXE) && defined(HRR_TEST_EXE) + +namespace { +// Small enough to be certainly not an address, and the value aiter actually +// passes. +constexpr uint64_t kSentinel = 1; +constexpr size_t kProbeBytes = 64 * 1024; +// Far enough past the probe allocation that no allocation of the replay's own +// can cover it, while staying inside device VA space so the value is +// address-shaped — which is the whole distinction under test. +constexpr size_t kFarOff = 1ull << 32; + +#define HRR_KARGS_MARKER "HRR_KARGS_LOST" +} // namespace + +__global__ void hrr_kargs_probe(int* out, int* sentinel, int* lost) { + // Neither `sentinel` nor `lost` is dereferenced. At capture one is a literal + // and the other points at nothing mapped; at replay the second arrives as + // null. The kernel only has to carry them in its kernarg segment for HRR to + // record and translate them. + (void)sentinel; + (void)lost; + if (threadIdx.x == 0) out[0] = 1; +} + +// =========================================================================== +// The captured workload: one launch carrying both cases in pointer slots. +// =========================================================================== +TEST_CASE("Unit_HRR_KernelArgs_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + + int* out = nullptr; + HRR_HIP_CHECK(hipMalloc(&out, sizeof(int))); + HRR_HIP_CHECK(hipMemset(out, 0, sizeof(int))); + + char* probe = nullptr; + HRR_HIP_CHECK(hipMalloc(&probe, kProbeBytes)); + + int* lost = reinterpret_cast(probe + kFarOff); + int* sentinel = reinterpret_cast(kSentinel); + + // The address has to reach the parent: nothing in the archive records an + // allocation for it, which is exactly what makes it the case under test. + printf(HRR_KARGS_MARKER " 0x%llx\n", + static_cast(reinterpret_cast(lost))); + fflush(stdout); + + hipLaunchKernelGGL(hrr_kargs_probe, dim3(1), dim3(64), 0, nullptr, out, + sentinel, lost); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipDeviceSynchronize()); + + int host = 0; + HRR_HIP_CHECK(hipMemcpy(&host, out, sizeof(int), hipMemcpyDeviceToHost)); + REQUIRE(host == 1); + + HRR_HIP_CHECK(hipFree(probe)); + HRR_HIP_CHECK(hipFree(out)); +} + +namespace { +// Capture the workload, keeping the child's stdout so the parent learns the +// address it passed. +inline uint64_t hrr_capture_direct_kargs(const fs::path& cap_path) { + std::string out; + { hrr::test::SpawnProc proc(HRR_TEST_EXE, /*capture_stdout=*/true); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap_path.string()); + set_proc_search_path(proc); + int ret = proc.run("\"Unit_HRR_KernelArgs_Direct\""); + out = proc.getOutput(); + INFO("Capture exit: " << ret << "\n" << out); + REQUIRE(ret == 0); } + + const size_t at = out.find(HRR_KARGS_MARKER); + if (at == std::string::npos) return 0; + unsigned long long lost = 0; + if (sscanf(out.c_str() + at, HRR_KARGS_MARKER " 0x%llx", &lost) != 1) + return 0; + return lost; +} + +// Recover the live value the ptr-dump paired with a recorded one. +// +// The dump prints the live side with %p, whose rendering is not portable: +// glibc emits "0x1" and "(nil)", the MSVC CRT emits zero-padded hex with no +// prefix and no null spelling ("0000000000000001", "0000000000000000"). Only +// the recorded side is formatted by HRR itself, so match on that and parse +// whatever the platform produced instead of comparing formatted text. +inline bool hrr_live_arg(const std::string& out, uint64_t recorded, + uint64_t* live) { + char needle[64]; + snprintf(needle, sizeof(needle), "recorded=0x%llx -> live=", + static_cast(recorded)); + const size_t at = out.find(needle); + if (at == std::string::npos) return false; + + const char* p = out.c_str() + at + strlen(needle); + if (strncmp(p, "(nil)", 5) == 0) { *live = 0; return true; } + // Base 16 accepts both the bare digits MSVC prints and glibc's 0x prefix. + *live = strtoull(p, nullptr, 16); + return true; +} +} // namespace + +// --------------------------------------------------------------------------- +// Both halves, from one archive and one replay. +// +// Before replay distinguished them, every unresolvable pointer took the same +// branch, so one of these two assertions failed whichever branch that was: the +// sentinel arrived as null, or the lost pointer arrived as the recorded +// address. +// --------------------------------------------------------------------------- +TEST_CASE("Unit_HRR_KernelArgs_SentinelKeptLostPointerNulled", "[.][hrr]") { + ScopedDir cap(fs::temp_directory_path() / "hrr_kernel_args.hrr"); + const uint64_t lost = hrr_capture_direct_kargs(cap.path); + REQUIRE(lost != 0); + + const fs::path archive = hrr_single_process_archive(cap.path); + + // Ordinal 1 is the workload's only launch. + auto [rc, out] = hrr_playback_merged( + archive, "--warn-untranslated-args", + {{"HIP_HRR_REPLAY_DUMP_PTRS_ORDINAL", "1"}}); + INFO("Replay:\n" << out); + CHECK(rc == 0); + + // A value no device VA could be keeps its meaning. + uint64_t sentinel_live = 0; + REQUIRE(hrr_live_arg(out, kSentinel, &sentinel_live)); + CHECK(sentinel_live == kSentinel); + CHECK(out.find("not an address, passing it through unchanged") != + std::string::npos); + + // An address-shaped value that resolves nowhere does not. + uint64_t lost_live = 1; + REQUIRE(hrr_live_arg(out, lost, &lost_live)); + CHECK(lost_live == 0); + CHECK(out.find("passing null so it faults at first use") != + std::string::npos); + + // Both were counted, and neither was quietly treated as translated. + CHECK(out.find("Untranslated : 0 ") == std::string::npos); +} + +#endif // HRR_PLAYBACK_EXE && HRR_TEST_EXE + +/** + * @} + */ diff --git a/projects/hrr/tests/integration/hrr_roundtrip_test.cc b/projects/hrr/tests/integration/hrr_roundtrip_test.cc index 5ffcbfc79265..56a25bff57b1 100644 --- a/projects/hrr/tests/integration/hrr_roundtrip_test.cc +++ b/projects/hrr/tests/integration/hrr_roundtrip_test.cc @@ -200,6 +200,120 @@ HRR_TEST_CASE(Unit_HRR_CaptureReplayRoundtrip) { hrr_run_playback(cap.path); } +/** + * Test Description + * ---------------- + * - Spawns Unit_HRR_GpuWorkload_Direct with HIP_HRR_CAPTURE_OUTPUT set and + * AMD_LOG_LEVEL=0, capturing stdout and stderr. The capture layer must print + * its start notice exactly once, naming the per-process archive directory + * and the base directory that child processes record to. + * - Repeats the capture with only stdout captured: the notice is on stderr, + * so stdout holds none. + * - Spawns the same workload without HIP_HRR_CAPTURE_OUTPUT: no notice. + */ +HRR_TEST_CASE(Unit_HRR_CaptureStartNotice) { + ScopedDir cap{fs::temp_directory_path() / "hrr_capture_notice"}; + const std::string kNotice = + "[HRR capture] Recording this process's HIP calls, with their host buffers, kernel " + "arguments and code objects, to "; + auto count_notices = [&](const std::string& out) { + size_t n = 0; + for (size_t at = out.find(kNotice); at != std::string::npos; + at = out.find(kNotice, at + kNotice.size())) + ++n; + return n; + }; + + { + hrr::test::SpawnProc proc(HRR_TEST_EXE, /*capture_stdout=*/true, /*capture_stderr=*/true); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + proc.setEnv("AMD_LOG_LEVEL", "0"); + set_proc_search_path(proc); + int ret = proc.run("\"Unit_HRR_GpuWorkload_Direct\""); + const std::string out = proc.getOutput(); + INFO("Capture subprocess output:\n" << out); + REQUIRE(ret == 0); + REQUIRE(count_notices(out) == 1); + + const size_t begin = out.find(kNotice); + std::string line = out.substr(begin, out.find('\n', begin) - begin); + if (!line.empty() && line.back() == '\r') line.pop_back(); + const std::string base = cap.path.string(); + const std::string pid_dir = hrr_single_process_archive(cap.path).filename().string(); + // The writer joins the pid- component with '/' on every platform. + const std::string expected = kNotice + base + "/" + pid_dir + + " (child processes record to their own pid-* directories in " + + base + ")"; + CHECK(line == expected); + } + + { + // Same capture with only stdout captured: the notice goes to stderr, so the + // program's own stdout stays exactly what it was without capture. + ScopedDir cap_stdout{fs::temp_directory_path() / "hrr_capture_notice_stdout"}; + hrr::test::SpawnProc proc(HRR_TEST_EXE, /*capture_stdout=*/true, /*capture_stderr=*/false); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap_stdout.path.string()); + proc.setEnv("AMD_LOG_LEVEL", "0"); + set_proc_search_path(proc); + int ret = proc.run("\"Unit_HRR_GpuWorkload_Direct\""); + const std::string out = proc.getOutput(); + INFO("Capture subprocess stdout:\n" << out); + REQUIRE(ret == 0); + CHECK(count_notices(out) == 0); + } + + { + hrr::test::SpawnProc proc(HRR_TEST_EXE, /*capture_stdout=*/true, /*capture_stderr=*/true); + proc.setEnv("AMD_LOG_LEVEL", "0"); + // CLR's flag parser turns HIP_HRR_CAPTURE_OUTPUT= into a single space, which + // still enables capture. Unset the variable so an inherited value cannot arm it. + proc.unsetEnv("HIP_HRR_CAPTURE_OUTPUT"); + set_proc_search_path(proc); + int ret = proc.run("\"Unit_HRR_GpuWorkload_Direct\""); + const std::string out = proc.getOutput(); + INFO("Subprocess output without capture:\n" << out); + REQUIRE(ret == 0); + CHECK(count_notices(out) == 0); + } +} + +/** + * Test Description + * ---------------- + * - Spawns Unit_HRR_GpuWorkload_Direct with HIP_HRR_CAPTURE_OUTPUT set and + * AMD_LOG_LEVEL=3, capturing stdout and stderr. The summary that + * hip_capture_shutdown() logs at exit must name the per-process pid- + * directory the archive was written to, the same one the start notice names, + * not the base directory. + */ +HRR_TEST_CASE(Unit_HRR_CaptureExitSummary) { + ScopedDir cap{fs::temp_directory_path() / "hrr_capture_exit_summary"}; + hrr::test::SpawnProc proc(HRR_TEST_EXE, /*capture_stdout=*/true, /*capture_stderr=*/true); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + // The summary goes through the CLR log at LOG_INFO with LOG_ALWAYS, which no + // mask filters out; AMD_LOG_MASK=0 keeps the per-API traces out of the output. + proc.setEnv("AMD_LOG_LEVEL", "3"); + proc.setEnv("AMD_LOG_MASK", "0"); + set_proc_search_path(proc); + int ret = proc.run("\"Unit_HRR_GpuWorkload_Direct\""); + const std::string out = proc.getOutput(); + INFO("Capture subprocess output:\n" << out); + REQUIRE(ret == 0); + + const std::string kSummary = "[HRR capture] Wrote "; + const size_t begin = out.find(kSummary); + REQUIRE(begin != std::string::npos); + CHECK(out.find(kSummary, begin + kSummary.size()) == std::string::npos); + std::string line = out.substr(begin, out.find('\n', begin) - begin); + if (!line.empty() && line.back() == '\r') line.pop_back(); + INFO("Summary line: " << line); + const std::string pid_dir = hrr_single_process_archive(cap.path).filename().string(); + // The writer joins the pid- component with '/' on every platform. + const std::string tail = " blobs to: " + cap.path.string() + "/" + pid_dir; + REQUIRE(line.size() >= tail.size()); + CHECK(line.compare(line.size() - tail.size(), tail.size(), tail) == 0); +} + /** * Test Description * ---------------- @@ -625,7 +739,8 @@ TEST_CASE("Unit_HRR_NullOptionalPtrRoundtrip", "[.][hrr-repro]") { * ---------------- * - Capture Unit_HRR_StreamWriteValue_Direct (hipStreamWriteValue32 / * hipStreamWriteValue64, including one hipExtStreamWriteValueIncrement - * write) and replay it. Replay must reproduce every written value. + * write when the headers define it) and replay it. Replay must + * reproduce every written value. * - Replay with HIP_HRR_D2H_EXACT=1. This is deliberate, not decoration: * with the default tolerant validator a lost 32-bit write is accepted as * "f64 within tolerance" on any blob whose length is a multiple of 8, and @@ -1193,6 +1308,54 @@ HRR_TEST_CASE(Unit_HRR_ModuleAPIRoundtrip) { hrr_run_roundtrip("Unit_HRR_ModuleAPI_Direct", cap.path); } +/** + * Test Description + * ---------------- + * - Capture Unit_HRR_ModuleLoadBundle_Direct, which loads one HIPRTC ELF with + * hipModuleLoadData and, wrapped in an offload bundle file, with + * hipModuleLoad, then launches rtc_fill from the bundle's module. + * - REQUIRE that the hipModuleLoad event records the device ELF the runtime + * loaded, not the bundle file: its code object is an ELF and has the same + * hash as the hipModuleLoadData event of the bare ELF. + * - REQUIRE that the launch from that module records the same code object as + * its load event, then replay and validate the D2H. + */ +HRR_TEST_CASE(Unit_HRR_ModuleLoadBundleRoundtrip) { + ScopedDir cap{fs::temp_directory_path() / "hrr_roundtrip_moduleloadbundle"}; + hrr_run_roundtrip("Unit_HRR_ModuleLoadBundle_Direct", cap.path); + + hrr::Archive arc; + REQUIRE(hrr::load_archive(cap.path.string(), arc)); + const hrr::Event* load_data = nullptr; + const hrr::Event* load_file = nullptr; + const hrr::Event* launch = nullptr; + for (const auto& ev : arc.events) { + const uint16_t type = ev.header().event_type; + if (type == HRR_API_HIPMODULELOADDATA) load_data = &ev; + if (type == HRR_API_HIPMODULELOAD) load_file = &ev; + if (ev.kernel_launch && + ev.kernel_launch->kernel_name.find("rtc_fill") != std::string::npos) + launch = &ev; + } + REQUIRE(load_data); + REQUIRE(load_file); + REQUIRE(launch); + + const auto& file_ev = load_file->module_load_ev; + INFO("hipModuleLoad code object: " << hrr::hash_hex(file_ev.hash_lo, file_ev.hash_hi)); + std::vector file_co; + REQUIRE(hrr::read_code_object(arc, file_ev.hash_lo, file_ev.hash_hi, file_co)); + INFO("hipModuleLoad code object size: " << file_co.size()); + REQUIRE(file_co.size() >= 4); + REQUIRE(std::memcmp(file_co.data(), "\x7f" "ELF", 4) == 0); + + const auto& data_ev = load_data->module_load_ev; + CHECK(file_ev.hash_lo == data_ev.hash_lo); + CHECK(file_ev.hash_hi == data_ev.hash_hi); + CHECK(launch->kernel_launch->co_hash_lo == file_ev.hash_lo); + CHECK(launch->kernel_launch->co_hash_hi == file_ev.hash_hi); +} + HRR_TEST_CASE(Unit_HRR_VMMRoundtrip) { ScopedDir cap{fs::temp_directory_path() / "hrr_roundtrip_vmm"}; hrr_run_roundtrip("Unit_HRR_VMM_Direct", cap.path); @@ -1485,3 +1648,159 @@ HRR_TEST_CASE(Unit_HRR_ReplaceKernelBadSpec) { REQUIRE(ret < 128); // ...with a clean error, not a crash } #endif // !_WIN32 + +// --------------------------------------------------------------------------- +/** + * Test Description + * ---------------- + * - Runs Unit_HRR_FailedMemcpy3D_Direct under capture: a host-to-device + * copy whose extent is far larger than both buffers, which the runtime + * rejects, made through hipMemcpy3D, hipMemcpy3DAsync and their _spt + * spellings. + * - The workload exits cleanly and the archive holds none of the four. + * Before the copies were success-gated, capture hashed extent-many bytes + * from the 4 KiB host buffer and faulted. + */ +HRR_TEST_CASE(Unit_HRR_FailedMemcpy3DNotRecorded) { + ScopedDir cap{fs::temp_directory_path() / "hrr_failed_memcpy3d"}; + { + hrr::test::SpawnProc proc(HRR_TEST_EXE); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + set_proc_search_path(proc); + const int ret = proc.run("\"Unit_HRR_FailedMemcpy3D_Direct\""); + INFO("Capture exit code: " << ret); + REQUIRE(ret == 0); + } + const auto counts = hrr_info_api_counts(cap.path); + const auto it = counts.find("hipMalloc3D"); + REQUIRE(it != counts.end()); // the capture was live + CHECK(it->second == 1); + CHECK(counts.count("hipMemcpy3D") == 0); + CHECK(counts.count("hipMemcpy3DAsync") == 0); + CHECK(counts.count("hipMemcpy3D_spt") == 0); + CHECK(counts.count("hipMemcpy3DAsync_spt") == 0); +} + +// --------------------------------------------------------------------------- +/** + * Test Description + * ---------------- + * - Runs Unit_HRR_LongKernelName_Direct under capture: a hipModuleLaunchKernel + * of a kernel whose 70,000-character name does not fit the uint16_t length + * on the wire. + * - The workload exits cleanly, the archive holds no launch and is marked + * incomplete. Before the length was checked it wrapped, and the launch was + * recorded under a truncated name in an archive marked complete. + */ +HRR_TEST_CASE(Unit_HRR_LongKernelNameNotRecorded) { + ScopedDir cap{fs::temp_directory_path() / "hrr_long_kernel_name"}; + { + hrr::test::SpawnProc proc(HRR_TEST_EXE); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + set_proc_search_path(proc); + const int ret = proc.run("\"Unit_HRR_LongKernelName_Direct\""); + INFO("Capture exit code: " << ret); + REQUIRE(ret == 0); + } + const auto counts = hrr_info_api_counts(cap.path); + const auto it = counts.find("hipModuleLoadData"); + REQUIRE(it != counts.end()); // the capture was live + CHECK(counts.count("hipModuleLaunchKernel") == 0); + + hrr::Archive arc; + REQUIRE(hrr::load_archive(cap.path.string(), arc)); + CHECK_FALSE(arc.complete); +} + +// --------------------------------------------------------------------------- +/** + * Test Description + * ---------------- + * - Runs Unit_HRR_OverflowingMemcpy3D_Direct under capture: an accepted + * host-to-device hipMemcpy3DAsync whose width * height * depth overflows + * size_t, so no blob can be sized for it. + * - The archive holds no hipMemcpy3DAsync, is marked incomplete and replays + * cleanly. Recorded without a blob, the copy took the device-to-device + * replay path with an unmapped host source, and replay stopped. + */ +HRR_TEST_CASE(Unit_HRR_OverflowingMemcpy3DNotRecorded) { + ScopedDir cap{fs::temp_directory_path() / "hrr_overflowing_memcpy3d"}; + { + hrr::test::SpawnProc proc(HRR_TEST_EXE); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + set_proc_search_path(proc); + const int ret = proc.run("\"Unit_HRR_OverflowingMemcpy3D_Direct\""); + INFO("Capture exit code: " << ret); + REQUIRE(ret == 0); + } + const auto counts = hrr_info_api_counts(cap.path); + const auto it = counts.find("hipStreamBeginCapture"); + REQUIRE(it != counts.end()); // the capture was live + CHECK(it->second == 1); + CHECK(counts.count("hipMemcpy3DAsync") == 0); + + hrr::Archive arc; + REQUIRE(hrr::load_archive(cap.path.string(), arc)); + CHECK_FALSE(arc.complete); + + auto [rc, out] = hrr_playback_merged(hrr_single_process_archive(cap.path)); + INFO("Replay output:\n" << out); + CHECK(rc == 0); +} + +// --------------------------------------------------------------------------- +/** + * Test Description + * ---------------- + * - Runs Unit_HRR_FailedShimCalls_Direct under capture: one rejected call for + * each hand-written shim that inlines a struct, plus an accepted and a + * rejected hipMemPoolSetAttribute. + * - The archive holds only the accepted calls and replays cleanly. Replay + * issues every recorded call again, so a recorded rejected call stops it. + * - The accepted 4-byte reuse policy is recorded as exactly 1: an 8-byte copy + * would also take the sentinel word the workload put after it. + */ +HRR_TEST_CASE(Unit_HRR_FailedShimCallsNotRecorded) { + ScopedDir cap{fs::temp_directory_path() / "hrr_failed_shim_calls"}; + { + hrr::test::SpawnProc proc(HRR_TEST_EXE); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + set_proc_search_path(proc); + const int ret = proc.run("\"Unit_HRR_FailedShimCalls_Direct\""); + INFO("Capture exit code: " << ret); + REQUIRE(ret == 0); + } + const auto counts = hrr_info_api_counts(cap.path); + const auto pools = counts.find("hipMemPoolCreate"); + REQUIRE(pools != counts.end()); // the capture was live + CHECK(pools->second == 1); + const auto attrs = counts.find("hipMemPoolSetAttribute"); + REQUIRE(attrs != counts.end()); + CHECK(attrs->second == 1); + for (const char* api : + {"hipMemPoolSetAccess", "hipMemSetAccess", "hipArrayCreate", "hipArray3DCreate", + "hipStreamSetAttribute", "hipMemGetAllocationGranularity"}) { + INFO("API: " << api); + CHECK(counts.count(api) == 0); + } + + // The accepted reuse policy is an int32_t, so capture copies 4 bytes and the + // sentinel the workload put after it stays out of the recorded value. + { + hrr::Archive arc; + REQUIRE(hrr::load_archive(cap.path.string(), arc)); + const auto ev = std::find_if(arc.events.begin(), arc.events.end(), [](const hrr::Event& e) { + return e.header().event_type == static_cast(HRR_API_HIPMEMPOOLSETATTRIBUTE); + }); + REQUIRE(ev != arc.events.end()); + hrr_args_hipMemPoolSetAttribute a{}; + REQUIRE(ev->raw_payload.size() >= sizeof(a)); + std::memcpy(&a, ev->raw_payload.data(), sizeof(a)); + CHECK(a.attr == static_cast(hipMemPoolReuseAllowOpportunistic)); + CHECK(a.value_u64 == 1); + } + + auto [rc, out] = hrr_playback_merged(hrr_single_process_archive(cap.path)); + INFO("Replay output:\n" << out); + CHECK(rc == 0); +} diff --git a/projects/hrr/tests/integration/hrr_secure_exec_test.cc b/projects/hrr/tests/integration/hrr_secure_exec_test.cc new file mode 100644 index 000000000000..76e29b942e3a --- /dev/null +++ b/projects/hrr/tests/integration/hrr_secure_exec_test.cc @@ -0,0 +1,226 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +/** + * @addtogroup HRR HRR Capture Gate + * @{ + * @ingroup HRRTest + * When HIP_HRR_CAPTURE_OUTPUT must not arm capture: + * + * Unit_HRR_BlankCaptureOutputCapturesNothing: + * An exported empty variable reaches CLR as a single space, and a value of + * other blanks is no path either. The workload runs with each and must leave + * no archive under a directory of that name. + * + * Unit_HRR_SecureExecIgnoresCaptureOutput (Linux): + * A set-group-ID copy of this binary, for one of the user's supplementary + * groups, starts in secure-execution mode. It must write no archive and + * print the notice, and print no notice when the variable is unset. A copy + * without the bit captures and prints no notice. No root is needed. + * Secure-execution mode ignores LD_LIBRARY_PATH, so the copy only tests this + * build when the loader finds the same libamdhip64 without it (an installed + * ROCm, a RUNPATH or a loader cache that lists the build). The case skips + * when it does not, when the filesystem is mounted nosuid, when no_new_privs + * is set, and when the kernel does not mark the copy secure. + */ + +#include "hrr_test_common.hh" +#include "hrr_test_process.hh" + +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#include +#include +#include +#include +#ifndef AT_SECURE +#define AT_SECURE 23 // Linux ABI value; the same fallback as hip_capture_metadata.cpp +#endif +#endif + +HRR_TEST_CASE(Unit_HRR_BlankCaptureOutputCapturesNothing) { + // The child inherits this working directory, so a blank value taken as a + // path would write under a directory of that name here. "" reaches CLR as " ". + for (const char* value : {"", " \t "}) { + INFO("HIP_HRR_CAPTURE_OUTPUT=\"" << value << "\""); + const fs::path blank = fs::current_path() / (*value != '\0' ? value : " "); + if (fs::exists(blank)) + HRR_SKIP("a directory named " << blank.filename() << " already exists here"); + ScopedDir guard{blank}; + + hrr::test::SpawnProc proc(HRR_TEST_EXE, /*capture_stdout=*/true, /*capture_stderr=*/true); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", value); + set_proc_search_path(proc); + const int ret = proc.run("\"Unit_HRR_GpuWorkload_Direct\""); + INFO("Workload output:\n" << proc.getOutput()); + REQUIRE(ret == 0); + CHECK(hrr_process_archives(blank).empty()); + CHECK_FALSE(fs::exists(blank)); + } +} + +#if defined(__linux__) +namespace { + +constexpr const char* kSecureExecNotice = + "[HRR capture] HIP_HRR_CAPTURE_OUTPUT ignored: the program was started in " + "secure-execution mode"; +constexpr const char* kProbeRuntime = "hrr-probe hip-runtime="; +constexpr const char* kProbeSecure = "hrr-probe at-secure="; + +// A group the file can be given that differs from the effective one, so a +// set-group-ID exec changes the effective group and the kernel sets AT_SECURE. +// The owner of a file may give it any of their supplementary groups; root may +// give it any group at all. +bool pick_other_group(gid_t* out) { + const gid_t egid = getegid(); + const int n = getgroups(0, nullptr); + if (n > 0) { + std::vector groups(static_cast(n)); + const int got = getgroups(n, groups.data()); + for (int i = 0; i < got; ++i) { + if (groups[i] != egid) { + *out = groups[i]; + return true; + } + } + } + if (geteuid() == 0) { + *out = egid == 65534 ? 65533 : 65534; + return true; + } + return false; +} + +// The libamdhip64 the loader gave this process, as a canonical path. +std::string loaded_hip_runtime() { + std::string found; + dl_iterate_phdr( + [](dl_phdr_info* info, size_t, void* data) { + if (info->dlpi_name == nullptr || std::strstr(info->dlpi_name, "libamdhip64") == nullptr) + return 0; + std::error_code ec; + *static_cast(data) = fs::canonical(info->dlpi_name, ec).string(); + return 1; + }, + &found); + return found; +} + +// The rest of the line after `key` in a child's output, or "" if absent. +std::string probe_value(const std::string& out, const char* key) { + const size_t at = out.find(key); + if (at == std::string::npos) return {}; + const size_t begin = at + std::strlen(key); + return out.substr(begin, out.find('\n', begin) - begin); +} + +struct CopyRun { + int ret; + std::string out; +}; + +// Runs the probe and the GPU workload in the copy, capturing to `cap` unless it +// is empty, with no LD_LIBRARY_PATH: secure-execution mode ignores it, so the +// control has to do without it too. +CopyRun run_copy(const fs::path& exe, const fs::path& cap) { + hrr::test::SpawnProc proc(exe.string(), /*capture_stdout=*/true, /*capture_stderr=*/true); + if (!cap.empty()) proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.string()); + proc.setEnv("LD_LIBRARY_PATH", ""); + set_proc_search_path(proc); + // Unquoted: Catch2 reads a quoted filter as one name, comma included. + const int ret = proc.run("Unit_HRR_SecureExecProbe_Direct,Unit_HRR_GpuWorkload_Direct"); + return {ret, proc.getOutput()}; +} + +} // namespace + +// Hidden: reports what the loader and the kernel gave this process, for the +// case below. It makes no HIP call. +TEST_CASE("Unit_HRR_SecureExecProbe_Direct", "[.][hrr-direct]") { + std::printf("%s%s\n%s%lu\n", kProbeRuntime, loaded_hip_runtime().c_str(), kProbeSecure, + getauxval(AT_SECURE)); + std::fflush(stdout); +} +#endif + +HRR_TEST_CASE(Unit_HRR_SecureExecIgnoresCaptureOutput) { +#if !defined(__linux__) + HRR_SKIP("secure-execution mode is Linux only"); +#else + ScopedDir dir{fs::temp_directory_path() / "hrr_secure_exec"}; + fs::create_directories(dir.path); + + struct statvfs vfs {}; + if (statvfs(dir.path.c_str(), &vfs) != 0 || (vfs.f_flag & ST_NOSUID)) + HRR_SKIP("the temporary directory is on a nosuid filesystem"); + if (prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) == 1) + HRR_SKIP("no_new_privs is set, so exec would ignore the set-group-ID bit"); + gid_t gid = 0; + if (!pick_other_group(&gid)) HRR_SKIP("the user has no supplementary group to use"); + + const fs::path exe = dir.path / "hrr-integration-tests"; + std::error_code ec; + fs::copy_file("/proc/self/exe", exe, ec); + if (ec) HRR_SKIP("cannot copy this test binary: " << ec.message()); + if (chown(exe.c_str(), static_cast(-1), gid) != 0 || chmod(exe.c_str(), 0755) != 0) + HRR_SKIP("cannot give the copy group " << gid); + + // Control: the same file without the set-group-ID bit. It has to load this + // build's runtime and capture, or a missing archive below would prove nothing. + // Every skip comes before the first assertion, so a pass means the gate ran. + const std::string runtime = loaded_hip_runtime(); + const fs::path control_cap = dir.path / "control"; + const CopyRun control = run_copy(exe, control_cap); + INFO("Control output:\n" << control.out); + const std::string control_loaded = probe_value(control.out, kProbeRuntime); + if (control_loaded.empty()) + HRR_SKIP("the copy does not start without LD_LIBRARY_PATH (exit " << control.ret << ")"); + if (control_loaded != runtime) + HRR_SKIP("without LD_LIBRARY_PATH the copy loads " << control_loaded << ", not " << runtime); + + struct stat st {}; + if (chmod(exe.c_str(), 02755) != 0 || stat(exe.c_str(), &st) != 0 || !(st.st_mode & S_ISGID)) + HRR_SKIP("cannot set the set-group-ID bit on the copy"); + + const fs::path cap = dir.path / "secure"; + const CopyRun run = run_copy(exe, cap); + INFO("Set-group-ID output:\n" << run.out); + const std::string loaded = probe_value(run.out, kProbeRuntime); + if (loaded.empty()) HRR_SKIP("the loader refused the set-group-ID copy (exit " << run.ret << ")"); + if (probe_value(run.out, kProbeSecure) != "1") + HRR_SKIP("the set-group-ID copy did not start in secure-execution mode"); + if (loaded != runtime) + HRR_SKIP("in secure-execution mode the copy loads " << loaded << ", not " << runtime); + + // Without the variable a secure process has nothing to be told. + const CopyRun unset = run_copy(exe, fs::path{}); + INFO("Set-group-ID output without HIP_HRR_CAPTURE_OUTPUT:\n" << unset.out); + + REQUIRE(control.ret == 0); + REQUIRE_FALSE(hrr_process_archives(control_cap).empty()); + CHECK(control.out.find(kSecureExecNotice) == std::string::npos); + REQUIRE(run.ret == 0); + CHECK(hrr_process_archives(cap).empty()); + CHECK_FALSE(fs::exists(cap)); + CHECK(run.out.find(kSecureExecNotice) != std::string::npos); + REQUIRE(unset.ret == 0); + CHECK(unset.out.find(kSecureExecNotice) == std::string::npos); +#endif +} + +/** + * End of HRR group + * @} + */ diff --git a/projects/hrr/tests/integration/hrr_workload_test.cc b/projects/hrr/tests/integration/hrr_workload_test.cc index 0a5d7e098dee..fdbf287af4b0 100644 --- a/projects/hrr/tests/integration/hrr_workload_test.cc +++ b/projects/hrr/tests/integration/hrr_workload_test.cc @@ -419,6 +419,9 @@ TEST_CASE("Unit_HRR_AllApis_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipMemPoolGetAttribute(pool, hipMemPoolAttrReleaseThreshold, &threshold)); threshold = static_cast(-1); // never release automatically HRR_HIP_CHECK(hipMemPoolSetAttribute(pool, hipMemPoolAttrReleaseThreshold, &threshold)); + // The reuse policies take an int, so capture records 4 bytes rather than 8. + int32_t opportunistic = 1; + HRR_HIP_CHECK(hipMemPoolSetAttribute(pool, hipMemPoolReuseAllowOpportunistic, &opportunistic)); int *d_pool = nullptr; HRR_HIP_CHECK(hipMallocFromPoolAsync(&d_pool, SZ, pool, s0)); @@ -926,7 +929,7 @@ TEST_CASE("Unit_HRR_StreamWriteValue_Direct", "[.][hrr-direct]") { // Increment slot: base value, then a read-modify-write increment on top of it. // 0x0A0A0A0A + 0xF4E3F0C4 wraps to 0xFEEDFACE. constexpr uint32_t kIncBase = 0x0A0A0A0Au; - constexpr uint32_t kIncDelta = 0xF4E3F0C4u; + [[maybe_unused]] constexpr uint32_t kIncDelta = 0xF4E3F0C4u; hipStream_t s; HRR_HIP_CHECK(hipStreamCreateWithFlags(&s, hipStreamNonBlocking)); @@ -955,11 +958,18 @@ TEST_CASE("Unit_HRR_StreamWriteValue_Direct", "[.][hrr-direct]") { // target, so tolerate a rejection: capture only records successful calls, so // on a target that rejects it the slot simply keeps kIncBase in both the // recorded blob and the replay. + // + // The flag is an in-tree HIP extension, and this suite is also built against + // SDK headers that predate it. Absent at compile time is the same situation + // as rejected at run time, so it takes the same path rather than failing the + // build. HRR_HIP_CHECK(hipStreamWriteValue32(s, inc, kIncBase, 0)); - hipError_t incErr = hipStreamWriteValue32(s, inc, kIncDelta, - hipExtStreamWriteValueIncrement); + hipError_t incErr = hipErrorNotSupported; +#ifdef hipExtStreamWriteValueIncrement + incErr = hipStreamWriteValue32(s, inc, kIncDelta, hipExtStreamWriteValueIncrement); REQUIRE((incErr == hipSuccess || incErr == hipErrorInvalidValue || incErr == hipErrorNotSupported)); +#endif HRR_HIP_CHECK(hipStreamSynchronize(s)); HRR_HIP_CHECK(hipDeviceSynchronize()); @@ -3279,6 +3289,31 @@ TEST_CASE("Unit_HRR_ModuleAPI_Direct", "[.][hrr][direct]") { } hipModule_t mod_file = nullptr; HRR_HIP_CHECK(hipModuleLoad(&mod_file, tmp_co.string().c_str())); + { + // Regression guard for a launch from a file-loaded module. Replay + // resolves the launch by its own code-object hash, so this does not + // check what the hipModuleLoad event itself loaded. + hipFunction_t fn_file = nullptr; + HRR_HIP_CHECK(hipModuleGetFunction(&fn_file, mod_file, "rtc_fill")); + int* d_file = nullptr; + HRR_HIP_CHECK(hipMalloc(&d_file, SZ)); + // Replay compares D2H buffers under a float tolerance, so a small + // integer would also pass against a buffer the kernel never wrote. + // 0x3F800000 is 1.0f, which is outside the tolerance of an unwritten buffer. + int val = 0x3F800000; + int n = N; + void* args[] = { &d_file, &val, &n }; + int blocks = (N + 255) / 256; + HRR_HIP_CHECK(hipModuleLaunchKernel(fn_file, + blocks, 1, 1, // grid + 256, 1, 1, // block + 0, s, args, nullptr)); + std::vector h_file(N); + HRR_HIP_CHECK(hipMemcpyAsync(h_file.data(), d_file, SZ, hipMemcpyDeviceToHost, s)); + HRR_HIP_CHECK(hipStreamSynchronize(s)); + for (int i = 0; i < N; ++i) REQUIRE(h_file[i] == val); + HRR_HIP_CHECK(hipFree(d_file)); + } HRR_HIP_CHECK(hipModuleUnload(mod_file)); // Ignore remove errors: on Windows the ROCm driver may keep the file // open after hipModuleUnload, making fs::remove throw. The temp @@ -3301,6 +3336,113 @@ TEST_CASE("Unit_HRR_ModuleAPI_Direct", "[.][hrr][direct]") { delete[] h; } +// --------------------------------------------------------------------------- +// Workload Z2 — hipModuleLoad of an offload bundle file +// --------------------------------------------------------------------------- +// Compiles rtc_fill with HIPRTC, loads the ELF once with hipModuleLoadData, +// then wraps the same ELF in an uncompressed clang offload bundle (a host entry +// plus one entry for this device), writes it to a file and loads that with +// hipModuleLoad. Unit_HRR_ModuleLoadBundleRoundtrip checks that both module +// events record the same code object, the device ELF, and not the bundle file. +// D2H blob value = 0x3F800000 (1.0f), written by rtc_fill from the bundle. +// --------------------------------------------------------------------------- +static void hrr_append_u64_le(std::vector& bytes, uint64_t value) { + for (unsigned i = 0; i < sizeof(value); ++i) + bytes.push_back(static_cast(value >> (i * 8))); +} + +TEST_CASE("Unit_HRR_ModuleLoadBundle_Direct", "[.][hrr][direct]") { + // Warm-up first HIP call so the hipMalloc below is captured (see MiscAPIs). + HRR_HIP_CHECK(hipSetDevice(0)); + int* d = nullptr; + HRR_HIP_CHECK(hipMalloc(&d, SZ)); + hipStream_t s; + HRR_HIP_CHECK(hipStreamCreate(&s)); + + hiprtcProgram prog = nullptr; + HRR_HIPRTC_CHECK(hiprtcCreateProgram(&prog, k_fill_src, "rtc_fill.hip", + 0, nullptr, nullptr)); + hiprtcResult compile_rc = hiprtcCompileProgram(prog, 0, nullptr); + if (compile_rc != HIPRTC_SUCCESS) { + size_t log_sz = 0; + (void)hiprtcGetProgramLogSize(prog, &log_sz); + std::string log(log_sz, '\0'); + (void)hiprtcGetProgramLog(prog, log.data()); + (void)hiprtcDestroyProgram(&prog); + FAIL("hiprtcCompileProgram failed: " + log); + } + size_t co_size = 0; + HRR_HIPRTC_CHECK(hiprtcGetCodeSize(prog, &co_size)); + std::vector co(co_size); + HRR_HIPRTC_CHECK(hiprtcGetCode(prog, co.data())); + HRR_HIPRTC_CHECK(hiprtcDestroyProgram(&prog)); + REQUIRE(co.size() >= 4); + REQUIRE(std::string(co.data(), 4) == "\x7f" "ELF"); + + // The in-memory ELF: the reference the bundle load is compared against. + hipModule_t mod_data = nullptr; + HRR_HIP_CHECK(hipModuleLoadData(&mod_data, co.data())); + + // Same layout as clang-offload-bundler --type=o: magic, entry count, then + // (offset, size, id length, id) per entry; the host entry is empty and the + // device ELF starts on the next 4 KiB boundary. + hipDeviceProp_t props{}; + HRR_HIP_CHECK(hipGetDeviceProperties(&props, 0)); + constexpr uint64_t kElfOffset = 4096; + static const char kMagic[] = "__CLANG_OFFLOAD_BUNDLE__"; + std::vector bundle(kMagic, kMagic + sizeof(kMagic) - 1); + auto add_entry = [&bundle](const std::string& id, uint64_t size) { + hrr_append_u64_le(bundle, kElfOffset); + hrr_append_u64_le(bundle, size); + hrr_append_u64_le(bundle, id.size()); + bundle.insert(bundle.end(), id.begin(), id.end()); + }; + hrr_append_u64_le(bundle, 2); + add_entry("host-x86_64-unknown-linux-gnu-", 0); + add_entry(std::string("hipv4-amdgcn-amd-amdhsa--") + props.gcnArchName, co.size()); + REQUIRE(bundle.size() <= kElfOffset); + bundle.resize(kElfOffset, '\0'); + bundle.insert(bundle.end(), co.begin(), co.end()); + + { + namespace fs = std::filesystem; + // Unique per run for the same reason as the ModuleAPI workload above. + auto tmp_bundle = fs::temp_directory_path() / + (std::string("hrr_rtc_fill_bundle_") + + std::to_string(reinterpret_cast(&bundle)) + ".co"); + { + std::ofstream f(tmp_bundle, std::ios::binary); + REQUIRE(f.is_open()); + f.write(bundle.data(), static_cast(bundle.size())); + } + hipModule_t mod_file = nullptr; + HRR_HIP_CHECK(hipModuleLoad(&mod_file, tmp_bundle.string().c_str())); + hipFunction_t fn = nullptr; + HRR_HIP_CHECK(hipModuleGetFunction(&fn, mod_file, "rtc_fill")); + // 1.0f as an int, so an unwritten buffer fails replay's float-tolerant + // D2H comparison (see the ModuleAPI workload). + int val = 0x3F800000; + int n = N; + void* args[] = { &d, &val, &n }; + int blocks = (N + 255) / 256; + HRR_HIP_CHECK(hipModuleLaunchKernel(fn, + blocks, 1, 1, // grid + 256, 1, 1, // block + 0, s, args, nullptr)); + std::vector h(N); + HRR_HIP_CHECK(hipMemcpyAsync(h.data(), d, SZ, hipMemcpyDeviceToHost, s)); + HRR_HIP_CHECK(hipStreamSynchronize(s)); + for (int i = 0; i < N; ++i) REQUIRE(h[i] == val); + HRR_HIP_CHECK(hipModuleUnload(mod_file)); + std::error_code ec; + fs::remove(tmp_bundle, ec); + } + + HRR_HIP_CHECK(hipModuleUnload(mod_data)); + HRR_HIP_CHECK(hipFree(d)); + HRR_HIP_CHECK(hipStreamDestroy(s)); +} + // --------------------------------------------------------------------------- // Workload AA — Virtual Memory Management (VMM) roundtrip // --------------------------------------------------------------------------- @@ -3410,3 +3552,187 @@ TEST_CASE("Unit_HRR_ChevronLaunch_Direct", "[.][hrr][direct]") { HRR_HIP_CHECK(hipFree(d)); HRR_HIP_CHECK(hipStreamDestroy(s)); } + +// --------------------------------------------------------------------------- +// A copy the runtime rejects, its extent far larger than both buffers, made +// through each of the four hipMemcpy3D entry points: each has its own success +// gate in capture. Unit_HRR_FailedMemcpy3DNotRecorded checks that capture +// records none of them. +// --------------------------------------------------------------------------- +TEST_CASE("Unit_HRR_FailedMemcpy3D_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + hipPitchedPtr dst{}; + HRR_HIP_CHECK(hipMalloc3D(&dst, make_hipExtent(64, 4, 1))); + std::vector host(4096, 1); + + hipMemcpy3DParms p{}; + p.srcPtr = make_hipPitchedPtr(host.data(), size_t{1} << 20, size_t{1} << 20, size_t{1} << 12); + p.dstPtr = dst; + p.extent = make_hipExtent(size_t{1} << 20, size_t{1} << 12, 1); // 4 GiB + p.kind = hipMemcpyHostToDevice; + REQUIRE(hipMemcpy3D(&p) == hipErrorInvalidValue); + (void)hipGetLastError(); + REQUIRE(hipMemcpy3DAsync(&p, nullptr) == hipErrorInvalidValue); + (void)hipGetLastError(); + REQUIRE(hipMemcpy3D_spt(&p) == hipErrorInvalidValue); + (void)hipGetLastError(); + REQUIRE(hipMemcpy3DAsync_spt(&p, hipStreamPerThread) == hipErrorInvalidValue); + (void)hipGetLastError(); + + HRR_HIP_CHECK(hipFree(dst.ptr)); +} + +// --------------------------------------------------------------------------- +// A hipModuleLaunchKernel of a hipRTC kernel whose name is 70,000 characters, +// longer than the uint16_t name length on the wire. The launch itself succeeds +// and its output is checked here; Unit_HRR_LongKernelNameNotRecorded checks +// that capture drops the launch and marks the archive incomplete. +// --------------------------------------------------------------------------- +TEST_CASE("Unit_HRR_LongKernelName_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + constexpr int LN = 256; + constexpr size_t LSZ = LN * sizeof(int); + + const std::string name = "k" + std::string(70000, 'a'); + const std::string src = "extern \"C\" __global__ void " + name + + "(int* out, int val, int n) {\n" + " int i = blockIdx.x * blockDim.x + threadIdx.x;\n" + " if (i < n) out[i] = val;\n" + "}\n"; + hiprtcProgram prog = nullptr; + HRR_HIPRTC_CHECK(hiprtcCreateProgram(&prog, src.c_str(), "long_name.hip", + 0, nullptr, nullptr)); + hiprtcResult crc = hiprtcCompileProgram(prog, 0, nullptr); + if (crc != HIPRTC_SUCCESS) { + size_t log_sz = 0; + (void)hiprtcGetProgramLogSize(prog, &log_sz); + std::string log(log_sz, '\0'); + (void)hiprtcGetProgramLog(prog, log.data()); + (void)hiprtcDestroyProgram(&prog); + FAIL("hiprtcCompileProgram failed: " + log); + } + size_t co_size = 0; + HRR_HIPRTC_CHECK(hiprtcGetCodeSize(prog, &co_size)); + std::vector co(co_size); + HRR_HIPRTC_CHECK(hiprtcGetCode(prog, co.data())); + HRR_HIPRTC_CHECK(hiprtcDestroyProgram(&prog)); + + hipModule_t mod = nullptr; + HRR_HIP_CHECK(hipModuleLoadData(&mod, co.data())); + hipFunction_t fn = nullptr; + HRR_HIP_CHECK(hipModuleGetFunction(&fn, mod, name.c_str())); + + int* d = nullptr; + HRR_HIP_CHECK(hipMalloc(&d, LSZ)); + int val = 77; + int n = LN; + void* args[] = { &d, &val, &n }; + HRR_HIP_CHECK(hipModuleLaunchKernel(fn, 1, 1, 1, LN, 1, 1, 0, nullptr, args, nullptr)); + HRR_HIP_CHECK(hipDeviceSynchronize()); + + std::vector h(LN, 0); + HRR_HIP_CHECK(hipMemcpy(h.data(), d, LSZ, hipMemcpyDeviceToHost)); + for (int i = 0; i < LN; ++i) REQUIRE(h[i] == 77); + + HRR_HIP_CHECK(hipFree(d)); + HRR_HIP_CHECK(hipModuleUnload(mod)); +} + +// --------------------------------------------------------------------------- +// A host-to-device hipMemcpy3DAsync in a stream capture whose width * height * +// depth overflows size_t. The runtime accepts it: it validates with the same +// wrapping product and only adds a graph node, which is never launched. +// Unit_HRR_OverflowingMemcpy3DNotRecorded checks that capture drops it. +// --------------------------------------------------------------------------- +TEST_CASE("Unit_HRR_OverflowingMemcpy3D_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + hipStream_t stream = nullptr; + HRR_HIP_CHECK(hipStreamCreate(&stream)); + void* dst = nullptr; + HRR_HIP_CHECK(hipMalloc(&dst, 1024)); + std::vector host(1024, 1); + + hipMemcpy3DParms p{}; + p.srcPtr = make_hipPitchedPtr(host.data(), 256, 256, 4); + p.dstPtr = make_hipPitchedPtr(dst, 256, 256, 4); + p.extent = make_hipExtent(256, (size_t{1} << 56) + 1, 1); // 2^64 + 256 bytes + p.kind = hipMemcpyHostToDevice; + HRR_HIP_CHECK(hipStreamBeginCapture(stream, hipStreamCaptureModeGlobal)); + HRR_HIP_CHECK(hipMemcpy3DAsync(&p, stream)); + hipGraph_t graph = nullptr; + HRR_HIP_CHECK(hipStreamEndCapture(stream, &graph)); + HRR_HIP_CHECK(hipGraphDestroy(graph)); + + HRR_HIP_CHECK(hipFree(dst)); + HRR_HIP_CHECK(hipStreamDestroy(stream)); +} + +// --------------------------------------------------------------------------- +// One call the runtime rejects for each hand-written shim that inlines a struct, +// next to accepted calls on the same pool and stream, one of them setting a +// 4-byte reuse-policy attribute. Unit_HRR_FailedShimCallsNotRecorded checks that +// capture records only the accepted calls. +// --------------------------------------------------------------------------- +TEST_CASE("Unit_HRR_FailedShimCalls_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + hipMemPoolProps props{}; + props.allocType = hipMemAllocationTypePinned; + props.location.type = hipMemLocationTypeDevice; + props.location.id = 0; + hipMemPool_t pool = nullptr; + HRR_HIP_CHECK(hipMemPoolCreate(&pool, &props)); + // The reuse policy is a 4-byte int. The word after it is a sentinel that an + // 8-byte copy in capture would record as the value's high half. + int32_t opportunistic[2] = {1, 0x5a5a5a5a}; + HRR_HIP_CHECK( + hipMemPoolSetAttribute(pool, hipMemPoolReuseAllowOpportunistic, &opportunistic[0])); + hipStream_t stream = nullptr; + HRR_HIP_CHECK(hipStreamCreate(&stream)); + + hipMemPoolProps bad_props = props; + bad_props.allocType = hipMemAllocationTypeInvalid; + hipMemPool_t bad_pool = nullptr; + REQUIRE(hipMemPoolCreate(&bad_pool, &bad_props) == hipErrorInvalidValue); + + uint64_t used_high = 1; // only 0 is accepted + REQUIRE(hipMemPoolSetAttribute(pool, hipMemPoolAttrUsedMemHigh, &used_high) == + hipErrorInvalidValue); + + hipMemAccessDesc bad_access{}; // location.type is hipMemLocationTypeInvalid + bad_access.flags = hipMemAccessFlagsProtReadWrite; + REQUIRE(hipMemPoolSetAccess(pool, &bad_access, 1) == hipErrorInvalidValue); + REQUIRE(hipMemSetAccess(nullptr, 4096, &bad_access, 1) == hipErrorInvalidValue); + + HIP_ARRAY_DESCRIPTOR array_desc{}; + array_desc.Width = 64; + array_desc.Height = 64; + array_desc.Format = HIP_AD_FORMAT_FLOAT; + array_desc.NumChannels = 3; // only 1, 2 or 4 + hipArray_t array = nullptr; + REQUIRE(hipArrayCreate(&array, &array_desc) == hipErrorInvalidValue); + + HIP_ARRAY3D_DESCRIPTOR array3d_desc{}; + array3d_desc.Width = 16; + array3d_desc.Height = 16; + array3d_desc.Depth = 16; + array3d_desc.Format = HIP_AD_FORMAT_FLOAT; + array3d_desc.NumChannels = 3; + REQUIRE(hipArray3DCreate(&array, &array3d_desc) == hipErrorInvalidValue); + + hipStreamAttrValue sync{}; + sync.syncPolicy = static_cast(hipSyncPolicyBlockingSync + 1); + REQUIRE(hipStreamSetAttribute(stream, hipStreamAttributeSynchronizationPolicy, &sync) == + hipErrorInvalidValue); + + hipMemAllocationProp alloc_prop{}; // type is hipMemAllocationTypeInvalid + alloc_prop.location.type = hipMemLocationTypeDevice; + alloc_prop.location.id = 0; + size_t granularity = 0; + REQUIRE(hipMemGetAllocationGranularity(&granularity, &alloc_prop, + hipMemAllocationGranularityMinimum) == + hipErrorInvalidValue); + (void)hipGetLastError(); + + HRR_HIP_CHECK(hipStreamDestroy(stream)); + HRR_HIP_CHECK(hipMemPoolDestroy(pool)); +} diff --git a/projects/hrr/tools/gen_hrr_api_args.py b/projects/hrr/tools/gen_hrr_api_args.py index 8fa416c1baa7..b6ea04991859 100644 --- a/projects/hrr/tools/gen_hrr_api_args.py +++ b/projects/hrr/tools/gen_hrr_api_args.py @@ -90,12 +90,27 @@ "hipExtModuleLaunchKernel", "hipLaunchKernel", "hipLaunchByPtr", + # The stream-per-thread and cooperative spellings by host stub. Same + # encoding as hipLaunchKernel, each under its own event id. + "hipLaunchKernel_spt", + "hipLaunchCooperativeKernel", + "hipLaunchCooperativeKernel_spt", + # Extensible launches — the launch descriptor is a const struct pointer + # whose attribute list is a second pointer; both are carried in the + # variable-length launch payload rather than as capture-time addresses. + "hipDrvLaunchKernelEx", + "hipLaunchKernelExC", + # Cooperative launch — void** kernelParams, same encoding as the others. + "hipModuleLaunchCooperativeKernel", # <<<>>> launch config — must save grid/block/shared/stream into TLS for hipLaunchByPtr "__hipPushCallConfiguration", # Module load — code object snapshotting to disk "hipModuleLoadData", "hipModuleLoadDataEx", "hipModuleLoad", + # Module unload — drop the program's code-object hash so a recycled + # amd::Program address cannot hit a stale g_prog_hash entry. + "hipModuleUnload", # Memcpy H2D variants — blob snapshotting of host src data to disk "hipMemcpy", "hipMemcpyAsync", @@ -136,6 +151,26 @@ "hipDrvMemcpy3D", "hipDrvMemcpy3DAsync", "hipDrvMemcpy2DUnaligned", + "hipMemcpyParam2D", + "hipMemcpyParam2DAsync", + # Graph kernel nodes — hipKernelNodeParams names a host function and points + # at a host argument array; both need the kernel-launch encoding as a tail. + "hipGraphAddKernelNode", + "hipGraphKernelNodeSetParams", + "hipGraphExecKernelNodeSetParams", + # Graph batch-memory-operation nodes — the op list is a second pointer hop + # off hipBatchMemOpNodeParams, appended as an array tail. + "hipGraphAddBatchMemOpNode", + "hipGraphBatchMemOpNodeSetParams", + "hipGraphExecBatchMemOpNodeSetParams", + # Symbol copy nodes — the host buffer on the far side of the symbol is a + # blob, and whether there is one at all depends on the copy kind. + "hipGraphAddMemcpyNodeToSymbol", + "hipGraphAddMemcpyNodeFromSymbol", + "hipGraphMemcpyNodeSetParamsToSymbol", + "hipGraphExecMemcpyNodeSetParamsToSymbol", + # JIT linker input — the image is a blob. + "hipLinkAddData", } # Alias for backward compat within the file (some helpers used MANUAL_APIS) @@ -148,6 +183,12 @@ "hipExtModuleLaunchKernel", "hipLaunchKernel", "hipLaunchByPtr", + "hipLaunchKernel_spt", + "hipLaunchCooperativeKernel", + "hipLaunchCooperativeKernel_spt", + "hipDrvLaunchKernelEx", + "hipLaunchKernelExC", + "hipModuleLaunchCooperativeKernel", } # Minimum bytes for a valid variable-length kernel launch payload: @@ -163,6 +204,38 @@ "hipModuleGetFunction", } +# --------------------------------------------------------------------------- +# Windows exports +# +# amdhip.def.in is the export list the Windows amdhip64 DLL is linked with, so +# an API missing from it has no import library entry and hrr-playback.exe fails +# to link against it — even though the HIP headers declare it. Read the list +# rather than hard-coding the gaps, so the guard disappears on its own once clr +# starts exporting the symbol. +# --------------------------------------------------------------------------- + +# This script lives at projects/hrr/tools/; the export list stays in CLR next +# to the amdhip64 sources it is linked into. +_AMDHIP_DEF_IN = (Path(__file__).resolve().parent.parent.parent + / "clr" / "hipamd" / "src" / "amdhip.def.in") + + +def _windows_exported_symbols() -> Set[str]: + if not _AMDHIP_DEF_IN.is_file(): + sys.exit(f"ERROR: Windows export list not found at {_AMDHIP_DEF_IN}. " + f"The playback shims need it to know which APIs cannot be " + f"called on Windows.") + out: Set[str] = set() + for line in _AMDHIP_DEF_IN.read_text().splitlines(): + sym = line.strip() + if sym and sym != "EXPORTS" and not sym.startswith(";"): + out.add(sym) + return out + + +WINDOWS_EXPORTED_APIS: Set[str] = _windows_exported_symbols() + + # --------------------------------------------------------------------------- # Playback manual APIs — implemented by hand in hip_playback.cpp # --------------------------------------------------------------------------- @@ -173,6 +246,18 @@ "hipExtModuleLaunchKernel", "hipLaunchKernel", "hipLaunchByPtr", + "hipLaunchKernel_spt", + # Cooperative launches by host stub — the cooperative entry point, for the + # same reason hipModuleLaunchCooperativeKernel needs it. + "hipLaunchCooperativeKernel", + "hipLaunchCooperativeKernel_spt", + # Extensible launches — same payload plus the recorded launch-attribute + # list, replayed through hipDrvLaunchKernelEx with a rebuilt descriptor. + "hipDrvLaunchKernelEx", + "hipLaunchKernelExC", + # Cooperative launch — must go back through the cooperative entry point, or + # a grid-wide barrier hangs instead of running. + "hipModuleLaunchCooperativeKernel", # Memcpy H2D — must load blob from disk using hash fields appended to struct "hipMemcpy", "hipMemcpyAsync", @@ -209,6 +294,14 @@ "hipStreamQuery_spt", # Fat binary registration — load blob as module so kernel names resolve "__hipRegisterFatBinary", + # Variable registration — resolve the recorded symbol name in the loaded + # modules and map both the host shadow and the capture-time device address + # onto it, which is what makes the symbol copy family replayable. + "__hipRegisterVar", + # Symbol queries — answered from that map rather than from a host shadow + # address this process does not have. + "hipGetSymbolAddress", + "hipGetSymbolSize", # Host memory registration — need handle map + blob restore + device ptr recording "hipHostRegister", "hipHostUnregister", @@ -252,6 +345,13 @@ "hipMemGetAllocationGranularity", "hipMemPoolSetAccess", "hipMemSetAccess", + # hipStreamBatchMemOp — the recorded op list needs a per-op device-pointer + # rewrite, which depends on the op type stored in each entry. + "hipStreamBatchMemOp", + # IPC handles — the live handle must be paired with the recorded one so a + # later import in the same archive resolves. + "hipIpcGetMemHandle", + "hipIpcOpenMemHandle", # VMM (Virtual Memory Management) — output handles / VAs must be tracked "hipMemAddressReserve", "hipMemAddressFree", @@ -266,57 +366,145 @@ "hipDrvMemcpy3D", "hipDrvMemcpy3DAsync", "hipDrvMemcpy2DUnaligned", + "hipMemcpyParam2D", + "hipMemcpyParam2DAsync", + # Batch 3D copy — each operand's address lives in a different union member + # depending on the tag beside it. + "hipMemcpy3DBatchAsync", + # Graph kernel nodes — resolve the kernel by name, rebuild the argument + # array from the recorded bytes, register the node handle. + "hipGraphAddKernelNode", + "hipGraphKernelNodeSetParams", + "hipGraphExecKernelNodeSetParams", + # Graph batch-memory-operation nodes — per-op device address rewrite, same + # as hipStreamBatchMemOp. + "hipGraphAddBatchMemOpNode", + "hipGraphBatchMemOpNodeSetParams", + "hipGraphExecBatchMemOpNodeSetParams", + # Symbol copy nodes — the symbol resolves through the symbol registry and + # the host side of the copy comes from a blob. + "hipGraphAddMemcpyNodeToSymbol", + "hipGraphAddMemcpyNodeFromSymbol", + "hipGraphMemcpyNodeSetParamsToSymbol", + "hipGraphMemcpyNodeSetParamsFromSymbol", + "hipGraphExecMemcpyNodeSetParamsToSymbol", + "hipGraphExecMemcpyNodeSetParamsFromSymbol", + # JIT linker input — the image comes back from a blob, and the linker state + # from the map hipLinkCreate now fills. + "hipLinkAddData", + # Graph memory-allocation nodes — the address the node hands out is chosen + # by the replay's pool, so it must land in alloc_map under the recorded one. + "hipGraphAddMemAllocNode", + # Driver-API memcpy nodes — HIP_MEMCPY3D distinguishes host from device + # memory by a member, and a host operand was never recorded. + "hipDrvGraphAddMemcpyNode", + "hipDrvGraphMemcpyNodeSetParams", + "hipDrvGraphExecMemcpyNodeSetParams", } # --------------------------------------------------------------------------- -# Explicit graph-construction APIs that HRR cannot replay. Unlike the generic -# NOOP handlers (which emit a vague once-per-process message), these emit a -# loud, attributable "explicit graph construction is not supported" warning -# naming the specific API (finding H1) — so when replay later fails, the cause -# is traceable. +# Graph work whose arguments the archive does not carry well enough to rebuild. +# Most of the node API now replays for real; what is left here needs something +# the recording does not have — a semaphore only another process can produce, a +# device symbol the archive has no name for, a union whose active member is not +# recoverable. Unlike the generic NOOP handlers (which emit a vague +# once-per-process message), these emit a loud, attributable warning naming the +# specific API (finding H1), so when replay later fails the cause is traceable. # -# These are intentionally NON-FATAL (they return hipSuccess): a program may -# legitimately create/clone/build graphs it never instantiates, or exercise -# these APIs for coverage. The HARD fail-loud happens at the point that actually -# matters — hipGraphInstantiate / hipGraphInstantiateWithFlags (manual handlers -# in hip_playback.cpp) return hipErrorNotSupported (fatal) when asked to -# instantiate a graph that is absent from graph_map, which can only be a -# node-API-built graph. That prevents a node-API graph from silently replaying -# as an empty graph with skipped launches, without aborting programs that merely -# touch the construction APIs. +# These are intentionally NON-FATAL (they return hipSuccess) but they poison the +# graph: ctx.mark_graph_incomplete records that the graph is now short a node, +# and hipGraphInstantiate / hipGraphInstantiateWithFlags refuse an incomplete +# graph with hipErrorNotSupported. That is the point at which a missing node +# would otherwise turn into a graph launch that silently does less work than the +# recording did, and it leaves programs that merely build graphs they never +# instantiate free to run. # -# Only true *construction* calls belong here. Harmless queries +# Only calls that build or mutate graph contents belong here. Harmless queries # (hipGraphGetNodes/Edges, hipStreamGetCaptureInfo_v2, ...) stay in # NOOP_PLAYBACK_APIS. The stream-capture chain never emits these APIs, so the -# supported hipStreamBeginCapture/EndCapture path is unaffected. +# hipStreamBeginCapture/EndCapture path is unaffected. # --------------------------------------------------------------------------- ERROR_STUB_PLAYBACK_APIS: Set[str] = { - "hipGraphCreate", - "hipGraphClone", - "hipGraphAddChildGraphNode", - "hipGraphAddDependencies", - "hipGraphAddEmptyNode", - "hipGraphAddEventRecordNode", - "hipGraphAddEventWaitNode", + # An external semaphore is imported from a handle only its owning process + # can produce, so a node that signals or waits on one has nothing to point + # at here. "hipGraphAddExternalSemaphoresSignalNode", "hipGraphAddExternalSemaphoresWaitNode", - "hipGraphAddHostNode", - "hipGraphAddKernelNode", - "hipGraphAddMemAllocNode", - "hipGraphAddMemFreeNode", - "hipGraphAddMemcpyNode", - "hipGraphAddMemcpyNode1D", - "hipGraphAddMemcpyNodeFromSymbol", - "hipGraphAddMemcpyNodeToSymbol", - "hipGraphAddMemsetNode", + # hipGraphNodeParams is a union over every node kind, so reconstructing it + # means reconstructing all of them through one 256-byte blob whose active + # member is only known from a type tag. The typed spellings above are + # replayed; this one is not. "hipGraphAddNode", - "hipGraphAddBatchMemOpNode", - "hipDrvGraphAddMemFreeNode", - "hipDrvGraphAddMemcpyNode", - "hipDrvGraphAddMemsetNode", + # hipGraphInstantiateParams carries an out node pointer and an error-log + # buffer belonging to the capturing process. "hipGraphInstantiateWithParams", } +# --------------------------------------------------------------------------- +# APIs whose effect cannot be reproduced at replay by anything HRR can do, as +# opposed to APIs that merely have not been implemented yet. +# +# A host callback is a function pointer into the capturing process; a fabric or +# IPC handle names an export only its owning process can make; a multi-device +# launch descriptor carries a host function address for each device. Recording +# more bytes does not help — the missing thing is not in the arguments. +# +# The alternative to declaring these is what HRR used to do: hand the recorded +# value to the real API and let it fail (or crash) somewhere inside the runtime, +# where the diagnosis is a HIP error code with no attribution. So they get a +# handler that says exactly which API is unreplayable and why, returns +# hipErrorNotSupported (fatal unless --continue-on-error), and is counted in the +# replay summary's unreplayable list. The capture shim also warns once, so the +# gap is visible when the recording is made. +# +# Maps API name -> the reason, which is printed verbatim at both ends. +# --------------------------------------------------------------------------- +_HOST_CALLBACK_REASON = ( + "the callback is a host function pointer belonging to the capturing " + "process, so there is no function here to enqueue" +) + +_MULTI_DEVICE_LAUNCH_REASON = ( + "each hipLaunchParams entry names its kernel by a host function address in " + "the capturing process, and a cooperative multi-device launch cannot be " + "decomposed into per-device launches without breaking the grid-wide " + "barrier it exists for" +) + +_SHAREABLE_IMPORT_REASON = ( + "the recorded argument is an OS handle (a POSIX fd or a Win32 HANDLE) " + "belonging to the process that exported it, and the same number in the " + "replaying process names a different object or nothing at all" +) + +UNREPLAYABLE_PLAYBACK_APIS: Dict[str, str] = { + "hipLaunchHostFunc": _HOST_CALLBACK_REASON, + "hipLaunchHostFunc_spt": _HOST_CALLBACK_REASON, + "hipStreamAddCallback": _HOST_CALLBACK_REASON, + "hipStreamAddCallback_spt": _HOST_CALLBACK_REASON, + "hipLaunchCooperativeKernelMultiDevice": _MULTI_DEVICE_LAUNCH_REASON, + "hipExtLaunchMultiKernelMultiDevice": _MULTI_DEVICE_LAUNCH_REASON, + # A host node is the graph spelling of the same thing: hipHostNodeParams + # is a function pointer plus a userData pointer, both into the capturing + # process. The graph it belongs to is marked incomplete so instantiation + # refuses rather than running a graph with the host work missing. + "hipGraphAddHostNode": _HOST_CALLBACK_REASON, + "hipGraphHostNodeSetParams": _HOST_CALLBACK_REASON, + "hipGraphExecHostNodeSetParams": _HOST_CALLBACK_REASON, + # The two shareable-handle imports. Replay does re-run the matching export, + # but into a buffer of its own: the exported handle is minted fresh and the + # recorded number is not it. Handing the recorded number to the runtime is + # what made hipMemPoolImportFromShareableHandle crash inside the pool + # import, so replay refuses it by name instead. + "hipMemImportFromShareableHandle": _SHAREABLE_IMPORT_REASON, + "hipMemPoolImportFromShareableHandle": _SHAREABLE_IMPORT_REASON, + # A user object exists to run its destructor when the last reference goes, + # and that destructor is a hipHostFn_t in the capturing process. Creating + # one at replay with no destructor would look like it worked and quietly + # drop the only thing the object does. + "hipUserObjectCreate": _HOST_CALLBACK_REASON, +} + # --------------------------------------------------------------------------- # APIs that get an inline no-op playback body (return hipSuccess; immediately) # rather than an extern declaration. Used for: @@ -336,7 +524,6 @@ "hipOccupancyMaxActiveClusters", "hipOccupancyMaxPotentialClusterSize", "hipPointerSetAttribute", - "hipMemGetHandleForAddressRange", # Category 3: hipDeviceptr_t output params — generator emits wrong cast "hipMemAllocPitch", "hipMemGetAddressRange", @@ -354,32 +541,28 @@ "hipGraphGetEdges", "hipGraphGetNodes", "hipGraphGetRootNodes", - "hipGraphNodeFindInClone", "hipGraphNodeGetDependencies", "hipGraphNodeGetDependentNodes", "hipGraphRemoveDependencies", - "hipStreamBeginCaptureToGraph", "hipStreamGetCaptureInfo_v2", "hipStreamGetCaptureInfo_v2_spt", "hipStreamUpdateCaptureDependencies", # Category 5: Other type mismatches (output handle params stored as void** by generator) - "hipCtxCreate", - "hipCtxGetCurrent", + # The hipCtx* family used to live here; it now runs against ctx_map, which + # is what makes hipCtxPushCurrent push a context that exists in this + # process. hipCtxGetDevice stays: it writes a hipDevice_t the replay has no + # use for, and the device it reports is the one hipSetDevice already chose. "hipCtxGetDevice", - "hipCtxPopCurrent", "hipDeviceGet", - "hipDevicePrimaryCtxRetain", "hipDrvGetErrorName", "hipDrvGetErrorString", "hipGetTextureReference", "hipKernelGetName", - "hipMemImportFromShareableHandle", "hipMemRetainAllocationHandle", # hipMemGetAllocationPropertiesFromHandle — handle is stale at playback; query not needed "hipMemGetAllocationPropertiesFromHandle", "hipModuleGetTexRef", "hipStreamGetDevice", - "hipUserObjectCreate", # Category 6: Misc — struct field issues, wrong return type casts, or missing types # hipDeviceGetByPCIBusId takes a char* PCI string (stale capture-time pointer) — noop "hipDeviceGetByPCIBusId", @@ -388,16 +571,11 @@ # These are query-only calls with no effect on replay correctness — noop. "hipDeviceGetName", "hipDeviceGetPCIBusId", - # hipChooseDevice takes a hipDeviceProp_t* (stale capture-time pointer) — noop - "hipChooseDevice", # hipDeviceGetGraphMemAttribute / hipDeviceSetGraphMemAttribute — void* value ptr (stale) — noop "hipDeviceGetGraphMemAttribute", "hipDeviceSetGraphMemAttribute", # hipDevicePrimaryCtxGetState — output pointers (stale) — noop "hipDevicePrimaryCtxGetState", - # hipGetSymbolAddress / hipGetSymbolSize — symbol name is a stale host ptr — noop - "hipGetSymbolAddress", - "hipGetSymbolSize", # hipPointerGetAttribute (singular) — void* output (stale) — noop "hipPointerGetAttribute", # hipDrvPointerGetAttributes — void** array of outputs (stale) — noop @@ -445,9 +623,6 @@ # hipMemcpyAtoH / hipMemcpyHtoA — hipArray_t not in array_map for generated shim — noop "hipMemcpyAtoH", "hipMemcpyHtoA", - # hipMemcpyParam2D / hipMemcpyParam2DAsync — hip_Memcpy2D struct* (stale) — noop - "hipMemcpyParam2D", - "hipMemcpyParam2DAsync", # hipMemcpyToSymbol / hipMemcpyFromSymbol / Async — symbol name is host ptr (stale) — noop "hipMemcpyToSymbol", "hipMemcpyFromSymbol", @@ -459,13 +634,10 @@ "hipMemset3D_spt", "hipMemset3DAsync_spt", # hipMallocPitch — already in MANUAL_PLAYBACK_APIS for the DrvMemcpy test; these are the _spt wrappers - # hipLaunchCooperativeKernel — variable args (void**); handled via regular kernel launch fallback — noop - "hipLaunchCooperativeKernel", # NOTE: hipExtMallocWithFlags is intentionally NOT noop'd. It is a real device # allocation (see _ALLOC_CREATE_APIS) and is handled by a manual playback handler # (MANUAL_PLAYBACK_APIS) so the returned device pointer lands in alloc_map and any # H2D/D2H/kernel-arg use of it translates correctly. - "hipChooseDeviceR0000", "hipGetDevicePropertiesR0000", "hipGetErrorName", "hipGetErrorString", @@ -486,8 +658,6 @@ # Category 8: Device extra — output ptr stale, dangerous context ops, or struct-ptr params # hipDeviceGetUuid — output hipUUID* stale "hipDeviceGetUuid", - # hipDeviceGetTexture1DLinearMaxWidth — hipChannelFormatDesc* param (stale struct ptr) - "hipDeviceGetTexture1DLinearMaxWidth", # Primary ctx ops that would destroy the context at playback "hipDevicePrimaryCtxRelease", "hipDevicePrimaryCtxReset", @@ -554,7 +724,8 @@ "hipKernelGetAttribute", "hipKernelSetAttribute", # Category 12: Misc — stale output pointers or wrong return type - "hipGetProcAddress", + # hipGetProcAddress and hipGetProcAddress_spt used to be here and to crash + # respectively; both now carry the symbol name and look it up for real. "hipGetDriverEntryPoint", "hipGetDriverEntryPoint_spt", # hipOccupancyAvailableDynamicSMemPerBlock — hipFunction_t stale handle @@ -563,10 +734,6 @@ "hipSetValidDevices", # hipMemPtrGetInfo — void* output size ptr stale "hipMemPtrGetInfo", - # hipMemGetMemPool — output hipMemPool_t* (type mismatch) + handle not tracked - "hipMemGetMemPool", - # hipMemGetAccess — hipMemLocation* (stale struct ptr) - "hipMemGetAccess", # hipMemRangeGetAttribute — segfaults on Linux ROCm 7.13; stale dev_ptr unsafe "hipMemRangeGetAttribute", # hipMemRangeGetAttributes — attribute arrays stale @@ -605,44 +772,31 @@ "hipMipmappedArrayGetLevel", "hipGetMipmappedArrayLevel", "hipFreeMipmappedArray", - # Category 15: Graph explicit APIs — stale hipGraph_t/hipGraphExec_t/hipGraphNode_t handles - # NOTE: hipGraphCreate / hipGraphClone / hipGraphInstantiateWithParams moved to - # ERROR_STUB_PLAYBACK_APIS (explicit construction now fails loudly, see H1). + # Category 15: Graph explicit APIs. + # The node-construction and node-mutation calls are replayed now that a + # recorded hipGraphNode_t resolves through graph_node_map and the node + # parameter structs reach the archive (DEREF_FIELDS). What is left here is + # the query half — a Get* that writes into a caller buffer changes nothing + # about the replayed graph — plus the handful whose arguments still name + # something only the capturing process had. "hipGraphDestroy", "hipGraphUpload", "hipGraphDebugDotPrint", "hipGraphNodeGetType", - "hipGraphNodeSetEnabled", "hipGraphNodeGetEnabled", "hipGraphKernelNodeGetParams", - "hipGraphKernelNodeSetParams", - "hipGraphKernelNodeCopyAttributes", "hipGraphKernelNodeGetAttribute", - "hipGraphKernelNodeSetAttribute", "hipGraphMemcpyNodeGetParams", - "hipGraphMemcpyNodeSetParams", - "hipGraphMemcpyNodeSetParams1D", "hipGraphMemsetNodeGetParams", - "hipGraphMemsetNodeSetParams", "hipGraphHostNodeGetParams", - "hipGraphHostNodeSetParams", "hipGraphExecDestroy", - "hipGraphExecKernelNodeSetParams", - "hipGraphExecMemcpyNodeSetParams", - "hipGraphExecMemcpyNodeSetParams1D", - "hipGraphExecMemsetNodeSetParams", - "hipGraphExecHostNodeSetParams", - "hipGraphExecChildGraphNodeSetParams", "hipGraphExecGetFlags", + # The generic hipGraphNodeParams spelling — see hipGraphAddNode. "hipGraphNodeSetParams", "hipGraphExecNodeSetParams", "hipGraphChildGraphNodeGetGraph", "hipGraphEventRecordNodeGetEvent", - "hipGraphEventRecordNodeSetEvent", "hipGraphEventWaitNodeGetEvent", - "hipGraphEventWaitNodeSetEvent", - "hipGraphExecEventRecordNodeSetEvent", - "hipGraphExecEventWaitNodeSetEvent", "hipGraphMemAllocNodeGetParams", "hipGraphMemFreeNodeGetParams", "hipUserObjectRetain", @@ -650,12 +804,7 @@ "hipGraphRetainUserObject", "hipGraphReleaseUserObject", "hipGraphBatchMemOpNodeGetParams", - "hipGraphBatchMemOpNodeSetParams", - "hipGraphExecBatchMemOpNodeSetParams", - "hipDrvGraphExecMemcpyNodeSetParams", - "hipDrvGraphExecMemsetNodeSetParams", "hipDrvGraphMemcpyNodeGetParams", - "hipDrvGraphMemcpyNodeSetParams", # Category 16: present in the internal dispatch table (hip_api_trace.hpp) but # NOT declared in the public HIP runtime headers the playback tool builds # against (hip::host / find_package(hip)). hip_playback_generated.cpp calls @@ -732,6 +881,31 @@ # hipHostRegister — snapshot of host memory at registration time "hipHostRegister": [("uint64_t", "blob_hash_lo", "sysmem blob hash lo"), ("uint64_t", "blob_hash_hi", "sysmem blob hash hi")], + # The graph symbol copies. The to-symbol spellings read a host buffer the + # recording owned, so the bytes go in a blob; the from-symbol spellings + # write one, and the blob is the expected output. + "hipGraphAddMemcpyNodeToSymbol": + [("uint64_t", "blob_hash_lo", "host src blob hash lo, 0 if src is a device pointer"), + ("uint64_t", "blob_hash_hi", "host src blob hash hi")], + "hipGraphAddMemcpyNodeFromSymbol": + [("uint64_t", "blob_hash_lo", "unused; the destination is written at replay"), + ("uint64_t", "blob_hash_hi", "unused")], + "hipGraphMemcpyNodeSetParamsToSymbol": + [("uint64_t", "blob_hash_lo", "host src blob hash lo, 0 if src is a device pointer"), + ("uint64_t", "blob_hash_hi", "host src blob hash hi")], + "hipGraphExecMemcpyNodeSetParamsToSymbol": + [("uint64_t", "blob_hash_lo", "host src blob hash lo, 0 if src is a device pointer"), + ("uint64_t", "blob_hash_hi", "host src blob hash hi")], + # hipLinkAddData — the code object being linked. Sizes run to hundreds of + # kilobytes, so it is a blob rather than inline bytes. + "hipLinkAddData": [("uint64_t", "blob_hash_lo", "linker input image blob hash lo"), + ("uint64_t", "blob_hash_hi", "linker input image blob hash hi")], + # __hipRegisterVar — the symbol's device address in the capturing process. + # Zero from the live shim, which runs at static-init time before any device + # exists; filled in by the post-registration sweep, which is where every + # registration this suite sees is actually recorded. + "__hipRegisterVar": [("uint64_t", "dev_addr", + "capture-time device address of the symbol, 0 if unresolved")], # Memcpy 1D variants — blob hash for H2D data "hipMemcpy": [("uint64_t", "blob_hash_lo", "H2D blob hash lo"), ("uint64_t", "blob_hash_hi", "H2D blob hash hi")], @@ -807,6 +981,18 @@ ("uint64_t", "blob_hash_hi", "H2D blob hash hi"), ("uint64_t", "d2h_hash_lo", "D2H expected-output blob hash lo (0 if not D2H)"), ("uint64_t", "d2h_hash_hi", "D2H expected-output blob hash hi")], + # Same descriptor, same treatment: hipMemcpyParam2D and its async spelling + # take the identical hip_Memcpy2D that hipDrvMemcpy2DUnaligned does. + "hipMemcpyParam2D": [("uint8_t", "drv2d_bytes[136]", "hip_Memcpy2D inline copy"), + ("uint64_t", "blob_hash_lo", "H2D blob hash lo (0 if not H2D)"), + ("uint64_t", "blob_hash_hi", "H2D blob hash hi"), + ("uint64_t", "d2h_hash_lo", "D2H expected-output blob hash lo (0 if not D2H)"), + ("uint64_t", "d2h_hash_hi", "D2H expected-output blob hash hi")], + "hipMemcpyParam2DAsync": [("uint8_t", "drv2d_bytes[136]", "hip_Memcpy2D inline copy"), + ("uint64_t", "blob_hash_lo", "H2D blob hash lo (0 if not H2D)"), + ("uint64_t", "blob_hash_hi", "H2D blob hash hi"), + ("uint64_t", "d2h_hash_lo", "D2H expected-output blob hash lo (0 if not D2H)"), + ("uint64_t", "d2h_hash_hi", "D2H expected-output blob hash hi")], "hipArrayCreate": [("uint8_t", "array_desc_bytes[24]", "HIP_ARRAY_DESCRIPTOR inline copy")], # hipArray3DCreate — HIP_ARRAY3D_DESCRIPTOR is 40 bytes; store inline. "hipArray3DCreate": [("uint8_t", "array3d_desc_bytes[40]", "HIP_ARRAY3D_DESCRIPTOR inline copy")], @@ -819,6 +1005,394 @@ "hipMemSetAccess": [("uint8_t", "access_desc_bytes[12]", "hipMemAccessDesc[0] inline copy")], } +# --------------------------------------------------------------------------- +# Dereferenced pointer arguments +# +# normalise_field_type() lowers every pointer to a uint64_t holding the +# capture-time address, so by default the pointee never reaches the archive. +# For an input struct that is silent data loss (section 8.3): replay is handed +# an address from a process that no longer exists. A Deref entry tells the +# generator to carry the pointee itself. +# +# Capture: the generated shim memcpy's `size` bytes (or `count` elements) out of +# the pointer into the event, and sets _present. +# Playback: the generated handler rebuilds a local copy from those bytes and +# passes its address, instead of casting the recorded address back to a pointer. +# +# Use direction="in" for an argument the API reads, "out" for one it writes +# (recorded for fidelity; replay still passes a local, since the recorded +# address means nothing here), and "inout" for both. +# +# An API whose pointee contains device pointers, handles or nested pointers +# needs those translated at replay, which no table entry can express: give it a +# Deref for the capture side and a hand-written handler in MANUAL_PLAYBACK_APIS +# for the replay side. `playback="manual"` says so, and suppresses the generic +# reconstruction. +# --------------------------------------------------------------------------- + +@dataclass(frozen=True) +class Deref: + param: str # parameter name in the hip_api_trace.hpp typedef + ctype: str # pointee type, e.g. "hipMemAllocationProp" + size: int # inline bytes per element; static_assert'd against sizeof(ctype) + direction: str = "in" # "in" | "out" | "inout" + count: str = "" # element-count parameter name (array arguments only) + max_count: int = 0 # inline capacity in elements, for array/string arguments + # A NUL-terminated string has no count parameter and no fixed pointee size: + # capture copies up to max_count - 1 characters plus the terminator, and + # replay passes the recorded bytes straight out of the event. + string: bool = False + playback: str = "auto" # "auto" (generic reconstruction) | "manual" + # Members of the pointee that hold recorded device addresses. Replay runs + # each through ctx.translate_ptr before the call; without that the struct + # would arrive intact but pointing into the capturing process. + ptr_members: Tuple[str, ...] = () + # Members of the pointee that hold recorded HIP handles, as + # (member, handle type). Same idea as ptr_members, through the handle map + # for that type instead of alloc_map. + handle_members: Tuple[Tuple[str, str], ...] = () + # For an array argument whose elements are themselves HIP handles (a graph + # dependency list is an array of hipGraphNode_t): the handle type each + # element is translated through. + elem_handle: str = "" + # For an array of bare device addresses (a batch copy's dsts[] / srcs[]): + # every element goes through translate_ptr, the way a ptr_member of a + # struct does. + elem_ptr: bool = False + + @property + def is_array(self) -> bool: + return bool(self.count) + + @property + def bytes_field(self) -> str: + return f"{self.param}_bytes" + + @property + def present_field(self) -> str: + return f"{self.param}_present" + + @property + def count_field(self) -> str: + return f"{self.param}_n" + + @property + def total_bytes(self) -> int: + if self.string: + return self.max_count + return self.size * self.max_count if self.is_array else self.size + + +# API name -> the pointer arguments whose pointee is carried in the event. +DEREF_FIELDS: Dict[str, List[Deref]] = { + # The allocation property decides the memory type and, for a heap that maps + # cross-node peers, which device the pages come from. Playback used to + # hardcode Pinned/device 0, which builds the wrong topology silently. + "hipMemCreate": [ + Deref("prop", "hipMemAllocationProp", 32, playback="manual"), + ], + # The op list is the whole call. Each entry's address is a device pointer, + # and which member holds it depends on the op type, so the reconstruction + # is hand-written rather than a ptr_members rewrite. + "hipStreamBatchMemOp": [ + Deref("paramArray", "hipStreamBatchMemOpParams", 48, count="count", + max_count=16, playback="manual"), + ], + # The IPC handle is 64 bytes. Recorded as one uint64_t it was 56 bytes of + # nothing, and replay passed a null pointer for the out buffer, which is + # why the call came back as invalid argument. Carrying the whole handle + # also gives the archive something an importer in another rank can be + # matched against (ipc_handle_map). + "hipIpcGetMemHandle": [ + Deref("handle", "hipIpcMemHandle_t", 64, direction="out", + playback="manual"), + ], + # Which device's view of the mapping is being asked about, and what the + # answer was. The location was a stale struct pointer, which is why this + # was a NOOP; carrying its eight bytes makes the query answerable, and the + # recorded flags make the recording self-describing. + "hipMemGetAccess": [ + Deref("location", "hipMemLocation", 8), + Deref("flags", "unsigned long long", 8, direction="out"), + ], + # The pre-chevron launch ABI pushes `size` bytes from a host address into + # the pending kernarg buffer. Recorded as a bare address, replay handed the + # runtime a pointer into the capturing process and the read segfaulted. + # 256 bytes covers any single kernel argument, struct arguments included. + "hipSetupArgument": [ + Deref("arg", "unsigned char", 1, count="size", max_count=256), + ], + # The symbol name is the entire question these two ask, and it was recorded + # as an address with none of the characters behind it: measured as a SIGSEGV + # inside hip::hipGetProcAddress_common (hip_device.cpp:880). + "hipGetProcAddress": [ + Deref("symbol", "char", 1, max_count=256, string=True), + ], + "hipGetProcAddress_spt": [ + Deref("symbol", "char", 1, max_count=256, string=True), + ], + # deviceVar is the symbol's name in the code object, and it is the only + # part of a registration that means anything in another process: `var` is + # a host shadow address and `modules` a fat-binary handle. Recording the + # name is what lets replay resolve the same global through + # hipModuleGetGlobal and map the recorded device address onto it. + "__hipRegisterVar": [ + Deref("deviceVar", "char", 1, max_count=256, string=True), + ], + # The JIT linker's option arrays. Both were recorded as addresses and then + # replayed as if they were out parameters, so a link was rebuilt with + # whatever the uninitialised locals held. Thirty-two options is more than + # any caller passes. + "hipLinkCreate": [ + Deref("options", "hipJitOption", 4, count="numOptions", max_count=32), + Deref("optionValues", "void*", 8, count="numOptions", max_count=32), + ], + "hipLinkAddData": [ + Deref("name", "char", 1, max_count=256, string=True), + Deref("options", "hipJitOption", 4, count="numOptions", max_count=32), + Deref("optionValues", "void*", 8, count="numOptions", max_count=32), + ], + "hipLinkAddFile": [ + Deref("path", "char", 1, max_count=256, string=True), + Deref("options", "hipJitOption", 4, count="numOptions", max_count=32), + Deref("optionValues", "void*", 8, count="numOptions", max_count=32), + ], + # The device properties being matched against are the whole question these + # two ask. 1472 bytes is a large event, but a program calls hipChooseDevice + # once at startup, not in a loop. + "hipChooseDevice": [ + Deref("prop", "hipDeviceProp_t", 1472), + ], + "hipChooseDeviceR0000": [ + Deref("properties", "hipDeviceProp_tR0000", 792), + ], + # The format decides the answer: the maximum width depends on the element + # size the caller asked about. + "hipDeviceGetTexture1DLinearMaxWidth": [ + Deref("fmtDesc", "hipChannelFormatDesc", 20), + Deref("maxWidthInElements", "size_t", 8, direction="out"), + ], + # The attribute value is the whole call, and the union is 64 bytes of it. + "hipGraphKernelNodeSetAttribute": [ + Deref("value", "hipKernelNodeAttrValue", 64), + ], + # Which device's pool is being redirected. Recorded as a stale struct + # pointer this returned hipErrorInvalidValue at replay; the pool handle + # beside it was already translated through mempool_map. + "hipMemSetMemPool": [ + Deref("location", "hipMemLocation", 8), + ], + "hipMemGetMemPool": [ + Deref("location", "hipMemLocation", 8), + ], + # The batch copies are five parallel arrays and an op list. Recorded as + # addresses they were five stale pointers, which is why the handler + # returned an error rather than copying anything. Sixteen entries inline + # covers the batches this suite issues and warns at capture beyond that. + "hipMemcpyBatchAsync": [ + Deref("dsts", "void*", 8, count="count", max_count=16, elem_ptr=True), + Deref("srcs", "void*", 8, count="count", max_count=16, elem_ptr=True), + Deref("sizes", "size_t", 8, count="count", max_count=16), + Deref("attrs", "hipMemcpyAttributes", 24, count="numAttrs", + max_count=16), + Deref("attrsIdxs", "size_t", 8, count="numAttrs", max_count=16), + Deref("failIdx", "size_t", 8, direction="out"), + ], + # Each operand is a tagged union of a pointer and an array handle, so + # which member holds the recorded address depends on the tag beside it — + # a rewrite no table entry can express. + "hipMemcpy3DBatchAsync": [ + Deref("opList", "hipMemcpy3DBatchOp", 112, count="numOps", + max_count=16, playback="manual"), + Deref("failIdx", "size_t", 8, direction="out", playback="manual"), + ], +} + +# --------------------------------------------------------------------------- +# Graph node construction and mutation +# +# Every hipGraphAdd*Node call takes the same two things — a dependency array of +# hipGraphNode_t and a node-parameter struct — and until now neither reached +# the archive: the array was recorded as one stale address and the struct not +# at all. That is why explicit graph construction was a blanket exclusion. +# +# Both are ordinary Deref shapes. The dependency array is an array of handles +# (elem_handle), the parameter structs are single pointees whose device +# addresses and array handles are rewritten at replay (ptr_members / +# handle_members). The node handle each call produces is registered in +# graph_node_map by _HANDLE_CREATE_APIS, which is what makes a later +# dependency reference or *NodeSetParams call resolvable. +# +# A recorded graph rarely has many roots feeding one node; 16 inline +# dependencies covers real graphs and the overflow warns at capture. +# --------------------------------------------------------------------------- +_MAX_DEPS = 16 + +def _deps(param: str, count: str = "numDependencies") -> Deref: + return Deref(param, "hipGraphNode_t", 8, count=count, max_count=_MAX_DEPS, + elem_handle="hipGraphNode_t") + + +# hipMemsetParams::dst and hipMemcpy3DParms' pitched pointers and array handles +# are the recorded-address members inside each node-parameter struct. +_MEMSET_PARAMS = dict(ctype="hipMemsetParams", size=48, ptr_members=("dst",)) +_MEMCPY3D_PARAMS = dict( + ctype="hipMemcpy3DParms", size=160, + ptr_members=("srcPtr.ptr", "dstPtr.ptr"), + handle_members=(("srcArray", "hipArray_t"), ("dstArray", "hipArray_t")), +) + +DEREF_FIELDS.update({ + # --- construction: dependency array only --- + "hipGraphAddEmptyNode": [_deps("pDependencies")], + "hipGraphAddEventRecordNode": [_deps("pDependencies")], + "hipGraphAddEventWaitNode": [_deps("pDependencies")], + "hipGraphAddChildGraphNode": [_deps("pDependencies")], + "hipGraphAddMemcpyNode1D": [_deps("pDependencies")], + # The symbol copies. Their src/dst is a host buffer on the H2D and D2H + # spellings, so it is carried as a blob by the manual shim rather than + # dereferenced here; only the dependency array is declarative. + "hipGraphAddMemcpyNodeToSymbol": [_deps("pDependencies")], + "hipGraphAddMemcpyNodeFromSymbol": [_deps("pDependencies")], + "hipGraphAddMemFreeNode": [_deps("pDependencies")], + "hipDrvGraphAddMemFreeNode": [_deps("dependencies")], + # from[] and to[] are parallel arrays under one count. + "hipGraphAddDependencies": [_deps("from"), _deps("to")], + # --- construction: dependency array plus a node-parameter struct --- + "hipGraphAddMemsetNode": [ + _deps("pDependencies"), + Deref("pMemsetParams", **_MEMSET_PARAMS), + ], + "hipDrvGraphAddMemsetNode": [ + _deps("dependencies"), + Deref("memsetParams", **_MEMSET_PARAMS), + ], + "hipGraphAddMemcpyNode": [ + _deps("pDependencies"), + Deref("pCopyParams", **_MEMCPY3D_PARAMS), + ], + # A kernel node names its kernel by host function address and carries a + # void** argument array, which is the kernel-launch encoder's problem, not + # a memcpy's. + "hipGraphAddKernelNode": [ + _deps("pDependencies"), + Deref("pNodeParams", "hipKernelNodeParams", 64, playback="manual"), + ], + # dptr is written by the call and must land in alloc_map, and poolProps + # names a device — hand-written on the replay side. + "hipGraphAddMemAllocNode": [ + _deps("pDependencies"), + Deref("pNodeParams", "hipMemAllocNodeParams", 120, direction="inout", + playback="manual"), + ], + # HIP_MEMCPY3D distinguishes host from device source by a member, so the + # host case needs a blob rather than a pointer rewrite. + "hipDrvGraphAddMemcpyNode": [ + _deps("dependencies"), + Deref("copyParams", "HIP_MEMCPY3D", 184, playback="manual"), + ], + # nodeParams.paramArray is a second pointer hop. + "hipGraphAddBatchMemOpNode": [ + _deps("dependencies"), + Deref("nodeParams", "hipBatchMemOpNodeParams", 32, playback="manual"), + ], + # --- mutation: node parameters, no dependency array --- + "hipGraphMemsetNodeSetParams": [Deref("pNodeParams", **_MEMSET_PARAMS)], + "hipGraphExecMemsetNodeSetParams": [Deref("pNodeParams", **_MEMSET_PARAMS)], + "hipDrvGraphExecMemsetNodeSetParams": [Deref("memsetParams", **_MEMSET_PARAMS)], + "hipGraphMemcpyNodeSetParams": [Deref("pNodeParams", **_MEMCPY3D_PARAMS)], + "hipGraphExecMemcpyNodeSetParams": [Deref("pNodeParams", **_MEMCPY3D_PARAMS)], + "hipGraphKernelNodeSetParams": [ + Deref("pNodeParams", "hipKernelNodeParams", 64, playback="manual"), + ], + "hipGraphExecKernelNodeSetParams": [ + Deref("pNodeParams", "hipKernelNodeParams", 64, playback="manual"), + ], + "hipDrvGraphMemcpyNodeSetParams": [ + Deref("nodeParams", "HIP_MEMCPY3D", 184, playback="manual"), + ], + "hipDrvGraphExecMemcpyNodeSetParams": [ + Deref("copyParams", "HIP_MEMCPY3D", 184, playback="manual"), + ], + "hipGraphBatchMemOpNodeSetParams": [ + Deref("nodeParams", "hipBatchMemOpNodeParams", 32, direction="inout", + playback="manual"), + ], + "hipGraphExecBatchMemOpNodeSetParams": [ + Deref("nodeParams", "hipBatchMemOpNodeParams", 32, playback="manual"), + ], + # Beginning a capture into an explicitly named graph. As a NOOP the begin + # never happened at replay, so the stream was not capturing when + # hipStreamEndCapture arrived and that call returned 401 — a silent no-op + # billing a real handler two events later. + "hipStreamBeginCaptureToGraph": [ + _deps("dependencies", count="numDependencies"), + Deref("dependencyData", "hipGraphEdgeData", 8, count="numDependencies", + max_count=_MAX_DEPS), + ], +}) + + +def deref_specs(api: str) -> List[Deref]: + return DEREF_FIELDS.get(api, []) + + +def deref_for_param(api: str, param: str) -> Optional[Deref]: + for d in deref_specs(api): + if d.param == param: + return d + return None + + +def deref_covered_params(api: str) -> Set[str]: + """Parameters whose pointee reaches the archive. + + Read by the matrix's payload-loss detector (derive_manifest.py): a + const-struct-pointer argument that is carried inline is no longer a loss. + """ + return {d.param for d in deref_specs(api)} + + +# --------------------------------------------------------------------------- +# Structs passed BY VALUE that do not fit in a uint64_t +# +# normalise_field_type() lowers an unrecognised by-value type to a single +# uint64_t and _fill_param() then records a zero for it, so a 24-byte hipExtent +# or a 64-byte hipIpcMemHandle_t reaches the archive as nothing at all. These +# are recorded as inline bytes instead: `uint8_t _bytes[N]`, memcpy'd at +# capture and memcpy'd back into a local at replay. +# +# ptr_members names the members holding recorded device addresses; replay +# translates each through ctx.translate_ptr, because a faithfully restored +# struct still points into the capturing process otherwise. +# --------------------------------------------------------------------------- + +@dataclass(frozen=True) +class ByValueStruct: + size: int + ptr_members: Tuple[str, ...] = () + + +BY_VALUE_STRUCTS: Dict[str, ByValueStruct] = { + "hipExtent": ByValueStruct(24), + "hipPos": ByValueStruct(24), + "hipPitchedPtr": ByValueStruct(32, ptr_members=("ptr",)), + "hipMemLocation": ByValueStruct(8), + "hipChannelFormatDesc": ByValueStruct(20), + "hipIpcMemHandle_t": ByValueStruct(64), + "hipIpcEventHandle_t": ByValueStruct(64), +} + + +def by_value_struct(raw_type: str) -> Optional[ByValueStruct]: + """The by-value inline-bytes spec for a parameter type, if it has one.""" + t = re.sub(r"\b(const|volatile|restrict|struct)\b", " ", raw_type) + if "*" in t: + return None + parts = t.split() + return BY_VALUE_STRUCTS.get(parts[0]) if parts else None + + # Maps the base name of a uint8_t inline-array field (e.g. "pool_props_bytes") # to the C struct type whose sizeof must match the declared array extent. # Used to emit static_assert(sizeof(CType) == N) immediately after each struct @@ -849,24 +1423,21 @@ "hipMemPool_t", "hipGraph_t", "hipGraphNode_t", "hipGraphExec_t", "hipUserObject_t", "hipMemGenericAllocationHandle_t", "hipExternalMemory_t", "hipExternalSemaphore_t", - "hipKernel_t", "hipLibrary_t", + "hipKernel_t", "hipLibrary_t", "hipLinkState_t", # HIP 7.14+ green context / device resource handles "hipExecutionCtx_t", "hipDevResourceDesc_t", } -# Types that cannot be cast to uint64_t (structs by value, function pointers) +# Types that cannot be cast to uint64_t and are recorded as zero. +# By-value structs are NOT in here: they are carried as inline bytes instead +# (BY_VALUE_STRUCTS). What is left is genuinely unrecordable — a function +# pointer or an opaque handle whose value means nothing outside the capturing +# process. Anything here that an API actually depends on belongs in +# UNREPLAYABLE_PLAYBACK_APIS, so replay says so instead of passing a null. _NON_CASTABLE_TYPES = { - "hipGraphicsResource_t", # struct* - "hipIpcEventHandle_t", # struct by value - "hipIpcMemHandle_t", # struct by value - "hipPitchedPtr", # struct by value - "hipExtent", # struct by value - "hipPos", # struct by value - "hipMemLocation", # struct with enum+int + "hipGraphicsResource_t", # opaque GL interop resource "hipStreamCallback_t", # function pointer "hipHostFn_t", # function pointer - "hipLinkState_t", # opaque ptr (treat as non-castable) - "hipChannelFormatDesc", # struct by value } # Enum types -> int32_t @@ -884,6 +1455,14 @@ "hipTextureFilterMode", "hipTextureAddressMode", "hipTextureMipmapFilterMode", "hipResourcetype", "hipResourceViewFormat", "hipChannelFormatKind", "hipKernelAttribute", + # Enums the generator used to lower to `uint64_t /* T */`, which reads as a + # dropped by-value struct to anything inspecting the generated types (the + # matrix's payload-loss detector counted all eight). Every one is a 4-byte + # enum whose value survives intact; naming them here says so. + "hipArray_Format", "hipDevResourceType", "hipFunction_attribute", + "hipGraphMemAttributeType", "hipJitInputType", + "hipMemAllocationHandleType", "hipMemRangeHandleType", + "hipPointer_attribute", "hipStreamAttrID", "hipGLDeviceList", } # Scalar type map: normalised type string -> C field type @@ -907,17 +1486,22 @@ ("double", "double"), ("bool", "uint8_t"), ("char", "int8_t"), + # OpenGL interop spellings of unsigned int (the GL headers are not included + # here, so these do not resolve through the scalar keywords above). + ("GLenum", "uint32_t"), + ("GLuint", "uint32_t"), ] def normalise_field_type(raw_type: str) -> str: """ Convert a C parameter type string to the field type used in the struct. - Returns a sentinel "__DIM3__" for dim3/uint3 (caller expands to x/y/z). + Returns a sentinel "__DIM3__" for dim3/uint3 (caller expands to x/y/z), and + "__BYVAL__" for a by-value struct carried as inline bytes (BY_VALUE_STRUCTS). """ t = raw_type.strip() # strip const/volatile - t_nc = re.sub(r'\b(const|volatile|restrict)\b', '', t).strip() + t_nc = re.sub(r'\b(const|volatile|restrict|struct)\b', '', t).strip() t_nc = re.sub(r'\s+', ' ', t_nc) # Any pointer -> uint64_t @@ -929,6 +1513,10 @@ def normalise_field_type(raw_type: str) -> str: if base in ("dim3", "uint3"): return "__DIM3__" + # By-value struct too large for a uint64_t -> inline bytes + if base in BY_VALUE_STRUCTS: + return "__BYVAL__" + # Opaque handle if base in _HANDLE_TYPES: return "uint64_t" @@ -960,6 +1548,10 @@ def field_lines(self) -> List[str]: f"uint32_t {safe}_y;", f"uint32_t {safe}_z;", ] + if ft == "__BYVAL__": + bv = by_value_struct(self.raw_type) + assert bv is not None + return [f"uint8_t {safe}_bytes[{bv.size}];"] return [f"{ft} {safe};"] @@ -972,6 +1564,7 @@ class ApiEntry: # Retired or unparsable dispatch-table slot. Occupies an hrr_api_id_t so # later members keep their IDs; no capture shim is installed. reserved: bool = False + reserved_reason: str = "" # --------------------------------------------------------------------------- @@ -1190,7 +1783,7 @@ def reserve(func_name: str, table: str, why: str) -> None: print(f"WARNING: {why} for {func_name}; reserving ID so later slots " f"do not shift", file=sys.stderr) entries.append(ApiEntry(name=func_name, ret_type="void", params=[], - table=table, reserved=True)) + table=table, reserved=True, reserved_reason=why)) def add_entry(func_name: str, table: str, reserved: bool = False) -> None: if reserved: @@ -1249,6 +1842,39 @@ def add_entry(func_name: str, table: str, reserved: bool = False) -> None: return entries +def validate_capture_coverage(entries: List[ApiEntry], + manual_capture_source: Path, + silent: bool = False) -> None: + """Validate reserved slots and hand-written capture shim definitions.""" + errors: List[str] = [] + if not manual_capture_source.is_file(): + errors.append(f"manual capture source is missing: {manual_capture_source}") + manual_capture_text = "" + else: + manual_capture_text = manual_capture_source.read_text(encoding="utf-8") + + for entry in entries: + if entry.reserved: + if entry.reserved_reason != "retired void* dispatch slot": + errors.append(f"{entry.table}:{entry.name}: {entry.reserved_reason}") + continue + if entry.name not in MANUAL_CAPTURE_APIS: + continue + definition = re.compile( + r"(?m)^[^#\n]*\bcapture_" + re.escape(entry.name) + r"\s*\(") + if not definition.search(manual_capture_text): + errors.append( + f"{entry.table}:{entry.name}: manual capture shim definition is " + f"missing from {manual_capture_source}") + + if errors: + sys.exit("ERROR: HRR capture coverage check failed:\n " + "\n ".join(errors)) + if not silent: + manual_count = sum( + not entry.reserved and entry.name in MANUAL_CAPTURE_APIS for entry in entries) + print(f" HRR capture coverage: {manual_count} manual shims verified") + + # --------------------------------------------------------------------------- # Header generation # --------------------------------------------------------------------------- @@ -1269,7 +1895,7 @@ def add_entry(func_name: str, table: str, reserved: bool = False) -> None: * One packed struct per HIP API covering both HipDispatchTable (runtime) and * HipCompilerDispatchTable (compiler stubs). * - * Archive format (v5): + * Archive format (v6): * events.bin: * [0..7] hrr_file_header { magic, version, reserved } * [8..] hrr_event_header (32 bytes) + payload bytes, repeated per event @@ -1294,6 +1920,14 @@ def add_entry(func_name: str, table: str, reserved: bool = False) -> None: * Extra fields on selected APIs (blob hashes, module_id) are appended AFTER * the normal parameters — see EXTRA_FIELDS in gen_hrr_api_args.py. * + * Dereferenced pointer arguments (DEREF_FIELDS in gen_hrr_api_args.py) add, + * for a pointer argument whose pointee must survive into the archive: + * - uint8_t _bytes[N] the pointee's bytes, copied at capture + * - uint8_t _present 1 when the argument was non-null + * - uint32_t _n element count, for array arguments only + * Without them a pointer argument reaches the archive as a capture-time host + * address and nothing else, which is the payload-loss class of section 8.3. + * * The structs use #pragma pack(1) so layout is identical on all platforms. * ============================================================================ */ @@ -1317,8 +1951,12 @@ def add_entry(func_name: str, table: str, reserved: bool = False) -> None: * HipDispatchTable member takes the first compiler ID and shifts the whole * compiler tail up by one. Compiler APIs do write events, so that second * case still needs a version bump. A retired dispatch-table slot (nulled - * void*) still occupies an ID. */ -#define HRR_VERSION ((uint16_t)5u) + * void*) still occupies an ID. + * v6: pointer arguments whose pointee used to be dropped now carry it inline + * (DEREF_FIELDS). Event payloads grew for ~50 APIs, so an archive written + * before v6 cannot be read by a v6 reader: re-capture rather than replay an + * old recording. */ +#define HRR_VERSION ((uint16_t)6u) /* Written once at byte 0 of events.bin. */ #pragma pack(push, 1) @@ -1424,9 +2062,17 @@ def generate_struct(entry: ApiEntry) -> str: lines.append(" uint64_t ret;") else: ft = normalise_field_type(entry.ret_type) - if ft == "__DIM3__": + if ft == "__BYVAL__": + # A by-value struct return (hipCreateChannelDesc). Never + # recorded — such an API is a capture pass-through — but the + # field has to have a size the archive can round-trip. + bv = by_value_struct(entry.ret_type) + lines.append(f" uint8_t ret_bytes[{bv.size}];") + ft = None + elif ft == "__DIM3__": ft = "uint64_t" - lines.append(f" {ft} ret;") + if ft is not None: + lines.append(f" {ft} ret;") # Parameters — special-case __hipRegisterFatBinary raw data* -> blob fields if entry.name == "__hipRegisterFatBinary": @@ -1444,9 +2090,23 @@ def generate_struct(entry: ApiEntry) -> str: if ft == "__DIM3__": for suffix in ('_x', '_y', '_z'): lines.append(f" uint32_t {safe}{suffix};") + elif ft == "__BYVAL__": + bv = by_value_struct(param.raw_type) + base = _get_base_type(param.raw_type) + lines.append(f" uint8_t {safe}_bytes[{bv.size}];" + f" /* {base} passed by value, inline copy */") else: lines.append(f" {ft} {safe};") + # Dereferenced pointer arguments — the pointee, carried inline + for d in deref_specs(entry.name): + what = (f"{d.ctype}[{d.max_count}] inline copy" if d.is_array + else f"{d.ctype} inline copy") + lines.append(f" uint8_t {d.bytes_field}[{d.total_bytes}]; /* {what} */") + lines.append(f" uint8_t {d.present_field}; /* 1 when {d.param} was non-null */") + if d.is_array: + lines.append(f" uint32_t {d.count_field}; /* elements copied from {d.param} */") + # Extra fields for (ftype, fname, fcomment) in EXTRA_FIELDS.get(entry.name, []): lines.append(f" {ftype} {fname}; /* {fcomment} */") @@ -1472,6 +2132,24 @@ def generate_struct(entry: ApiEntry) -> str: f"static_assert(sizeof({ctype}) <= {n}," f' "hrr_args_{entry.name}::{fname} too small for {ctype}");' ) + # A Deref or by-value field that reserves fewer bytes than the value needs + # would truncate silently at capture, which is the failure the whole + # mechanism exists to remove. Catch it in the compiler instead. + for d in deref_specs(entry.name): + asserts.append( + f"static_assert(sizeof({d.ctype}) <= {d.size}," + f' "hrr_args_{entry.name}::{d.bytes_field} too small for {d.ctype}");' + ) + for param in entry.params: + if normalise_field_type(param.raw_type) != "__BYVAL__": + continue + bv = by_value_struct(param.raw_type) + base = _get_base_type(param.raw_type) + asserts.append( + f"static_assert(sizeof({base}) <= {bv.size}," + f' "hrr_args_{entry.name}::{param.name or "unnamed"}_bytes' + f' too small for {base}");' + ) if asserts: lines.append("#ifdef HIP_INCLUDE_HIP_HIP_RUNTIME_H") lines.extend(asserts) @@ -1562,9 +2240,18 @@ def generate_header(entries: List[ApiEntry]) -> str: #include "hrr/hrr_api_args.h" #include "hip/amd_detail/hip_api_trace.hpp" +// LogPrintfWarning — capture diagnostics go through amd's log-level machinery, +// never raw stderr. debug.hpp's ClPrint macro reads the AMD_LOG_LEVEL and +// AMD_LOG_MASK flag variables but does not declare them, so flags.hpp must come +// with it, and flags.hpp spells its defaults with top.hpp's unit suffixes (Mi) +// and typedefs (uint). All three, in this order. +#include "top.hpp" +#include "utils/flags.hpp" +#include "utils/debug.hpp" #include #include +#include #include // These global tables are defined (non-static) in hip_capture.cpp @@ -1624,6 +2311,11 @@ def _ret_type_to_error_value(ret: str) -> Optional[str]: def _is_output_ptr_param(param: Param) -> bool: """True if this param is an output pointer (T** or handle type* or int-typedef*).""" t = param.raw_type.strip() + # A single-level pointer to const is something the API reads, whatever it + # points at. Without this, a `const hipGraphNode_t*` dependency array reads + # as an output handle and both ends treat an array as one node. + if t.count('*') == 1 and re.match(r'^\s*const\b', t): + return False if t.count('*') >= 2: return True # Single pointer to any handle type → output @@ -1634,6 +2326,7 @@ def _is_output_ptr_param(param: Param) -> bool: 'hipSurfaceObject_t', 'hipTextureObject_t', 'hipCtx_t', 'hipUserObject_t', 'hipMemGenericAllocationHandle_t', + 'hipLinkState_t', ) for handle in _OUTPUT_HANDLE_TYPES: if handle + '*' in t or handle + ' *' in t: @@ -1662,6 +2355,8 @@ def _fill_param(lines: List[str], p: Param, name: str, ft: str) -> None: lines.append(f" a.{name}_x = {name}.x;") lines.append(f" a.{name}_y = {name}.y;") lines.append(f" a.{name}_z = {name}.z;") + elif ft == "__BYVAL__": + lines.append(f" std::memcpy(a.{name}_bytes, &{name}, sizeof({name}));") elif base in _NON_CASTABLE_TYPES: lines.append(f" a.{name} = 0; // non-castable type skipped") elif base == 'hipDevice_t': @@ -1678,6 +2373,76 @@ def _fill_param(lines: List[str], p: Param, name: str, ft: str) -> None: lines.append(f" a.{name} = static_cast({name});") +def _fill_derefs(lines: List[str], entry: ApiEntry) -> None: + """Emit the memcpy that carries each dereferenced pointee into the event. + + Runs after the real call, alongside the other fills: the arguments a HIP + API reads are still valid there, and an output argument has been written + by then. + + A pointee too large for its fixed-size field is clipped. The warning alone + is not enough: AMD_LOG_LEVEL defaults to 0, so nothing durable would record + that the payload is partial and replay would treat it as faithful. Each + truncation therefore also calls note_unreplayable(), which lists the API, + with the clipped argument as the reason, under manifest.unreplayable_apis + where replay can see it. + + note_unreplayable(), not mark_incomplete(): the event is written and the + archive stays well-formed — only the clipped argument makes the call + impossible to reproduce exactly. mark_incomplete() means an event was + dropped or torn and suppresses the clean-shutdown trailer, which is a + different and stronger claim. See hip_capture_writer.h. + """ + for d in deref_specs(entry.name): + if d.string: + lines.append(f" if ({d.param}) {{") + lines.append(f" size_t _n = std::strlen({d.param});") + lines.append(f" if (_n > {d.max_count - 1}u) {{") + lines.append(f" static bool warned_{d.param} = false;") + lines.append(f" if (!warned_{d.param}) {{") + lines.append(f" warned_{d.param} = true;") + lines.append(f" LogPrintfWarning(") + lines.append(f" \"[HRR] {entry.name}: {d.param} is %zu characters; \"") + lines.append(f" \"recording the first {d.max_count - 1} only\", _n);") + lines.append(f" hrr_cap::writer::note_unreplayable(\"{entry.name}\",") + lines.append(f" \"{d.param} was truncated to {d.max_count - 1} characters at capture \"") + lines.append(f" \"time; replay would pass a shortened string\");") + lines.append(f" }}") + lines.append(f" _n = {d.max_count - 1}u;") + lines.append(f" }}") + lines.append(f" std::memcpy(a.{d.bytes_field}, {d.param}, _n);") + lines.append(f" a.{d.bytes_field}[_n] = 0;") + lines.append(f" a.{d.present_field} = 1;") + lines.append(f" }}") + elif d.is_array: + lines.append(f" if ({d.param} && {d.count} > 0) {{") + lines.append(f" uint32_t _n = static_cast({d.count});") + lines.append(f" if (_n > {d.max_count}u) {{") + lines.append(f" static bool warned_{d.param} = false;") + lines.append(f" if (!warned_{d.param}) {{") + lines.append(f" warned_{d.param} = true;") + lines.append(f" LogPrintfWarning(") + lines.append(f" \"[HRR] {entry.name}: recording only the first {d.max_count} \"") + lines.append(f" \"of %u {d.param} entries; replay of this call will be \"") + lines.append(f" \"incomplete\", _n);") + lines.append(f" hrr_cap::writer::note_unreplayable(\"{entry.name}\",") + lines.append(f" \"only the first {d.max_count} {d.param} entries were recorded at \"") + lines.append(f" \"capture time; replay would pass a partial array\");") + lines.append(f" }}") + lines.append(f" _n = {d.max_count}u;") + lines.append(f" }}") + lines.append(f" std::memcpy(a.{d.bytes_field}, {d.param}," + f" static_cast(_n) * sizeof({d.ctype}));") + lines.append(f" a.{d.count_field} = _n;") + lines.append(f" a.{d.present_field} = 1;") + lines.append(f" }}") + else: + lines.append(f" if ({d.param}) {{") + lines.append(f" std::memcpy(a.{d.bytes_field}, {d.param}, sizeof({d.ctype}));") + lines.append(f" a.{d.present_field} = 1;") + lines.append(f" }}") + + def _fill_output_param_post(lines: List[str], p: Param, name: str) -> None: """Emit lines that fill a.name AFTER the real call for output pointer params.""" t = p.raw_type.strip() @@ -1828,7 +2593,9 @@ def generate_shim(entry: ApiEntry) -> str: # Post-call output params for p, name, ft in output_params: _fill_output_param_post(lines, p, name) + _fill_derefs(lines, entry) enum_name = "HRR_API_" + entry.name.lstrip('_').upper() + _note_unreplayable(lines, entry) lines.append(f" hrr_cap::writer::write_event_raw({enum_name}, &a.hdr, sizeof(a));") lines.append(f" }}") else: @@ -1859,7 +2626,9 @@ def generate_shim(entry: ApiEntry) -> str: # Post-call: fill output ptr fields from dereferenced values for p, name, ft in output_params: _fill_output_param_post(lines, p, name) + _fill_derefs(lines, entry) enum_name = "HRR_API_" + entry.name.lstrip('_').upper() + _note_unreplayable(lines, entry) lines.append(f" hrr_cap::writer::write_event_raw({enum_name}, &a.hdr, sizeof(a));") lines.append(f" }}") lines.append(f" return r;") @@ -1869,6 +2638,16 @@ def generate_shim(entry: ApiEntry) -> str: return "\n".join(lines) +def _note_unreplayable(lines: List[str], entry: ApiEntry) -> None: + """An API replay cannot reproduce is worth knowing about while the recording + is being made, not only when someone later tries to replay it. Emitted next + to write_event_raw() so manifest.json lists the API only when an event was + actually recorded — a failed call writes no event.""" + if entry.name in UNREPLAYABLE_PLAYBACK_APIS: + lines.append(f' hrr_cap::writer::note_unreplayable("{entry.name}",') + lines.append(f' "{UNREPLAYABLE_PLAYBACK_APIS[entry.name]}");') + + def generate_build_table(entries: List[ApiEntry]) -> str: """Generate hip_capture_build_table() and hip_capture_build_compiler_table().""" runtime_entries = [e for e in entries if e.table == "runtime"] @@ -1992,11 +2771,45 @@ def generate_capture_cpp(entries: List[ApiEntry]) -> str: 'hipArray_t': ('ctx.translate_array', 'ctx.record_array', 'ctx.remove_array'), 'hipMipmappedArray_t': ('ctx.translate_mipmapped', 'ctx.record_mipmapped', 'ctx.remove_mipmapped'), 'hipGraph_t': ('ctx.translate_graph', 'ctx.record_graph', 'ctx.remove_graph'), + 'hipGraphNode_t': ('ctx.translate_graph_node','ctx.record_graph_node','ctx.remove_graph_node'), 'hipGraphExec_t': ('ctx.translate_graph_exec','ctx.record_graph_exec','ctx.remove_graph_exec'), 'hipSurfaceObject_t': ('ctx.translate_surface', 'ctx.record_surface', 'ctx.remove_surface'), 'hipTextureObject_t': ('ctx.translate_texture', 'ctx.record_texture', 'ctx.remove_texture'), + 'hipCtx_t': ('ctx.translate_ctx', 'ctx.record_ctx', 'ctx.remove_ctx'), + 'hipLinkState_t': ('ctx.translate_link_state','ctx.record_link_state','ctx.remove_link_state'), } _PLAYBACK_HANDLE_TRANSLATE = {k: v[0] for k, v in _PLAYBACK_HANDLE_INFO.items()} +# The VMM handle has its own map (PlaybackContext::vmm_handle_map, filled by the +# hand-written hipMemCreate handler) but no record/remove pair, so it is not in +# the table above. Generated handlers still have to translate it: passing the +# recorded value names an allocation in the capturing process. +_PLAYBACK_HANDLE_TRANSLATE['hipMemGenericAllocationHandle_t'] = \ + 'ctx.translate_vmm_handle' + +# `void*` parameters that are really out buffers — the callee writes a handle +# through them. Nothing in the type says so, so the generator used to treat the +# recorded value as an ordinary opaque input and hand the runtime an address in +# the capturing process; measured outcome was a SIGSEGV inside +# hipMemExportToShareableHandle, which also cost every event recorded after it. +# The value is the number of bytes the callee writes, and the handler passes a +# zeroed local of that size. +# +# What the callee writes stays local to the replay: a POSIX fd or a Win32 +# HANDLE is meaningful only in the process that received it, and the archive +# has no way to hand it to another one. hipMemImportFromShareableHandle is a +# NOOP for the same reason, so nothing downstream consumes it. +_OUT_BUFFER_PARAMS: Dict[str, Dict[str, int]] = { + # An fd is an int and a Win32 HANDLE is a pointer; eight bytes covers both. + 'hipMemExportToShareableHandle': {'shareableHandle': 8}, + 'hipMemPoolExportToShareableHandle': {'shared_handle': 8}, + 'hipMemGetHandleForAddressRange': {'handle': 8}, + # A `char*` the callee fills rather than a string it reads: hipDeviceGetLuid + # memcpys the low and high parts of the adapter LUID into luid[0..7]. Without + # a size here the pointee rule would give it a single char to write eight + # bytes into. Only the Windows path writes; Linux returns hipErrorNotSupported + # before touching the buffer. + 'hipDeviceGetLuid': {'luid': 8}, +} # APIs that create device allocations: API name -> (rec_ptr_param, size_param) # rec_ptr_param is the name of the output param storing the address (in the struct) @@ -2037,6 +2850,9 @@ def generate_capture_cpp(entries: List[ApiEntry]) -> str: 'hipGraphExecDestroy': ('graphExec', 'hipGraphExec_t'), 'hipDestroySurfaceObject': ('surfaceObject', 'hipSurfaceObject_t'), 'hipDestroyTextureObject': ('textureObject', 'hipTextureObject_t'), + 'hipGraphDestroyNode': ('node', 'hipGraphNode_t'), + 'hipCtxDestroy': ('ctx', 'hipCtx_t'), + 'hipLinkDestroy': ('state', 'hipLinkState_t'), } # APIs that create handles: API name -> list of (output param name, handle type) @@ -2048,9 +2864,17 @@ def generate_capture_cpp(entries: List[ApiEntry]) -> str: 'hipStreamCreateWithPriority': [('stream', 'hipStream_t')], 'hipEventCreate': [('event', 'hipEvent_t')], 'hipEventCreateWithFlags': [('event', 'hipEvent_t')], - # Memory pools + # Memory pools. The two query spellings hand out a pool they did not + # create — usually the device default — and registering it is what lets a + # later hipMemSetMemPool or allocation name that pool. 'hipMemPoolCreate': [('mem_pool', 'hipMemPool_t')], - 'hipMemPoolImportFromShareableHandle': [('mem_pool', 'hipMemPool_t')], + 'hipDeviceGetMemPool': [('mem_pool', 'hipMemPool_t')], + 'hipDeviceGetDefaultMemPool': [('mem_pool', 'hipMemPool_t')], + 'hipMemGetMemPool': [('pool', 'hipMemPool_t')], + # JIT linker state. Without this the state hipLinkCreate hands out was + # dropped and hipLinkComplete / hipLinkDestroy were passed the capturing + # process's pointer, which is why both failed however good the inputs were. + 'hipLinkCreate': [('stateOut', 'hipLinkState_t')], # Arrays — param names from hip_api_trace.hpp 'hipArrayCreate': [('pHandle', 'hipArray_t')], 'hipArray3DCreate': [('array', 'hipArray_t')], @@ -2062,9 +2886,85 @@ def generate_capture_cpp(entries: List[ApiEntry]) -> str: 'hipGraphClone': [('pGraphClone', 'hipGraph_t')], 'hipGraphInstantiate': [('pGraphExec', 'hipGraphExec_t')], 'hipGraphInstantiateWithFlags': [('pGraphExec', 'hipGraphExec_t')], + # Graph nodes. Registering the node each construction call produces is + # what lets a later dependency list, *NodeSetParams or hipGraphDestroyNode + # name it; without this the node API replays as a set of unconnected calls. + 'hipGraphAddEmptyNode': [('pGraphNode', 'hipGraphNode_t')], + 'hipGraphAddEventRecordNode': [('pGraphNode', 'hipGraphNode_t')], + 'hipGraphAddEventWaitNode': [('pGraphNode', 'hipGraphNode_t')], + 'hipGraphAddChildGraphNode': [('pGraphNode', 'hipGraphNode_t')], + 'hipGraphAddMemsetNode': [('pGraphNode', 'hipGraphNode_t')], + 'hipGraphAddMemcpyNode': [('pGraphNode', 'hipGraphNode_t')], + 'hipGraphAddMemcpyNode1D': [('pGraphNode', 'hipGraphNode_t')], + 'hipGraphAddMemFreeNode': [('pGraphNode', 'hipGraphNode_t')], + 'hipDrvGraphAddMemsetNode': [('phGraphNode', 'hipGraphNode_t')], + 'hipDrvGraphAddMemFreeNode': [('phGraphNode', 'hipGraphNode_t')], + 'hipGraphNodeFindInClone': [('pNode', 'hipGraphNode_t')], # Texture / surface objects 'hipCreateSurfaceObject': [('pSurfObject', 'hipSurfaceObject_t')], 'hipCreateTextureObject': [('pTexObject', 'hipTextureObject_t')], + # Driver contexts. Every call that hands one out registers it, including + # the two that only report a context they did not create: a recording + # pushes back what it popped, and the pop is where replay learns which + # live context that recorded value stands for. + 'hipCtxCreate': [('ctx', 'hipCtx_t')], + 'hipCtxPopCurrent': [('ctx', 'hipCtx_t')], + 'hipCtxGetCurrent': [('ctx', 'hipCtx_t')], + 'hipDevicePrimaryCtxRetain': [('pctx', 'hipCtx_t')], +} + + +def _deref_translate_lines(var: str, ptr_members: Tuple[str, ...], + indent: str = " ", + handle_members: Tuple[Tuple[str, str], ...] = (), + ) -> List[str]: + """Rewrite the recorded device addresses and handles in a restored struct.""" + out = [] + for member in ptr_members: + out.append(f"{indent}{var}.{member} = ctx.translate_ptr(" + f"reinterpret_cast({var}.{member}));") + for (member, htype) in handle_members: + fn = _PLAYBACK_HANDLE_TRANSLATE[htype] + out.append(f"{indent}{var}.{member} = ({htype}){fn}(" + f"reinterpret_cast({var}.{member}));") + return out + + +def _graph_param(entry: ApiEntry) -> str: + """The name of the hipGraph_t this API adds to, or "" if it has none.""" + for p in entry.params: + t = p.raw_type.strip() + if '*' not in t and _get_base_type(t) == 'hipGraph_t' and p.name: + return p.name + return "" + + +# hipCtx_t arguments of the driver-API graph node calls. The recorded value is +# an address in the capturing process and nothing here maps it, so passing it +# through would hand the runtime a stale pointer. Null is not the answer +# either: hipDrvGraphAddMemcpyNode rejects a null context outright +# (hip_graph.cpp:1445). hrr_live_ctx substitutes the context the replaying +# thread is on, and preserves a recorded null so a call that failed that way at +# capture fails the same way here. +_LIVE_CTX_PLAYBACK_APIS: Set[str] = { + "hipDrvGraphAddMemsetNode", + "hipDrvGraphExecMemsetNodeSetParams", +} + +# Lines emitted just before the handler returns, for post-call bookkeeping the +# tables above cannot express. `_r` holds the call's result. +_POST_CALL_EXTRA: Dict[str, List[str]] = { + # A clone of a graph HRR could not rebuild is equally unbuildable, and + # nothing about the clone call itself would reveal that. + "hipGraphClone": [ + "if (_r == hipSuccess && ctx.graph_is_incomplete(a->originalGraph))", + " ctx.mark_graph_incomplete(a->pGraphClone, \"hipGraphClone\");", + ], + # Likewise a parent graph inherits whatever its child could not rebuild. + "hipGraphAddChildGraphNode": [ + "if (_r == hipSuccess && ctx.graph_is_incomplete(a->childGraph))", + " ctx.mark_graph_incomplete(a->graph, \"hipGraphAddChildGraphNode\");", + ], } @@ -2108,6 +3008,14 @@ def _apply_arg_bridge(bridge: Dict[str, object], entry: 'ApiEntry', call_args: List[str]) -> Tuple[List[str], List[str]]: """Return (pre_lines, call_args) with the bridged argument substituted.""" + # The bridge substitutes by position, so call_args must stay 1:1 with + # entry.params. generate_playback_shim appends exactly one argument per + # parameter today; assert it rather than silently bridging the wrong one. + if len(call_args) != len(entry.params): + sys.exit(f"ERROR: _PLAYBACK_ARG_BRIDGES[{entry.name!r}] cannot be applied: " + f"the generated call has {len(call_args)} arguments for " + f"{len(entry.params)} dispatch-table parameters, so arguments no " + f"longer map to parameters by position. Update _apply_arg_bridge.") pname = str(bridge['param']) idx = next((i for i, p in enumerate(entry.params) if p.name == pname), None) if idx is None: @@ -2120,12 +3028,110 @@ def _apply_arg_bridge(bridge: Dict[str, object], return pre, args -def _playback_arg(p: Param, name: str, pre_lines: List[str]) -> str: +def _playback_arg(entry: ApiEntry, p: Param, name: str, + pre_lines: List[str]) -> str: """Return the expression to pass to the real HIP API during playback.""" t = p.raw_type.strip() base = _get_base_type(t) is_output = _is_output_ptr_param(p) + # A declared out buffer — the recorded address belongs to the capturing + # process, so the callee writes into a local of the right size instead. + out_bytes = _OUT_BUFFER_PARAMS.get(entry.name, {}).get(name) + if out_bytes: + pre_lines.append(f" alignas(8) unsigned char _outbuf_{name}" + f"[{out_bytes}]{{}};") + return f"({t})_outbuf_{name}" + + # A by-value struct carried as inline bytes — restore it into a local. + bv = by_value_struct(t) + if bv is not None: + pre_lines.append(f" {base} _v_{name}{{}};") + pre_lines.append(f" std::memcpy(&_v_{name}, a->{name}_bytes," + f" sizeof(_v_{name}));") + pre_lines.extend(_deref_translate_lines(f"_v_{name}", bv.ptr_members)) + return f"_v_{name}" + + # A dereferenced pointer argument — rebuild the pointee and pass its address + # rather than casting the recorded capture-time address back to a pointer. + d = deref_for_param(entry.name, name) + if d is not None and d.playback == "auto": + if d.string: + # The recorded characters are already NUL-terminated inside the + # event, so the string is read straight out of the payload. + return (f"(a->{d.present_field} ? ({t})a->{d.bytes_field}" + f" : ({t})nullptr)") + if d.is_array: + pre_lines.append(f" {d.ctype} _d_{name}[{d.max_count}]{{}};") + pre_lines.append(f" uint32_t _d_{name}_n = a->{d.count_field} >" + f" {d.max_count}u ? {d.max_count}u : a->{d.count_field};") + pre_lines.append(f" (void)_d_{name}_n;") + if d.direction != "out": + pre_lines.append(f" if (a->{d.present_field})") + pre_lines.append(f" std::memcpy(_d_{name}, a->{d.bytes_field}," + f" static_cast(_d_{name}_n) * sizeof({d.ctype}));") + for member in d.ptr_members: + pre_lines.append(f" for (uint32_t _i = 0; _i < _d_{name}_n; ++_i)") + pre_lines.append(f" _d_{name}[_i].{member} = ctx.translate_ptr(" + f"reinterpret_cast(_d_{name}[_i].{member}));") + for (member, htype) in d.handle_members: + hfn = _PLAYBACK_HANDLE_TRANSLATE[htype] + pre_lines.append(f" for (uint32_t _i = 0; _i < _d_{name}_n; ++_i)") + pre_lines.append(f" _d_{name}[_i].{member} = ({htype}){hfn}(" + f"reinterpret_cast(_d_{name}[_i].{member}));") + if d.elem_ptr: + pre_lines.append(f" for (uint32_t _i = 0; _i < _d_{name}_n; ++_i)") + pre_lines.append(f" _d_{name}[_i] = ctx.translate_ptr(" + f"reinterpret_cast(_d_{name}[_i]));") + # An array of handles: every element is itself a recorded + # handle. A graph dependency list arrives as node addresses + # from the capturing process; passed through untranslated the + # runtime would either reject them or, worse, wire the node to + # whatever lives at that address here. + if d.elem_handle: + efn = _PLAYBACK_HANDLE_TRANSLATE[d.elem_handle] + gparam = _graph_param(entry) + pre_lines.append(f" for (uint32_t _i = 0; _i < _d_{name}_n; ++_i) {{") + pre_lines.append(f" {d.elem_handle} _live = ({d.elem_handle}){efn}(" + f"reinterpret_cast(_d_{name}[_i]));") + pre_lines.append(f" if (!_live && _d_{name}[_i]) {{") + pre_lines.append(f" fprintf(stderr, \"[HRR] {entry.name}: dependency \"") + pre_lines.append(f" \"0x%llx was never built at replay, so this call is \"") + pre_lines.append(f" \"skipped and the graph it belongs to is marked \"") + pre_lines.append(f" \"incomplete; instantiating that graph fails loudly \"") + pre_lines.append(f" \"rather than running one that is missing an \"") + pre_lines.append(f" \"ordering constraint.\\n\",") + pre_lines.append(f" (unsigned long long)reinterpret_cast(_d_{name}[_i]));") + if gparam: + pre_lines.append(f" ctx.mark_graph_incomplete(a->{gparam}," + f" \"{entry.name}\");") + pre_lines.append(f" return hipSuccess;") + pre_lines.append(f" }}") + pre_lines.append(f" _d_{name}[_i] = _live;") + pre_lines.append(f" }}") + # An empty array is passed as a null pointer, not as the address of + # an empty local: hipDrvGraphAddMemFreeNode (hip_graph.cpp:3502) + # rejects a non-null dependency pointer with a zero count outright, + # and a zero count is what every node added without dependencies + # has. + return f"({t})(_d_{name}_n ? _d_{name} : nullptr)" + pre_lines.append(f" {d.ctype} _d_{name}{{}};") + # An output pointee is written by the call, so what matters is that the + # buffer is the right size — the recorded value is carried for fidelity + # and comparison, not to be handed back to the runtime. Passing a local + # of the pointee's own type is also what stops the generic + # output-pointer path from handing a 64-byte write an 8-byte void*. + if d.direction == "out": + return f"({t})&_d_{name}" + pre_lines.append(f" if (a->{d.present_field})") + pre_lines.append(f" std::memcpy(&_d_{name}, a->{d.bytes_field}," + f" sizeof(_d_{name}));") + pre_lines.extend(_deref_translate_lines(f"_d_{name}", d.ptr_members, + handle_members=d.handle_members)) + if d.direction == "inout": + return f"({t})&_d_{name}" + return f"(a->{d.present_field} ? ({t})&_d_{name} : ({t})nullptr)" + # Output pointer — declare local, pass address of local if is_output: # Check all known handle types @@ -2144,6 +3150,9 @@ def _playback_arg(p: Param, name: str, pre_lines: List[str]) -> str: pre_lines.append(f" void* _out_{name} = nullptr;") return f"(void**)&_out_{name}" + if base == 'hipCtx_t' and entry.name in _LIVE_CTX_PLAYBACK_APIS: + return f"hrr_live_ctx(a->{name})" + # Handle types — translate recorded handle to live handle if base in _PLAYBACK_HANDLE_TRANSLATE: fn = _PLAYBACK_HANDLE_TRANSLATE[base] @@ -2206,29 +3215,56 @@ def generate_playback_shim(entry: ApiEntry) -> str: f" return hipSuccess;\n" f"}}\n") - # Error-stub playback APIs: explicit graph construction that HRR cannot - # replay. Emit a loud, attributable (per-API) warning, but return hipSuccess - # — these are non-fatal. The HARD failure is at hipGraphInstantiate / - # hipGraphInstantiateWithFlags, which abort (hipErrorNotSupported) only when - # actually asked to instantiate a non-replayable node-API graph (finding H1). - # This keeps replay alive for programs that merely create/clone/build graphs - # they never instantiate in an unsupported way. Message is once/process. + # Error-stub playback APIs: graph work whose arguments the archive does not + # carry well enough to rebuild. Emit a loud, attributable (per-API) warning + # and poison the graph, but return hipSuccess — these are non-fatal on their + # own. The HARD failure is at hipGraphInstantiate / + # hipGraphInstantiateWithFlags, which refuse an incomplete graph (finding + # H1). This keeps replay alive for programs that merely create, clone or + # build graphs they never instantiate. Message is once/process. if entry.name in ERROR_STUB_PLAYBACK_APIS: + gparam = _graph_param(entry) + mark = (f" const auto* a = reinterpret_cast(payload);\n" + f" ctx.mark_graph_incomplete(a->{gparam}, \"{entry.name}\");\n" + if gparam else " (void)ctx; (void)payload;\n") return (f"static hipError_t {fname}" f"(PlaybackContext& ctx, const uint8_t* payload) {{\n" - f" (void)ctx; (void)payload;\n" + f"{mark}" f" static bool warned = false;\n" f" if (!warned) {{\n" f" warned = true;\n" - f" fprintf(stderr, \"[HRR] {entry.name}: explicit (node-API) graph \"\n" - f" \"construction is NOT supported by HRR replay. Only \"\n" - f" \"stream-capture graphs (hipStreamBeginCapture/EndCapture) \"\n" - f" \"are replayable; instantiating a node-API-built graph will \"\n" - f" \"fail loudly. This call is skipped.\\n\");\n" + f" fprintf(stderr, \"[HRR] {entry.name}: not reconstructable at \"\n" + f" \"replay, so the call is skipped and the graph it belongs \"\n" + f" \"to is marked incomplete; instantiating that graph fails \"\n" + f" \"loudly rather than running a graph that is missing \"\n" + f" \"work.\\n\");\n" f" }}\n" f" return hipSuccess;\n" f"}}\n") + # Unreplayable APIs: name the API and the reason, mark the replay as + # having skipped something it cannot reproduce, and return an error. Loud + # and attributable beats letting the real API fail somewhere inside the + # runtime with a bare error code, which is what these used to do. + if entry.name in UNREPLAYABLE_PLAYBACK_APIS: + reason = UNREPLAYABLE_PLAYBACK_APIS[entry.name] + gparam = _graph_param(entry) + # Unlike ERROR_STUB_PLAYBACK_APIS this returns an error, so by default + # the replay stops right here. Under --continue-on-error it goes on, + # and the graph that should hold this node is now short one: marking + # it makes the later instantiate refuse it instead of running a graph + # missing work. + mark = (f" const auto* a = reinterpret_cast(payload);\n" + f" ctx.mark_graph_incomplete(a->{gparam}, \"{entry.name}\");\n" + if gparam else " (void)payload;\n") + return (f"static hipError_t {fname}" + f"(PlaybackContext& ctx, const uint8_t* payload) {{\n" + f"{mark}" + f" hrr_note_unreplayable(ctx, \"{entry.name}\",\n" + f" \"{reason}\");\n" + f" return hipErrorNotSupported;\n" + f"}}\n") + # No-op playback APIs: emit a one-time warning then return hipSuccess. # The static bool ensures the message fires once per process, not once per event, # so replays with thousands of events don't spam stderr. @@ -2309,6 +3345,30 @@ def generate_playback_shim(entry: ApiEntry) -> str: lines.append(f" return hipSuccess;") lines.append(f" }}") + # A node or executable graph that was never built here cannot be mutated. + # Handing the runtime a null handle would come back as an error, and + # dispatch treats any handler error as fatal — so a graph HRR already + # refused to instantiate (loudly, at the instantiate call) would take the + # rest of the replay down with it through its follow-up mutations. The + # useful complaint has already been made; skip these. + for p in entry.params: + pt = p.raw_type.strip() + if '*' in pt or not p.name: + continue + if _get_base_type(pt) not in ('hipGraphNode_t', 'hipGraphExec_t'): + continue + tfn = _PLAYBACK_HANDLE_TRANSLATE[_get_base_type(pt)] + lines.append(f" if (a->{p.name} != 0 && {tfn}(a->{p.name}) == nullptr) {{") + lines.append(f" static bool warned = false;") + lines.append(f" if (!warned) {{") + lines.append(f" warned = true;") + lines.append(f" fprintf(stderr, \"[HRR] {entry.name}: {p.name} 0x%llx was \"") + lines.append(f" \"never built at replay, so this call is skipped.\\n\",") + lines.append(f" (unsigned long long)a->{p.name});") + lines.append(f" }}") + lines.append(f" return hipSuccess;") + lines.append(f" }}") + # For alloc-free and handle-destroy APIs: grab the recorded key before the call if is_alloc_free: rec_param = _ALLOC_FREE_APIS[entry.name] @@ -2350,7 +3410,17 @@ def generate_playback_shim(entry: ApiEntry) -> str: call_args.append(f"({hdl_type})a->{name}") continue - call_args.append(_playback_arg(p, name, pre_lines)) + # The element count of a deref'd array is however many elements the + # event actually carries, which is not the recorded count when the + # array overflowed the inline capacity. + array_owner = next((d for d in deref_specs(entry.name) + if d.is_array and d.count == name + and d.playback == "auto"), None) + if array_owner is not None: + call_args.append(f"({p.raw_type.strip()})_d_{array_owner.param}_n") + continue + + call_args.append(_playback_arg(entry, p, name, pre_lines)) # Emit pre-call locals for pl in pre_lines: @@ -2381,6 +3451,17 @@ def _emit_call(target_lines: List[str], a_str: str) -> None: _emit_call(lines, args_str) ret_expr = "hipSuccess" if void_ret else "_r" + # A call that failed at capture and fails identically here was reproduced, + # not botched: return success so dispatch_event does not abort the replay + # over it. Only the hand-written capture shims record failing calls at all, + # so for everything else a->ret is 0 and this is dead weight. + if not void_ret and _get_base_type(entry.ret_type) == 'hipError_t': + lines.append(f" if (_r != hipSuccess && a->ret != 0 &&" + f" static_cast(_r) == a->ret) {{") + lines.append(f" hrr_note_recorded_error(ctx, \"{entry.name}\", a->ret);") + lines.append(f" return hipSuccess;") + lines.append(f" }}") + # Post-call: register/unregister allocs and handles success_cond = "true" if void_ret else "_r == hipSuccess" @@ -2415,11 +3496,43 @@ def _emit_call(target_lines: List[str], a_str: str) -> None: lines.append(f" {remove_fn}(_rec_hdl);") lines.append(f" }}") + for extra in _POST_CALL_EXTRA.get(entry.name, []): + lines.append(f" {extra}") + lines.append(f" return {ret_expr};") lines.append("}") + + if entry.name not in WINDOWS_EXPORTED_APIS: + return _win_unexported_shim(entry.name, lines) + return "\n".join(lines) + "\n" +def _win_unexported_shim(api: str, lines: List[str]) -> str: + """Wrap a generated playback body so Windows gets a warning instead of a call. + + Same contract as NOOP_PLAYBACK_APIS: warn once per process and report + success, so one unexportable API does not abort an otherwise fine replay. + """ + return "\n".join([ + lines[0], # signature and opening brace + "#ifdef _WIN32", + " (void)ctx; (void)payload;", + " static bool warned = false;", + " if (!warned) {", + " warned = true;", + f" fprintf(stderr, \"[HRR] {api} is not exported by amdhip64 on \"", + " \"Windows; skipping it during replay, results may differ \"", + " \"from capture.\\n\");", + " }", + " return hipSuccess;", + "#else", + *lines[1:-1], # the real call + "#endif", + lines[-1], # closing brace + ]) + "\n" + + def generate_dispatch_table(entries: List[ApiEntry]) -> str: """Generate the hrr_playback_dispatch array indexed by hrr_api_id_t.""" lines = [] @@ -2604,7 +3717,7 @@ def line_of(pos: int) -> int: def validate_playback_signatures(entries: List[ApiEntry], public: Dict[str, List[PublicDecl]], - public_path: Path) -> None: + public_path: Path, silent: bool = False) -> None: """Fail generation if a playback cast would disagree with its callee. An API is safe when the public header offers a declaration that both @@ -2660,8 +3773,9 @@ def validate_playback_signatures(entries: List[ApiEntry], "\n\nResolve each by adding an entry to _PLAYBACK_ARG_BRIDGES with the\n" "guard and conversion the public header requires.") - print(f" Public API signatures: {checked} verified against " - f"{public_path.name}, {skipped} declared elsewhere (skipped)") + if not silent: + print(f" Public API signatures: {checked} verified against " + f"{public_path.name}, {skipped} declared elsewhere (skipped)") # --------------------------------------------------------------------------- @@ -2683,6 +3797,7 @@ def main() -> None: default_header = hrr_project_dir / "include" / "hrr" / "hrr_api_args.h" default_capture = clr_hrr_dir / "hip_capture_generated.cpp" default_playback = hrr_project_dir / "playback" / "hip_playback_generated.cpp" + default_manual_capture = clr_hrr_dir / "hip_capture.cpp" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input", default=str(default_input), @@ -2699,18 +3814,29 @@ def main() -> None: help="Path to generated hip_capture_generated.cpp") parser.add_argument("--output-playback", default=str(default_playback), help="Path to generated hip_playback_generated.cpp") + parser.add_argument("--check-hrr-coverage", action="store_true", + help="Validate parseable dispatch slots and manual capture shims") + parser.add_argument("--manual-capture-source", default=str(default_manual_capture), + help="Path to hand-written hip_capture.cpp") + parser.add_argument("--silent", action="store_true", + help="Suppress normal progress output; warnings and errors remain visible") args = parser.parse_args() + def log(message: str) -> None: + if not args.silent: + print(message) + in_path = Path(args.input) public_path = Path(args.public_header) header_path = Path(args.output_header) capture_path = Path(args.output_capture) playback_path = Path(args.output_playback) + manual_capture_path = Path(args.manual_capture_source) if not in_path.exists(): sys.exit(f"ERROR: input file not found: {in_path}") - print(f"Parsing {in_path} ...") + log(f"Parsing {in_path} ...") entries = parse_hip_api_trace(in_path) n_compiler = sum(1 for e in entries if e.table == "compiler") n_runtime = sum(1 for e in entries if e.table == "runtime") @@ -2718,8 +3844,8 @@ def main() -> None: n_manual_cap = sum(1 for e in entries if e.name in MANUAL_CAPTURE_APIS) n_manual_play = sum(1 for e in entries if e.name in MANUAL_PLAYBACK_APIS) n_noop_play = sum(1 for e in entries if e.name in NOOP_PLAYBACK_APIS) - print(f" Found {n_compiler} compiler + {n_runtime} runtime = {len(entries)} total" - f" ({n_reserved} reserved slots)") + log(f" Found {n_compiler} compiler + {n_runtime} runtime = {len(entries)} total" + f" ({n_reserved} reserved slots)") # ------------------------------------------------------------------------- # Cross-validate classification sets against parsed API names. @@ -2737,6 +3863,8 @@ def main() -> None: ("SKIP_IF_UNMAPPED_DST_PLAYBACK_APIS", set(SKIP_IF_UNMAPPED_DST_PLAYBACK_APIS.keys())), ("EXTRA_FIELDS", set(EXTRA_FIELDS.keys())), + ("DEREF_FIELDS", set(DEREF_FIELDS.keys())), + ("UNREPLAYABLE_PLAYBACK_APIS", set(UNREPLAYABLE_PLAYBACK_APIS.keys())), ]: bad = sorted(n for n in api_set if n not in parsed_names) if bad: @@ -2761,11 +3889,106 @@ def main() -> None: for n in shadowed: print(f" '{n}'") sys.exit(1) - print(f" Manual capture (hand-written in hip_capture.cpp): {n_manual_cap}") - print(f" Manual playback (hand-written in hip_playback.cpp): {n_manual_play}") - print(f" No-op playback (inline hipSuccess stubs): {n_noop_play}") - print(f" Generated capture shims: {len(entries) - n_manual_cap}") - print(f" Generated playback shims: {len(entries) - n_manual_play - n_noop_play}") + + # An API's replay class is exactly one of these sets, and + # generate_playback_shim() tests them in this order, so an API in two of + # them silently gets the earlier one's handler while the later one's is + # left unreachable. That is what a merge resolving a rename against an + # edited set literal produces when it keeps an addition but drops the + # matching removal, and nothing downstream notices until a replay-class + # test disagrees. + replay_classes = [ + ("ERROR_STUB_PLAYBACK_APIS", ERROR_STUB_PLAYBACK_APIS), + ("UNREPLAYABLE_PLAYBACK_APIS", set(UNREPLAYABLE_PLAYBACK_APIS)), + ("NOOP_PLAYBACK_APIS", NOOP_PLAYBACK_APIS), + ("CUSTOM_PLAYBACK_BODIES", set(CUSTOM_PLAYBACK_BODIES)), + ("MANUAL_PLAYBACK_APIS", MANUAL_PLAYBACK_APIS), + ] + overlaps: List[str] = [] + for i, (first_name, first_set) in enumerate(replay_classes): + for later_name, later_set in replay_classes[i + 1:]: + for n in sorted(first_set & later_set): + overlaps.append(f"'{n}' is in both {first_name} and " + f"{later_name}; it will be replayed as " + f"{first_name} and the {later_name} handler " + f"is dead code") + if overlaps: + print("\nERROR: an API may name only one replay classification set:") + for problem in overlaps: + print(f" {problem}") + sys.exit(1) + + # A Deref that names a parameter the API does not have would silently do + # nothing, which is indistinguishable from the payload loss it is there to + # fix. The same goes for an array whose count parameter does not exist. + by_api = {e.name: e for e in entries} + deref_problems: List[str] = [] + for api, specs in sorted(DEREF_FIELDS.items()): + params = {p.name for p in by_api[api].params} + for d in specs: + if d.param not in params: + deref_problems.append( + f"{api}: Deref names parameter '{d.param}', which is not in " + f"the signature ({', '.join(sorted(params))})") + if d.direction not in ("in", "out", "inout"): + deref_problems.append( + f"{api}.{d.param}: direction '{d.direction}' is not " + "in/out/inout") + if d.playback not in ("auto", "manual"): + deref_problems.append( + f"{api}.{d.param}: playback '{d.playback}' is not " + "auto/manual") + if d.is_array: + if d.count not in params: + deref_problems.append( + f"{api}.{d.param}: count parameter '{d.count}' is not " + "in the signature") + if d.max_count <= 0: + deref_problems.append( + f"{api}.{d.param}: an array Deref needs max_count > 0") + if deref_problems: + print("\nERROR: DEREF_FIELDS does not match the parsed signatures:") + for p in deref_problems: + print(f" {p}") + sys.exit(1) + + # An unreplayable API whose capture shim is hand-written never emits the + # capture-time warning, so the gap would only show up at replay. + silent_unreplayable = sorted( + set(UNREPLAYABLE_PLAYBACK_APIS) & MANUAL_CAPTURE_APIS) + if silent_unreplayable: + print("\nERROR: these UNREPLAYABLE_PLAYBACK_APIS have hand-written " + "capture shims, so no capture-time warning is emitted. Call " + "hrr_cap::writer::note_unreplayable() from the shim in " + "hip_capture.cpp, or drop the API from MANUAL_CAPTURE_APIS:") + for n in silent_unreplayable: + print(f" '{n}'") + sys.exit(1) + + # generate_playback_shim() applies the classes in a fixed order, so an API + # in two of them silently gets whichever comes first. + for a_name, a_set, b_name, b_set in [ + ("UNREPLAYABLE_PLAYBACK_APIS", set(UNREPLAYABLE_PLAYBACK_APIS), + "ERROR_STUB_PLAYBACK_APIS", ERROR_STUB_PLAYBACK_APIS), + ("UNREPLAYABLE_PLAYBACK_APIS", set(UNREPLAYABLE_PLAYBACK_APIS), + "NOOP_PLAYBACK_APIS", NOOP_PLAYBACK_APIS), + ("UNREPLAYABLE_PLAYBACK_APIS", set(UNREPLAYABLE_PLAYBACK_APIS), + "MANUAL_PLAYBACK_APIS", MANUAL_PLAYBACK_APIS), + ("ERROR_STUB_PLAYBACK_APIS", ERROR_STUB_PLAYBACK_APIS, + "NOOP_PLAYBACK_APIS", NOOP_PLAYBACK_APIS), + ]: + both = sorted(a_set & b_set) + if both: + print(f"\nERROR: these APIs are in both {a_name} and {b_name}; " + "only the first would take effect:") + for n in both: + print(f" '{n}'") + sys.exit(1) + log(f" Manual capture (hand-written in hip_capture.cpp): {n_manual_cap}") + log(f" Manual playback (hand-written in hip_playback.cpp): {n_manual_play}") + log(f" No-op playback (inline hipSuccess stubs): {n_noop_play}") + log(f" Generated capture shims: {len(entries) - n_manual_cap}") + log(f" Generated playback shims: {len(entries) - n_manual_play - n_noop_play}") # ------------------------------------------------------------------------- # Playback calls the public API, not the dispatch table. Refuse to generate @@ -2782,25 +4005,29 @@ def main() -> None: f"generate anyway (unsafe).") else: public_decls = parse_public_api_signatures(public_path) - validate_playback_signatures(entries, public_decls, public_path) + validate_playback_signatures(entries, public_decls, public_path, args.silent) + + capture_cpp = generate_capture_cpp(entries) + if args.check_hrr_coverage: + validate_capture_coverage(entries, manual_capture_path, args.silent) header_path.parent.mkdir(parents=True, exist_ok=True) header = generate_header(entries) - header_path.write_text(header, encoding='utf-8') - print(f"Written header -> {header_path}") + header_path.write_text(header, encoding="utf-8", newline="\n") + log(f"Written header -> {header_path}") capture_path.parent.mkdir(parents=True, exist_ok=True) - capture_cpp = generate_capture_cpp(entries) - capture_path.write_text(capture_cpp, encoding='utf-8') - print(f"Written capture -> {capture_path}") + capture_path.write_text(capture_cpp, encoding="utf-8", newline="\n") + log(f"Written capture -> {capture_path}") playback_path.parent.mkdir(parents=True, exist_ok=True) playback_cpp = generate_playback_cpp(entries) - playback_path.write_text(playback_cpp, encoding='utf-8') - print(f"Written playback -> {playback_path}") + playback_path.write_text(playback_cpp, encoding="utf-8", newline="\n") + log(f"Written playback -> {playback_path}") # Spot-check a few important structs - _spot_check(entries) + if not args.silent: + _spot_check(entries) def _spot_check(entries: List[ApiEntry]) -> None: diff --git a/projects/rccl-tests/src/all_reduce.cu b/projects/rccl-tests/src/all_reduce.cu index 7adb1494ef56..c6d49589d371 100644 --- a/projects/rccl-tests/src/all_reduce.cu +++ b/projects/rccl-tests/src/all_reduce.cu @@ -593,7 +593,7 @@ testResult_t AllReduceRunTest(struct threadArgs* args, int root, ncclDataType_t for (int i=0; i= NCCL_VERSION(2,28,0)) +// Skip when common.h already included nccl_device.h. +#if !((defined(ENABLE_DEVICE_API) && NCCL_VERSION_CODE >= NCCL_VERSION(2,28,0)) || NCCL_VERSION_CODE >= NCCL_VERSION(2,29,0)) struct ncclDevComm; // Device API from 2.28 struct ncclDevCommRequirements; ncclResult_t ncclDevCommCreate(ncclComm_t, ncclDevCommRequirements*, ncclDevComm*) __attribute__((weak)); diff --git a/projects/rccl-tests/src/rccl_float8.h b/projects/rccl-tests/src/rccl_float8.h index bc08b7ebd018..472d51b916ca 100644 --- a/projects/rccl-tests/src/rccl_float8.h +++ b/projects/rccl-tests/src/rccl_float8.h @@ -49,7 +49,7 @@ typedef struct defined(__gfx942__) || defined(__gfx950__) || \ defined(__gfx1100__) || defined(__gfx1101__) || \ defined(__gfx1200__) || defined(__gfx1201__) || \ - defined(__gfx1250__)) + defined(__gfx1250__) || defined(__gfx1250_strict__)) #include diff --git a/projects/rccl-tests/src/reduce.cu b/projects/rccl-tests/src/reduce.cu index dfed8d7858f5..7b79478d87d8 100644 --- a/projects/rccl-tests/src/reduce.cu +++ b/projects/rccl-tests/src/reduce.cu @@ -111,7 +111,7 @@ testResult_t ReduceRunTest(struct threadArgs* args, int root, ncclDataType_t typ for (int i=0; i= 2 ranks/GPUs for GIN-SDMA AllGather") - - rc, out = 1, "" - for attempt in range(1, max(1, AG_CONN_RETRIES) + 1): - rc, out = _launch_ag_gin_sdma(request, total_bytes, dtype, **launch_kw) - if rc == 0: - return rc, out - # A real wrong-element mismatch must fail now, never retried. - if _DATA_FAIL_RE.search(out or ""): - return rc, out - # Intermittent gfx950 cuMem-VMM connectivity-gate abort: settle + retry. - if _CONN_GATE_RE.search(out or "") and attempt < AG_CONN_RETRIES: - print("=== connectivity-gate abort (attempt {}/{}); re-launching after settle ===".format( - attempt, AG_CONN_RETRIES)) - time.sleep(3) - continue - return rc, out - return rc, out + return run_with_conn_gate_retry( + lambda: _launch_ag_gin_sdma(request, total_bytes, dtype, **launch_kw), + AG_CONN_RETRIES, + ) # (items 1 + 2) Multi-segment loop and the 1 GiB single-copy boundary. per_rank diff --git a/projects/rccl-tests/test/test_AllToAll.py b/projects/rccl-tests/test/test_AllToAll.py index f2af20d8520c..8bd2a65341ce 100644 --- a/projects/rccl-tests/test/test_AllToAll.py +++ b/projects/rccl-tests/test/test_AllToAll.py @@ -46,9 +46,10 @@ # RCCL_TESTS_MPI_LAUNCHER launcher binary (default: mpirun) # RCCL_TESTS_MPI_OPTS extra launcher opts (e.g. --allow-run-as-root -mca ...) # RCCL_TESTS_A2A_XENV extra "-x K=V" env the backend needs on this cluster -# (e.g. HSA_FORCE_FINE_GRAIN_PCIE=1 NCCL_CUMEM_ENABLE=1) # RCCL_TESTS_A2A_EXE path to alltoall_perf (default: ../build/alltoall_perf) +# RCCL_TESTS_A2A_CTAS device CTA count (-V) (default: 16) # RCCL_TESTS_A2A_TIMEOUT_S per-run hang timeout in seconds (default: 900) +# RCCL_TESTS_A2A_CONN_RETRIES connectivity-gate retries (default: 5) # # Verified on 8x MI355X (ROCm 7.13, NCCL_GIN_TYPE=7, force1ch): 256 MiB/peer and # 2 GiB/peer int32 and the 2 GiB-total guard all pass with #wrong=0, no hang @@ -56,128 +57,99 @@ import os import shlex -import signal -import subprocess import pytest -MiB = 1024 * 1024 -GiB = 1024 * MiB +from .gin_sdma_harness import ( + GiB, + MiB, + detect_ngpus, + env_int, + gin_env_xflags, + gin_hang_msg, + gin_perf_argv, + launch_mpi_shell, + mpi_launch_prefix, + run_with_conn_gate_retry, +) path = os.path.dirname(os.path.abspath(__file__)) -# Path to alltoall_perf. Override with RCCL_TESTS_A2A_EXE when the binary is not -# at the default build/ location (e.g. an installed/container path). executable = os.environ.get( - "RCCL_TESTS_A2A_EXE", os.path.join(path, "..", "build", "alltoall_perf")) + "RCCL_TESTS_A2A_EXE", os.path.join(path, "..", "build", "alltoall_perf") +) -# Opt-in: only run where the GIN-SDMA backend + hardware are available. -_enabled = os.environ.get("RCCL_TESTS_GIN_SDMA_A2A", "") not in ("", "0", "false", "False") +_enabled = os.environ.get("RCCL_TESTS_GIN_SDMA_A2A", "") not in ( + "", + "0", + "false", + "False", +) - -def _detect_ngpus(): - if os.environ.get("ROCR_VISIBLE_DEVICES") is not None: - return len(os.environ["ROCR_VISIBLE_DEVICES"].split(",")) - if os.environ.get("HIP_VISIBLE_DEVICES") is not None: - return len(os.environ["HIP_VISIBLE_DEVICES"].split(",")) - try: - out = subprocess.check_output( - 'rocminfo | grep "Device Type:.\\s*.GPU" | wc -l', shell=True) - return int(out) - except Exception: - return 0 - - -NP = int(os.environ.get("RCCL_TESTS_A2A_NP", "0")) or _detect_ngpus() +NP = env_int("RCCL_TESTS_A2A_NP", 0) or (detect_ngpus() if _enabled else 0) LAUNCHER = os.environ.get("RCCL_TESTS_MPI_LAUNCHER", "mpirun") CTAS = os.environ.get("RCCL_TESTS_A2A_CTAS", "16") -# Generous per-call timeout; a real hang spins forever, so a finite cap is what -# converts item (3) into a detectable failure. Scaled up for the multi-GiB case. -TIMEOUT_S = int(os.environ.get("RCCL_TESTS_A2A_TIMEOUT_S", "900")) - -# Extra launcher options (e.g. "--allow-run-as-root -mca pml ob1 ...") and any -# deployment-specific env the GIN-SDMA backend needs beyond the essentials below -# (e.g. HSA_FORCE_FINE_GRAIN_PCIE=1 NCCL_CUMEM_ENABLE=1 RCCL_ENABLE_INTRANET=1). -# GIN enablement itself is site-specific, so it is injected here rather than -# hard-coded, keeping the test portable across clusters. +TIMEOUT_S = env_int("RCCL_TESTS_A2A_TIMEOUT_S", 900) +CONN_RETRIES = env_int("RCCL_TESTS_A2A_CONN_RETRIES", 5) MPI_OPTS = shlex.split(os.environ.get("RCCL_TESTS_MPI_OPTS", "")) XENV = shlex.split(os.environ.get("RCCL_TESTS_A2A_XENV", "")) pytestmark = pytest.mark.skipif( not _enabled, reason="GIN-SDMA AllToAll tests are opt-in; set RCCL_TESTS_GIN_SDMA_A2A=1 on " - "a GIN-SDMA-capable (e.g. 8x MI355X) node to enable.") + "a GIN-SDMA-capable (e.g. 8x MI355X) node to enable.", +) + + +def _launch_a2a(request, total_bytes, dtype): + """Launch alltoall_perf -D 3 once at a fixed total (per-rank) size.""" + size = str(int(total_bytes)) + gin_env = gin_env_xflags( + [ + "NCCL_GIN_ENABLE=1", + "NCCL_GIN_TYPE=7", + "NCCL_GIN_ANVIL_SDMA_THRESHOLD=0", + "NCCL_GIN_ANVIL_SDMA_THRESHOLD_ALLTOALL=0", + ] + + XENV + ) + + args = ( + mpi_launch_prefix(request, LAUNCHER, NP, MPI_OPTS) + + gin_env + + gin_perf_argv(executable, size, dtype, CTAS) + ) + cmd = " ".join(shlex.quote(a) for a in args) + hang_msg = gin_hang_msg( + "AllToAll", + TIMEOUT_S, + size, + dtype, + "{} MiB/peer".format(total_bytes // NP // MiB), + ) + return launch_mpi_shell(cmd, TIMEOUT_S, hang_msg) def _run_a2a(request, total_bytes, dtype): - """Run one all-to-all at a fixed total (per-rank) size, forcing the GIN-SDMA - tier. Returns (returncode, stdout). A timeout (hang) fails the test.""" + """Run one all-to-all with connectivity-gate retry on gfx950 VMM aborts.""" if NP < 2: pytest.skip("need >= 2 ranks/GPUs for AllToAll") + return run_with_conn_gate_retry( + lambda: _launch_a2a(request, total_bytes, dtype), CONN_RETRIES + ) - size = str(int(total_bytes)) - # Essentials to exercise the GIN-SDMA put path; force the SDMA (large) tier - # for every size. Deployment-specific env comes from RCCL_TESTS_A2A_XENV. - gin_env = [] - for kv in ["NCCL_GIN_ENABLE=1", "NCCL_GIN_TYPE=7", - "NCCL_GIN_ANVIL_SDMA_THRESHOLD=0", - "NCCL_GIN_ANVIL_SDMA_THRESHOLD_ALLTOALL=0"] + XENV: - gin_env += ["-x", kv] - - hostfile = request.config.getoption("--hostfile") - launch = [LAUNCHER, "-np", str(NP)] + MPI_OPTS - if hostfile: - launch += ["-host", hostfile] - - args = launch + gin_env + [ - executable, - "-b", size, "-e", size, - "-f", "2", - "-g", "1", - "-R", "2", - "-D", "3", - "-V", CTAS, - "-d", dtype, - "-c", "1", # data check: nonzero exit on any wrong element - "-w", "1", - "-n", "3", - ] - cmd = " ".join(shlex.quote(a) for a in args) - print(cmd) - # Run in a new session so a hang can be killed as a whole process group -- - # otherwise a wedged launcher leaves orphaned ranks pinning the GPUs (and - # exhausting SDMA queues for later tests). - proc = subprocess.Popen(cmd, shell=True, universal_newlines=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - start_new_session=True) - try: - out, _ = proc.communicate(timeout=TIMEOUT_S) - except subprocess.TimeoutExpired: - try: - os.killpg(os.getpgid(proc.pid), signal.SIGKILL) - except ProcessLookupError: - pass - out, _ = proc.communicate() - pytest.fail( - "AllToAll GIN-SDMA HANG: no completion within {}s at total={} bytes " - "({} MiB/peer), dtype={}. Output tail:\n{}".format( - TIMEOUT_S, size, total_bytes // NP // MiB, dtype, (out or "")[-2000:])) - print(out) - return proc.returncode, out - - -# (item 1 + 2) Multi-segment loop and the 1 GiB truncation boundary. per_peer is -# the transfer to each peer; total per-rank bytes = per_peer * NP. -@pytest.mark.parametrize("per_peer_mib", [256, 2048]) # 256 MiB (2 seg), 2 GiB (16 seg) + +@pytest.mark.parametrize("per_peer_mib", [256, 2048]) @pytest.mark.parametrize("dtype", ["int32", "int64", "uint8"]) def test_AllToAllGinSdmaLargeSegmented(request, per_peer_mib, dtype): total = per_peer_mib * MiB * NP rc, _ = _run_a2a(request, total, dtype) - assert rc == 0, "AllToAll data check failed (nonzero exit) at {} MiB/peer, dtype={}".format( - per_peer_mib, dtype) + assert rc == 0, ( + "AllToAll data check failed (nonzero exit) at {} MiB/peer, dtype={}".format( + per_peer_mib, dtype + ) + ) -# (item 3) The original 2 GiB-total hang repro, as a completion guard. At 8 ranks -# this is 256 MiB/peer -- the size that hung as a single unclamped put. def test_AllToAllGinSdma2GiBTotalHangGuard(request): rc, _ = _run_a2a(request, 2 * GiB, "int32") assert rc == 0, "AllToAll 2 GiB-total data check failed (nonzero exit)" diff --git a/projects/rccl-tests/test/test_Broadcast.py b/projects/rccl-tests/test/test_Broadcast.py index 6e18da5aa227..1f9fa0434ada 100644 --- a/projects/rccl-tests/test/test_Broadcast.py +++ b/projects/rccl-tests/test/test_Broadcast.py @@ -20,30 +20,33 @@ ################################################################################ import os -import re -import glob import shlex -import shutil -import signal import subprocess import itertools import math -import tempfile -import time import pytest -ngpus = 0 -if os.environ.get("ROCR_VISIBLE_DEVICES") is not None: - ngpus = len(os.environ["ROCR_VISIBLE_DEVICES"].split(",")) -elif os.environ.get("HIP_VISIBLE_DEVICES") is not None: - ngpus = len(os.environ["HIP_VISIBLE_DEVICES"].split(",")) -else: - ngpus = int( - subprocess.check_output( - 'rocminfo | grep "Device Type:.\\s*.GPU" | wc -l', shell=True - ) - ) +from .gin_sdma_harness import ( + GiB, + MiB, + assert_bcast_ok, + bcast_data_failed, + bcast_rows, + bcast_tiers, + detect_ngpus, + env_int, + gin_perf_argv, + launch_bcast_gin_sdma, + run_bcast_with_conn_gate_retry, + run_with_conn_gate_retry, +) + +try: + _detected_ngpus = detect_ngpus() +except RuntimeError: + _detected_ngpus = 0 +ngpus = max(1, _detected_ngpus) log_ngpus = int(math.log2(ngpus)) nthreads = ["1"] @@ -144,9 +147,6 @@ def test_BroadcastSingleProcess( # - 6 : #9924 (131884a2ba) [NCCL v2.30.7] to #10785 (d0f7d1966a) [NCCL v2.31.2] # - 5 : #7826 (bc6a304a9c) [SDMA support] to #9924 (131884a2ba) [NCCL v2.30.7] -MiB = 1024 * 1024 -GiB = 1024 * MiB - _bcast_enabled = os.environ.get("RCCL_TESTS_GIN_SDMA_BCAST", "") not in ( "", "0", @@ -154,19 +154,9 @@ def test_BroadcastSingleProcess( "False", ) - -def _env_int(name, default): - """Parse an integer env var; bad values fall back so collection stays usable.""" - raw = os.environ.get(name) - if raw is None or raw == "": - return default - try: - return int(raw) - except ValueError: - return default - - -BCAST_NP = _env_int("RCCL_TESTS_BCAST_NP", 0) or ngpus +BCAST_NP = env_int("RCCL_TESTS_BCAST_NP", 0) or ( + detect_ngpus() if _bcast_enabled else _detected_ngpus +) BCAST_LAUNCHER = os.environ.get("RCCL_TESTS_MPI_LAUNCHER", "mpirun") BCAST_CTAS = os.environ.get("RCCL_TESTS_BCAST_CTAS", "8") BCAST_GIN_TYPE = os.environ.get("RCCL_TESTS_BCAST_GIN_TYPE", "7") @@ -174,8 +164,8 @@ def _env_int(name, default): # HW_CASES * retries * TIMEOUT_S stays under GIN_PYTEST_TIMEOUT; otherwise # GNU timeout kills pytest and leaves the mpirun session (start_new_session) # orphaned because killpg only runs in this TimeoutExpired handler. -BCAST_TIMEOUT_S = _env_int("RCCL_TESTS_BCAST_TIMEOUT_S", 900) -BCAST_CONN_RETRIES = _env_int("RCCL_TESTS_BCAST_CONN_RETRIES", 5) +BCAST_TIMEOUT_S = env_int("RCCL_TESTS_BCAST_TIMEOUT_S", 900) +BCAST_CONN_RETRIES = env_int("RCCL_TESTS_BCAST_CONN_RETRIES", 5) BCAST_MPI_OPTS = shlex.split(os.environ.get("RCCL_TESTS_MPI_OPTS", "")) BCAST_XENV = shlex.split(os.environ.get("RCCL_TESTS_BCAST_XENV", "")) BCAST_EXE = os.environ.get( @@ -188,267 +178,18 @@ def _env_int(name, default): "a GIN-SDMA-capable (e.g. 8x MI355X) node to enable.", ) -_CONN_GATE_RE = re.compile( - r"LSA signal connectivity gate failed|unhandled system error", re.I -) - -# The GIN Anvil-SDMA backend has to actually bind for any of this to mean -# something: a run that falls back to another transport still exits 0 and still -# prints a full results table. NCCL_DEBUG_SUBSYS=INIT,NET below is what puts the -# bind line in the captured output. -_PLUGIN_RE = re.compile(r"gin-anvil-sdma", re.I) - -# One measured rccl-tests results row: -# size count type redop root | time algbw busbw #wrong | time algbw busbw #wrong -# out-of-place columns first, then in-place. Timings come from getFloatStr, which -# falls back to scientific notation when a value will not fit its width, so the -# numeric fields have to admit exponents. The row is not end-anchored: an algo / -# proto / nchannels group and a timestamp may follow the in-place columns, and -# concurrent rank output can splice itself onto the end of the line. -_NUM = r"[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?" -_WRONG = r"(?:{}|N/A)".format(_NUM) -_ROW_RE = re.compile( - r"^\s*(\d+)\s+(\d+)\s+\S+\s+\S+\s+(\d+)" - r"\s+{n}\s+{n}\s+{n}\s+({w})" - r"\s+{n}\s+{n}\s+{n}\s+({w})".format(n=_NUM, w=_WRONG) -) -_OOB_RE = re.compile(r"Out of bounds values\s*:\s*(\d+)") - -# "#[bcast-tier] ", emitted once per distinct tier by BroadcastRunColl when -# RCCL_TESTS_BCAST_TIER_LOG is set. Named tiers are the contract between the -D 3 -# dispatch and the tier assertions below; see bcastReportTier in broadcast.cu. -_TIER_RE = re.compile(r"^#\[bcast-tier\]\s+(\S+)", re.M) - - -def _bcast_tiers(out): - """Set of -D 3 tier names the run reported.""" - return set(_TIER_RE.findall(out or "")) - - -def _bcast_rows(out): - """Measured rows as (size, root, wrong_out_of_place, wrong_in_place).""" - rows = [] - for line in (out or "").splitlines(): - m = _ROW_RE.match(line) - if m: - rows.append((int(m.group(1)), int(m.group(3)), m.group(4), m.group(5))) - return rows - - -def _wrong_count(field): - """Numeric #wrong, or None when the column is N/A because checks were off.""" - try: - return float(field) - except ValueError: - return None - -def _data_failed(out): - """True when the run produced demonstrably wrong results. - - Deliberately not a text match on "Wrong": rccl-tests prints a "#wrong" column - header on every run that reaches the results table, so a bare pattern matched - every healthy run and made the connectivity retry below unreachable. N/A is - not a failure here -- it means checking was off, which _assert_bcast_ok - rejects separately rather than burning retries on. - """ - for _, _, oop, ip in _bcast_rows(out): - if any(_wrong_count(f) not in (None, 0.0) for f in (oop, ip)): - return True - m = _OOB_RE.search(out or "") - return bool(m and m.group(1) != "0") - - -def _read_debug_logs(debug_dir): - """Merged per-rank NCCL debug output written under debug_dir.""" - chunks = [] - for log_path in sorted(glob.glob(os.path.join(debug_dir, "nccl-debug.*"))): - try: - with open(log_path, errors="replace") as fh: - chunks.append(fh.read()) - except OSError: - pass - return "\n".join(chunks) - - -def _assert_bcast_ok(rc, out, debug, ctx, expect_tier=None): - """Assert the run bound GIN-SDMA, measured something, and checked clean. - - Exit status alone is not enough: it stays 0 for a run that binds a different - backend, and for one that produces no measured rows at all. - - Nor is the data check enough to say which tier ran. Every -D 3 tier produces - a correct broadcast, so #wrong stays 0 when a tier is deleted and its - messages fall through to the next one -- the test keeps passing while the - kernel it named is no longer reached. expect_tier pins the tier that has to - have run, so removing it fails here instead of silently going uncovered. - """ - tail = (out or "")[-2000:] - assert rc == 0, "Broadcast {} failed (exit {}). Output tail:\n{}".format( - ctx, rc, tail - ) - assert _PLUGIN_RE.search(debug or "") or _PLUGIN_RE.search(out or ""), ( - "Broadcast {} never bound the GIN Anvil-SDMA backend, so the run proves " - "nothing about the GIN path. Output tail:\n{}".format(ctx, tail) - ) - rows = _bcast_rows(out) - assert rows, "Broadcast {} produced no measured rows. Output tail:\n{}".format( - ctx, tail - ) - unchecked = [ - (s, r) - for s, r, oop, ip in rows - if _wrong_count(oop) is None or _wrong_count(ip) is None - ] - assert not unchecked, ( - "Broadcast {} reported #wrong as N/A at (size, root) {}, so the data check " - "never ran and the result is unverified.".format(ctx, unchecked) - ) - bad = [ - (s, r, oop, ip) - for s, r, oop, ip in rows - if _wrong_count(oop) != 0.0 or _wrong_count(ip) != 0.0 - ] - assert ( - not bad - ), "Broadcast {} reported nonzero #wrong (size, root, oop, ip): {}".format(ctx, bad) - m = _OOB_RE.search(out or "") - assert m and m.group(1) == "0", "Broadcast {} out-of-bounds count is {}".format( - ctx, m.group(1) if m else "absent" - ) - if expect_tier is not None: - # A string pins one tier; a collection allows a set of equivalent ones. - # The ring tier needs the latter: it reports ring-table where an - # edge-disjoint decomposition exists and ring-multi where none does - # (N=4 and N=6), and BCAST_NP follows whatever GPUs the host has. - # - # The ring also requires the LSA team to cover the world, so a multi-node - # launch takes scatter+allgather instead and fails here. That is the - # intent: such a run is not exercising the ring the caller asked for. - allowed = ( - {expect_tier} if isinstance(expect_tier, str) else set(expect_tier) - ) - tiers = _bcast_tiers(out) - assert tiers, ( - "Broadcast {} reported no #[bcast-tier] line, so which tier ran is " - "unverified. Is RCCL_TESTS_BCAST_TIER_LOG set for this launch, and is " - "the binary new enough to emit it? Output tail:\n{}".format(ctx, tail) - ) - # Exactly one tier, and an allowed one. These launches pin a single size, - # so a second tier means some roots took a different kernel; merely - # intersecting `allowed` would let that through. - assert len(tiers) == 1 and tiers <= allowed, ( - "Broadcast {} ran tier(s) {} but expected exactly one of {}. The tier " - "gate moved, or the tier under test fell through to another " - "kernel.".format(ctx, sorted(tiers), sorted(allowed)) - ) - - -def _launch_bcast_gin_sdma( - request, msg_bytes, dtype, *, force_flat_gin=False, force_sag_gin=False -): - """Launch broadcast_perf -D 3 at a fixed message size. When force_flat_gin, - disable the SAG/ring large tiers so the root flat gin.put() path is used. - When force_sag_gin, disable only the ring tier so scatter+allgather runs. - - Returns (returncode, stdout, debug_text); debug_text is the merged per-rank - NCCL debug log, kept off stdout so the results table stays parseable.""" - size = str(int(msg_bytes)) - debug_dir = tempfile.mkdtemp(prefix="bcast-gin-dbg-") - gin_env = [ - "NCCL_GIN_ENABLE=1", - "NCCL_GIN_TYPE={}".format(BCAST_GIN_TYPE), - "NCCL_GIN_ANVIL_SDMA_THRESHOLD=0", - "NCCL_GIN_ANVIL_SDMA_THRESHOLD_BROADCAST=0", - # Debug output is what lets _PLUGIN_RE confirm which backend actually - # bound. It goes to per-rank files rather than stdout because ranks - # interleave: on stdout an INFO line splices itself into the middle of a - # results row, which corrupts the very table we need to parse. - "NCCL_DEBUG=INFO", - "NCCL_DEBUG_SUBSYS=INIT,NET", - "NCCL_DEBUG_FILE={}/nccl-debug.%h.%p.log".format(debug_dir), - # Makes BroadcastRunColl name the tier it launched, which is what lets - # _assert_bcast_ok tell the -D 3 tiers apart. One line from the main - # proc, so the results table stays parseable. - "RCCL_TESTS_BCAST_TIER_LOG=1", - ] - if force_flat_gin: - gin_env += [ - "NCCL_GIN_ANVIL_BCAST_SCATTER_AG_MIN_BYTES=0", - "NCCL_GIN_ANVIL_BCAST_RING_MIN_BYTES=0", - ] - elif force_sag_gin: - # Ring is checked before SAG; disable ring only so SAG is exercised. - gin_env += ["NCCL_GIN_ANVIL_BCAST_RING_MIN_BYTES=0"] - gin_env += BCAST_XENV - gin_env = sum([["-x", kv] for kv in gin_env], []) - - hostfile = request.config.getoption("--hostfile") - launch = [BCAST_LAUNCHER, "-np", str(BCAST_NP)] + BCAST_MPI_OPTS - if hostfile: - launch += ["-host", hostfile] - - args = ( - launch - + gin_env - + [ - BCAST_EXE, - "-b", - size, - "-e", - size, - "-f", - "2", - "-g", - "1", - "-R", - "2", - "-D", - "3", # GinHybridBroadcastKernel - "-A", - "1", - "-V", - BCAST_CTAS, - "-d", - dtype, - "-c", - "1", - "-w", - "1", - "-n", - "3", - ] - ) - cmd = " ".join(shlex.quote(a) for a in args) - print(cmd) - try: - proc = subprocess.Popen( - cmd, - shell=True, - universal_newlines=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - try: - out, _ = proc.communicate(timeout=BCAST_TIMEOUT_S) - except subprocess.TimeoutExpired: - try: - os.killpg(os.getpgid(proc.pid), signal.SIGKILL) - except ProcessLookupError: - pass - out, _ = proc.communicate() - pytest.fail( - "Broadcast GIN-SDMA HANG: no completion within {}s at msg={} bytes " - "({} MiB), dtype={}. Output tail:\n{}".format( - BCAST_TIMEOUT_S, size, msg_bytes // MiB, dtype, (out or "")[-2000:] - ) - ) - debug = _read_debug_logs(debug_dir) - finally: - shutil.rmtree(debug_dir, ignore_errors=True) - print(out) - return proc.returncode, out, debug +def _bcast_launch_kwargs(): + return { + "exe": BCAST_EXE, + "np": BCAST_NP, + "launcher": BCAST_LAUNCHER, + "mpi_opts": BCAST_MPI_OPTS, + "xenv": BCAST_XENV, + "ctas": BCAST_CTAS, + "gin_type": BCAST_GIN_TYPE, + "timeout_s": BCAST_TIMEOUT_S, + } def _run_bcast_gin_sdma( @@ -456,30 +197,17 @@ def _run_bcast_gin_sdma( ): if BCAST_NP < 2: pytest.skip("need >= 2 ranks/GPUs for GIN-SDMA Broadcast") - - rc, out, debug = 1, "", "" - for attempt in range(1, max(1, BCAST_CONN_RETRIES) + 1): - rc, out, debug = _launch_bcast_gin_sdma( + return run_bcast_with_conn_gate_retry( + lambda: launch_bcast_gin_sdma( request, msg_bytes, dtype, force_flat_gin=force_flat_gin, force_sag_gin=force_sag_gin, - ) - if rc == 0: - return rc, out, debug - if _data_failed(out): - return rc, out, debug - if _CONN_GATE_RE.search(out or "") and attempt < BCAST_CONN_RETRIES: - print( - "=== connectivity-gate abort (attempt {}/{}); re-launching after settle ===".format( - attempt, BCAST_CONN_RETRIES - ) - ) - time.sleep(3) - continue - return rc, out, debug - return rc, out, debug + **_bcast_launch_kwargs(), + ), + BCAST_CONN_RETRIES, + ) @_bcast_skip @@ -490,7 +218,7 @@ def test_BroadcastGinSdmaLargeSegmented(request, msg_mib, dtype): rc, out, debug = _run_bcast_gin_sdma( request, msg_mib * MiB, dtype, force_flat_gin=True ) - _assert_bcast_ok( + assert_bcast_ok( rc, out, debug, @@ -505,7 +233,7 @@ def test_BroadcastGinSdmaScatterAllgather(request): rc, out, debug = _run_bcast_gin_sdma( request, 256 * MiB, "int32", force_sag_gin=True ) - _assert_bcast_ok( + assert_bcast_ok( rc, out, debug, "SAG 256 MiB", expect_tier="scatter-allgather" ) @@ -516,7 +244,7 @@ def test_BroadcastGinSdma2GiBHangGuard(request): rc, out, debug = _run_bcast_gin_sdma( request, 2 * GiB, "int32", force_flat_gin=False ) - _assert_bcast_ok( + assert_bcast_ok( rc, out, debug, @@ -531,7 +259,7 @@ def test_BroadcastGinSdma4GiBHangGuard(request): rc, out, debug = _run_bcast_gin_sdma( request, 4 * GiB, "int32", force_flat_gin=False ) - _assert_bcast_ok( + assert_bcast_ok( rc, out, debug, @@ -551,7 +279,7 @@ def test_BroadcastGinSdmaRowRegexAcceptsScientificNotation(): " 1.23e+02 1.00e+02 1.00e+02 0" " 1.23e+02 1.00e+02 1.00e+02 0" ) - rows = _bcast_rows(line) + rows = bcast_rows(line) assert rows == [(134217728, 0, "0", "0")] @@ -560,7 +288,7 @@ def test_BroadcastGinSdmaDataFailedIgnoresColumnHeader(): "# size count type redop root" " time algbw busbw #wrong" ) - assert not _data_failed(header) + assert not bcast_data_failed(header) def test_BroadcastGinSdmaDataFailedDetectsNonzeroWrong(): @@ -569,7 +297,7 @@ def test_BroadcastGinSdmaDataFailedDetectsNonzeroWrong(): " 12.34 1.00 1.00 1" " 12.34 1.00 1.00 0" ) - assert _data_failed(line) + assert bcast_data_failed(line) def test_BroadcastGinSdmaDataFailedTreatsNaAsUncheckedNotFailed(): @@ -578,7 +306,77 @@ def test_BroadcastGinSdmaDataFailedTreatsNaAsUncheckedNotFailed(): " 12.34 1.00 1.00 N/A" " 12.34 1.00 1.00 0" ) - assert not _data_failed(line) + assert not bcast_data_failed(line) + + +def test_BroadcastGinSdmaConnRetryReturnsCleanPassWithoutRetry(): + calls = [] + + def launch(): + calls.append(None) + return 0, "clean" + + assert run_with_conn_gate_retry(launch, 5, settle_s=0) == (0, "clean") + assert len(calls) == 1 + + +def test_BroadcastGinSdmaConnRetryExhaustsConnectivityAborts(): + calls = [] + + def launch(): + calls.append(None) + return 1, "LSA signal connectivity gate failed" + + assert run_with_conn_gate_retry(launch, 3, settle_s=0) == ( + 1, + "LSA signal connectivity gate failed", + ) + assert len(calls) == 3 + + +def test_BroadcastGinSdmaConnRetryNeverRetriesDataFailure(): + calls = [] + + def launch(): + calls.append(None) + return 1, "unhandled system error: #wrong = 1" + + assert run_with_conn_gate_retry(launch, 5, settle_s=0) == ( + 1, + "unhandled system error: #wrong = 1", + ) + assert len(calls) == 1 + + +def test_BroadcastGinSdmaConnRetryDoesNotRetryNonGateError(): + calls = [] + + def launch(): + calls.append(None) + return 1, "some other error" + + assert run_with_conn_gate_retry(launch, 5, settle_s=0) == ( + 1, + "some other error", + ) + assert len(calls) == 1 + + +def test_BroadcastGinSdmaDetectNgpusRequiresRocminfo(monkeypatch): + monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising=False) + monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False) + from . import gin_sdma_harness as harness + + monkeypatch.setattr(harness.shutil, "which", lambda _cmd: None) + with pytest.raises(RuntimeError, match="rocminfo"): + harness.detect_ngpus() + + +def test_BroadcastGinSdmaPerfArgvMakesAverageCountGapExplicit(): + ag = gin_perf_argv("all_gather_perf", 8, "int32", "8", extra=["-A", "1"]) + a2a = gin_perf_argv("alltoall_perf", 8, "int32", "16") + assert ag[ag.index("-D") + 2 : ag.index("-V")] == ["-A", "1"] + assert a2a[a2a.index("-D") + 2 : a2a.index("-V")] == [] def _clean_bcast_output(tier): @@ -599,16 +397,16 @@ def _clean_bcast_output(tier): # expands its args unquoted, so a filter with a space in it is not available. def test_BroadcastGinSdmaTierGuardLineIsNotAResultsRow(): """The tier line shares stdout with the table it must not corrupt.""" - assert _bcast_rows("#[bcast-tier] scatter-allgather") == [] - assert _bcast_tiers(_clean_bcast_output("ring-table")) == {"ring-table"} - assert _bcast_rows(_clean_bcast_output("ring-table")) + assert bcast_rows("#[bcast-tier] scatter-allgather") == [] + assert bcast_tiers(_clean_bcast_output("ring-table")) == {"ring-table"} + assert bcast_rows(_clean_bcast_output("ring-table")) def test_BroadcastGinSdmaTierGuardAcceptsExpectedTier(): out = _clean_bcast_output("scatter-allgather") - _assert_bcast_ok(0, out, "gin-anvil-sdma", "guard", expect_tier="scatter-allgather") + assert_bcast_ok(0, out, "gin-anvil-sdma", "guard", expect_tier="scatter-allgather") # Ring passes against either decomposition outcome. - _assert_bcast_ok( + assert_bcast_ok( 0, _clean_bcast_output("ring-multi"), "gin-anvil-sdma", @@ -626,9 +424,9 @@ def test_BroadcastGinSdmaTierGuardRejectsFallThrough(): would go green while covering nothing. """ fell_through = _clean_bcast_output("flat-gin") - _assert_bcast_ok(0, fell_through, "gin-anvil-sdma", "guard", expect_tier="flat-gin") + assert_bcast_ok(0, fell_through, "gin-anvil-sdma", "guard", expect_tier="flat-gin") with pytest.raises(AssertionError, match="expected exactly one of"): - _assert_bcast_ok( + assert_bcast_ok( 0, fell_through, "gin-anvil-sdma", "guard", expect_tier="scatter-allgather" ) @@ -637,7 +435,7 @@ def test_BroadcastGinSdmaTierGuardRejectsMissingTierLine(): """A binary predating the tier log must fail loudly, not pass vacuously.""" no_tier = "\n".join(_clean_bcast_output("flat-gin").splitlines()[1:]) with pytest.raises(AssertionError, match="no .*bcast-tier.* line"): - _assert_bcast_ok( + assert_bcast_ok( 0, no_tier, "gin-anvil-sdma", "guard", expect_tier="flat-gin" ) diff --git a/projects/rccl/.clang-format-ignore b/projects/rccl/.clang-format-ignore index 0c6a7534c119..b056f7440680 100644 --- a/projects/rccl/.clang-format-ignore +++ b/projects/rccl/.clang-format-ignore @@ -3,6 +3,8 @@ external/fmt/** src/include/nvtx3/** src/transport/net_ib/gdaki/doca-gpunetio/** src/transport/net_efa_gda/efa-dp-direct/** +src/transport/net_nd/nddef.h +src/transport/net_nd/ndspi.h src/include/gdrwrap.h src/include/mlx5/mlx5dvcore.h src/include/ibvcore.h diff --git a/projects/rccl/.gitignore b/projects/rccl/.gitignore index f1726239ab26..f8f960cdcc7a 100644 --- a/projects/rccl/.gitignore +++ b/projects/rccl/.gitignore @@ -45,3 +45,4 @@ examples/**/rmsnorm_lsa examples/**/rmsnorm_multimem examples/**/rmsnorm_gin examples/**/rmsnorm_hybrid +docs/examples/**/ring_exchange diff --git a/projects/rccl/CHANGELOG.md b/projects/rccl/CHANGELOG.md index a78faa69c8b3..a5dba4d34b43 100644 --- a/projects/rccl/CHANGELOG.md +++ b/projects/rccl/CHANGELOG.md @@ -2,7 +2,54 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https://rccl.readthedocs.io) -## RCCL 2.31.2 for ROCm 10.0.0 (Unreleased) +## RCCL 2.32.3 for ROCm 10.2.0 (Unreleased) + +### Added +* Compatibility with NCCL 2.32.3. +* `ncclSetEncryption()` and `ncclEncryptionConfig_t` (initialize with `NCCL_ENCRYPTION_CONFIG_INITIALIZER`) to encrypt NCCL-owned TCP socket traffic with TLS using a pre-shared key of at least 32 bytes. Requires a build configured with `-DTLS_BACKEND=OPENSSL3`; `libssl.so.3` is loaded at run time. Encryption is off by default and covers only traffic sent through RCCL's TCP sockets, not RDMA or GPU-to-GPU transfers. +* `launchCompletionEvent` in the per-collective config (`ncclCollConfig_t`, now `ncclCollConfig_v23200`): a caller-owned event that RCCL records for the collective's kernel launch. Every rank must pass an event, or every rank must pass `NULL`. HIP has no launch-completion launch attribute, so on ROCm the event is recorded on the stream immediately before the kernel launch. +* `NCCL_WIN_GIN_ONLY` window registration flag, which registers a window for GIN access only. +* GIN device-API C entry points for the timeout waits (`ncclGinWaitTimeout`, `ncclGinWaitSignalTimeout`, `ncclGinWaitSignalTimeoutVA`, `ncclGinWaitCounterTimeout`, `ncclGinFlushTimeout`), which return `ncclTimeout` after `timeoutCycles` `clock64()` cycles, and for `ncclGinFlushAsync`, `ncclGinWait` and the VA-signal operations (`ncclGinReadSignalVA`, `ncclGinResetSignalVA`, `ncclGinWaitSignalVA`). The equivalent C++ `ncclGin` members were already available. +* ReduceSum and ReduceCopy device APIs (`ncclLsaReduceSum`, `ncclLsaCopy`, `ncclLsaReduceSumCopy`, `ncclLocalReduceSumCopy`) in the LLVM bitcode library (`EMIT_LLVM_IR=ON`, `librccl_device.bc`) for 8-, 32- and 64-bit integer, FP16, BF16, FP32 and FP64 types. +* `NCCL_DEBUG_LEVELS` and the `ATTN` log level. `ATTN` reports non-fatal conditions that need attention, such as configuration fallbacks and plugin initialization failures. `NCCL_DEBUG_LEVELS` adds individual levels (`VERSION`, `WARN`, `ATTN`, `INFO`, `ABORT`, `TRACE`) to the selection made by `NCCL_DEBUG`. +* Device progress counters (`NCCL_PROGRESS_COUNTERS`, default `0`; also requires `NCCL_RAS_ENABLE=1`, the default) mirrored to host memory, and a RAS progress monitor that warns when a communicator's counters stop advancing (`NCCL_PROGRESS_COUNTER_MONITOR_POLL_MS`, `NCCL_PROGRESS_COUNTER_MONITOR_STALE_MS`, `NCCL_PROGRESS_COUNTER_MONITOR_STALE_WARN_SEC`). +* New RAS diagnostics checks (`NCCL_RUN_RAS_DIAGNOSTICS`) for PCI topology (`rdma_topo check`), the IOMMU mode and groups and the ATS state of GPU/NIC pairs, and the path types each communicator uses. +* An IB write-bandwidth check in the communicator init diagnostics (`NCCL_RUN_DIAGNOSTICS`). It runs perftest's `ib_write_bw` between hosts of multi-node communicators, with a per-run timeout set by `NCCL_DIAGNOSTICS_IB_BW_TIMEOUT` (default 5 seconds). It measures from GPU memory only when the installed `ib_write_bw` supports `--use_cuda`; otherwise it measures from host memory. +* Communicator initialization reports, at the `ATTN` level, ranks that run RCCL builds from different git revisions. +* Inspector plugin ring-buffer drop counters: a single warning per process when events are dropped, a `dump_stats` record with operation and drop counts in JSON output, and `NCCL_INSPECTOR_PROM_DUMP_STATS` (default `0`) to add the same counters to Prometheus output. +* `NCCL_CE_CHUNK_SIZE` (default 8 MiB): chunk size for the round-robin chunking that host RMA Copy Engine batches use when per-peer transfer sizes differ. +* `NCCL_CE_INTRA_GPU_MEMCPY_ENABLE` (default `1`): controls whether batched Copy Engine collective copies pass `hipMemcpyFlagPreferOverlapWithCompute`. RCCL previously always set this flag, so the default keeps the previous behavior; set `0` to omit it. +* `NCCL_HIER_CE_COLL_AG_RAIL_RING_ENABLE` (default `-1`): a positive value selects a ring for the inter-node rail phase of hierarchical Copy Engine `ncclAllGather`. The default keeps the direct path. +* `NCCL_IB_SORT_MERGE_NICS`: sorts the sub-devices of a merged IB device by plane ID. RCCL defaults it to `0` (NCCL defaults to `1`), so merged-device order and names are unchanged. +* nccl4py: per-call collective configuration (`NCCLCollConfig`, `VendorOption`) including the launch completion event, communicator properties (`NCCLCommProperties`), and the `GIN_ONLY` window flag. On ROCm, the HIP `Event` shim provides only an event handle; `record()`, `sync()` and `query` are not implemented. + +### Changed +* Host-side device API declarations (`ncclDevCommCreate`, `ncclDevCommDestroy`, `ncclCommQueryProperties`, `ncclGetPeerDevicePointer`, the `*CreateRequirement` helpers, the `ncclDevCommRequirements` and `ncclCommProperties` structs and their initializers) moved to `nccl_device/host.h`. `nccl_device.h` still includes it; code that includes individual `nccl_device/*.h` headers directly must also include `nccl_device/host.h`. +* Profiler `ncclProfileKernelPhase` events are emitted only by symmetric kernels, matching NCCL 2.32. Regular and P2P kernels report only `ncclProfileKernelCh` start and stop. +* Updated the RMA plugin interface to v16, which adds a per-communicator `getRmaProperties` query. Plugins built against v13, v14 or v15 still load. +* `NCCL_DEBUG_TIMESTAMP_LEVELS` now timestamps `ATTN` messages as well as `WARN` messages by default. +* The internal IB RMA proxy chains work requests and rings the doorbell once per batch of up to `NCCL_RMA_IB_WR_BATCHSIZE` (default `64`) requests, which raises the small-message rate. +* Communicator initialization at large scale is faster because the proxy no longer scans inactive poll descriptors. +* Network devices for each GPU are now chosen by rail and plane assignment, replacing the previous start-device scattering. +* `NCCL_MLOPART_RDMA_ENABLE` (default `0`) is kept. NCCL 2.32 removed it and treats buffers on partitioned GPUs as RDMA-capable on all non-ARM hosts; RCCL keeps network buffer registration for partitioned (CPX/DPX) GPUs opt-in. + +### Removed +* `NCCL_TOPO_SCATTER_START_NET`, which selected how the first network device was scattered across GPUs. Rail and plane assignment replaces it. + +### Resolved issues +* Fixed profiler overhead when no profiler plugin is loaded or only kernel-channel events are enabled. +* Fixed profiler API events reporting rank 0 instead of the originating communicator rank. +* Fixed P2P IPC registration reuse producing out-of-bounds remote addresses when a registered allocation spans multiple cuMem segments. +* Fixed non-thread-safe token parsing that could corrupt configuration parsed concurrently from multiple threads. +* Fixed tuner plugins receiving uninitialized cost-model constants. +* Fixed GIN proxy descriptor shared-memory sizing and alignment. +* Fixed virtual address space exhaustion when symmetric windows backed by the same physical allocation are registered repeatedly. + +### Known issues +* The FP8 ReduceSum and ReduceCopy device APIs are not exported in the LLVM bitcode library. +* On the Anvil SDMA (`NCCL_GIN_TYPE=7`) and rocSHMEM GDA (`NCCL_GIN_TYPE=6`) GIN backends, `ncclGinFlushTimeout` ignores the timeout and blocks until the flush completes; `ncclGinWaitTimeout` does the same on Anvil SDMA. rocSHMEM GDA does not support `ncclGinFlushAsync` or waiting on a GIN request, and those calls trap on the device. The signal and counter wait timeouts work on all backends. + +## RCCL 2.31.2 for ROCm 10.2.0 (Unreleased) ### Added * `RCCL_CE_AR_MAX_MSG_BYTES` (default `-1`): overrides the 2-shot AllReduce size cap from the arch table. Set to a positive value to override `ceNonRegMax[AR]`. @@ -13,6 +60,7 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https: * New `rcclAddonAlgos_t` values: `RCCL_CE_SCRATCH` (CE via DDA scratch, distinct from `RCCL_CE_REGISTERED`), `RCCL_A2A_PIVOT`, `RCCL_A2A_GDA`, `RCCL_A2A_GIN_SDMA`, and `RCCL_DIRECT_ALLTOALL`, with corresponding `rcclGetAlgoName()` labels. * rccl-tests: AlltoAll now reports algo/protocol in VERSION output, consistent with other collectives. * Compatibility with NCCL 2.31.2. +* `net_ib_cast` refreshes local GIDs after `IBV_EVENT_GID_CHANGE` and applies them to data, flush, probing and port recovery QPs during IB port recovery, matching `net_ib` in NCCL 2.31.2. * Per-collective configuration APIs (`ncclCollConfig_t` / `nccl*Config()` entry points) and the `ncclConfigExt_t` vendor extension list. Initialize configs with `NCCL_COLLCONFIG_INITIALIZER`. * Communicator config (`ncclConfig_v23100`) fields for implicit launch ordering (`launchOrderImplicit`), RMA signal count (`numRmaSig`), eager RMA init (`rmaEagerInit`), and host collective fault tolerance (`hostCftMode`). * NCCL profiler plugin API v7, with per-call user profiler tags and symmetric-kernel phase events. @@ -47,6 +95,7 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https: * Widened the LL128 per-thread shared-memory slice from 8 to 32 elements on gfx1250 (MI450/MI455); other architectures are unchanged. Forced Ring/LL128 bandwidth roughly doubles at 16 MB and above. Per-block LDS rises from 36448 to 85472 bytes, and because the scratch region is shared across protocols this applies to every kernel launch, not only LL128 ones. The tuner's protocol selection is unchanged, so the gain is only visible when LL128 is selected explicitly. ### Resolved issues +* Fixed FP8 E4M3 and E5M2 `ncclAvg` returning wrong values for every element of every reduction collective on gfx942, and on architectures that use the software FP8 fallback. The `1/nRanks` scale was packed as an FP8 value in the host's OCP encoding, which those devices decode as FNUZ, so every result came out half as large. The scale is now carried as `float` on all architectures, which also keeps it exact at rank counts FP8 cannot represent: in OCP E4M3, `1/384` was off by 25% and `1/1024` rounded to zero. Single-rank FP8 reductions on gfx942 no longer fail to launch the `oneRankReduce` kernel, and rccl-tests no longer skips FP8 `avg` for `all_reduce`, `all_reduce_bias` and `reduce`. * Restored topo tuning-model init (`ncclTopoTuneModel`) after the 2.31 `ncclTuningInit` switch so multi-node kernels do not launch with `blockDim.x=0`. * Grouped multi-rank finalize to match the v2.31 teardown barrier. * Fixed Copy Engine `ncclAllGather`, `ncclAlltoAll`, `ncclScatter` and `ncclGather` returning incorrect data, and no error, when `NCCL_LSA_TEAM_SIZE` was set below the number of ranks on the node. These routines index both their peer list and their buffer offsets by LSA rank, so a communicator wider than its LSA team exchanged data only within that team and placed it at LSA rather than communicator offsets, leaving the slices owned by every other rank unwritten. Copy Engine selection now requires the LSA team to cover the whole communicator on the scratch (`RCCL_FORCE_CE`) path as well as the registered-window path, so such communicators use the kernel path instead. Runs that leave `NCCL_LSA_TEAM_SIZE` unset are unaffected, because the LSA team then spans every rank on the node. @@ -55,7 +104,7 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https: ### Known issues * The following NCCL 2.31 features are NVIDIA-specific and are not available in RCCL: Compute Fabric Transport, the GDAKI and EFA GDA GIN backends, PAT combined with NVLS, NVLink-multicast AllGather, and the CuTeDSL and `nccl4rust` bindings. -## RCCL 2.30.7 for ROCm 10.1.0 (Unreleased) +## RCCL 2.30.7 for ROCm 10.1.0 ### Added * Compatibility with NCCL 2.30.7. @@ -89,6 +138,7 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https: * Fixed DDA fabric AllToAll validation race by staging send data into scratch with a host-launched `cudaMemcpyAsync` before the peer exchange kernel. * Fixed DDA fabric barrier publication race causing sporadic validation errors by using release-acquire semantics on the prologue barrier. * Fixed `ncclCommWindowRegister` after `ncclDevCommCreate` aborting on the GIN Anvil SDMA backend (`NCCL_GIN_TYPE=7`) with `could not resolve LSA flat addr`. Symmetric-window memory is now linked onto the device-runtime list before GIN/RMA registration so plugins that resolve the user VA through that list can see the in-flight window. +* Fixed the DDA, Copy-Engine 2-shot, GIN SDMA and hierarchical AllGather/ReduceScatter backends not making the communicator's device current and not ordering a new stream against the previous collective. A collective issued while another device was current failed with `ncclUnhandledCudaError`, and consecutive collectives on different streams could overlap. ### Known issues * On gfx90a (MI210/MI250/MI250X) with ROCm 7.13 or later, per-launch scratch-memory reclaim in the runtime degrades RCCL performance. Set `HSA_NO_SCRATCH_RECLAIM=1` to restore performance. diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index ef14f7bf4cea..c15ad00dd5ed 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -175,9 +175,6 @@ include(cmake/CheckSymbolExistsNoWarn.cmake) # Include rocSHMEM build module only if enabled if(ENABLE_ROCSHMEM OR ENABLE_ROCSHMEM_GIN) include(cmake/ROCSHMEM.cmake) - if(RCCL_POISON_HIP_ATOMICS) - message(WARNING "ENABLE_ROCSHMEM and ENABLE_ROCSHMEM_GIN are incompatible with RCCL_POISON_HIP_ATOMICS") - endif() endif() list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -199,8 +196,10 @@ endif() set(GPU_TARGETS "${DEFAULT_GPUS}" CACHE STRING "Target default GPUs if GPU_TARGETS is not defined.") -# Modify GPU architectures for Address Sanitizer builds by appending "xnack+" +# Adjust GPU architectures for Address Sanitizer builds; per-family rules are +# in cmake/AsanGpuTargets.cmake. if (BUILD_ADDRESS_SANITIZER) + include(cmake/AsanGpuTargets.cmake) set(_suffix "asan") set(_clang_rt_names "libclang_rt.${_suffix}.so" "libclang_rt.${_suffix}-x86_64.so") foreach(_clang_rt_name ${_clang_rt_names}) @@ -220,18 +219,13 @@ if (BUILD_ADDRESS_SANITIZER) endif() message(STATUS "ASAN runtime directory: ${ASAN_RUNTIME_DIR}") - SET(amdgpu_targets "") - foreach(amdgpu_target IN LISTS GPU_TARGETS) - if(NOT amdgpu_target STREQUAL "") - string(FIND "${amdgpu_target}" ":xnack+" HAS_XNACK_SUFFIX) - if(HAS_XNACK_SUFFIX EQUAL -1) - list(APPEND amdgpu_targets "${amdgpu_target}:xnack+") - else() - list(APPEND amdgpu_targets "${amdgpu_target}") - endif() - endif() - endforeach() - SET(GPU_TARGETS "${amdgpu_targets}") + # Guarded so that "no targets at all" still reaches the curated diagnostic + # below. rocm_local_targets() reports failure as the literal "NOTFOUND", + # which if() treats as false along with the empty case. + if(GPU_TARGETS) + rccl_asan_adjust_gpu_targets(GPU_TARGETS ${GPU_TARGETS}) + message(STATUS "ASAN: GPU targets = ${GPU_TARGETS}") + endif() endif() # Check if clang compiler can offload to GPU_TARGETS @@ -243,6 +237,10 @@ else() set(SUPPORTED_GPUS ${DEFAULT_GPUS}) endif() +# TLS backend for socket encryption (empty = disabled; OPENSSL3 = headers only, libssl is dlopen'd at runtime) +set(TLS_BACKEND "$ENV{TLS_BACKEND}" CACHE STRING "TLS backend (empty or OPENSSL3)") +set_property(CACHE TLS_BACKEND PROPERTY STRINGS "" OPENSSL3) + set(GPU_TARGETS "${SUPPORTED_GPUS}") # Fail fast if we ended up with no GPU targets at all @@ -630,6 +628,17 @@ set(USE_AMDSMI ON CACHE INTERNAL "Force enable AMD SMI path since we use lazy lo # endif () # endif() +## Check for TLS backend (socket encryption, see src/misc/crypt.cc) +set(NCCL_TLS_INCLUDE_DIRS) +if(TLS_BACKEND STREQUAL "OPENSSL3") + find_package(OpenSSL 3.0 REQUIRED COMPONENTS SSL Crypto) + add_compile_definitions(NCCL_TLS_BACKEND_OPENSSL3=1) + list(APPEND NCCL_TLS_INCLUDE_DIRS ${OPENSSL_INCLUDE_DIR}) + include_directories(${NCCL_TLS_INCLUDE_DIRS}) +elseif(NOT TLS_BACKEND STREQUAL "") + message(FATAL_ERROR "Unsupported TLS_BACKEND '${TLS_BACKEND}'; expected OPENSSL3 or empty") +endif() + ## Check for BFD library if custom backtrace is requested if(BUILD_BFD) enable_language(C) @@ -848,6 +857,16 @@ rocm_install(FILES ${PROJECT_BINARY_DIR}/include/nccl.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) rocm_install(DIRECTORY ${PROJECT_BINARY_DIR}/include/nccl_device/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nccl_device) +# Install SDMA device headers so consumers can compile GIN SDMA device templates +# without depending on ROCSHMEM_SOURCE_DIR. +if(ENABLE_ROCSHMEM_GIN AND ROCSHMEM_SOURCE_DIR) + rocm_install(FILES + ${ROCSHMEM_SOURCE_DIR}/src/sdma/anvil_device.hpp + ${ROCSHMEM_SOURCE_DIR}/src/sdma/sdma_opcodes.h + ${ROCSHMEM_SOURCE_DIR}/src/sdma/sdma_pkt_struct.h + ${ROCSHMEM_SOURCE_DIR}/src/sdma/sdma_pkt_struct_mi4.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nccl_device/gin/anvil_sdma/sdma) +endif() ## Optional: install LLVM bitcode artifacts when EMIT_LLVM_IR is enabled. ## librccl_device.bc, nccl_device_wrapper.h, hipified nccl_device headers, ## and nccl.h (same as rccl.h; nccl_device #includes ) are produced @@ -932,6 +951,7 @@ if(BUILD_TESTS) add_subdirectory(plugins/env/json) add_subdirectory(plugins/gin/example) add_subdirectory(plugins/mixed/example) + add_subdirectory(plugins/rma/example) endif() add_subdirectory(test) diff --git a/projects/rccl/bindings/ir/CMakeLists.txt b/projects/rccl/bindings/ir/CMakeLists.txt index 2dc4603f61e9..fd695794cbd3 100644 --- a/projects/rccl/bindings/ir/CMakeLists.txt +++ b/projects/rccl/bindings/ir/CMakeLists.txt @@ -71,6 +71,12 @@ message(STATUS "RCCL IR: llvm-dis = ${LLVM_DIS_EXECUTABLE}") message(STATUS "RCCL IR: llvm-as = ${LLVM_AS_EXECUTABLE}") message(STATUS "RCCL IR: target = ${BITCODE_LIB_ARCH} (${BITCODE_CXX_STD})") +##### Device API defines ##### +# Enable all ReduceCopy type and block-size combinations in generated IR. +set(NCCL_DEVICE_DEFINES + -DNCCL_DEVICE_PERMIT_EXPERIMENTAL_CODE=1 +) + ##### Include paths ##### # Point at the hipify staging directory so this build sees exactly the # same headers librccl.so is compiled from. HIPIFY_DIR is defined in @@ -94,6 +100,8 @@ set(ROCM_INCLUDES ##### Compiler flags ##### # HIP device-only compilation. --offload-device-only is the modern, # target-agnostic spelling of NCCL's --cuda-device-only. +# Clang supports extended device lambdas but does not define nvcc's feature +# macro; the ReduceCopy device APIs require it. set(COMMON_CLANG_FLAGS -std=${BITCODE_CXX_STD} -x hip @@ -103,6 +111,8 @@ set(COMMON_CLANG_FLAGS ${RCCL_INCLUDES} ${ROCM_INCLUDES} -D__clang_llvm_bitcode_lib__ + -D__CUDACC_EXTENDED_LAMBDA__ + ${NCCL_DEVICE_DEFINES} -D__HIP_PLATFORM_AMD__=1 ) @@ -128,9 +138,10 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E make_directory ${OBJDIR} COMMAND ${CLANG_EXECUTABLE} ${CLANG_FLAGS} ${LLVM_SRC} -o ${UNOPTIMIZED_BC} # Track both the impl TU (clang's direct input) and the staged wrapper - # header it transitively includes. Without the WRAPPER_HEADER dep, - # editing nccl_device_wrapper.h would not retrigger this command. - DEPENDS ${LLVM_SRC} ${WRAPPER_HEADER} + # header it transitively includes (plus util.h's type lists). Without + # the WRAPPER_HEADER dep, editing nccl_device_wrapper.h would not + # retrigger this command. + DEPENDS ${LLVM_SRC} ${WRAPPER_HEADER} ${CMAKE_CURRENT_SOURCE_DIR}/util.h COMMENT "Generating unoptimized LLVM bitcode for ${BITCODE_LIB_ARCH}..." VERBATIM ) @@ -166,6 +177,20 @@ set(RCCL_DEVICE_BC_PUBLIC_API ncclBarrierSessionInit ncclBarrierSessionSync ) +# [D] ReduceCopy: one thunk per op and type, matching NCCL_IR_TYPES / +# NCCL_IR_MULTIMEM_TYPES in util.h. +set(RCCL_IR_ALL_TYPES I8 U8 I32 U32 I64 U64 F16 F32 F64 BF16) +set(RCCL_IR_MULTIMEM_TYPES I32 U32 I64 U64 F16 F32 F64 BF16) +foreach(_op ncclLsaReduceSum ncclLsaCopy ncclLsaReduceSumCopy ncclLocalReduceSumCopy) + foreach(_ty ${RCCL_IR_ALL_TYPES}) + list(APPEND RCCL_DEVICE_BC_PUBLIC_API ${_op}_${_ty}) + endforeach() +endforeach() +foreach(_op ncclMultimemReduceSum ncclMultimemCopy ncclMultimemReduceSumCopy) + foreach(_ty ${RCCL_IR_MULTIMEM_TYPES}) + list(APPEND RCCL_DEVICE_BC_PUBLIC_API ${_op}_${_ty}) + endforeach() +endforeach() string(REPLACE ";" "," RCCL_DEVICE_BC_PUBLIC_API_CSV "${RCCL_DEVICE_BC_PUBLIC_API}") add_custom_command( diff --git a/projects/rccl/bindings/ir/Makefile b/projects/rccl/bindings/ir/Makefile index 654c7983d82e..e8f1eb3bd84c 100644 --- a/projects/rccl/bindings/ir/Makefile +++ b/projects/rccl/bindings/ir/Makefile @@ -66,6 +66,10 @@ FINAL_BC := $(LIBDIR)/librccl_device.bc WRAPPER_HEADER := $(INCDIR)/nccl_device_wrapper.h +##### Device API defines ##### +# Enable all ReduceCopy type and block-size combinations in generated IR. +NCCL_DEVICE_DEFINES := -DNCCL_DEVICE_PERMIT_EXPERIMENTAL_CODE=1 + ##### Include paths ##### # Match bindings/ir/CMakeLists.txt: compile against the hipify-staged # headers librccl.so is built from, not raw src/include. @@ -81,6 +85,8 @@ ROCM_INCLUDES := \ ##### Compiler flags ##### # HIP device-only compilation. --offload-device-only is the modern # (target-agnostic) spelling of NCCL's --cuda-device-only. +# Clang supports extended device lambdas but does not define nvcc's feature +# macro; the ReduceCopy device APIs require it. COMMON_CLANG_FLAGS := \ -std=$(BITCODE_CXX_STD) \ -x hip \ @@ -89,6 +95,8 @@ COMMON_CLANG_FLAGS := \ --offload-arch=$(BITCODE_LIB_ARCH) \ $(RCCL_INCLUDES) $(ROCM_INCLUDES) \ -D__clang_llvm_bitcode_lib__ \ + -D__CUDACC_EXTENDED_LAMBDA__ \ + $(NCCL_DEVICE_DEFINES) \ -D__HIP_PLATFORM_AMD__=1 CLANG_FLAGS := -c -emit-llvm -O1 $(COMMON_CLANG_FLAGS) @@ -98,7 +106,16 @@ CLANG_FLAGS := -c -emit-llvm -O1 $(COMMON_CLANG_FLAGS) # dead globals. opt's internalize pass takes EXACT symbol names, not globs; # keep this list in sync with RCCL_DEVICE_BC_PUBLIC_API in CMakeLists.txt # and the NCCL_IR_EXPORT entries in nccl_device_wrapper.h. -RCCL_DEVICE_BC_PUBLIC_API := ncclGetPeerPointerTeam,ncclCoopAnyInitThread,ncclCoopAnyInitWarp,ncclCoopAnyInitLanes,ncclCoopAnyInitWarpSpan,ncclCoopAnyInitCta,ncclCoopThreadRank,ncclCoopSize,ncclCoopNumThreads,ncclCoopSync,ncclLsaBarrierSessionInit,ncclLsaBarrierSessionArrive,ncclLsaBarrierSessionWait,ncclLsaBarrierSessionSync,ncclGinBarrierSessionInit,ncclGinBarrierSessionSync,ncclBarrierSessionInit,ncclBarrierSessionSync +# ReduceCopy thunks: one per op and type, matching util.h. +RCCL_IR_ALL_TYPES := I8 U8 I32 U32 I64 U64 F16 F32 F64 BF16 +RCCL_IR_MULTIMEM_TYPES := I32 U32 I64 U64 F16 F32 F64 BF16 +RCCL_IR_REDUCE_COPY_API := \ + $(foreach op,ncclLsaReduceSum ncclLsaCopy ncclLsaReduceSumCopy ncclLocalReduceSumCopy,$(foreach t,$(RCCL_IR_ALL_TYPES),$(op)_$(t))) \ + $(foreach op,ncclMultimemReduceSum ncclMultimemCopy ncclMultimemReduceSumCopy,$(foreach t,$(RCCL_IR_MULTIMEM_TYPES),$(op)_$(t))) +RCCL_IR_EMPTY := +RCCL_IR_SPACE := $(RCCL_IR_EMPTY) $(RCCL_IR_EMPTY) +RCCL_IR_COMMA := , +RCCL_DEVICE_BC_PUBLIC_API := ncclGetPeerPointerTeam,ncclCoopAnyInitThread,ncclCoopAnyInitWarp,ncclCoopAnyInitLanes,ncclCoopAnyInitWarpSpan,ncclCoopAnyInitCta,ncclCoopThreadRank,ncclCoopSize,ncclCoopNumThreads,ncclCoopSync,ncclLsaBarrierSessionInit,ncclLsaBarrierSessionArrive,ncclLsaBarrierSessionWait,ncclLsaBarrierSessionSync,ncclGinBarrierSessionInit,ncclGinBarrierSessionSync,ncclBarrierSessionInit,ncclBarrierSessionSync,$(subst $(RCCL_IR_SPACE),$(RCCL_IR_COMMA),$(strip $(RCCL_IR_REDUCE_COPY_API))) OPT_PASSES := --passes=internalize,inline,globaldce \ -internalize-public-api-list=$(RCCL_DEVICE_BC_PUBLIC_API) @@ -120,7 +137,7 @@ $(WRAPPER_HEADER): nccl_device_wrapper.h @echo "Copying nccl_device_wrapper.h -> $@" cp -f $< $@ -$(UNOPTIMIZED_BC): $(LLVM_SRC) $(WRAPPER_HEADER) check_hipify +$(UNOPTIMIZED_BC): $(LLVM_SRC) $(WRAPPER_HEADER) util.h check_hipify @mkdir -p $(OBJDIR) @echo "Generating unoptimized LLVM bitcode for $(BITCODE_LIB_ARCH)..." $(CLANG) $(CLANG_FLAGS) $(LLVM_SRC) -o $@ diff --git a/projects/rccl/bindings/ir/nccl_device_wrapper.h b/projects/rccl/bindings/ir/nccl_device_wrapper.h index ef5fa544b185..d09fcb890666 100644 --- a/projects/rccl/bindings/ir/nccl_device_wrapper.h +++ b/projects/rccl/bindings/ir/nccl_device_wrapper.h @@ -33,6 +33,24 @@ * bitcode can resolve NCCL Device API implementations from libnccl_device.bc. */ +/* + * Emit __activemask() as inline asm, like CUDA's sm_30_intrinsics.hpp does. + * Clang's version returns __nvvm_activemask(), which libNVVM 13.3.27 fails to + * lower: the PTX calls an .extern llvm.nvvm.activemask that ptxas rejects. + * + * Must precede the NCCL and cooperative_groups includes below so all call sites + * expand. Device pass only: CUDA declares __activemask in the host pass. + */ +#if defined(__clang__) && defined(__CUDA_ARCH__) +#undef __activemask +#define __activemask() \ + (__extension__({ \ + unsigned _nccl_activemask_ret; \ + asm volatile("activemask.b32 %0;" : "=r"(_nccl_activemask_ret)); \ + _nccl_activemask_ret; \ + })) +#endif + /* * Production's __forceinline__ (__inline__ __attribute__((always_inline))) is * linkonce_odr and emits no symbol. Drop __inline__ so the device API has @@ -249,6 +267,9 @@ struct ncclGinBarrierSession_C { struct ncclBarrierSession_C { ncclBarrierSession bar; }; +// Collectively finalize the placement-new object and persist its epoch. Does not free storage. +NCCL_IR_EXTERN_C __device__ +void ncclLsaBarrierSessionDestroy(ncclLsaBarrierSession_C* session); /* GIN Barrier Session APIs */ NCCL_IR_EXPORT void ncclGinBarrierSessionInit( @@ -264,6 +285,10 @@ NCCL_IR_EXPORT void ncclGinBarrierSessionSync( ncclCoopAny coop, cuda::memory_order order, ncclGinFenceLevel fence = ncclGinFenceLevel::Put | ncclGinFenceLevel::Get); +// Finalize the placement-new object. Currently a no-op -- the underlying destructor is +// empty -- but required for lifetime symmetry with Init. Does not free storage. +NCCL_IR_EXTERN_C __device__ +void ncclGinBarrierSessionDestroy(ncclGinBarrierSession_C* session); /* Composite (LSA + GIN) Barrier Session APIs */ NCCL_IR_EXPORT void ncclBarrierSessionInit( @@ -283,5 +308,242 @@ NCCL_IR_EXPORT void ncclBarrierSessionSync( ncclCoopAny coop, cuda::memory_order order, ncclGinFenceLevel fence = ncclGinFenceLevel::Put | ncclGinFenceLevel::Get); +// Collectively finalize all nested sessions and persist the inner LSA epoch. Does not free storage. +NCCL_IR_EXTERN_C __device__ +void ncclBarrierSessionDestroy(ncclBarrierSession_C* session); + +/* ReduceCopy APIs */ +#if defined(__HIPCC__) +/* [RCCL] This header is not hipified: spell bf16 the HIP way. RCCL's device + * ReduceSum/Copy has no fp8 support, so the F8E4M3/F8E5M2 entry points are CUDA-only. */ +#include +#define NCCL_IR_BF16_T __hip_bfloat16 +#else +#define NCCL_IR_BF16_T __nv_bfloat16 +#endif +NCCL_IR_EXPORT void ncclLsaReduceSum_I8( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + int8_t* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_U8( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + uint8_t* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_I32( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + int32_t* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_U32( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + uint32_t* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_I64( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + int64_t* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_U64( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + uint64_t* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_F16( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + half* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_F32( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + float* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_F64( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + double* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_BF16( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + NCCL_IR_BF16_T* dst, size_t count, ncclTeam team); +#if !defined(__HIPCC__) +NCCL_IR_EXPORT void ncclLsaReduceSum_F8E4M3( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + __nv_fp8_e4m3* dst, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSum_F8E5M2( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + __nv_fp8_e5m2* dst, size_t count, ncclTeam team); +#endif + +NCCL_IR_EXPORT void ncclMultimemReduceSum_I32( + ncclCoopAny coop, int32_t* mcSrc, int32_t* dst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSum_U32( + ncclCoopAny coop, uint32_t* mcSrc, uint32_t* dst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSum_I64( + ncclCoopAny coop, int64_t* mcSrc, int64_t* dst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSum_U64( + ncclCoopAny coop, uint64_t* mcSrc, uint64_t* dst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSum_F16( + ncclCoopAny coop, half* mcSrc, half* dst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSum_F32( + ncclCoopAny coop, float* mcSrc, float* dst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSum_F64( + ncclCoopAny coop, double* mcSrc, double* dst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSum_BF16( + ncclCoopAny coop, NCCL_IR_BF16_T* mcSrc, NCCL_IR_BF16_T* dst, size_t count); +#if !defined(__HIPCC__) +NCCL_IR_EXPORT void ncclMultimemReduceSum_F8E4M3( + ncclCoopAny coop, __nv_fp8_e4m3* mcSrc, __nv_fp8_e4m3* dst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSum_F8E5M2( + ncclCoopAny coop, __nv_fp8_e5m2* mcSrc, __nv_fp8_e5m2* dst, size_t count); +#endif + +NCCL_IR_EXPORT void ncclLsaCopy_I8( + ncclCoopAny coop, int8_t* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_U8( + ncclCoopAny coop, uint8_t* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_I32( + ncclCoopAny coop, int32_t* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_U32( + ncclCoopAny coop, uint32_t* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_I64( + ncclCoopAny coop, int64_t* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_U64( + ncclCoopAny coop, uint64_t* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_F16( + ncclCoopAny coop, half* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_F32( + ncclCoopAny coop, float* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_F64( + ncclCoopAny coop, double* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_BF16( + ncclCoopAny coop, NCCL_IR_BF16_T* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +#if !defined(__HIPCC__) +NCCL_IR_EXPORT void ncclLsaCopy_F8E4M3( + ncclCoopAny coop, __nv_fp8_e4m3* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaCopy_F8E5M2( + ncclCoopAny coop, __nv_fp8_e5m2* src, ncclWindow_t dstWindow, + size_t dstOffset, size_t count, ncclTeam team); +#endif + +NCCL_IR_EXPORT void ncclMultimemCopy_I32( + ncclCoopAny coop, int32_t* src, int32_t* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemCopy_U32( + ncclCoopAny coop, uint32_t* src, uint32_t* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemCopy_I64( + ncclCoopAny coop, int64_t* src, int64_t* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemCopy_U64( + ncclCoopAny coop, uint64_t* src, uint64_t* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemCopy_F16( + ncclCoopAny coop, half* src, half* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemCopy_F32( + ncclCoopAny coop, float* src, float* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemCopy_F64( + ncclCoopAny coop, double* src, double* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemCopy_BF16( + ncclCoopAny coop, NCCL_IR_BF16_T* src, NCCL_IR_BF16_T* mcDst, size_t count); +#if !defined(__HIPCC__) +NCCL_IR_EXPORT void ncclMultimemCopy_F8E4M3( + ncclCoopAny coop, __nv_fp8_e4m3* src, __nv_fp8_e4m3* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemCopy_F8E5M2( + ncclCoopAny coop, __nv_fp8_e5m2* src, __nv_fp8_e5m2* mcDst, size_t count); +#endif + +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_I8( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_U8( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_I32( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_U32( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_I64( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_U64( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_F16( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_F32( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_F64( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_BF16( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +#if !defined(__HIPCC__) +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_F8E4M3( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +NCCL_IR_EXPORT void ncclLsaReduceSumCopy_F8E5M2( + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team); +#endif + +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_I32( + ncclCoopAny coop, int32_t* mcSrc, int32_t* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_U32( + ncclCoopAny coop, uint32_t* mcSrc, uint32_t* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_I64( + ncclCoopAny coop, int64_t* mcSrc, int64_t* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_U64( + ncclCoopAny coop, uint64_t* mcSrc, uint64_t* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_F16( + ncclCoopAny coop, half* mcSrc, half* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_F32( + ncclCoopAny coop, float* mcSrc, float* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_F64( + ncclCoopAny coop, double* mcSrc, double* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_BF16( + ncclCoopAny coop, NCCL_IR_BF16_T* mcSrc, NCCL_IR_BF16_T* mcDst, size_t count); +#if !defined(__HIPCC__) +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_F8E4M3( + ncclCoopAny coop, __nv_fp8_e4m3* mcSrc, __nv_fp8_e4m3* mcDst, size_t count); +NCCL_IR_EXPORT void ncclMultimemReduceSumCopy_F8E5M2( + ncclCoopAny coop, __nv_fp8_e5m2* mcSrc, __nv_fp8_e5m2* mcDst, size_t count); +#endif + +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_I8( + ncclCoopAny coop, int nSrc, int8_t* srcBase, size_t srcDispl, + int nDst, int8_t* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_U8( + ncclCoopAny coop, int nSrc, uint8_t* srcBase, size_t srcDispl, + int nDst, uint8_t* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_I32( + ncclCoopAny coop, int nSrc, int32_t* srcBase, size_t srcDispl, + int nDst, int32_t* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_U32( + ncclCoopAny coop, int nSrc, uint32_t* srcBase, size_t srcDispl, + int nDst, uint32_t* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_I64( + ncclCoopAny coop, int nSrc, int64_t* srcBase, size_t srcDispl, + int nDst, int64_t* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_U64( + ncclCoopAny coop, int nSrc, uint64_t* srcBase, size_t srcDispl, + int nDst, uint64_t* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_F16( + ncclCoopAny coop, int nSrc, half* srcBase, size_t srcDispl, + int nDst, half* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_F32( + ncclCoopAny coop, int nSrc, float* srcBase, size_t srcDispl, + int nDst, float* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_F64( + ncclCoopAny coop, int nSrc, double* srcBase, size_t srcDispl, + int nDst, double* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_BF16( + ncclCoopAny coop, int nSrc, NCCL_IR_BF16_T* srcBase, size_t srcDispl, + int nDst, NCCL_IR_BF16_T* dstBase, size_t dstDispl, size_t count); +#if !defined(__HIPCC__) +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_F8E4M3( + ncclCoopAny coop, int nSrc, __nv_fp8_e4m3* srcBase, size_t srcDispl, + int nDst, __nv_fp8_e4m3* dstBase, size_t dstDispl, size_t count); +NCCL_IR_EXPORT void ncclLocalReduceSumCopy_F8E5M2( + ncclCoopAny coop, int nSrc, __nv_fp8_e5m2* srcBase, size_t srcDispl, + int nDst, __nv_fp8_e5m2* dstBase, size_t dstDispl, size_t count); +#endif #endif /* _NCCL_DEVICE_WRAPPER_H_ */ diff --git a/projects/rccl/bindings/ir/nccl_device_wrapper__impl.h b/projects/rccl/bindings/ir/nccl_device_wrapper__impl.h index 7c41d9b08c24..365f95198f9a 100755 --- a/projects/rccl/bindings/ir/nccl_device_wrapper__impl.h +++ b/projects/rccl/bindings/ir/nccl_device_wrapper__impl.h @@ -15,7 +15,14 @@ // (always_inline, external) that the device-API definitions in nccl_device.h // below are then compiled with, so the library emits real symbols. #include "nccl_device_wrapper.h" +// The bitcode library is device-only: suppress "nccl_device/host.h" by +// pre-defining its include guard, so the host entrypoints never enter this TU. +#define _NCCL_DEVICE_HOST_H_ #include "nccl_device.h" +#ifdef NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER +#error "nccl_device/host.h leaked in: its include guard was renamed, update the #define above" +#endif +#include "util.h" #include #if defined(__CUDACC__) || defined(__HIPCC__) @@ -96,6 +103,11 @@ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLsaBarrierSessionSync(ncclLsaBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order) { session->bar.sync(coop, order); } +NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE +void ncclLsaBarrierSessionDestroy(ncclLsaBarrierSession_C* session) { + using Session = ncclLsaBarrierSession; + session->bar.~Session(); +} /* GIN barrier session */ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinBarrierSessionInit( @@ -125,6 +137,11 @@ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinBarrierSessionSync( ncclGinFenceLevel fence) { session->bar.sync(coop, order, fence); } +NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE +void ncclGinBarrierSessionDestroy(ncclGinBarrierSession_C* session) { + using Session = ncclGinBarrierSession; + session->bar.~Session(); +} /* Barrier Session*/ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclBarrierSessionInit( @@ -148,6 +165,67 @@ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclBarrierSessionSync( ncclGinFenceLevel fence) { session->bar.sync(coop, order, fence); } +NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE +void ncclBarrierSessionDestroy(ncclBarrierSession_C* session) { + using Session = ncclBarrierSession; + session->bar.~Session(); +} + +// ReduceCopy APIs +#define NCCL_IR_DEFINE_ncclLsaReduceSum(suffix, type) \ + NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLsaReduceSum_##suffix( \ + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, \ + type* dst, size_t count, ncclTeam team) { \ + ncclLsaReduceSum( \ + coop, srcWindow, srcOffset, dst, count, team); \ + } +#define NCCL_IR_DEFINE_ncclMultimemReduceSum(suffix, type) \ + NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclMultimemReduceSum_##suffix( \ + ncclCoopAny coop, type* mcSrc, type* dst, size_t count) { \ + ncclMultimemReduceSum( \ + coop, mcSrc, dst, count); \ + } +#define NCCL_IR_DEFINE_ncclLsaCopy(suffix, type) \ + NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLsaCopy_##suffix( \ + ncclCoopAny coop, type* src, ncclWindow_t dstWindow, \ + size_t dstOffset, size_t count, ncclTeam team) { \ + ncclLsaCopy( \ + coop, src, dstWindow, dstOffset, count, team); \ + } +#define NCCL_IR_DEFINE_ncclMultimemCopy(suffix, type) \ + NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclMultimemCopy_##suffix( \ + ncclCoopAny coop, type* src, type* mcDst, size_t count) { \ + ncclMultimemCopy( \ + coop, src, mcDst, count); \ + } +#define NCCL_IR_DEFINE_ncclLsaReduceSumCopy(suffix, type) \ + NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLsaReduceSumCopy_##suffix( \ + ncclCoopAny coop, ncclWindow_t srcWindow, size_t srcOffset, \ + ncclWindow_t dstWindow, size_t dstOffset, size_t count, ncclTeam team) { \ + ncclLsaReduceSumCopy( \ + coop, srcWindow, srcOffset, dstWindow, dstOffset, count, team); \ + } +#define NCCL_IR_DEFINE_ncclMultimemReduceSumCopy(suffix, type) \ + NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclMultimemReduceSumCopy_##suffix( \ + ncclCoopAny coop, type* mcSrc, type* mcDst, size_t count) { \ + ncclMultimemReduceSumCopy( \ + coop, mcSrc, mcDst, count); \ + } +#define NCCL_IR_DEFINE_ncclLocalReduceSumCopy(suffix, type) \ + NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLocalReduceSumCopy_##suffix( \ + ncclCoopAny coop, int nSrc, type* srcBase, size_t srcDispl, \ + int nDst, type* dstBase, size_t dstDispl, size_t count) { \ + ncclLocalReduceSumCopy( \ + coop, nSrc, srcBase, srcDispl, nDst, dstBase, dstDispl, count); \ + } + +NCCL_IR_DEFINE_API_ALL_TYPES(ncclLsaReduceSum) +NCCL_IR_DEFINE_API_MULTIMEM_TYPES(ncclMultimemReduceSum) +NCCL_IR_DEFINE_API_ALL_TYPES(ncclLsaCopy) +NCCL_IR_DEFINE_API_MULTIMEM_TYPES(ncclMultimemCopy) +NCCL_IR_DEFINE_API_ALL_TYPES(ncclLsaReduceSumCopy) +NCCL_IR_DEFINE_API_MULTIMEM_TYPES(ncclMultimemReduceSumCopy) +NCCL_IR_DEFINE_API_ALL_TYPES(ncclLocalReduceSumCopy) #endif // __CUDACC__ #endif // _NCCL_DEVICE_WRAPPER__IMPL_H_ diff --git a/projects/rccl/bindings/ir/test/IR_test.cpp b/projects/rccl/bindings/ir/test/IR_test.cpp index aab56dad816d..c212592cb164 100644 --- a/projects/rccl/bindings/ir/test/IR_test.cpp +++ b/projects/rccl/bindings/ir/test/IR_test.cpp @@ -389,6 +389,41 @@ TEST_F(IRDeviceTest, B3d_CoopInitLanes_UpperLaneMask) { [](int t){ return (t == 63) ? 0 : -1; }); } +TEST_F(IRDeviceTest, B3e_CoopInitLanes_IgnoresNonexistentLanes) { + const uint64_t mask = 0xffffffff00000000ull; + const int sz = warpSize_ > 32 ? 32 : 0; + auto h = launch_coop_r(k_coop_lanes_r, warpSize_, mask); + check_coop(h, + [sz](int) { return sz; }, + [](int) { return -1; }); +} + +/* ncclCoopTile::laneMask() is not reached by the ncclCoopAny tests above. + * AMD shims discard the mask today; this pins the value the tile returns. */ +__global__ void k_coop_tile4_lane_mask(uint64_t* out) { + ncclCoopTile<4> tile; + int tid = blockIdx.x * blockDim.x + threadIdx.x; + out[tid] = static_cast(tile.laneMask()); +} + +TEST_F(IRDeviceTest, B3f_CoopTileLaneMask_AlignedGroupOfFour) { + const int n = warpSize_; + uint64_t* d = nullptr; + HIP_ASSERT(hipMalloc(&d, sizeof(uint64_t) * n)); + k_coop_tile4_lane_mask<<<1, n>>>(d); + HIP_ASSERT(hipGetLastError()); + HIP_ASSERT(hipDeviceSynchronize()); + std::vector got(n); + HIP_ASSERT(hipMemcpy(got.data(), d, sizeof(uint64_t) * n, hipMemcpyDeviceToHost)); + HIP_ASSERT(hipFree(d)); + + const uint64_t window = ~uint64_t(0) >> (64 - n); + for (int lane = 0; lane < n; ++lane) { + const uint64_t expect = (window >> (n - 4)) << (lane & -4); + EXPECT_EQ(got[lane], expect) << "lane " << lane; + } +} + /* ===================================================================== * [B4] ncclCoopAnyInitWarpSpan — 1-warp and 2-warp spans * rank = threadIdx.x - WARP_SIZE * warp0 (WARP_SIZE == device warpSize) diff --git a/projects/rccl/bindings/ir/util.h b/projects/rccl/bindings/ir/util.h new file mode 100644 index 000000000000..4f35845690a5 --- /dev/null +++ b/projects/rccl/bindings/ir/util.h @@ -0,0 +1,57 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + ************************************************************************/ +#ifndef _NCCL_IR_UTIL_H_ +#define _NCCL_IR_UTIL_H_ + +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +// RCCL's device ReduceSum/Copy has no fp8 support, +// so the IR exposes no FP8 entry points. +#define NCCL_IR_BF16_TYPES(F) \ + F(BF16, __hip_bfloat16) + +#define NCCL_IR_FP8_TYPES(F) +#else +// BF16 requires CUDA 11.0; FP8 requires CUDA 11.8. +#define NCCL_IR_BF16_TYPES(F) \ + F(BF16, __nv_bfloat16) + +#define NCCL_IR_FP8_TYPES(F) \ + F(F8E4M3, __nv_fp8_e4m3) \ + F(F8E5M2, __nv_fp8_e5m2) +#endif + +#define NCCL_IR_MULTIMEM_TYPES(F) \ + F(I32, int32_t) \ + F(U32, uint32_t) \ + F(I64, int64_t) \ + F(U64, uint64_t) \ + F(F16, half) \ + F(F32, float) \ + F(F64, double) \ + NCCL_IR_BF16_TYPES(F) \ + NCCL_IR_FP8_TYPES(F) + +#define NCCL_IR_TYPES(F) \ + F(I8, int8_t) \ + F(U8, uint8_t) \ + F(I32, int32_t) \ + F(U32, uint32_t) \ + F(I64, int64_t) \ + F(U64, uint64_t) \ + F(F16, half) \ + F(F32, float) \ + F(F64, double) \ + NCCL_IR_BF16_TYPES(F) \ + NCCL_IR_FP8_TYPES(F) + +#define NCCL_IR_DEFINE_API_ALL_TYPES(func) \ + NCCL_IR_TYPES(NCCL_IR_DEFINE_##func) + +#define NCCL_IR_DEFINE_API_MULTIMEM_TYPES(func) \ + NCCL_IR_MULTIMEM_TYPES(NCCL_IR_DEFINE_##func) + +#endif // _NCCL_IR_UTIL_H_ diff --git a/projects/rccl/bindings/nccl4py/.git_archival.txt b/projects/rccl/bindings/nccl4py/.git_archival.txt new file mode 100644 index 000000000000..5a59c4065af8 --- /dev/null +++ b/projects/rccl/bindings/nccl4py/.git_archival.txt @@ -0,0 +1,3 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=nccl4py-v*[0-9]*.*[0-9]*)$ diff --git a/projects/rccl/bindings/nccl4py/.gitattributes b/projects/rccl/bindings/nccl4py/.gitattributes new file mode 100644 index 000000000000..a94cb2f8c249 --- /dev/null +++ b/projects/rccl/bindings/nccl4py/.gitattributes @@ -0,0 +1 @@ +.git_archival.txt export-subst diff --git a/projects/rccl/bindings/nccl4py/MANIFEST.in b/projects/rccl/bindings/nccl4py/MANIFEST.in index 4e300495d23f..04af669ab679 100644 --- a/projects/rccl/bindings/nccl4py/MANIFEST.in +++ b/projects/rccl/bindings/nccl4py/MANIFEST.in @@ -1,15 +1,19 @@ -# Include nccl package with all source files -recursive-include nccl *.pyx *.pxd +# Include all Cython extension sources and dependencies +recursive-include nccl/bindings *.pyx *.pxd + +# Include examples in the source distribution +graft examples # Exclude directories not needed in sdist -prune examples -prune *.egg-info +prune ci -# Exclude development files from root and subdirectories +# Exclude development files from the project root +exclude .gitlab-ci.yml +exclude .git_archival.txt +exclude CMakeLists.txt exclude Makefile exclude uv.lock -global-exclude .gitignore .gitattributes +global-exclude .gitignore .gitattributes *.py[cod] -# Exclude generated Cython build outputs in nccl/bindings/ -recursive-exclude nccl/bindings *.so -global-exclude *.cpp *.c +# Exclude generated Cython build outputs +recursive-exclude nccl/bindings *.so *.cpp *.c diff --git a/projects/rccl/bindings/nccl4py/README.md b/projects/rccl/bindings/nccl4py/README.md index 45878488dd58..5c3a48a2b9ca 100644 --- a/projects/rccl/bindings/nccl4py/README.md +++ b/projects/rccl/bindings/nccl4py/README.md @@ -53,6 +53,20 @@ Pythonic API for RCCL collective operations. pip install . ``` +Verify the installation and inspect the loaded component versions: + +```python +import nccl.core as nccl + +nccl.show_versions() +``` + +`nccl.core` ships inline type information for +[PEP 561](https://peps.python.org/pep-0561/)-compatible type checkers. + +See [`examples/01_basic`](examples/01_basic) for MPI-based collective and +point-to-point examples. + > **Note:** The `Makefile` and `CMakeLists.txt` shipped in this > directory are inherited from upstream NVIDIA `nccl4py` and target > NVIDIA hardware (they require `CUDA_HOME` / `nvcc` and pull @@ -99,6 +113,17 @@ mpirun -np 4 python examples/01_basic/01_allreduce.py mpirun -np 2 python examples/01_basic/02_send_recv.py ``` +## Experimental Cython Support + +The package includes `nccl/bindings/cynccl.pxd` as an experimental Cython API: + +```cython +from nccl.bindings cimport cynccl +``` + +This allows Cython extensions to call NCCL functions with minimal Python +overhead. + ## Layout - [`nccl/bindings/`](nccl/bindings/) - Cython bindings to the diff --git a/projects/rccl/bindings/nccl4py/examples/cute/main.py b/projects/rccl/bindings/nccl4py/examples/cute/00_basic.py similarity index 88% rename from projects/rccl/bindings/nccl4py/examples/cute/main.py rename to projects/rccl/bindings/nccl4py/examples/cute/00_basic.py index 1e55bfb9e06d..bb42843cb726 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/main.py +++ b/projects/rccl/bindings/nccl4py/examples/cute/00_basic.py @@ -1,3 +1,10 @@ +#!/usr/bin/env python3 +# +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# See LICENSE.txt for more license information +# """1 MiB NCCL GIN put example using CuTeDSL ``cute.Tensor`` views. Demonstrates the canonical workflow for the nccl4py CuTeDSL device API: @@ -19,11 +26,13 @@ against its annotation before it looks for an adapter, so annotating the converted type rejects a raw resource outright. -Run with two MPI ranks:: +Run with exactly two MPI ranks; the transfer is hardcoded from rank 0 to +rank 1:: - mpirun -n 2 python main.py + mpirun -n 2 python 00_basic.py """ +import os import sys try: @@ -50,6 +59,9 @@ import nccl.core as nccl import nccl.core.device.cute as nccl_cute + +NAME = os.path.basename(__file__) + # 1 MiB transfer: 131072 Int64 elements * 8 bytes = 1,048,576 bytes. NUM_ELEMS = 1024 * 1024 // 8 DST_RANK = 1 @@ -138,8 +150,8 @@ def test_nccl_put( ): """Launch the kernel, taking arguments the caller already converted. - The caller wraps each resource — nccl_cute.DevComm(resource) / - nccl_cute.Window(resource) — so no adapter is needed. Costs a line + The caller wraps each resource -- nccl_cute.DevComm(resource) / + nccl_cute.Window(resource) -- so no adapter is needed. Costs a line per argument but gives better IDE completion and keeps static analysis honest, since the annotations name the types the body actually sees. @@ -183,17 +195,26 @@ def test_nccl_put_resources( def main(): - """Run a 1 MiB GIN put + signal demo across two MPI ranks. + """Run a 1 MiB GIN put + signal demo across exactly two MPI ranks. Returns: - Exit code; 0 on success. Host-side validation on the receiver - prints ``[SUCCESS]`` or ``[ERROR N / NUM_ELEMS mismatches]``. + Exit code; 1 on a wrong rank count or a payload mismatch, otherwise 0. + Host-side validation prints ``[SUCCESS]`` or + ``[ERROR N / NUM_ELEMS mismatches]``. """ comm_mpi = MPI.COMM_WORLD rank = comm_mpi.Get_rank() nranks = comm_mpi.Get_size() root = 0 + if nranks != 2: + if rank == root: + print(f"\n[{NAME}] ERROR: needs exactly 2 ranks, got {nranks}") + return 1 + + if rank == root: + print(f"\n===== {NAME} =====", flush=True) + device = Device(rank % system.get_num_devices()) device.set_current() @@ -212,7 +233,8 @@ def main(): return 0 if rank == root: - print(f"Running with {nranks} ranks, transferring {NUM_ELEMS * 8} bytes...") + print(f"Running with {nranks} ranks, transferring {NUM_ELEMS * 8} bytes " + f"from rank 0 to rank {DST_RANK}...") # Rank 0 fills send_buf with a pattern; rank 1's recv_buf starts zeroed # so the transfer is visible. Each rank registers both windows because @@ -265,7 +287,8 @@ def main(): device.sync() comm_mpi.Barrier() - # Host-side validation on the receiver — compare the full 1 MiB payload. + # Host-side validation on the receiver -- compare the full 1 MiB payload. + mismatches = 0 if rank == DST_RANK: expected = cp.arange(NUM_ELEMS, dtype='int64') mismatches = int((recv_buf != expected).sum().item()) @@ -274,12 +297,16 @@ def main(): else: print(f"[rank {rank}] [ERROR] {mismatches} / {NUM_ELEMS} mismatches") + # Only the destination rank validates, so share the verdict for the exit + # code. + mismatches = comm_mpi.bcast(mismatches, root=DST_RANK) + dev_comm_resource.close() send_win_resource.close() recv_win_resource.close() nccl_comm.destroy() - return 0 + return 0 if mismatches == 0 else 1 if __name__ == "__main__": diff --git a/projects/rccl/bindings/nccl4py/examples/cute/02_lsa_allreduce.py b/projects/rccl/bindings/nccl4py/examples/cute/02_lsa_allreduce.py index 9ecbab8be795..3766bb2995e6 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/02_lsa_allreduce.py +++ b/projects/rccl/bindings/nccl4py/examples/cute/02_lsa_allreduce.py @@ -10,7 +10,7 @@ Every rank stores ``rank + 1`` into its send window; each rank then reads every LSA peer's send window directly over NVLink/peer-access and writes the sum into its own receive window. Two LSA barriers bracket the -reduction — the first split into its ``arrive`` / ``wait`` phases, the +reduction -- the first split into its ``arrive`` / ``wait`` phases, the second a single ``sync``. All ranks must be LSA peers, so run on a single node:: @@ -109,6 +109,7 @@ def lsa_allreduce_kernel( # arrive + wait in one call; release makes the reduction visible first. bar.sync(coop, nccl_cute.MemoryOrder.RELEASE) + bar.destroy() @cute.jit @@ -138,13 +139,19 @@ def main(): """Run the LSA all-reduce and validate the result host-side. Returns: - Exit code; 0 on success, 1 if the payload mismatched. + Exit code; 0 on success, 1 on a wrong rank count, topology error, + or payload mismatch. """ comm_mpi = MPI.COMM_WORLD rank = comm_mpi.Get_rank() nranks = comm_mpi.Get_size() root = 0 + if nranks < 2: + if rank == root: + print(f"\n[{NAME}] ERROR: needs at least 2 ranks, got {nranks}") + return 1 + if rank == root: print(f"\n===== {NAME} =====", flush=True) diff --git a/projects/rccl/bindings/nccl4py/examples/cute/03_multimem.py b/projects/rccl/bindings/nccl4py/examples/cute/03_multimem.py index c0ee8a5b8ce9..ea928f4a19de 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/03_multimem.py +++ b/projects/rccl/bindings/nccl4py/examples/cute/03_multimem.py @@ -9,8 +9,8 @@ A multimem (multicast) mapping lets one store reach every LSA peer's copy of a window. This requests one for the LSA team and shows the two places -the handle comes from — embedded in the devcomm, or returned to the host -by ``create_dev_comm`` — and the three places it is consumed. +the handle comes from -- embedded in the devcomm, or returned to the host +by ``create_dev_comm`` -- and the three places it is consumed. The CuTeDSL layer exposes multimem *addresses*; the multimem load-reduce and store instructions themselves are PTX and are not wrapped here. @@ -87,12 +87,14 @@ def multimem_kernel( bar = nccl_cute.lsa_default( coop, dev_comm, index=BARRIER_INDEX, multimem=True, mm_handle=embedded_mm) bar.sync(coop, nccl_cute.MemoryOrder.ACQ_REL) + bar.destroy() # Same barrier, with team and handle named by the caller. explicit = nccl_cute.lsa_session( coop, dev_comm, dev_comm.team_lsa, dev_comm.lsa_barrier, index=BARRIER_INDEX, multimem=True, mm_handle=host_mm) explicit.sync(coop, nccl_cute.MemoryOrder.ACQ_REL) + explicit.destroy() if 0 == tidx: cute.printf(f"rank {dev_comm.rank}: multimem barriers passed") @@ -122,13 +124,19 @@ def main(): """Request an LSA multimem mapping and exercise it from a kernel. Returns: - Exit code; 0 on success, 1 if multimem is unavailable. + Exit code; 1 on a wrong rank count or topology error, otherwise 0. + Missing multicast support is reported as a skipped run. """ comm_mpi = MPI.COMM_WORLD rank = comm_mpi.Get_rank() nranks = comm_mpi.Get_size() root = 0 + if nranks < 2: + if rank == root: + print(f"\n[{NAME}] ERROR: needs at least 2 ranks, got {nranks}") + return 1 + if rank == root: print(f"\n===== {NAME} =====", flush=True) diff --git a/projects/rccl/bindings/nccl4py/examples/cute/04_gin_ops.py b/projects/rccl/bindings/nccl4py/examples/cute/04_gin_ops.py index f0c5493c24a9..189a94516071 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/04_gin_ops.py +++ b/projects/rccl/bindings/nccl4py/examples/cute/04_gin_ops.py @@ -106,7 +106,7 @@ def gin_ops_kernel( recv_tail = recv_win.tensor(cutlass.Int64, scalar, offset=(NUM_ELEMS - 1) * 8) if SRC_RANK == dev_comm.rank: - # is_counter bumps a *local* counter on local completion — the + # is_counter bumps a *local* counter on local completion -- the # sender-side notification, unlike the receiver-side signal below. # The release scopes say what the caller has released, and what the # transfer needs released before it reads the source. @@ -146,7 +146,7 @@ def gin_ops_kernel( f"rank 1: signal={gin.read_signal(signal=SIGNAL_ID)} " f"recv[0]={recv[0]} tail={recv_tail[0]}") - # A user-managed uint64 next to the signal — NCCL never writes + # A user-managed uint64 next to the signal -- NCCL never writes # it, and reset_signal zeroes both. The natural place to record # how much of the signal this rank has consumed. shadow = cute.make_tensor( diff --git a/projects/rccl/bindings/nccl4py/examples/cute/05_barriers.py b/projects/rccl/bindings/nccl4py/examples/cute/05_barriers.py index 37b4e10fb6cc..9c086229ad26 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/05_barriers.py +++ b/projects/rccl/bindings/nccl4py/examples/cute/05_barriers.py @@ -14,7 +14,7 @@ The factories that take a :class:`Gin` need NCCL 2.31.0, where the C entry points behind them changed from taking ``ncclGin_C`` by value to -taking a pointer. Below that they are skipped — passing a pointer to the +taking a pointer. Below that they are skipped -- passing a pointer to the older entry point corrupts memory instead of failing to link. ``GIN_ALL_CONTEXTS`` goes through a different, unchanged entry point and always runs. @@ -53,7 +53,7 @@ # ncclGinBarrierSessionInit / ncclBarrierSessionInit took ncclGin_C by value up # to 2.30.7 and take a pointer from 2.31.0. The bindings pass a pointer, which -# an older library reads as the struct's first field — silent corruption, so it +# an older library reads as the struct's first field -- silent corruption, so it # must be guarded rather than caught. ncclGinBarrierSessionInitAllContexts is # unchanged, so the GIN_ALL_CONTEXTS form works either way. # get_version() reports the loaded libnccl, which is what the device @@ -97,8 +97,9 @@ def barriers_kernel( # === LSA === # Convenience form: team_lsa + dev_comm.lsa_barrier. - nccl_cute.lsa_default(coop, dev_comm, index=INDEX).sync( - coop, nccl_cute.MemoryOrder.ACQ_REL) + default_lsa = nccl_cute.lsa_default(coop, dev_comm, index=INDEX) + default_lsa.sync(coop, nccl_cute.MemoryOrder.ACQ_REL) + default_lsa.destroy() # Explicit form on the separately requested handle. Only the LSA session # exposes arrive and wait as separate phases. @@ -106,40 +107,52 @@ def barriers_kernel( coop, dev_comm, dev_comm.team_lsa, lsa_handle, index=INDEX) explicit_lsa.arrive(coop, nccl_cute.MemoryOrder.RELEASE) explicit_lsa.wait(coop, nccl_cute.MemoryOrder.ACQUIRE) + explicit_lsa.destroy() # === GIN === # GIN_ALL_CONTEXTS fences every context on the comm rather than the one - # `gin` is bound to — needed when puts span several, at a flush per + # `gin` is bound to -- needed when puts span several, at a flush per # (context, peer). The fence level says what the barrier drains besides # synchronizing: NONE nothing, PUT inbound puts, GET this rank's gets. - nccl_cute.world_gin( - coop, nccl_cute.GIN_ALL_CONTEXTS, dev_comm, index=INDEX - ).sync(coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.PUT) + all_contexts = nccl_cute.world_gin( + coop, nccl_cute.GIN_ALL_CONTEXTS, dev_comm, index=INDEX) + all_contexts.sync( + coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.PUT) + all_contexts.destroy() if NCCL_GIN_SESSION_TAKES_PTR: - nccl_cute.world_gin(coop, gin, dev_comm, index=INDEX).sync( - coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.PUT) + world = nccl_cute.world_gin(coop, gin, dev_comm, index=INDEX) + world.sync(coop, nccl_cute.MemoryOrder.ACQ_REL, + nccl_cute.GinFenceLevel.PUT) + world.destroy() - nccl_cute.rail_gin(coop, gin, dev_comm, index=INDEX).sync( + rail = nccl_cute.rail_gin(coop, gin, dev_comm, index=INDEX) + rail.sync( coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.PUT | nccl_cute.GinFenceLevel.GET) + rail.destroy() # Explicit form: any team, any GIN barrier handle. - nccl_cute.gin_session( - coop, gin, dev_comm, dev_comm.team_world, gin_handle, index=INDEX - ).sync(coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.NONE) + explicit_gin = nccl_cute.gin_session( + coop, gin, dev_comm, dev_comm.team_world, gin_handle, index=INDEX) + explicit_gin.sync( + coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.NONE) + explicit_gin.destroy() # === Hybrid === # LSA within the node, GIN across nodes: one rank per node carries # the outer stage. - nccl_cute.world_hybrid(coop, gin, dev_comm, index=INDEX).sync( + world_hybrid = nccl_cute.world_hybrid( + coop, gin, dev_comm, index=INDEX) + world_hybrid.sync( coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.PUT) + world_hybrid.destroy() # Both stages named explicitly; the handles come from the devcomm's # hybrid pair, sized by requirements.barrier_count. - nccl_cute.hybrid_session( + explicit_hybrid = nccl_cute.hybrid_session( coop, dev_comm.team_lsa, dev_comm.team_world, @@ -147,7 +160,10 @@ def barriers_kernel( dev_comm.hybrid_lsa_barrier, dev_comm.hybrid_rail_gin_barrier, index=INDEX, - ).sync(coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.NONE) + ) + explicit_hybrid.sync( + coop, nccl_cute.MemoryOrder.ACQ_REL, nccl_cute.GinFenceLevel.NONE) + explicit_hybrid.destroy() if 0 == tidx: cute.printf(f"rank {dev_comm.rank}: barrier sessions completed") diff --git a/projects/rccl/bindings/nccl4py/examples/cute/06_resource_buffers.py b/projects/rccl/bindings/nccl4py/examples/cute/06_resource_buffers.py index 6096c5fe7a10..0f378866d37c 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/06_resource_buffers.py +++ b/projects/rccl/bindings/nccl4py/examples/cute/06_resource_buffers.py @@ -92,19 +92,21 @@ def resource_buffers_kernel( cute.printf(f"resource buffer multimem={mm} lsa_multimem={lsa_mm}") cute.printf(f"local == lsa(self): {local == lsa}, local == peer(self): {local == peer}") - # Live barrier storage: reading it around a barrier shows NCCL updating - # it. Never write through it. + # Live barrier storage; never write through it. For this non-multimem + # session, destroy() writes the uint32 epoch at n_barriers + index. state = cute.make_tensor( - cute.make_ptr(cutlass.Uint64, dev_comm.resource_buffer_local_pointer(handle)), - cute.make_layout(1)) - before = state[0] + cute.make_ptr(cutlass.Uint32, dev_comm.resource_buffer_local_pointer(handle)), + cute.make_layout(2 * lsa_handle.n_barriers)) + epoch_slot = lsa_handle.n_barriers + INDEX + before = state[epoch_slot] - nccl_cute.lsa_session( - coop, dev_comm, dev_comm.team_lsa, lsa_handle, index=INDEX - ).sync(coop, nccl_cute.MemoryOrder.ACQ_REL) + bar = nccl_cute.lsa_session( + coop, dev_comm, dev_comm.team_lsa, lsa_handle, index=INDEX) + bar.sync(coop, nccl_cute.MemoryOrder.ACQ_REL) + bar.destroy() if 0 == tidx: - cute.printf(f"barrier state before={before} after={state[0]}") + cute.printf(f"barrier epoch before={before} after={state[epoch_slot]}") @cute.jit @@ -129,13 +131,19 @@ def main(): """Request an LSA barrier resource and translate its handle. Returns: - Exit code; 0 on success, 1 if the topology or multimem is missing. + Exit code; 1 on a wrong rank count or topology error, otherwise 0. + Missing multicast support is reported as a skipped run. """ comm_mpi = MPI.COMM_WORLD rank = comm_mpi.Get_rank() nranks = comm_mpi.Get_size() root = 0 + if nranks < 2: + if rank == root: + print(f"\n[{NAME}] ERROR: needs at least 2 ranks, got {nranks}") + return 1 + if rank == root: print(f"\n===== {NAME} =====", flush=True) diff --git a/projects/rccl/bindings/nccl4py/examples/cute/07_compile_with_fake_args.py b/projects/rccl/bindings/nccl4py/examples/cute/07_compile_with_fake_args.py new file mode 100644 index 000000000000..60e417dbd1de --- /dev/null +++ b/projects/rccl/bindings/nccl4py/examples/cute/07_compile_with_fake_args.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# See LICENSE.txt for more license information +# +"""Compile a CuTeDSL NCCL function before creating its host resources. + +Demonstrates the canonical workflow for the nccl4py CuTeDSL device API: +register two windows (send / recv) on the host, construct ``cute.Tensor`` +views over them inside the kernel via Window.tensor, issue a +single Gin.put with a completion signal, wait on the signal on +the destination rank, and validate the payload host-side. + +The ``@cute.jit`` function is compiled with type-only placeholders before a +communicator or registered window exists. The resulting callable is invoked +later against the real NCCL resources, which supply the runtime values for the +already-compiled signature. The placeholders belong to ``cute.compile`` alone: +passing one to the compiled callable launches with garbage rather than +raising. + +Run with exactly two MPI ranks; the transfer is hardcoded from rank 0 to +rank 1:: + + mpirun -n 2 python 07_compile_with_fake_args.py +""" + +import os +import sys + +try: + from mpi4py import MPI +except ImportError: + print("ERROR: mpi4py required. Install with: pip install mpi4py", flush=True) + sys.exit(1) + +try: + from cuda.core import Device, system +except ImportError: + print("ERROR: cuda.core required. Install with: pip install cuda-core", flush=True) + sys.exit(1) + +try: + import cupy as cp +except ImportError: + print("ERROR: cupy required. Install with: pip install cupy-cuda13x (or cupy-cuda12x)", flush=True) + sys.exit(1) + +import cutlass +import cutlass.cute as cute +from cutlass.cute.arch.nvvm_wrappers import WARP_SIZE +import nccl.core as nccl +import nccl.core.device.cute as nccl_cute + + +NAME = os.path.basename(__file__) + +# 1 MiB transfer: 131072 Int64 elements * 8 bytes = 1,048,576 bytes. +NUM_ELEMS = 1024 * 1024 // 8 +DST_RANK = 1 +SIGNAL_ID = 1 + +@cute.kernel +def test_nccl_put_kernel( + dev_comm: nccl_cute.DevComm, + send_win: nccl_cute.Window, + recv_win: nccl_cute.Window, + signal_id, +): + """Issue a 1 MiB GIN put from rank 0 to rank 1 via ``cute.Tensor`` views. + + Runs with exactly 2 ranks. Two separate windows make the data flow + explicit: + + * ``send_win`` is rank 0's source buffer (pre-filled with + ``arange`` on the host). + * ``recv_win`` is rank 1's destination buffer (validated host-side + after sync). + + NCCL window registrations are collective, so both ranks register both + windows; rank 0's ``recv_win`` and rank 1's ``send_win`` exist but go + unused. + + Args: + dev_comm: Value-mode nccl_cute.DevComm reconstructed while tracing + from the placeholder's MLIR argument. + send_win: CuTeDSL view of the registered source window. + recv_win: CuTeDSL view of the registered destination window. + signal_id: GIN signal slot this launch uses; a Python int, so it + is baked in as a compile-time constant. + """ + tidx, _, _ = cute.arch.thread_idx() + team = dev_comm.team_world + gin = dev_comm.gin(nccl_cute.GinBackendMask.ALL, 0) + coop = nccl_cute.cta() + + # cute.Tensor views spanning the full 1 MiB of each window. + send = send_win.tensor(cutlass.Int64, cute.make_layout(NUM_ELEMS)) + recv = recv_win.tensor(cutlass.Int64, cute.make_layout(NUM_ELEMS)) + + if team.nRanks >= 2: + if 0 == team.rank: + if 0 == tidx: + cute.printf(f"Before Put: send[0]={send[0]} send[{NUM_ELEMS - 1}]={send[NUM_ELEMS - 1]}") + gin.put( + team, + DST_RANK, + recv_win, recv, # destination window + tensor (lives on the peer) + send_win, send, # source window + tensor (local) + coop, + is_signal=True, + signal_id=signal_id, + signal_op=0, + signal_op_arg=1, + ) + if 1 == team.rank: + gin.wait_signal(coop, signal=signal_id, least=1) + if 0 == tidx: + cute.printf(f"After Put: recv[0]={recv[0]} recv[{NUM_ELEMS - 1}]={recv[NUM_ELEMS - 1]}") + + +# The annotations name the views reconstructed while tracing, and the +# compile-only placeholders satisfy them without owning NCCL resources. As in +# 00_basic.py, they also fix the argument form: the caller passes nccl_cute +# objects, not raw resources. + + +@cute.jit +def test_nccl_put( + dev_comm: nccl_cute.DevComm, + send_win: nccl_cute.Window, + recv_win: nccl_cute.Window, + ): + """Launch the kernel with a signature described by device views. + + This example compiles the function with type-only placeholders before + any NCCL resources exist, then invokes the compiled callable with views + wrapping the real resources. + + Args: + dev_comm: CuTeDSL view of the NCCL device communicator. + send_win: CuTeDSL view of the registered source window. + recv_win: CuTeDSL view of the registered destination window. + """ + test_nccl_put_kernel(dev_comm, send_win, recv_win, SIGNAL_ID).launch( + grid=[1, 1, 1], + block=[cute.size(WARP_SIZE, mode=[0]), 1, 1], + cooperative=True + ) + + +def main(): + """Run a 1 MiB GIN put + signal demo across exactly two MPI ranks. + + Returns: + Exit code; 1 on a wrong rank count or a payload mismatch, otherwise 0. + Host-side validation prints ``[SUCCESS]`` or + ``[ERROR N / NUM_ELEMS mismatches]``. + """ + comm_mpi = MPI.COMM_WORLD + rank = comm_mpi.Get_rank() + nranks = comm_mpi.Get_size() + root = 0 + + if nranks != 2: + if rank == root: + print(f"\n[{NAME}] ERROR: needs exactly 2 ranks, got {nranks}") + return 1 + + if rank == root: + print(f"\n===== {NAME} =====", flush=True) + + device = Device(rank % system.get_num_devices()) + device.set_current() + + # Compile before any NCCL communicator or registered window exists. These + # placeholders contribute only MLIR argument types; the compiled callable + # receives real resources below. + compiled_test_nccl_put = cute.compile( + test_nccl_put, + nccl_cute.runtime.make_fake_dev_comm(), + nccl_cute.runtime.make_fake_window(), + nccl_cute.runtime.make_fake_window(), + ) + + unique_id = nccl.get_unique_id() if rank == root else None + unique_id = comm_mpi.bcast(unique_id, root=root) + + nccl_comm = nccl.Communicator.init(nranks=nranks, rank=rank, unique_id=unique_id) + + # The put below needs a GIN transport. + if not nccl_comm.device_api_support or nccl_comm.gin_type == nccl.NcclGinType.NONE: + if rank == root: + print(f"Gin.put needs a GIN transport, which this platform does not " + f"provide (device_api_support={nccl_comm.device_api_support}, " + f"gin_type={nccl_comm.gin_type.name}); nothing to run") + nccl_comm.destroy() + return 0 + + if rank == root: + print(f"Running with {nranks} ranks, transferring {NUM_ELEMS * 8} bytes " + f"from rank 0 to rank {DST_RANK}...") + + # Rank 0 fills send_buf with a pattern; rank 1's recv_buf starts zeroed + # so the transfer is visible. Each rank registers both windows because + # registration is collective, so one of them goes unused. + send_buf = nccl.cupy.empty(NUM_ELEMS, dtype='int64') + recv_buf = nccl.cupy.empty(NUM_ELEMS, dtype='int64') + if rank == 0: + send_buf[:] = cp.arange(NUM_ELEMS, dtype='int64') + else: + send_buf[:] = 0 + recv_buf[:] = 0 + device.sync() # make host-side fill visible before the kernel runs + + send_win_resource = nccl_comm.register_window(send_buf) + recv_win_resource = nccl_comm.register_window(recv_buf) + assert send_win_resource is not None and send_win_resource.is_valid + assert recv_win_resource is not None and recv_win_resource.is_valid + + reqs = nccl.NCCLDevCommRequirements( + gin_connection_type=nccl.NcclGinConnectionType.FULL, + gin_signal_count=SIGNAL_ID + 1, + ) + dev_comm_resource = nccl_comm.create_dev_comm(requirements=reqs) + assert dev_comm_resource.is_valid + assert dev_comm_resource.ptr != 0 + + if rank == root: + print("Launching a function compiled before NCCL resources existed") + compiled_test_nccl_put( + nccl_cute.DevComm(dev_comm_resource), + nccl_cute.Window(send_win_resource), + nccl_cute.Window(recv_win_resource), + ) + device.sync() + comm_mpi.Barrier() + + # Host-side validation on the receiver -- compare the full 1 MiB payload. + mismatches = 0 + if rank == DST_RANK: + expected = cp.arange(NUM_ELEMS, dtype='int64') + mismatches = int((recv_buf != expected).sum().item()) + if mismatches == 0: + print(f"[rank {rank}] [SUCCESS] {NUM_ELEMS * 8} bytes transferred correctly") + else: + print(f"[rank {rank}] [ERROR] {mismatches} / {NUM_ELEMS} mismatches") + + # Only the destination rank validates, so share the verdict for the exit + # code. + mismatches = comm_mpi.bcast(mismatches, root=DST_RANK) + + dev_comm_resource.close() + send_win_resource.close() + recv_win_resource.close() + nccl_comm.destroy() + + return 0 if mismatches == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/rccl/bindings/nccl4py/examples/cute/08_reduce_copy.py b/projects/rccl/bindings/nccl4py/examples/cute/08_reduce_copy.py new file mode 100644 index 000000000000..4eb0f8765986 --- /dev/null +++ b/projects/rccl/bindings/nccl4py/examples/cute/08_reduce_copy.py @@ -0,0 +1,127 @@ +"""LSA ReduceSum example using the nccl4py CuTeDSL device API. + +Each rank contributes its LSA rank plus one. The kernel reduces the +registered source windows across each LSA team and writes the sum to each +rank's local destination buffer. + +Run with two or more MPI ranks:: + + mpirun -n 2 python 08_reduce_copy.py +""" + +import sys + +try: + from mpi4py import MPI +except ImportError: + print("ERROR: mpi4py required. Install with: pip install mpi4py") + sys.exit(1) + +try: + from cuda.core import Device, system +except ImportError: + print("ERROR: cuda.core required. Install with: pip install cuda-core") + sys.exit(1) + +import cutlass +import cutlass.cute as cute +from cutlass.cute.arch.nvvm_wrappers import WARP_SIZE + +import nccl.core as nccl +import nccl.core.device.cute as nccl_cute + + +NUM_ELEMS = 16 + + +@cute.kernel +def lsa_reduce_sum_kernel( + src_window: nccl_cute.Window, + dst_window: nccl_cute.Window, + lsa_rank: cutlass.Int32, + lsa_size: cutlass.Int32, +): + team = nccl_cute.Team( + nRanks=lsa_size, + rank=lsa_rank, + stride=cutlass.Int32(1), + ) + layout = cute.make_layout(NUM_ELEMS) + src = src_window.tensor(cutlass.Float32, layout) + dst = dst_window.tensor(cutlass.Float32, layout) + nccl_cute.lsa_reduce_sum( + nccl_cute.cta(), + src_window, + src, + dst, + NUM_ELEMS, + team=team, + ) + + +@cute.jit +def run_lsa_reduce_sum( + src_window: nccl_cute.Window, + dst_window: nccl_cute.Window, + lsa_rank: cutlass.Int32, + lsa_size: cutlass.Int32, +): + lsa_reduce_sum_kernel(src_window, dst_window, lsa_rank, lsa_size).launch( + grid=[1, 1, 1], + block=[cute.size(WARP_SIZE, mode=[0]), 1, 1], + cooperative=True, + ) + + +def main() -> int: + comm_mpi = MPI.COMM_WORLD + rank = comm_mpi.Get_rank() + nranks = comm_mpi.Get_size() + root = 0 + + local_comm = comm_mpi.Split_type(MPI.COMM_TYPE_SHARED) + local_rank = local_comm.Get_rank() + + device = Device(local_rank % system.get_num_devices()) + device.set_current() + + unique_id = nccl.get_unique_id() if rank == root else None + unique_id = comm_mpi.bcast(unique_id, root=root) + nccl_comm = nccl.Communicator.init( + nranks=nranks, + rank=rank, + unique_id=unique_id, + ) + lsa_team = nccl_comm.team_lsa + + src_buf = nccl.cupy.empty(NUM_ELEMS, dtype="float32") + dst_buf = nccl.cupy.empty(NUM_ELEMS, dtype="float32") + src_buf[:] = lsa_team.rank + 1 + dst_buf[:] = 0 + device.sync() + + src_win_resource = nccl_comm.register_window(src_buf) + dst_win_resource = nccl_comm.register_window(dst_buf) + assert src_win_resource is not None and src_win_resource.is_valid + assert dst_win_resource is not None and dst_win_resource.is_valid + + src_window = nccl_cute.Window(src_win_resource) + dst_window = nccl_cute.Window(dst_win_resource) + + run_lsa_reduce_sum( + src_window, + dst_window, + lsa_team.rank, + lsa_team.n_ranks, + ) + device.sync() + + src_win_resource.close() + dst_win_resource.close() + nccl_comm.destroy() + local_comm.Free() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/rccl/bindings/nccl4py/examples/cute/README.md b/projects/rccl/bindings/nccl4py/examples/cute/README.md index 3ce7d2a8bc67..84e956279275 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/README.md +++ b/projects/rccl/bindings/nccl4py/examples/cute/README.md @@ -17,39 +17,70 @@ installed `nvidia-nccl-cuXX` wheel. | File | Topic | Ranks | Needs | |---|---|---|---| -| `main.py` | Start here: 1 MiB `Gin.put` with a completion signal, launched through both `@cute.jit` argument forms | **exactly 2** | GIN network | -| `01_coop_and_teams.py` | Cooperative groups, devcomm fields, teams and rank translation | any | — | -| `02_lsa_allreduce.py` | Window address translation + LSA barriers, as a device-side all-reduce | ≥2, one node | peer access | -| `03_multimem.py` | Multimem handles, multimem window addresses, multimem barrier arrive | ≥2, one node | multicast | +| `00_basic.py` | Start here: 1 MiB `Gin.put` with a completion signal, launched through both `@cute.jit` argument forms | **exactly 2** | GIN network | +| `01_coop_and_teams.py` | Cooperative groups, devcomm fields, teams and rank translation | any | - | +| `02_lsa_allreduce.py` | Window address translation + LSA barriers, as a device-side all-reduce | >=2, one node | peer access | +| `03_multimem.py` | Multimem handles, multimem window addresses, multimem barrier arrive | >=2, one node | multicast | | `04_gin_ops.py` | Every `Gin` operation: put / put_value / get / signal / counters / flush | **exactly 2** | GIN network, 2.31.1+ for `get` | -| `05_barriers.py` | All three barrier session types, both factory forms, all fence levels | ≥2 | GIN network, 2.31.0+ for the `Gin` factories | -| `06_resource_buffers.py` | The five `resource_buffer_*` address translations | ≥2, one node | peer access, multicast | +| `05_barriers.py` | All three barrier session types, both factory forms, all fence levels | >=2 | GIN network, 2.31.0+ for the `Gin` factories | +| `06_resource_buffers.py` | The five `resource_buffer_*` address translations | >=2, one node | peer access, multicast | +| `07_compile_with_fake_args.py` | Compile with type-only arguments before creating NCCL resources, then invoke with real resources | **exactly 2** | GIN network | +| `08_reduce_copy.py` | Device-side `ReduceCopy`: `lsa_reduce_sum` across registered windows into each rank's local buffer | >=2, one node | peer access | +"Exactly 2" means the example hardcodes a rank 0 -> rank 1 transfer and +rejects any other count rather than idling the extra ranks. "One node" means every rank must be in the same LSA team, since those examples read peer memory directly. GIN examples want two nodes for a real network path, but a single node works when NCCL can set up loopback GIN connections. -Run under MPI, one rank per GPU — substitute `srun --mpi=pmix -n ` +Run under MPI, one rank per GPU -- substitute `srun --mpi=pmix -n ` under Slurm. Recommended command per example: ```sh -mpirun -n 2 -N 1 python main.py # two nodes, one rank each +mpirun -n 2 -N 1 python 00_basic.py # two nodes, one rank each mpirun -n 2 python 01_coop_and_teams.py # any rank count mpirun -n 8 python 02_lsa_allreduce.py # one node, all GPUs mpirun -n 8 python 03_multimem.py # one node, all GPUs mpirun -n 2 -N 1 python 04_gin_ops.py # two nodes, one rank each mpirun -n 2 -N 1 python 05_barriers.py # two nodes, one rank each mpirun -n 8 python 06_resource_buffers.py # one node, all GPUs +mpirun -n 2 -N 1 python 07_compile_with_fake_args.py # two nodes, one rank each +mpirun -n 8 python 08_reduce_copy.py # one node, all GPUs ``` `-n 8` stands for however many GPUs the node has; the one-node examples -scale to any count ≥ 2. +scale to any count >= 2. Nothing faults on an unsuitable machine: a wrong rank count or topology is an `ERROR` with a non-zero exit, and a missing capability is reported as skipped. +## Compile before creating NCCL resources + +`DevComm` and `Window` are dynamic arguments, so an application may compile a +`@cute.jit` function before it creates a communicator or registers memory: + +```python +compiled = cute.compile( + fn, + nccl_cute.runtime.make_fake_dev_comm(), + nccl_cute.runtime.make_fake_window(), +) + +# Create the real NCCL resources later, then supply their runtime values. +compiled(nccl_cute.DevComm(dev_comm_resource), nccl_cute.Window(window_resource)) +``` + +The fake objects describe only the JIT argument types. They carry no NCCL +resource or runtime pointer, are not `Constexpr` values, and must only be +passed to `cute.compile`. Passing one to the compiled callable contributes no +runtime argument, which shifts every argument after it and launches with +garbage rather than raising, so invoke with host-mode views as above -- or with +the resources themselves, if the parameters are unannotated (see 00). The same +module provides compile-only factories for `MultimemHandle`, +`LsaBarrierHandle`, and `GinBarrierHandle` arguments. + ## Output The numbered examples name themselves on startup, warn when an API they @@ -70,7 +101,7 @@ affected sections check `nccl.get_version()` and skip rather than fail: `ncclGinGet` only gained C linkage in **2.31.1**, and the barrier-session entry points started taking `ncclGin_C` by pointer in **2.31.0**. The check reads the loaded `libnccl.so` and assumes `libnccl_device.bc` came from the -same install — which is how the wheel and a source build ship them. If you +same install -- which is how the wheel and a source build ship them. If you `LD_PRELOAD` one, point `NCCL_HOME` at the other's build. ## API coverage @@ -84,21 +115,25 @@ same install — which is how the wheel and a source build ship them. If you | `DevComm.lsa_barrier` / `rail_gin_barrier` / `world_gin_barrier` / `hybrid_*` | 05 | | `DevComm.lsa_multimem` | 03, 06 | | `DevComm.resource_buffer_*` (all five) | 06 | -| `Window.tensor` | 02, 04, main | +| `Window.tensor` | 00, 02, 04, 07 | | `Window.local_pointer` / `lsa_pointer` / `peer_pointer` | 02 | | `Window.multimem_pointer` / `lsa_multimem_pointer` | 03 | -| `Gin.put` | 04, main | +| `Gin.put` | 00, 04, 07 | | `Gin.put_value` / `get` / `signal` / `flush` | 04 | -| `Gin.read_signal` / `wait_signal` / `reset_signal` / `signal_shadow_pointer` | 04, main | +| `Gin.wait_signal` | 00, 04, 07 | +| `Gin.read_signal` / `reset_signal` / `signal_shadow_pointer` | 04 | | `Gin.read_counter` / `wait_counter` / `reset_counter` | 04 | | `lsa_session` / `lsa_default`, `arrive` / `wait` / `sync` | 02, 03, 05, 06 | | `gin_session` / `world_gin` / `rail_gin`, `GIN_ALL_CONTEXTS` | 05 | | `hybrid_session` / `world_hybrid` | 05 | | `MultimemHandle` / `LsaBarrierHandle` / `GinBarrierHandle` and their fields | 03, 05, 06 | -| `MemoryOrder`, `ThreadScope`, `GinFenceLevel`, `GinBackendMask`, `GinResourceSharingMode` | 01–06 | +| `MemoryOrder` | 02-06 | +| `ThreadScope` / `GinResourceSharingMode` | 04 | +| `GinFenceLevel` | 05 | +| `GinBackendMask` | 00, 04, 05, 07 | Not exercised: `Gin.value()` and the raw `.ptr` accessors (binding plumbing); the `is_descriptor` / `descriptor_ptr` arguments, which need an `ncclGinDescriptorSmem` this layer does not wrap; and pass-through knobs -left at their defaults — `opt_flags`, `signal_op=1` (Add), the non-GPU +left at their defaults -- `opt_flags`, `signal_op=1` (Add), the non-GPU resource-sharing modes, and the individual `GinBackendMask` bits. diff --git a/projects/rccl/bindings/nccl4py/examples/cute/requirements-cu12.txt b/projects/rccl/bindings/nccl4py/examples/cute/requirements-cu12.txt index e384a0e714e2..2bc5c8590646 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/requirements-cu12.txt +++ b/projects/rccl/bindings/nccl4py/examples/cute/requirements-cu12.txt @@ -1,4 +1,5 @@ # CUDA 12. nccl4py[cu12] brings in cuda.core and nvidia-cutlass-dsl. nccl4py[cu12] -cupy-cuda12x +# CuPy 14's NVRTC path needs the CUDA toolkit components supplied by the ctk extra. +cupy-cuda12x[ctk]>=14.0 mpi4py diff --git a/projects/rccl/bindings/nccl4py/examples/cute/requirements-cu13.txt b/projects/rccl/bindings/nccl4py/examples/cute/requirements-cu13.txt index ddd12f201a47..75151ceaa796 100644 --- a/projects/rccl/bindings/nccl4py/examples/cute/requirements-cu13.txt +++ b/projects/rccl/bindings/nccl4py/examples/cute/requirements-cu13.txt @@ -1,4 +1,5 @@ # CUDA 13. nccl4py[cu13] brings in cuda.core and nvidia-cutlass-dsl. nccl4py[cu13] -cupy-cuda13x +# CuPy 14's NVRTC path needs the CUDA toolkit components supplied by the ctk extra. +cupy-cuda13x[ctk]>=14.0 mpi4py diff --git a/projects/rccl/bindings/nccl4py/nccl/__init__.py b/projects/rccl/bindings/nccl4py/nccl/__init__.py index 538cd080e91b..80badfa58c69 100644 --- a/projects/rccl/bindings/nccl4py/nccl/__init__.py +++ b/projects/rccl/bindings/nccl4py/nccl/__init__.py @@ -14,9 +14,6 @@ and custom reduction operators. """ -from nccl._version import __version__ -from nccl._show_versions import LibraryInfo, VersionInfo, get_version, show_versions - # Register the local HIP-backed cuda.core shim under the `cuda.core` # namespace via sys.modules. Idempotent and gated on # `"cuda.core" not in sys.modules`, so it is a no-op if a real @@ -26,12 +23,16 @@ # top-level `cuda` package on disk (a regular non-PEP-420 # `cuda/__init__.py` would shadow co-installed distributions that # legitimately contribute to `cuda.*`), so users must `import nccl` -# before reaching `cuda.core`. +# before reaching `cuda.core`. It runs first: importing nccl.core +# (even for _version) pulls in cuda.core. from nccl._hip_compat.cuda_core_shim import _register_as_cuda_core as _register_cuda_core_shim _register_cuda_core_shim() del _register_cuda_core_shim +from nccl.core._version import __version__ +from nccl._show_versions import LibraryInfo, VersionInfo, get_version, show_versions + # Re-export get_version() lazily. Importing nccl alone does not load # librccl.so — that happens only on first access of ``nccl.get_version`` diff --git a/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/__init__.py b/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/__init__.py index 8c1c68a6adc5..0e65a294adc8 100644 --- a/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/__init__.py +++ b/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/__init__.py @@ -13,7 +13,7 @@ ``nvidia-*`` packages, or ``hip-python-as-cuda``. Scope: exactly the symbols imported by ``nccl/core/*.py`` (``Device``, -``Stream``, ``Buffer``, ``MemoryResource``, ``system``, ``IsStreamType``, +``Stream``, ``Event``, ``Buffer``, ``MemoryResource``, ``system``, ``IsStreamType``, ``DevicePointerType``, ``StridedMemoryView``, ``args_viewable_as_strided_memory``), each reachable under every spelling cuda.core has used for it. Anything outside that surface is @@ -60,6 +60,10 @@ def _resolve_cuda_core(name: str) -> Any: from ._stream import Stream return Stream + if name == "Event": + from ._event import Event + + return Event if name == "Buffer": from ._memory import Buffer diff --git a/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/_device.py b/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/_device.py index 343e8a962107..072982ee2f84 100644 --- a/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/_device.py +++ b/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/_device.py @@ -12,8 +12,9 @@ .set_current() -> hipSetDevice .default_stream -> Stream wrapping the HIP NULL stream .create_stream(obj) -> Stream wrapping a foreign __cuda_stream__ object + .create_event() -> Event with timing disabled (hipEventCreateWithFlags) -``DeviceProperties`` / contexts / events / graphs / kernels / launch are +``DeviceProperties`` / contexts / graphs / kernels / launch are intentionally NOT ported. ``nccl/core`` imports none of them. """ @@ -23,6 +24,7 @@ from typing import Optional from ._hip import check_hip, hip +from ._event import Event, EventOptions from ._stream import Stream, StreamOptions, default_stream _tls = threading.local() @@ -92,6 +94,10 @@ def create_stream(self, obj=None, options: Optional[StreamOptions] = None) -> St """ return Stream._init(obj=obj, options=options, device_id=self._id) + def create_event(self, options: Optional[EventOptions] = None) -> Event: + """Create a HIP event (timing disabled by default).""" + return Event._init(options=options) + def __repr__(self) -> str: return f"" diff --git a/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/_event.py b/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/_event.py new file mode 100644 index 000000000000..9737024eb995 --- /dev/null +++ b/projects/rccl/bindings/nccl4py/nccl/_hip_compat/cuda_core_shim/_event.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``cuda.core.Event`` shim: only the surface ``nccl/core`` uses. + +``launch_completion_event`` only needs a handle; NCCL records the event. +``record()``, ``sync()``, ``query`` and timing are not implemented. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from ._hip import check_hip, hip + + +@dataclass +class EventOptions: + """Options for :class:`Event` creation; only ``enable_timing`` is honored.""" + + enable_timing: bool = False + + +class Event: + """HIP event; create via :meth:`Device.create_event` or :meth:`Event.from_handle`.""" + + __slots__ = ("_handle", "_owner") + + def __init__(self, *args, **kwargs): + raise RuntimeError( + "Event objects cannot be instantiated directly. " + "Please use Device APIs (create_event) or other Event APIs (from_handle)." + ) + + @classmethod + def _init(cls, options: Optional[EventOptions] = None) -> "Event": + opts = options if options is not None else EventOptions() + flags = int(hip.hipEventDefault) if opts.enable_timing else int(hip.hipEventDisableTiming) + + self = object.__new__(cls) + self._owner = None + self._handle = int( + check_hip( + hip.hipEventCreateWithFlags(flags), + "hipEventCreateWithFlags", + ) + ) + return self + + @classmethod + def from_handle(cls, handle: int) -> "Event": + """Wrap an existing raw event handle (no ownership transferred).""" + self = object.__new__(cls) + self._handle = int(handle) + self._owner = "external" + return self + + @property + def handle(self) -> int: + """Raw HIP event pointer as a Python int.""" + return self._handle + + def close(self) -> None: + """Destroy the event if we own it; no-op for borrowed events.""" + if self._owner is None and self._handle: + try: + check_hip(hip.hipEventDestroy(self._handle), "hipEventDestroy") + except Exception: + # Avoid raising during finalization + pass + self._handle = 0 + + def __del__(self): + try: + self.close() + except Exception: + pass + + def __repr__(self) -> str: + return f"" diff --git a/projects/rccl/bindings/nccl4py/nccl/_show_versions.py b/projects/rccl/bindings/nccl4py/nccl/_show_versions.py index af7155309f54..a25a9d473d1e 100644 --- a/projects/rccl/bindings/nccl4py/nccl/_show_versions.py +++ b/projects/rccl/bindings/nccl4py/nccl/_show_versions.py @@ -23,7 +23,7 @@ from packaging.version import Version as _Version -from nccl._version import __version__ +from nccl.core._version import __version__ from nccl.bindings import nccl as _nccl_bindings # nccl_ep is not built on ROCm; version reporting already treats it as optional. diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/__init__.py b/projects/rccl/bindings/nccl4py/nccl/bindings/__init__.py index 57072b42cafd..4df05a3981d9 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/__init__.py +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/__init__.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +"""Low-level Python bindings for NCCL.""" + from .nccl import * # Hand-written wrappers for RCCL-only collectives diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/__init__.py b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/__init__.py index 65e4f6438a57..28bcfc2b0807 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/__init__.py +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/__init__.py @@ -2,3 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information + +"""Internal implementation modules for the NCCL bindings.""" diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/nccl.pxd b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/nccl.pxd index cd7eace5c68e..0222a7486d87 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/nccl.pxd +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/nccl.pxd @@ -2,7 +2,24 @@ # # SPDX-License-Identifier: Apache-2.0 # -# This code was automatically generated with version 2.30.7. Do not modify it directly. +# This code was automatically generated with version 2.32.3. Do not modify it directly. + + +# <<<< PREAMBLE CONTENT >>>> + +from libc.stdint cimport ( + int16_t, + int32_t, + int64_t, + int8_t, + uint16_t, + uint32_t, + uint64_t, + uint8_t, +) + + +# <<<< END OF PREAMBLE CONTENT >>>> from ..cynccl cimport * @@ -52,6 +69,14 @@ cdef ncclResult_t _ncclAllGather(const void* sendbuff, void* recvbuff, size_t se cdef ncclResult_t _ncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGather(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclScatter(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclAllReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclBroadcastConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclAllGatherConfig(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclReduceScatterConfig(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclAlltoAllConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclGatherConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclScatterConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclPutSignal(const void* localbuff, size_t count, ncclDataType_t datatype, int peer, ncclWindow_t peerWin, size_t peerWinOffset, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil @@ -60,6 +85,7 @@ cdef ncclResult_t _ncclWaitSignal(int nDesc, ncclWaitSignalDesc_t* signalDescs, cdef ncclResult_t _ncclGroupStart() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGroupEnd() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclSetEncryption(const ncclEncryptionConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclParamBind(ncclParamHandle_t* out, const char* key) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclParamGetI8(ncclParamHandle_t h, int8_t* out) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclParamGetI16(ncclParamHandle_t h, int16_t* out) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil @@ -78,15 +104,20 @@ cdef ncclResult_t _ncclCommQueryProperties(ncclComm_t comm, ncclCommProperties_t cdef ncclResult_t _ncclDevCommCreate(ncclComm_t comm, const ncclDevCommRequirements_t* reqs, ncclDevComm_t* outDevComm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclDevCommDestroy(ncclComm_t comm, const ncclDevComm_t* devComm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGetLsaMultimemDevicePointer(ncclWindow_t window, size_t offset, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil -cdef ncclResult_t _ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGetMultimemDevicePointer(ncclWindow_t window, size_t offset, ncclMultimemHandle_t multimem, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGetPeerDevicePointer(ncclWindow_t window, size_t offset, int peer, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclGetMultimemDeviceLeInfo(ncclWindow_t window, size_t offset, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclGetCftDeviceLeInfo(ncclWindow_t window, size_t offset, int peerCft, ncclTeam_t cftTeam, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t _ncclGetPeerDeviceLeInfo(ncclWindow_t window, size_t offset, int peerWorld, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclTeam_t _ncclTeamWorld(ncclComm_t comm) except* nogil cdef ncclTeam_t _ncclTeamLsa(ncclComm_t comm) except* nogil -cdef ncclTeam_t _ncclTeamRail(ncclComm_t comm) except* nogil +cdef ncclTeam_t _ncclTeamCft(ncclComm_t comm, ncclCftTeamMode_t mode) except* nogil +cdef ncclTeam_t _ncclTeamCftMultimem(ncclComm_t comm) except* nogil cdef int _ncclTeamRankToWorld(ncclComm_t comm, ncclTeam_t team, int rank) except?-42 nogil cdef int _ncclTeamRankToLsa(ncclComm_t comm, ncclTeam_t team, int rank) except?-42 nogil +cdef ncclTeam_t _ncclTeamRail(ncclComm_t comm) except* nogil cdef ncclResult_t _ncclLsaBarrierCreateRequirement(ncclTeam_t team, int nBarriers, ncclLsaBarrierHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGinBarrierCreateRequirement(ncclComm_t comm, ncclTeam_t team, int nBarriers, ncclGinBarrierHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil -cdef ncclResult_t _ncclLLA2ACreateRequirement(int nBlocks, int nSlots, ncclLLA2AHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef int _ncclLLA2ACalcSlots(int maxElts, int maxEltSize) except?-42 nogil +cdef ncclResult_t _ncclLLA2ACreateRequirement(int nBlocks, int nSlots, ncclLLA2AHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/nccl_linux.pyx b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/nccl_linux.pyx index 055b6e7aecfa..60dc75157352 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/nccl_linux.pyx +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/nccl_linux.pyx @@ -3,54 +3,82 @@ # # SPDX-License-Identifier: Apache-2.0 # -# This code was automatically generated with version 2.30.7. Do not modify it directly. - -from libc.stdint cimport intptr_t, uintptr_t - -import os -import threading +# This code was automatically generated with version 2.32.3. Do not modify it directly. + + + +# <<<< PREAMBLE CONTENT >>>> + +cdef extern from * nogil: + """ + #if defined(_MSC_VER) && !defined(__clang__) + #include + static __forceinline int atomic_int_load(int *p) { + int v = *(int volatile *)p; _ReadBarrier(); return v; + } + static __forceinline void atomic_int_store(int *p, int v) { + _WriteBarrier(); *(int volatile *)p = v; + } + #elif defined(__cplusplus) + /* GCC/Clang __atomic builtins work in any C++ standard without headers */ + static inline int atomic_int_load(int *p) { + return __atomic_load_n(p, __ATOMIC_ACQUIRE); + } + static inline void atomic_int_store(int *p, int v) { + __atomic_store_n(p, v, __ATOMIC_RELEASE); + } + #else + #include + static inline int atomic_int_load(int *p) { + return (int)atomic_load_explicit((atomic_int *)p, memory_order_acquire); + } + static inline void atomic_int_store(int *p, int v) { + atomic_store_explicit((atomic_int *)p, v, memory_order_release); + } + #endif + + """ + cdef int _cyb_atomic_int_load "atomic_int_load"(int *p) nogil + cdef void _cyb_atomic_int_store "atomic_int_store"(int *p, int v) nogil + +cdef extern from "": + void* _cyb_dlsym "dlsym"(void*, const char*) nogil + const void * _cyb_RTLD_DEFAULT "RTLD_DEFAULT" + +cimport cython as _cyb_cython +from libc.stdint cimport ( + int16_t, + int32_t, + int64_t, + int8_t, + intptr_t, + uint16_t, + uint32_t, + uint64_t, + uint8_t, +) + +import threading as _cyb_threading + +cdef int _cyb___py_nccl_init = 0 +cdef dict _cyb_func_ptrs = None +cdef object _cyb_symbol_lock = _cyb_threading.Lock() + +# <<<< END OF PREAMBLE CONTENT >>>> + +from libc.stdint cimport uintptr_t from .utils import FunctionNotFoundError, NotSupportedError - -############################################################################### -# Extern -############################################################################### - -# You must 'from .utils import NotSupportedError' before using this template - +# Patched for RCCL: cuda.pathfinder is not used on ROCm; load_library() below +# dlopen()s librccl.so directly, so declare the dlfcn entry points it needs. cdef extern from "" nogil: void* dlopen(const char*, int) char* dlerror() - void* dlsym(void*, const char*) - int dlclose(void*) enum: - RTLD_LAZY RTLD_NOW RTLD_GLOBAL - RTLD_LOCAL - - const void* RTLD_DEFAULT 'RTLD_DEFAULT' - -cdef int get_cuda_version(): - cdef void* handle = NULL - cdef int err, driver_ver = 0 - - # Load driver to check version - handle = dlopen('libcuda.so.1', RTLD_NOW | RTLD_GLOBAL) - if handle == NULL: - err_msg = dlerror() - raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') - cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") - if cuDriverGetVersion == NULL: - raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') - err = (cuDriverGetVersion)(&driver_ver) - if err != 0: - raise RuntimeError(f'cuDriverGetVersion returned error code {err}') - - return driver_ver - cdef extern from "" nogil: ctypedef struct Dl_info: @@ -65,8 +93,6 @@ cdef extern from "" nogil: # Wrapper init ############################################################################### -cdef object __symbol_lock = threading.Lock() -cdef bint __py_nccl_init = False cdef void* __ncclMemAlloc = NULL cdef void* __ncclMemFree = NULL @@ -109,6 +135,14 @@ cdef void* __ncclAllGather = NULL cdef void* __ncclAlltoAll = NULL cdef void* __ncclGather = NULL cdef void* __ncclScatter = NULL +cdef void* __ncclAllReduceConfig = NULL +cdef void* __ncclBroadcastConfig = NULL +cdef void* __ncclReduceConfig = NULL +cdef void* __ncclAllGatherConfig = NULL +cdef void* __ncclReduceScatterConfig = NULL +cdef void* __ncclAlltoAllConfig = NULL +cdef void* __ncclGatherConfig = NULL +cdef void* __ncclScatterConfig = NULL cdef void* __ncclSend = NULL cdef void* __ncclRecv = NULL cdef void* __ncclPutSignal = NULL @@ -117,6 +151,7 @@ cdef void* __ncclWaitSignal = NULL cdef void* __ncclGroupStart = NULL cdef void* __ncclGroupEnd = NULL cdef void* __ncclGroupSimulateEnd = NULL +cdef void* __ncclSetEncryption = NULL cdef void* __ncclParamBind = NULL cdef void* __ncclParamGetI8 = NULL cdef void* __ncclParamGetI16 = NULL @@ -135,643 +170,698 @@ cdef void* __ncclCommQueryProperties = NULL cdef void* __ncclDevCommCreate = NULL cdef void* __ncclDevCommDestroy = NULL cdef void* __ncclGetLsaMultimemDevicePointer = NULL -cdef void* __ncclGetLsaDevicePointer = NULL cdef void* __ncclGetMultimemDevicePointer = NULL +cdef void* __ncclGetLsaDevicePointer = NULL cdef void* __ncclGetPeerDevicePointer = NULL +cdef void* __ncclGetMultimemDeviceLeInfo = NULL +cdef void* __ncclGetCftDeviceLeInfo = NULL +cdef void* __ncclGetPeerDeviceLeInfo = NULL cdef void* __ncclTeamWorld = NULL cdef void* __ncclTeamLsa = NULL -cdef void* __ncclTeamRail = NULL +cdef void* __ncclTeamCft = NULL +cdef void* __ncclTeamCftMultimem = NULL cdef void* __ncclTeamRankToWorld = NULL cdef void* __ncclTeamRankToLsa = NULL +cdef void* __ncclTeamRail = NULL cdef void* __ncclLsaBarrierCreateRequirement = NULL cdef void* __ncclGinBarrierCreateRequirement = NULL -cdef void* __ncclLLA2ACreateRequirement = NULL cdef void* __ncclLLA2ACalcSlots = NULL +cdef void* __ncclLLA2ACreateRequirement = NULL - -cdef void* load_library() except* with gil: - # Patched for RCCL: replace upstream cuda.pathfinder lookup with a - # ROCm-native search. - # 1. NCCL_LIBRARY env var, if set, is used verbatim (path or SONAME). - # 2. Otherwise: ctypes.util.find_library('rccl') (ldconfig cache), - # then $ROCM_PATH/lib/librccl.so and $HIP_PATH/lib/librccl.so, - # then bare 'librccl.so' (resolved via LD_LIBRARY_PATH at dlopen). - import os - import ctypes.util - - cdef void* handle = NULL - cdef bytes lib_bytes - cdef const char* err_ptr - - explicit = os.environ.get("NCCL_LIBRARY") - if explicit: - candidates = [explicit] - else: - candidates = [] - resolved = ctypes.util.find_library("rccl") - if resolved is not None: - candidates.append(resolved) - for env_var in ("ROCM_PATH", "HIP_PATH"): - root = os.environ.get(env_var) - if root: - candidates.append(os.path.join(root, "lib", "librccl.so")) - candidates.append("librccl.so") - - for cand in candidates: - lib_bytes = cand.encode("utf-8") - handle = dlopen(lib_bytes, RTLD_NOW | RTLD_GLOBAL) - if handle != NULL: - return handle - - err_ptr = dlerror() - err_str = err_ptr.decode() if err_ptr != NULL else "no diagnostic" - raise NotSupportedError( - f"Could not load RCCL library. Tried: {candidates}. " - f"Last dlerror: {err_str}. Set NCCL_LIBRARY or ensure " - "ROCM_PATH/HIP_PATH points to the ROCm install." - ) - - -cdef int _check_or_init_nccl() except -1 nogil: - global __py_nccl_init - if __py_nccl_init: - return 0 - +cdef int _init_nccl() except -1 nogil: + global _cyb___py_nccl_init cdef void* handle = NULL + with gil, _cyb_symbol_lock: + if _cyb___py_nccl_init: return 0 - with gil, __symbol_lock: - # Recheck the flag after obtaining the locks - if __py_nccl_init: - return 0 - - # Load function global __ncclMemAlloc - __ncclMemAlloc = dlsym(RTLD_DEFAULT, 'ncclMemAlloc') + __ncclMemAlloc = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclMemAlloc') if __ncclMemAlloc == NULL: if handle == NULL: handle = load_library() - __ncclMemAlloc = dlsym(handle, 'ncclMemAlloc') + __ncclMemAlloc = _cyb_dlsym(handle, 'ncclMemAlloc') global __ncclMemFree - __ncclMemFree = dlsym(RTLD_DEFAULT, 'ncclMemFree') + __ncclMemFree = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclMemFree') if __ncclMemFree == NULL: if handle == NULL: handle = load_library() - __ncclMemFree = dlsym(handle, 'ncclMemFree') + __ncclMemFree = _cyb_dlsym(handle, 'ncclMemFree') global __ncclGetVersion - __ncclGetVersion = dlsym(RTLD_DEFAULT, 'ncclGetVersion') + __ncclGetVersion = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetVersion') if __ncclGetVersion == NULL: if handle == NULL: handle = load_library() - __ncclGetVersion = dlsym(handle, 'ncclGetVersion') + __ncclGetVersion = _cyb_dlsym(handle, 'ncclGetVersion') global __ncclGetUniqueId - __ncclGetUniqueId = dlsym(RTLD_DEFAULT, 'ncclGetUniqueId') + __ncclGetUniqueId = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetUniqueId') if __ncclGetUniqueId == NULL: if handle == NULL: handle = load_library() - __ncclGetUniqueId = dlsym(handle, 'ncclGetUniqueId') + __ncclGetUniqueId = _cyb_dlsym(handle, 'ncclGetUniqueId') global __ncclCommInitRankConfig - __ncclCommInitRankConfig = dlsym(RTLD_DEFAULT, 'ncclCommInitRankConfig') + __ncclCommInitRankConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommInitRankConfig') if __ncclCommInitRankConfig == NULL: if handle == NULL: handle = load_library() - __ncclCommInitRankConfig = dlsym(handle, 'ncclCommInitRankConfig') + __ncclCommInitRankConfig = _cyb_dlsym(handle, 'ncclCommInitRankConfig') global __ncclCommInitRank - __ncclCommInitRank = dlsym(RTLD_DEFAULT, 'ncclCommInitRank') + __ncclCommInitRank = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommInitRank') if __ncclCommInitRank == NULL: if handle == NULL: handle = load_library() - __ncclCommInitRank = dlsym(handle, 'ncclCommInitRank') + __ncclCommInitRank = _cyb_dlsym(handle, 'ncclCommInitRank') global __ncclCommInitAll - __ncclCommInitAll = dlsym(RTLD_DEFAULT, 'ncclCommInitAll') + __ncclCommInitAll = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommInitAll') if __ncclCommInitAll == NULL: if handle == NULL: handle = load_library() - __ncclCommInitAll = dlsym(handle, 'ncclCommInitAll') + __ncclCommInitAll = _cyb_dlsym(handle, 'ncclCommInitAll') global __ncclCommFinalize - __ncclCommFinalize = dlsym(RTLD_DEFAULT, 'ncclCommFinalize') + __ncclCommFinalize = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommFinalize') if __ncclCommFinalize == NULL: if handle == NULL: handle = load_library() - __ncclCommFinalize = dlsym(handle, 'ncclCommFinalize') + __ncclCommFinalize = _cyb_dlsym(handle, 'ncclCommFinalize') global __ncclCommDestroy - __ncclCommDestroy = dlsym(RTLD_DEFAULT, 'ncclCommDestroy') + __ncclCommDestroy = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommDestroy') if __ncclCommDestroy == NULL: if handle == NULL: handle = load_library() - __ncclCommDestroy = dlsym(handle, 'ncclCommDestroy') + __ncclCommDestroy = _cyb_dlsym(handle, 'ncclCommDestroy') global __ncclCommAbort - __ncclCommAbort = dlsym(RTLD_DEFAULT, 'ncclCommAbort') + __ncclCommAbort = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommAbort') if __ncclCommAbort == NULL: if handle == NULL: handle = load_library() - __ncclCommAbort = dlsym(handle, 'ncclCommAbort') + __ncclCommAbort = _cyb_dlsym(handle, 'ncclCommAbort') global __ncclCommRevoke - __ncclCommRevoke = dlsym(RTLD_DEFAULT, 'ncclCommRevoke') + __ncclCommRevoke = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommRevoke') if __ncclCommRevoke == NULL: if handle == NULL: handle = load_library() - __ncclCommRevoke = dlsym(handle, 'ncclCommRevoke') + __ncclCommRevoke = _cyb_dlsym(handle, 'ncclCommRevoke') global __ncclCommSplit - __ncclCommSplit = dlsym(RTLD_DEFAULT, 'ncclCommSplit') + __ncclCommSplit = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommSplit') if __ncclCommSplit == NULL: if handle == NULL: handle = load_library() - __ncclCommSplit = dlsym(handle, 'ncclCommSplit') + __ncclCommSplit = _cyb_dlsym(handle, 'ncclCommSplit') global __ncclCommShrink - __ncclCommShrink = dlsym(RTLD_DEFAULT, 'ncclCommShrink') + __ncclCommShrink = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommShrink') if __ncclCommShrink == NULL: if handle == NULL: handle = load_library() - __ncclCommShrink = dlsym(handle, 'ncclCommShrink') + __ncclCommShrink = _cyb_dlsym(handle, 'ncclCommShrink') global __ncclCommGetUniqueId - __ncclCommGetUniqueId = dlsym(RTLD_DEFAULT, 'ncclCommGetUniqueId') + __ncclCommGetUniqueId = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommGetUniqueId') if __ncclCommGetUniqueId == NULL: if handle == NULL: handle = load_library() - __ncclCommGetUniqueId = dlsym(handle, 'ncclCommGetUniqueId') + __ncclCommGetUniqueId = _cyb_dlsym(handle, 'ncclCommGetUniqueId') global __ncclCommGrow - __ncclCommGrow = dlsym(RTLD_DEFAULT, 'ncclCommGrow') + __ncclCommGrow = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommGrow') if __ncclCommGrow == NULL: if handle == NULL: handle = load_library() - __ncclCommGrow = dlsym(handle, 'ncclCommGrow') + __ncclCommGrow = _cyb_dlsym(handle, 'ncclCommGrow') global __ncclCommInitRankScalable - __ncclCommInitRankScalable = dlsym(RTLD_DEFAULT, 'ncclCommInitRankScalable') + __ncclCommInitRankScalable = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommInitRankScalable') if __ncclCommInitRankScalable == NULL: if handle == NULL: handle = load_library() - __ncclCommInitRankScalable = dlsym(handle, 'ncclCommInitRankScalable') + __ncclCommInitRankScalable = _cyb_dlsym(handle, 'ncclCommInitRankScalable') global __ncclGetErrorString - __ncclGetErrorString = dlsym(RTLD_DEFAULT, 'ncclGetErrorString') + __ncclGetErrorString = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetErrorString') if __ncclGetErrorString == NULL: if handle == NULL: handle = load_library() - __ncclGetErrorString = dlsym(handle, 'ncclGetErrorString') + __ncclGetErrorString = _cyb_dlsym(handle, 'ncclGetErrorString') global __ncclGetLastError - __ncclGetLastError = dlsym(RTLD_DEFAULT, 'ncclGetLastError') + __ncclGetLastError = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetLastError') if __ncclGetLastError == NULL: if handle == NULL: handle = load_library() - __ncclGetLastError = dlsym(handle, 'ncclGetLastError') + __ncclGetLastError = _cyb_dlsym(handle, 'ncclGetLastError') global __ncclCommGetAsyncError - __ncclCommGetAsyncError = dlsym(RTLD_DEFAULT, 'ncclCommGetAsyncError') + __ncclCommGetAsyncError = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommGetAsyncError') if __ncclCommGetAsyncError == NULL: if handle == NULL: handle = load_library() - __ncclCommGetAsyncError = dlsym(handle, 'ncclCommGetAsyncError') + __ncclCommGetAsyncError = _cyb_dlsym(handle, 'ncclCommGetAsyncError') global __ncclCommCount - __ncclCommCount = dlsym(RTLD_DEFAULT, 'ncclCommCount') + __ncclCommCount = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommCount') if __ncclCommCount == NULL: if handle == NULL: handle = load_library() - __ncclCommCount = dlsym(handle, 'ncclCommCount') + __ncclCommCount = _cyb_dlsym(handle, 'ncclCommCount') global __ncclCommCuDevice - __ncclCommCuDevice = dlsym(RTLD_DEFAULT, 'ncclCommCuDevice') + __ncclCommCuDevice = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommCuDevice') if __ncclCommCuDevice == NULL: if handle == NULL: handle = load_library() - __ncclCommCuDevice = dlsym(handle, 'ncclCommCuDevice') + __ncclCommCuDevice = _cyb_dlsym(handle, 'ncclCommCuDevice') global __ncclCommUserRank - __ncclCommUserRank = dlsym(RTLD_DEFAULT, 'ncclCommUserRank') + __ncclCommUserRank = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommUserRank') if __ncclCommUserRank == NULL: if handle == NULL: handle = load_library() - __ncclCommUserRank = dlsym(handle, 'ncclCommUserRank') + __ncclCommUserRank = _cyb_dlsym(handle, 'ncclCommUserRank') global __ncclCommRegister - __ncclCommRegister = dlsym(RTLD_DEFAULT, 'ncclCommRegister') + __ncclCommRegister = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommRegister') if __ncclCommRegister == NULL: if handle == NULL: handle = load_library() - __ncclCommRegister = dlsym(handle, 'ncclCommRegister') + __ncclCommRegister = _cyb_dlsym(handle, 'ncclCommRegister') global __ncclCommDeregister - __ncclCommDeregister = dlsym(RTLD_DEFAULT, 'ncclCommDeregister') + __ncclCommDeregister = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommDeregister') if __ncclCommDeregister == NULL: if handle == NULL: handle = load_library() - __ncclCommDeregister = dlsym(handle, 'ncclCommDeregister') + __ncclCommDeregister = _cyb_dlsym(handle, 'ncclCommDeregister') global __ncclCommSuspend - __ncclCommSuspend = dlsym(RTLD_DEFAULT, 'ncclCommSuspend') + __ncclCommSuspend = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommSuspend') if __ncclCommSuspend == NULL: if handle == NULL: handle = load_library() - __ncclCommSuspend = dlsym(handle, 'ncclCommSuspend') + __ncclCommSuspend = _cyb_dlsym(handle, 'ncclCommSuspend') global __ncclCommResume - __ncclCommResume = dlsym(RTLD_DEFAULT, 'ncclCommResume') + __ncclCommResume = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommResume') if __ncclCommResume == NULL: if handle == NULL: handle = load_library() - __ncclCommResume = dlsym(handle, 'ncclCommResume') + __ncclCommResume = _cyb_dlsym(handle, 'ncclCommResume') global __ncclCommMemStats - __ncclCommMemStats = dlsym(RTLD_DEFAULT, 'ncclCommMemStats') + __ncclCommMemStats = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommMemStats') if __ncclCommMemStats == NULL: if handle == NULL: handle = load_library() - __ncclCommMemStats = dlsym(handle, 'ncclCommMemStats') + __ncclCommMemStats = _cyb_dlsym(handle, 'ncclCommMemStats') global __ncclCommWindowRegister - __ncclCommWindowRegister = dlsym(RTLD_DEFAULT, 'ncclCommWindowRegister') + __ncclCommWindowRegister = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommWindowRegister') if __ncclCommWindowRegister == NULL: if handle == NULL: handle = load_library() - __ncclCommWindowRegister = dlsym(handle, 'ncclCommWindowRegister') + __ncclCommWindowRegister = _cyb_dlsym(handle, 'ncclCommWindowRegister') global __ncclCommWindowDeregister - __ncclCommWindowDeregister = dlsym(RTLD_DEFAULT, 'ncclCommWindowDeregister') + __ncclCommWindowDeregister = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommWindowDeregister') if __ncclCommWindowDeregister == NULL: if handle == NULL: handle = load_library() - __ncclCommWindowDeregister = dlsym(handle, 'ncclCommWindowDeregister') + __ncclCommWindowDeregister = _cyb_dlsym(handle, 'ncclCommWindowDeregister') global __ncclWinGetUserPtr - __ncclWinGetUserPtr = dlsym(RTLD_DEFAULT, 'ncclWinGetUserPtr') + __ncclWinGetUserPtr = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclWinGetUserPtr') if __ncclWinGetUserPtr == NULL: if handle == NULL: handle = load_library() - __ncclWinGetUserPtr = dlsym(handle, 'ncclWinGetUserPtr') + __ncclWinGetUserPtr = _cyb_dlsym(handle, 'ncclWinGetUserPtr') global __ncclRedOpCreatePreMulSum - __ncclRedOpCreatePreMulSum = dlsym(RTLD_DEFAULT, 'ncclRedOpCreatePreMulSum') + __ncclRedOpCreatePreMulSum = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclRedOpCreatePreMulSum') if __ncclRedOpCreatePreMulSum == NULL: if handle == NULL: handle = load_library() - __ncclRedOpCreatePreMulSum = dlsym(handle, 'ncclRedOpCreatePreMulSum') + __ncclRedOpCreatePreMulSum = _cyb_dlsym(handle, 'ncclRedOpCreatePreMulSum') global __ncclRedOpDestroy - __ncclRedOpDestroy = dlsym(RTLD_DEFAULT, 'ncclRedOpDestroy') + __ncclRedOpDestroy = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclRedOpDestroy') if __ncclRedOpDestroy == NULL: if handle == NULL: handle = load_library() - __ncclRedOpDestroy = dlsym(handle, 'ncclRedOpDestroy') + __ncclRedOpDestroy = _cyb_dlsym(handle, 'ncclRedOpDestroy') global __ncclReduce - __ncclReduce = dlsym(RTLD_DEFAULT, 'ncclReduce') + __ncclReduce = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclReduce') if __ncclReduce == NULL: if handle == NULL: handle = load_library() - __ncclReduce = dlsym(handle, 'ncclReduce') + __ncclReduce = _cyb_dlsym(handle, 'ncclReduce') global __ncclBcast - __ncclBcast = dlsym(RTLD_DEFAULT, 'ncclBcast') + __ncclBcast = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclBcast') if __ncclBcast == NULL: if handle == NULL: handle = load_library() - __ncclBcast = dlsym(handle, 'ncclBcast') + __ncclBcast = _cyb_dlsym(handle, 'ncclBcast') global __ncclBroadcast - __ncclBroadcast = dlsym(RTLD_DEFAULT, 'ncclBroadcast') + __ncclBroadcast = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclBroadcast') if __ncclBroadcast == NULL: if handle == NULL: handle = load_library() - __ncclBroadcast = dlsym(handle, 'ncclBroadcast') + __ncclBroadcast = _cyb_dlsym(handle, 'ncclBroadcast') global __ncclAllReduce - __ncclAllReduce = dlsym(RTLD_DEFAULT, 'ncclAllReduce') + __ncclAllReduce = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclAllReduce') if __ncclAllReduce == NULL: if handle == NULL: handle = load_library() - __ncclAllReduce = dlsym(handle, 'ncclAllReduce') + __ncclAllReduce = _cyb_dlsym(handle, 'ncclAllReduce') global __ncclReduceScatter - __ncclReduceScatter = dlsym(RTLD_DEFAULT, 'ncclReduceScatter') + __ncclReduceScatter = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclReduceScatter') if __ncclReduceScatter == NULL: if handle == NULL: handle = load_library() - __ncclReduceScatter = dlsym(handle, 'ncclReduceScatter') + __ncclReduceScatter = _cyb_dlsym(handle, 'ncclReduceScatter') global __ncclAllGather - __ncclAllGather = dlsym(RTLD_DEFAULT, 'ncclAllGather') + __ncclAllGather = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclAllGather') if __ncclAllGather == NULL: if handle == NULL: handle = load_library() - __ncclAllGather = dlsym(handle, 'ncclAllGather') + __ncclAllGather = _cyb_dlsym(handle, 'ncclAllGather') global __ncclAlltoAll - __ncclAlltoAll = dlsym(RTLD_DEFAULT, 'ncclAlltoAll') + __ncclAlltoAll = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclAlltoAll') if __ncclAlltoAll == NULL: if handle == NULL: handle = load_library() - __ncclAlltoAll = dlsym(handle, 'ncclAlltoAll') + __ncclAlltoAll = _cyb_dlsym(handle, 'ncclAlltoAll') global __ncclGather - __ncclGather = dlsym(RTLD_DEFAULT, 'ncclGather') + __ncclGather = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGather') if __ncclGather == NULL: if handle == NULL: handle = load_library() - __ncclGather = dlsym(handle, 'ncclGather') + __ncclGather = _cyb_dlsym(handle, 'ncclGather') global __ncclScatter - __ncclScatter = dlsym(RTLD_DEFAULT, 'ncclScatter') + __ncclScatter = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclScatter') if __ncclScatter == NULL: if handle == NULL: handle = load_library() - __ncclScatter = dlsym(handle, 'ncclScatter') + __ncclScatter = _cyb_dlsym(handle, 'ncclScatter') + + global __ncclAllReduceConfig + __ncclAllReduceConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclAllReduceConfig') + if __ncclAllReduceConfig == NULL: + if handle == NULL: + handle = load_library() + __ncclAllReduceConfig = _cyb_dlsym(handle, 'ncclAllReduceConfig') + + global __ncclBroadcastConfig + __ncclBroadcastConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclBroadcastConfig') + if __ncclBroadcastConfig == NULL: + if handle == NULL: + handle = load_library() + __ncclBroadcastConfig = _cyb_dlsym(handle, 'ncclBroadcastConfig') + + global __ncclReduceConfig + __ncclReduceConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclReduceConfig') + if __ncclReduceConfig == NULL: + if handle == NULL: + handle = load_library() + __ncclReduceConfig = _cyb_dlsym(handle, 'ncclReduceConfig') + + global __ncclAllGatherConfig + __ncclAllGatherConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclAllGatherConfig') + if __ncclAllGatherConfig == NULL: + if handle == NULL: + handle = load_library() + __ncclAllGatherConfig = _cyb_dlsym(handle, 'ncclAllGatherConfig') + + global __ncclReduceScatterConfig + __ncclReduceScatterConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclReduceScatterConfig') + if __ncclReduceScatterConfig == NULL: + if handle == NULL: + handle = load_library() + __ncclReduceScatterConfig = _cyb_dlsym(handle, 'ncclReduceScatterConfig') + + global __ncclAlltoAllConfig + __ncclAlltoAllConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclAlltoAllConfig') + if __ncclAlltoAllConfig == NULL: + if handle == NULL: + handle = load_library() + __ncclAlltoAllConfig = _cyb_dlsym(handle, 'ncclAlltoAllConfig') + + global __ncclGatherConfig + __ncclGatherConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGatherConfig') + if __ncclGatherConfig == NULL: + if handle == NULL: + handle = load_library() + __ncclGatherConfig = _cyb_dlsym(handle, 'ncclGatherConfig') + + global __ncclScatterConfig + __ncclScatterConfig = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclScatterConfig') + if __ncclScatterConfig == NULL: + if handle == NULL: + handle = load_library() + __ncclScatterConfig = _cyb_dlsym(handle, 'ncclScatterConfig') global __ncclSend - __ncclSend = dlsym(RTLD_DEFAULT, 'ncclSend') + __ncclSend = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclSend') if __ncclSend == NULL: if handle == NULL: handle = load_library() - __ncclSend = dlsym(handle, 'ncclSend') + __ncclSend = _cyb_dlsym(handle, 'ncclSend') global __ncclRecv - __ncclRecv = dlsym(RTLD_DEFAULT, 'ncclRecv') + __ncclRecv = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclRecv') if __ncclRecv == NULL: if handle == NULL: handle = load_library() - __ncclRecv = dlsym(handle, 'ncclRecv') + __ncclRecv = _cyb_dlsym(handle, 'ncclRecv') global __ncclPutSignal - __ncclPutSignal = dlsym(RTLD_DEFAULT, 'ncclPutSignal') + __ncclPutSignal = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclPutSignal') if __ncclPutSignal == NULL: if handle == NULL: handle = load_library() - __ncclPutSignal = dlsym(handle, 'ncclPutSignal') + __ncclPutSignal = _cyb_dlsym(handle, 'ncclPutSignal') global __ncclSignal - __ncclSignal = dlsym(RTLD_DEFAULT, 'ncclSignal') + __ncclSignal = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclSignal') if __ncclSignal == NULL: if handle == NULL: handle = load_library() - __ncclSignal = dlsym(handle, 'ncclSignal') + __ncclSignal = _cyb_dlsym(handle, 'ncclSignal') global __ncclWaitSignal - __ncclWaitSignal = dlsym(RTLD_DEFAULT, 'ncclWaitSignal') + __ncclWaitSignal = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclWaitSignal') if __ncclWaitSignal == NULL: if handle == NULL: handle = load_library() - __ncclWaitSignal = dlsym(handle, 'ncclWaitSignal') + __ncclWaitSignal = _cyb_dlsym(handle, 'ncclWaitSignal') global __ncclGroupStart - __ncclGroupStart = dlsym(RTLD_DEFAULT, 'ncclGroupStart') + __ncclGroupStart = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGroupStart') if __ncclGroupStart == NULL: if handle == NULL: handle = load_library() - __ncclGroupStart = dlsym(handle, 'ncclGroupStart') + __ncclGroupStart = _cyb_dlsym(handle, 'ncclGroupStart') global __ncclGroupEnd - __ncclGroupEnd = dlsym(RTLD_DEFAULT, 'ncclGroupEnd') + __ncclGroupEnd = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGroupEnd') if __ncclGroupEnd == NULL: if handle == NULL: handle = load_library() - __ncclGroupEnd = dlsym(handle, 'ncclGroupEnd') + __ncclGroupEnd = _cyb_dlsym(handle, 'ncclGroupEnd') global __ncclGroupSimulateEnd - __ncclGroupSimulateEnd = dlsym(RTLD_DEFAULT, 'ncclGroupSimulateEnd') + __ncclGroupSimulateEnd = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGroupSimulateEnd') if __ncclGroupSimulateEnd == NULL: if handle == NULL: handle = load_library() - __ncclGroupSimulateEnd = dlsym(handle, 'ncclGroupSimulateEnd') + __ncclGroupSimulateEnd = _cyb_dlsym(handle, 'ncclGroupSimulateEnd') + + global __ncclSetEncryption + __ncclSetEncryption = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclSetEncryption') + if __ncclSetEncryption == NULL: + if handle == NULL: + handle = load_library() + __ncclSetEncryption = _cyb_dlsym(handle, 'ncclSetEncryption') global __ncclParamBind - __ncclParamBind = dlsym(RTLD_DEFAULT, 'ncclParamBind') + __ncclParamBind = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamBind') if __ncclParamBind == NULL: if handle == NULL: handle = load_library() - __ncclParamBind = dlsym(handle, 'ncclParamBind') + __ncclParamBind = _cyb_dlsym(handle, 'ncclParamBind') global __ncclParamGetI8 - __ncclParamGetI8 = dlsym(RTLD_DEFAULT, 'ncclParamGetI8') + __ncclParamGetI8 = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetI8') if __ncclParamGetI8 == NULL: if handle == NULL: handle = load_library() - __ncclParamGetI8 = dlsym(handle, 'ncclParamGetI8') + __ncclParamGetI8 = _cyb_dlsym(handle, 'ncclParamGetI8') global __ncclParamGetI16 - __ncclParamGetI16 = dlsym(RTLD_DEFAULT, 'ncclParamGetI16') + __ncclParamGetI16 = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetI16') if __ncclParamGetI16 == NULL: if handle == NULL: handle = load_library() - __ncclParamGetI16 = dlsym(handle, 'ncclParamGetI16') + __ncclParamGetI16 = _cyb_dlsym(handle, 'ncclParamGetI16') global __ncclParamGetI32 - __ncclParamGetI32 = dlsym(RTLD_DEFAULT, 'ncclParamGetI32') + __ncclParamGetI32 = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetI32') if __ncclParamGetI32 == NULL: if handle == NULL: handle = load_library() - __ncclParamGetI32 = dlsym(handle, 'ncclParamGetI32') + __ncclParamGetI32 = _cyb_dlsym(handle, 'ncclParamGetI32') global __ncclParamGetI64 - __ncclParamGetI64 = dlsym(RTLD_DEFAULT, 'ncclParamGetI64') + __ncclParamGetI64 = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetI64') if __ncclParamGetI64 == NULL: if handle == NULL: handle = load_library() - __ncclParamGetI64 = dlsym(handle, 'ncclParamGetI64') + __ncclParamGetI64 = _cyb_dlsym(handle, 'ncclParamGetI64') global __ncclParamGetU8 - __ncclParamGetU8 = dlsym(RTLD_DEFAULT, 'ncclParamGetU8') + __ncclParamGetU8 = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetU8') if __ncclParamGetU8 == NULL: if handle == NULL: handle = load_library() - __ncclParamGetU8 = dlsym(handle, 'ncclParamGetU8') + __ncclParamGetU8 = _cyb_dlsym(handle, 'ncclParamGetU8') global __ncclParamGetU16 - __ncclParamGetU16 = dlsym(RTLD_DEFAULT, 'ncclParamGetU16') + __ncclParamGetU16 = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetU16') if __ncclParamGetU16 == NULL: if handle == NULL: handle = load_library() - __ncclParamGetU16 = dlsym(handle, 'ncclParamGetU16') + __ncclParamGetU16 = _cyb_dlsym(handle, 'ncclParamGetU16') global __ncclParamGetU32 - __ncclParamGetU32 = dlsym(RTLD_DEFAULT, 'ncclParamGetU32') + __ncclParamGetU32 = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetU32') if __ncclParamGetU32 == NULL: if handle == NULL: handle = load_library() - __ncclParamGetU32 = dlsym(handle, 'ncclParamGetU32') + __ncclParamGetU32 = _cyb_dlsym(handle, 'ncclParamGetU32') global __ncclParamGetU64 - __ncclParamGetU64 = dlsym(RTLD_DEFAULT, 'ncclParamGetU64') + __ncclParamGetU64 = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetU64') if __ncclParamGetU64 == NULL: if handle == NULL: handle = load_library() - __ncclParamGetU64 = dlsym(handle, 'ncclParamGetU64') + __ncclParamGetU64 = _cyb_dlsym(handle, 'ncclParamGetU64') global __ncclParamGetStr - __ncclParamGetStr = dlsym(RTLD_DEFAULT, 'ncclParamGetStr') + __ncclParamGetStr = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetStr') if __ncclParamGetStr == NULL: if handle == NULL: handle = load_library() - __ncclParamGetStr = dlsym(handle, 'ncclParamGetStr') + __ncclParamGetStr = _cyb_dlsym(handle, 'ncclParamGetStr') global __ncclParamGet - __ncclParamGet = dlsym(RTLD_DEFAULT, 'ncclParamGet') + __ncclParamGet = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGet') if __ncclParamGet == NULL: if handle == NULL: handle = load_library() - __ncclParamGet = dlsym(handle, 'ncclParamGet') + __ncclParamGet = _cyb_dlsym(handle, 'ncclParamGet') global __ncclParamGetParameter - __ncclParamGetParameter = dlsym(RTLD_DEFAULT, 'ncclParamGetParameter') + __ncclParamGetParameter = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetParameter') if __ncclParamGetParameter == NULL: if handle == NULL: handle = load_library() - __ncclParamGetParameter = dlsym(handle, 'ncclParamGetParameter') + __ncclParamGetParameter = _cyb_dlsym(handle, 'ncclParamGetParameter') global __ncclParamGetAllParameterKeys - __ncclParamGetAllParameterKeys = dlsym(RTLD_DEFAULT, 'ncclParamGetAllParameterKeys') + __ncclParamGetAllParameterKeys = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamGetAllParameterKeys') if __ncclParamGetAllParameterKeys == NULL: if handle == NULL: handle = load_library() - __ncclParamGetAllParameterKeys = dlsym(handle, 'ncclParamGetAllParameterKeys') + __ncclParamGetAllParameterKeys = _cyb_dlsym(handle, 'ncclParamGetAllParameterKeys') global __ncclParamDumpAll - __ncclParamDumpAll = dlsym(RTLD_DEFAULT, 'ncclParamDumpAll') + __ncclParamDumpAll = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclParamDumpAll') if __ncclParamDumpAll == NULL: if handle == NULL: handle = load_library() - __ncclParamDumpAll = dlsym(handle, 'ncclParamDumpAll') + __ncclParamDumpAll = _cyb_dlsym(handle, 'ncclParamDumpAll') global __ncclCommQueryProperties - __ncclCommQueryProperties = dlsym(RTLD_DEFAULT, 'ncclCommQueryProperties') + __ncclCommQueryProperties = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclCommQueryProperties') if __ncclCommQueryProperties == NULL: if handle == NULL: handle = load_library() - __ncclCommQueryProperties = dlsym(handle, 'ncclCommQueryProperties') + __ncclCommQueryProperties = _cyb_dlsym(handle, 'ncclCommQueryProperties') global __ncclDevCommCreate - __ncclDevCommCreate = dlsym(RTLD_DEFAULT, 'ncclDevCommCreate') + __ncclDevCommCreate = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclDevCommCreate') if __ncclDevCommCreate == NULL: if handle == NULL: handle = load_library() - __ncclDevCommCreate = dlsym(handle, 'ncclDevCommCreate') + __ncclDevCommCreate = _cyb_dlsym(handle, 'ncclDevCommCreate') global __ncclDevCommDestroy - __ncclDevCommDestroy = dlsym(RTLD_DEFAULT, 'ncclDevCommDestroy') + __ncclDevCommDestroy = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclDevCommDestroy') if __ncclDevCommDestroy == NULL: if handle == NULL: handle = load_library() - __ncclDevCommDestroy = dlsym(handle, 'ncclDevCommDestroy') + __ncclDevCommDestroy = _cyb_dlsym(handle, 'ncclDevCommDestroy') global __ncclGetLsaMultimemDevicePointer - __ncclGetLsaMultimemDevicePointer = dlsym(RTLD_DEFAULT, 'ncclGetLsaMultimemDevicePointer') + __ncclGetLsaMultimemDevicePointer = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetLsaMultimemDevicePointer') if __ncclGetLsaMultimemDevicePointer == NULL: if handle == NULL: handle = load_library() - __ncclGetLsaMultimemDevicePointer = dlsym(handle, 'ncclGetLsaMultimemDevicePointer') + __ncclGetLsaMultimemDevicePointer = _cyb_dlsym(handle, 'ncclGetLsaMultimemDevicePointer') - global __ncclGetLsaDevicePointer - __ncclGetLsaDevicePointer = dlsym(RTLD_DEFAULT, 'ncclGetLsaDevicePointer') - if __ncclGetLsaDevicePointer == NULL: + global __ncclGetMultimemDevicePointer + __ncclGetMultimemDevicePointer = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetMultimemDevicePointer') + if __ncclGetMultimemDevicePointer == NULL: if handle == NULL: handle = load_library() - __ncclGetLsaDevicePointer = dlsym(handle, 'ncclGetLsaDevicePointer') + __ncclGetMultimemDevicePointer = _cyb_dlsym(handle, 'ncclGetMultimemDevicePointer') - global __ncclGetMultimemDevicePointer - __ncclGetMultimemDevicePointer = dlsym(RTLD_DEFAULT, 'ncclGetMultimemDevicePointer') - if __ncclGetMultimemDevicePointer == NULL: + global __ncclGetLsaDevicePointer + __ncclGetLsaDevicePointer = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetLsaDevicePointer') + if __ncclGetLsaDevicePointer == NULL: if handle == NULL: handle = load_library() - __ncclGetMultimemDevicePointer = dlsym(handle, 'ncclGetMultimemDevicePointer') + __ncclGetLsaDevicePointer = _cyb_dlsym(handle, 'ncclGetLsaDevicePointer') global __ncclGetPeerDevicePointer - __ncclGetPeerDevicePointer = dlsym(RTLD_DEFAULT, 'ncclGetPeerDevicePointer') + __ncclGetPeerDevicePointer = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetPeerDevicePointer') if __ncclGetPeerDevicePointer == NULL: if handle == NULL: handle = load_library() - __ncclGetPeerDevicePointer = dlsym(handle, 'ncclGetPeerDevicePointer') + __ncclGetPeerDevicePointer = _cyb_dlsym(handle, 'ncclGetPeerDevicePointer') + + global __ncclGetMultimemDeviceLeInfo + __ncclGetMultimemDeviceLeInfo = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetMultimemDeviceLeInfo') + if __ncclGetMultimemDeviceLeInfo == NULL: + if handle == NULL: + handle = load_library() + __ncclGetMultimemDeviceLeInfo = _cyb_dlsym(handle, 'ncclGetMultimemDeviceLeInfo') + + global __ncclGetCftDeviceLeInfo + __ncclGetCftDeviceLeInfo = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetCftDeviceLeInfo') + if __ncclGetCftDeviceLeInfo == NULL: + if handle == NULL: + handle = load_library() + __ncclGetCftDeviceLeInfo = _cyb_dlsym(handle, 'ncclGetCftDeviceLeInfo') + + global __ncclGetPeerDeviceLeInfo + __ncclGetPeerDeviceLeInfo = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGetPeerDeviceLeInfo') + if __ncclGetPeerDeviceLeInfo == NULL: + if handle == NULL: + handle = load_library() + __ncclGetPeerDeviceLeInfo = _cyb_dlsym(handle, 'ncclGetPeerDeviceLeInfo') global __ncclTeamWorld - __ncclTeamWorld = dlsym(RTLD_DEFAULT, 'ncclTeamWorld') + __ncclTeamWorld = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclTeamWorld') if __ncclTeamWorld == NULL: if handle == NULL: handle = load_library() - __ncclTeamWorld = dlsym(handle, 'ncclTeamWorld') + __ncclTeamWorld = _cyb_dlsym(handle, 'ncclTeamWorld') global __ncclTeamLsa - __ncclTeamLsa = dlsym(RTLD_DEFAULT, 'ncclTeamLsa') + __ncclTeamLsa = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclTeamLsa') if __ncclTeamLsa == NULL: if handle == NULL: handle = load_library() - __ncclTeamLsa = dlsym(handle, 'ncclTeamLsa') + __ncclTeamLsa = _cyb_dlsym(handle, 'ncclTeamLsa') - global __ncclTeamRail - __ncclTeamRail = dlsym(RTLD_DEFAULT, 'ncclTeamRail') - if __ncclTeamRail == NULL: + global __ncclTeamCft + __ncclTeamCft = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclTeamCft') + if __ncclTeamCft == NULL: if handle == NULL: handle = load_library() - __ncclTeamRail = dlsym(handle, 'ncclTeamRail') + __ncclTeamCft = _cyb_dlsym(handle, 'ncclTeamCft') + + global __ncclTeamCftMultimem + __ncclTeamCftMultimem = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclTeamCftMultimem') + if __ncclTeamCftMultimem == NULL: + if handle == NULL: + handle = load_library() + __ncclTeamCftMultimem = _cyb_dlsym(handle, 'ncclTeamCftMultimem') global __ncclTeamRankToWorld - __ncclTeamRankToWorld = dlsym(RTLD_DEFAULT, 'ncclTeamRankToWorld') + __ncclTeamRankToWorld = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclTeamRankToWorld') if __ncclTeamRankToWorld == NULL: if handle == NULL: handle = load_library() - __ncclTeamRankToWorld = dlsym(handle, 'ncclTeamRankToWorld') + __ncclTeamRankToWorld = _cyb_dlsym(handle, 'ncclTeamRankToWorld') global __ncclTeamRankToLsa - __ncclTeamRankToLsa = dlsym(RTLD_DEFAULT, 'ncclTeamRankToLsa') + __ncclTeamRankToLsa = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclTeamRankToLsa') if __ncclTeamRankToLsa == NULL: if handle == NULL: handle = load_library() - __ncclTeamRankToLsa = dlsym(handle, 'ncclTeamRankToLsa') + __ncclTeamRankToLsa = _cyb_dlsym(handle, 'ncclTeamRankToLsa') + + global __ncclTeamRail + __ncclTeamRail = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclTeamRail') + if __ncclTeamRail == NULL: + if handle == NULL: + handle = load_library() + __ncclTeamRail = _cyb_dlsym(handle, 'ncclTeamRail') global __ncclLsaBarrierCreateRequirement - __ncclLsaBarrierCreateRequirement = dlsym(RTLD_DEFAULT, 'ncclLsaBarrierCreateRequirement') + __ncclLsaBarrierCreateRequirement = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclLsaBarrierCreateRequirement') if __ncclLsaBarrierCreateRequirement == NULL: if handle == NULL: handle = load_library() - __ncclLsaBarrierCreateRequirement = dlsym(handle, 'ncclLsaBarrierCreateRequirement') + __ncclLsaBarrierCreateRequirement = _cyb_dlsym(handle, 'ncclLsaBarrierCreateRequirement') global __ncclGinBarrierCreateRequirement - __ncclGinBarrierCreateRequirement = dlsym(RTLD_DEFAULT, 'ncclGinBarrierCreateRequirement') + __ncclGinBarrierCreateRequirement = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclGinBarrierCreateRequirement') if __ncclGinBarrierCreateRequirement == NULL: if handle == NULL: handle = load_library() - __ncclGinBarrierCreateRequirement = dlsym(handle, 'ncclGinBarrierCreateRequirement') + __ncclGinBarrierCreateRequirement = _cyb_dlsym(handle, 'ncclGinBarrierCreateRequirement') - global __ncclLLA2ACreateRequirement - __ncclLLA2ACreateRequirement = dlsym(RTLD_DEFAULT, 'ncclLLA2ACreateRequirement') - if __ncclLLA2ACreateRequirement == NULL: + global __ncclLLA2ACalcSlots + __ncclLLA2ACalcSlots = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclLLA2ACalcSlots') + if __ncclLLA2ACalcSlots == NULL: if handle == NULL: handle = load_library() - __ncclLLA2ACreateRequirement = dlsym(handle, 'ncclLLA2ACreateRequirement') + __ncclLLA2ACalcSlots = _cyb_dlsym(handle, 'ncclLLA2ACalcSlots') - global __ncclLLA2ACalcSlots - __ncclLLA2ACalcSlots = dlsym(RTLD_DEFAULT, 'ncclLLA2ACalcSlots') - if __ncclLLA2ACalcSlots == NULL: + global __ncclLLA2ACreateRequirement + __ncclLLA2ACreateRequirement = _cyb_dlsym(_cyb_RTLD_DEFAULT, 'ncclLLA2ACreateRequirement') + if __ncclLLA2ACreateRequirement == NULL: if handle == NULL: handle = load_library() - __ncclLLA2ACalcSlots = dlsym(handle, 'ncclLLA2ACalcSlots') - __py_nccl_init = True + __ncclLLA2ACreateRequirement = _cyb_dlsym(handle, 'ncclLLA2ACreateRequirement') + + _cyb_atomic_int_store(&_cyb___py_nccl_init, 1) return 0 +cdef inline int _check_or_init_nccl() except -1 nogil: + if _cyb_atomic_int_load(&_cyb___py_nccl_init): + return 0 -cdef dict func_ptrs = None + return _init_nccl() cpdef dict _inspect_function_pointers(): - global func_ptrs - if func_ptrs is not None: - return func_ptrs + global _cyb_func_ptrs + if _cyb_func_ptrs is not None: + return _cyb_func_ptrs _check_or_init_nccl() cdef dict data = {} - global __ncclMemAlloc data["__ncclMemAlloc"] = __ncclMemAlloc @@ -895,6 +985,30 @@ cpdef dict _inspect_function_pointers(): global __ncclScatter data["__ncclScatter"] = __ncclScatter + global __ncclAllReduceConfig + data["__ncclAllReduceConfig"] = __ncclAllReduceConfig + + global __ncclBroadcastConfig + data["__ncclBroadcastConfig"] = __ncclBroadcastConfig + + global __ncclReduceConfig + data["__ncclReduceConfig"] = __ncclReduceConfig + + global __ncclAllGatherConfig + data["__ncclAllGatherConfig"] = __ncclAllGatherConfig + + global __ncclReduceScatterConfig + data["__ncclReduceScatterConfig"] = __ncclReduceScatterConfig + + global __ncclAlltoAllConfig + data["__ncclAlltoAllConfig"] = __ncclAlltoAllConfig + + global __ncclGatherConfig + data["__ncclGatherConfig"] = __ncclGatherConfig + + global __ncclScatterConfig + data["__ncclScatterConfig"] = __ncclScatterConfig + global __ncclSend data["__ncclSend"] = __ncclSend @@ -919,6 +1033,9 @@ cpdef dict _inspect_function_pointers(): global __ncclGroupSimulateEnd data["__ncclGroupSimulateEnd"] = __ncclGroupSimulateEnd + global __ncclSetEncryption + data["__ncclSetEncryption"] = __ncclSetEncryption + global __ncclParamBind data["__ncclParamBind"] = __ncclParamBind @@ -973,23 +1090,35 @@ cpdef dict _inspect_function_pointers(): global __ncclGetLsaMultimemDevicePointer data["__ncclGetLsaMultimemDevicePointer"] = __ncclGetLsaMultimemDevicePointer - global __ncclGetLsaDevicePointer - data["__ncclGetLsaDevicePointer"] = __ncclGetLsaDevicePointer - global __ncclGetMultimemDevicePointer data["__ncclGetMultimemDevicePointer"] = __ncclGetMultimemDevicePointer + global __ncclGetLsaDevicePointer + data["__ncclGetLsaDevicePointer"] = __ncclGetLsaDevicePointer + global __ncclGetPeerDevicePointer data["__ncclGetPeerDevicePointer"] = __ncclGetPeerDevicePointer + global __ncclGetMultimemDeviceLeInfo + data["__ncclGetMultimemDeviceLeInfo"] = __ncclGetMultimemDeviceLeInfo + + global __ncclGetCftDeviceLeInfo + data["__ncclGetCftDeviceLeInfo"] = __ncclGetCftDeviceLeInfo + + global __ncclGetPeerDeviceLeInfo + data["__ncclGetPeerDeviceLeInfo"] = __ncclGetPeerDeviceLeInfo + global __ncclTeamWorld data["__ncclTeamWorld"] = __ncclTeamWorld global __ncclTeamLsa data["__ncclTeamLsa"] = __ncclTeamLsa - global __ncclTeamRail - data["__ncclTeamRail"] = __ncclTeamRail + global __ncclTeamCft + data["__ncclTeamCft"] = __ncclTeamCft + + global __ncclTeamCftMultimem + data["__ncclTeamCftMultimem"] = __ncclTeamCftMultimem global __ncclTeamRankToWorld data["__ncclTeamRankToWorld"] = __ncclTeamRankToWorld @@ -997,33 +1126,81 @@ cpdef dict _inspect_function_pointers(): global __ncclTeamRankToLsa data["__ncclTeamRankToLsa"] = __ncclTeamRankToLsa + global __ncclTeamRail + data["__ncclTeamRail"] = __ncclTeamRail + global __ncclLsaBarrierCreateRequirement data["__ncclLsaBarrierCreateRequirement"] = __ncclLsaBarrierCreateRequirement global __ncclGinBarrierCreateRequirement data["__ncclGinBarrierCreateRequirement"] = __ncclGinBarrierCreateRequirement - global __ncclLLA2ACreateRequirement - data["__ncclLLA2ACreateRequirement"] = __ncclLLA2ACreateRequirement - global __ncclLLA2ACalcSlots data["__ncclLLA2ACalcSlots"] = __ncclLLA2ACalcSlots - func_ptrs = data + global __ncclLLA2ACreateRequirement + data["__ncclLLA2ACreateRequirement"] = __ncclLLA2ACreateRequirement + _cyb_func_ptrs = data return data cpdef _inspect_function_pointer(str name): - global func_ptrs - if func_ptrs is None: - func_ptrs = _inspect_function_pointers() - return func_ptrs[name] + global _cyb_func_ptrs + if _cyb_func_ptrs is None: + _cyb_func_ptrs = _inspect_function_pointers() + return _cyb_func_ptrs[name] + + + + +cdef void* load_library() except* with gil: + # Patched for RCCL: replace upstream cuda.pathfinder lookup with a + # ROCm-native search. + # 1. NCCL_LIBRARY env var, if set, is used verbatim (path or SONAME). + # 2. Otherwise: ctypes.util.find_library('rccl') (ldconfig cache), + # then $ROCM_PATH/lib/librccl.so and $HIP_PATH/lib/librccl.so, + # then bare 'librccl.so' (resolved via LD_LIBRARY_PATH at dlopen). + import os + import ctypes.util + + cdef void* handle = NULL + cdef bytes lib_bytes + cdef const char* err_ptr + + explicit = os.environ.get("NCCL_LIBRARY") + if explicit: + candidates = [explicit] + else: + candidates = [] + resolved = ctypes.util.find_library("rccl") + if resolved is not None: + candidates.append(resolved) + for env_var in ("ROCM_PATH", "HIP_PATH"): + root = os.environ.get(env_var) + if root: + candidates.append(os.path.join(root, "lib", "librccl.so")) + candidates.append("librccl.so") + + for cand in candidates: + lib_bytes = cand.encode("utf-8") + handle = dlopen(lib_bytes, RTLD_NOW | RTLD_GLOBAL) + if handle != NULL: + return handle + + err_ptr = dlerror() + err_str = err_ptr.decode() if err_ptr != NULL else "no diagnostic" + raise NotSupportedError( + f"Could not load RCCL library. Tried: {candidates}. " + f"Last dlerror: {err_str}. Set NCCL_LIBRARY or ensure " + "ROCM_PATH/HIP_PATH points to the ROCm install." + ) cdef object __nccl_loaded_so_path = None cpdef object _inspect_loaded_library_path(): + import os # Path of the .so backing the loaded symbols, via dladdr() on a # resolved entry point. None if it cannot be determined. global __nccl_loaded_so_path @@ -1461,6 +1638,86 @@ cdef ncclResult_t _ncclScatter(const void* sendbuff, void* recvbuff, size_t coun sendbuff, recvbuff, count, datatype, root, comm, stream) +cdef ncclResult_t _ncclAllReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclAllReduceConfig + _check_or_init_nccl() + if __ncclAllReduceConfig == NULL: + with gil: + raise FunctionNotFoundError("function ncclAllReduceConfig is not found") + return (__ncclAllReduceConfig)( + sendbuff, recvbuff, count, datatype, op, comm, stream, config) + + +cdef ncclResult_t _ncclBroadcastConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclBroadcastConfig + _check_or_init_nccl() + if __ncclBroadcastConfig == NULL: + with gil: + raise FunctionNotFoundError("function ncclBroadcastConfig is not found") + return (__ncclBroadcastConfig)( + sendbuff, recvbuff, count, datatype, root, comm, stream, config) + + +cdef ncclResult_t _ncclReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclReduceConfig + _check_or_init_nccl() + if __ncclReduceConfig == NULL: + with gil: + raise FunctionNotFoundError("function ncclReduceConfig is not found") + return (__ncclReduceConfig)( + sendbuff, recvbuff, count, datatype, op, root, comm, stream, config) + + +cdef ncclResult_t _ncclAllGatherConfig(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclAllGatherConfig + _check_or_init_nccl() + if __ncclAllGatherConfig == NULL: + with gil: + raise FunctionNotFoundError("function ncclAllGatherConfig is not found") + return (__ncclAllGatherConfig)( + sendbuff, recvbuff, sendcount, datatype, comm, stream, config) + + +cdef ncclResult_t _ncclReduceScatterConfig(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclReduceScatterConfig + _check_or_init_nccl() + if __ncclReduceScatterConfig == NULL: + with gil: + raise FunctionNotFoundError("function ncclReduceScatterConfig is not found") + return (__ncclReduceScatterConfig)( + sendbuff, recvbuff, recvcount, datatype, op, comm, stream, config) + + +cdef ncclResult_t _ncclAlltoAllConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclAlltoAllConfig + _check_or_init_nccl() + if __ncclAlltoAllConfig == NULL: + with gil: + raise FunctionNotFoundError("function ncclAlltoAllConfig is not found") + return (__ncclAlltoAllConfig)( + sendbuff, recvbuff, count, datatype, comm, stream, config) + + +cdef ncclResult_t _ncclGatherConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclGatherConfig + _check_or_init_nccl() + if __ncclGatherConfig == NULL: + with gil: + raise FunctionNotFoundError("function ncclGatherConfig is not found") + return (__ncclGatherConfig)( + sendbuff, recvbuff, count, datatype, root, comm, stream, config) + + +cdef ncclResult_t _ncclScatterConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclScatterConfig + _check_or_init_nccl() + if __ncclScatterConfig == NULL: + with gil: + raise FunctionNotFoundError("function ncclScatterConfig is not found") + return (__ncclScatterConfig)( + sendbuff, recvbuff, count, datatype, root, comm, stream, config) + + cdef ncclResult_t _ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclSend _check_or_init_nccl() @@ -1541,6 +1798,16 @@ cdef ncclResult_t _ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) except?_NCCLRESU simInfo) +cdef ncclResult_t _ncclSetEncryption(const ncclEncryptionConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclSetEncryption + _check_or_init_nccl() + if __ncclSetEncryption == NULL: + with gil: + raise FunctionNotFoundError("function ncclSetEncryption is not found") + return (__ncclSetEncryption)( + config) + + cdef ncclResult_t _ncclParamBind(ncclParamHandle_t* out, const char* key) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclParamBind _check_or_init_nccl() @@ -1671,6 +1938,7 @@ cdef ncclResult_t _ncclParamGetAllParameterKeys(const char*** table, int* tableL table, tableLen) +@_cyb_cython.show_performance_hints(False) cdef void _ncclParamDumpAll() except* nogil: global __ncclParamDumpAll _check_or_init_nccl() @@ -1721,16 +1989,6 @@ cdef ncclResult_t _ncclGetLsaMultimemDevicePointer(ncclWindow_t window, size_t o window, offset, outPtr) -cdef ncclResult_t _ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: - global __ncclGetLsaDevicePointer - _check_or_init_nccl() - if __ncclGetLsaDevicePointer == NULL: - with gil: - raise FunctionNotFoundError("function ncclGetLsaDevicePointer is not found") - return (__ncclGetLsaDevicePointer)( - window, offset, lsaRank, outPtr) - - cdef ncclResult_t _ncclGetMultimemDevicePointer(ncclWindow_t window, size_t offset, ncclMultimemHandle_t multimem, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclGetMultimemDevicePointer _check_or_init_nccl() @@ -1741,6 +1999,16 @@ cdef ncclResult_t _ncclGetMultimemDevicePointer(ncclWindow_t window, size_t offs window, offset, multimem, outPtr) +cdef ncclResult_t _ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclGetLsaDevicePointer + _check_or_init_nccl() + if __ncclGetLsaDevicePointer == NULL: + with gil: + raise FunctionNotFoundError("function ncclGetLsaDevicePointer is not found") + return (__ncclGetLsaDevicePointer)( + window, offset, lsaRank, outPtr) + + cdef ncclResult_t _ncclGetPeerDevicePointer(ncclWindow_t window, size_t offset, int peer, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclGetPeerDevicePointer _check_or_init_nccl() @@ -1751,6 +2019,36 @@ cdef ncclResult_t _ncclGetPeerDevicePointer(ncclWindow_t window, size_t offset, window, offset, peer, outPtr) +cdef ncclResult_t _ncclGetMultimemDeviceLeInfo(ncclWindow_t window, size_t offset, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclGetMultimemDeviceLeInfo + _check_or_init_nccl() + if __ncclGetMultimemDeviceLeInfo == NULL: + with gil: + raise FunctionNotFoundError("function ncclGetMultimemDeviceLeInfo is not found") + return (__ncclGetMultimemDeviceLeInfo)( + window, offset, leId, leOffset) + + +cdef ncclResult_t _ncclGetCftDeviceLeInfo(ncclWindow_t window, size_t offset, int peerCft, ncclTeam_t cftTeam, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclGetCftDeviceLeInfo + _check_or_init_nccl() + if __ncclGetCftDeviceLeInfo == NULL: + with gil: + raise FunctionNotFoundError("function ncclGetCftDeviceLeInfo is not found") + return (__ncclGetCftDeviceLeInfo)( + window, offset, peerCft, cftTeam, leId, leOffset) + + +cdef ncclResult_t _ncclGetPeerDeviceLeInfo(ncclWindow_t window, size_t offset, int peerWorld, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclGetPeerDeviceLeInfo + _check_or_init_nccl() + if __ncclGetPeerDeviceLeInfo == NULL: + with gil: + raise FunctionNotFoundError("function ncclGetPeerDeviceLeInfo is not found") + return (__ncclGetPeerDeviceLeInfo)( + window, offset, peerWorld, leId, leOffset) + + cdef ncclTeam_t _ncclTeamWorld(ncclComm_t comm) except* nogil: global __ncclTeamWorld _check_or_init_nccl() @@ -1771,13 +2069,23 @@ cdef ncclTeam_t _ncclTeamLsa(ncclComm_t comm) except* nogil: comm) -cdef ncclTeam_t _ncclTeamRail(ncclComm_t comm) except* nogil: - global __ncclTeamRail +cdef ncclTeam_t _ncclTeamCft(ncclComm_t comm, ncclCftTeamMode_t mode) except* nogil: + global __ncclTeamCft _check_or_init_nccl() - if __ncclTeamRail == NULL: + if __ncclTeamCft == NULL: with gil: - raise FunctionNotFoundError("function ncclTeamRail is not found") - return (__ncclTeamRail)( + raise FunctionNotFoundError("function ncclTeamCft is not found") + return (__ncclTeamCft)( + comm, mode) + + +cdef ncclTeam_t _ncclTeamCftMultimem(ncclComm_t comm) except* nogil: + global __ncclTeamCftMultimem + _check_or_init_nccl() + if __ncclTeamCftMultimem == NULL: + with gil: + raise FunctionNotFoundError("function ncclTeamCftMultimem is not found") + return (__ncclTeamCftMultimem)( comm) @@ -1801,6 +2109,16 @@ cdef int _ncclTeamRankToLsa(ncclComm_t comm, ncclTeam_t team, int rank) except?- comm, team, rank) +cdef ncclTeam_t _ncclTeamRail(ncclComm_t comm) except* nogil: + global __ncclTeamRail + _check_or_init_nccl() + if __ncclTeamRail == NULL: + with gil: + raise FunctionNotFoundError("function ncclTeamRail is not found") + return (__ncclTeamRail)( + comm) + + cdef ncclResult_t _ncclLsaBarrierCreateRequirement(ncclTeam_t team, int nBarriers, ncclLsaBarrierHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclLsaBarrierCreateRequirement _check_or_init_nccl() @@ -1821,16 +2139,6 @@ cdef ncclResult_t _ncclGinBarrierCreateRequirement(ncclComm_t comm, ncclTeam_t t comm, team, nBarriers, outHandle, outReq) -cdef ncclResult_t _ncclLLA2ACreateRequirement(int nBlocks, int nSlots, ncclLLA2AHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: - global __ncclLLA2ACreateRequirement - _check_or_init_nccl() - if __ncclLLA2ACreateRequirement == NULL: - with gil: - raise FunctionNotFoundError("function ncclLLA2ACreateRequirement is not found") - return (__ncclLLA2ACreateRequirement)( - nBlocks, nSlots, outHandle, outReq) - - cdef int _ncclLLA2ACalcSlots(int maxElts, int maxEltSize) except?-42 nogil: global __ncclLLA2ACalcSlots _check_or_init_nccl() @@ -1839,3 +2147,13 @@ cdef int _ncclLLA2ACalcSlots(int maxElts, int maxEltSize) except?-42 nogil: raise FunctionNotFoundError("function ncclLLA2ACalcSlots is not found") return (__ncclLLA2ACalcSlots)( maxElts, maxEltSize) + + +cdef ncclResult_t _ncclLLA2ACreateRequirement(int nBlocks, int nSlots, ncclLLA2AHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + global __ncclLLA2ACreateRequirement + _check_or_init_nccl() + if __ncclLLA2ACreateRequirement == NULL: + with gil: + raise FunctionNotFoundError("function ncclLLA2ACreateRequirement is not found") + return (__ncclLLA2ACreateRequirement)( + nBlocks, nSlots, outHandle, outReq) diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/utils.pxd b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/utils.pxd index 39cc3e3921e7..f7dd97de1d2c 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/utils.pxd +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/utils.pxd @@ -167,4 +167,3 @@ cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, obj cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1 cdef bint is_nested_sequence(data) -cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=*) except* diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/utils.pyx b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/utils.pyx index 1f5fc7fa0400..fc0a37670c7e 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/utils.pyx +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/_internal/utils.pyx @@ -20,39 +20,6 @@ cdef bint is_nested_sequence(data): return True -cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=True) except*: - """The caller must ensure ``buf`` is alive when the returned pointer is in use.""" - cdef void* bufPtr - cdef int flags = cpython.PyBUF_ANY_CONTIGUOUS - if not readonly: - flags |= cpython.PyBUF_WRITABLE - cdef int status = -1 - cdef cpython.Py_buffer view - - if isinstance(buf, int): - bufPtr = buf - else: # try buffer protocol - try: - status = cpython.PyObject_GetBuffer(buf, &view, flags) - # when the caller does not provide a size, it is set to -1 at generate-time by cybind - if size != -1: - assert view.len == size - assert view.ndim == 1 - except Exception as e: - adj = "writable " if not readonly else "" - raise ValueError( - "buf must be either a Python int representing the pointer " - f"address to a valid buffer, or a 1D contiguous {adj}" - "buffer, of size bytes") from e - else: - bufPtr = view.buf - finally: - if status == 0: - cpython.PyBuffer_Release(&view) - - return bufPtr - - # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, # so we need a dummy (__unused) input argument to help it cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/cynccl.pxd b/projects/rccl/bindings/nccl4py/nccl/bindings/cynccl.pxd index 046c9b57ee82..705bdb5cc37f 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/cynccl.pxd +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/cynccl.pxd @@ -3,10 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 # -# This code was automatically generated with version 2.30.7. Do not modify it directly. - - -from libc.stdint cimport int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t, uint32_t, uint64_t +# This code was automatically generated with version 2.32.3. Do not modify it directly. ############################################################################### @@ -14,6 +11,24 @@ from libc.stdint cimport int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t, u ############################################################################### # enums + +# <<<< PREAMBLE CONTENT >>>> + +from libc.stdint cimport ( + int16_t, + int32_t, + int64_t, + int8_t, + uint16_t, + uint32_t, + uint64_t, + uint8_t, +) +from libcpp cimport bool as _cyb_bool + + +# <<<< END OF PREAMBLE CONTENT >>>> + ctypedef enum ncclResult_t "ncclResult_t": ncclSuccess "ncclSuccess" = 0 ncclUnhandledCudaError "ncclUnhandledCudaError" = 1 @@ -27,6 +42,19 @@ ctypedef enum ncclResult_t "ncclResult_t": ncclNumResults "ncclNumResults" = 9 _NCCLRESULT_T_INTERNAL_LOADING_ERROR "_NCCLRESULT_T_INTERNAL_LOADING_ERROR" = -42 +ctypedef enum ncclHostCftMode_t "ncclHostCftMode_t": + ncclHostCftDefault "ncclHostCftDefault" = -(2147483648) + ncclHostCftEnable "ncclHostCftEnable" = 1 + ncclHostCftDisable "ncclHostCftDisable" = 2 + ncclHostCftFallback "ncclHostCftFallback" = 3 + +ctypedef enum ncclNvlsHostMode_t "ncclNvlsHostMode_t": + ncclNvlsHostModeDefault "ncclNvlsHostModeDefault" = -(2147483648) + ncclNvlsHostModeEnable "ncclNvlsHostModeEnable" = 0 + ncclNvlsHostModeDisableTransport "ncclNvlsHostModeDisableTransport" = (1 << 0) + ncclNvlsHostModeDisableSymmetricMultimem "ncclNvlsHostModeDisableSymmetricMultimem" = (1 << 1) + ncclNvlsHostModeDisable "ncclNvlsHostModeDisable" = 2147483647 + ctypedef enum ncclCommMemStat_t "ncclCommMemStat_t": ncclStatGpuMemSuspend "ncclStatGpuMemSuspend" = 0 ncclStatGpuMemSuspended "ncclStatGpuMemSuspended" = 1 @@ -69,31 +97,56 @@ ctypedef enum ncclScalarResidence_t "ncclScalarResidence_t": ncclScalarDevice "ncclScalarDevice" = 0 ncclScalarHostImmediate "ncclScalarHostImmediate" = 1 +ctypedef enum ncclEncryptionMode_t "ncclEncryptionMode_t": + NCCL_ENCRYPTION_MODE_NONE "NCCL_ENCRYPTION_MODE_NONE" = 0 + NCCL_ENCRYPTION_MODE_PSK "NCCL_ENCRYPTION_MODE_PSK" = 1 + +ctypedef enum ncclGinConnectionType_t "ncclGinConnectionType_t": + NCCL_GIN_CONNECTION_NONE "NCCL_GIN_CONNECTION_NONE" + NCCL_GIN_CONNECTION_FULL "NCCL_GIN_CONNECTION_FULL" + NCCL_GIN_CONNECTION_RAIL "NCCL_GIN_CONNECTION_RAIL" + NCCL_GIN_CONNECTION_CUSTOM_STRIDE "NCCL_GIN_CONNECTION_CUSTOM_STRIDE" + ctypedef enum ncclGinType_t "ncclGinType_t": NCCL_GIN_TYPE_NONE "NCCL_GIN_TYPE_NONE" = 0 NCCL_GIN_TYPE_PROXY "NCCL_GIN_TYPE_PROXY" = 2 NCCL_GIN_TYPE_GDAKI "NCCL_GIN_TYPE_GDAKI" = 3 NCCL_GIN_TYPE_GPI "NCCL_GIN_TYPE_GPI" = 4 + NCCL_GIN_TYPE_EFA_GDA "NCCL_GIN_TYPE_EFA_GDA" = 5 + NCCL_GIN_MAX_TYPES "NCCL_GIN_MAX_TYPES" = 6 -ctypedef enum ncclGinConnectionType_t "ncclGinConnectionType_t": - NCCL_GIN_CONNECTION_NONE "NCCL_GIN_CONNECTION_NONE" = 0 - NCCL_GIN_CONNECTION_FULL "NCCL_GIN_CONNECTION_FULL" = 1 - NCCL_GIN_CONNECTION_RAIL "NCCL_GIN_CONNECTION_RAIL" = 2 +ctypedef enum ncclCftTeamMode_t "ncclCftTeamMode_t": + NCCL_CFT_TEAM_FLAT "NCCL_CFT_TEAM_FLAT" + NCCL_CFT_TEAM_HIER_MULTIMEM "NCCL_CFT_TEAM_HIER_MULTIMEM" + NCCL_CFT_TEAM_HIER_LSA "NCCL_CFT_TEAM_HIER_LSA" + +ctypedef enum ncclCftCap_t "ncclCftCap_t": + NCCL_CFT_NONE "NCCL_CFT_NONE" = 0x0 + NCCL_CFT "NCCL_CFT" = 0x1 + NCCL_CFT_MULTIMEM "NCCL_CFT_MULTIMEM" = 0x2 + +ctypedef enum ncclGinSignalOp_t "ncclGinSignalOp_t": + ncclGinSignalInc "ncclGinSignalInc" = 0 + ncclGinSignalAdd "ncclGinSignalAdd" # types # Patched for RCCL: removed CUDA driver-types include block. cudaStream_t -# is treated as an opaque void* (HIP's hipStream_t is also void*); we never -# dereference it, only pass it through to RCCL functions resolved by dlsym. +# and cudaEvent_t are treated as opaque void* (HIP's hipStream_t and +# hipEvent_t are also pointers); we never dereference them, only pass them +# through to RCCL functions resolved by dlsym. ctypedef void* cudaStream_t +ctypedef void* cudaEvent_t -ctypedef uint32_t ncclDevResourceHandle_t 'ncclDevResourceHandle_t' +ctypedef uint32_t ncclDevResourceHandle 'ncclDevResourceHandle' ctypedef uint32_t ncclGinSignal_t 'ncclGinSignal_t' ctypedef uint32_t ncclGinCounter_t 'ncclGinCounter_t' +ctypedef uint32_t ncclCftLeId 'ncclCftLeId' + ctypedef void* ncclComm_t 'ncclComm_t' ctypedef void* ncclWindow_t 'ncclWindow_t' @@ -127,6 +180,20 @@ ctypedef struct ncclConfig_t 'ncclConfig_t': int numRmaCtx int maxP2pPeers int graphStreamOrdering + int launchOrderImplicit + int numRmaSig + int rmaEagerInit + int hostCftMode + int nvlsHostMode + +ctypedef struct nccl_bindings_nccl__anon_pod0: + int vendorId + int optionId + +ctypedef union nccl_bindings_nccl__anon_pod1: + int i + char* s + void* raw ctypedef struct ncclSimInfo_t 'ncclSimInfo_t': size_t size @@ -140,6 +207,26 @@ ctypedef struct ncclWaitSignalDesc_t 'ncclWaitSignalDesc_t': int sigIdx int ctx +ctypedef struct ncclEncryptionConfig_t 'ncclEncryptionConfig_t': + size_t size + unsigned int magic + unsigned int version + int mode + char* psk + +ctypedef struct ncclResourceWindow_vidmem_t 'ncclResourceWindow_vidmem_t': + char* lsaFlatBase + uint32_t stride4G + uint32_t mcOffset4K + +ctypedef struct ncclMultimemHandle_t 'ncclMultimemHandle_t': + void* mcBasePtr + +ctypedef struct ncclTeam_t 'ncclTeam_t': + int nRanks + int rank + int stride + ctypedef struct ncclCommProperties_t 'ncclCommProperties_t': size_t size unsigned int magic @@ -148,48 +235,56 @@ ctypedef struct ncclCommProperties_t 'ncclCommProperties_t': int nRanks int cudaDev int nvmlDev - uint8_t deviceApiSupport - uint8_t multimemSupport + _cyb_bool deviceApiSupport + _cyb_bool multimemSupport ncclGinType_t ginType int nLsaTeams - uint8_t hostRmaSupport + _cyb_bool hostRmaSupport ncclGinType_t railedGinType uint64_t commHash int ginMinStride ncclGinConnectionType_t ginConnectionType - uint8_t ginSupport[64] + _cyb_bool ginSupport[64] size_t devCommRuntimeVersionSize + _cyb_bool cftSupport + _cyb_bool cftMulticastSupport + _cyb_bool cftCountedSupport -ctypedef struct ncclTeam_t 'ncclTeam_t': - int nRanks - int rank - int stride +ctypedef ncclDevResourceHandle ncclDevResourceHandle_t 'ncclDevResourceHandle_t' -ctypedef struct ncclMultimemHandle_t 'ncclMultimemHandle_t': - void* mcBasePtr +ctypedef ncclCftLeId ncclCftLeId_t 'ncclCftLeId_t' -ctypedef struct ncclResourceWindow_vidmem_t 'ncclResourceWindow_vidmem_t': - char reserved1[8] - char* lsaFlatBase - char reserved2[8] - uint32_t stride4G - uint32_t mcOffset4K - char reserved3[32] +cdef struct ncclConfigExt: + ncclConfigExt* next + nccl_bindings_nccl__anon_pod0 key + nccl_bindings_nccl__anon_pod1 val +ctypedef ncclConfigExt ncclConfigExt_t + +cdef struct ncclTeamRequirements: + ncclTeamRequirements* next + ncclTeam_t team + _cyb_bool multimem + ncclMultimemHandle_t* outMultimemHandle +ctypedef ncclTeamRequirements ncclTeamRequirements_t + +ctypedef struct ncclGinBarrierHandle_t 'ncclGinBarrierHandle_t': + ncclGinSignal_t signal0 + ncclDevResourceHandle_t unused ctypedef struct ncclLsaBarrierHandle_t 'ncclLsaBarrierHandle_t': ncclDevResourceHandle_t bufHandle int nBarriers +ctypedef struct ncclCftBarrierHandle_t 'ncclCftBarrierHandle_t': + ncclDevResourceHandle_t bufHandle + int nBarriers + ctypedef struct ncclLLA2AHandle_t 'ncclLLA2AHandle_t': ncclDevResourceHandle_t bufHandle uint32_t nSlots -ctypedef struct ncclGinBarrierHandle_t 'ncclGinBarrierHandle_t': - ncclGinSignal_t signal0 - ncclDevResourceHandle_t unused - -ctypedef struct ncclDevResourceRequirements_t 'ncclDevResourceRequirements_t': - void* next +cdef struct ncclDevResourceRequirements: + ncclDevResourceRequirements* next size_t bufferSize size_t bufferAlign ncclDevResourceHandle_t* outBufferHandle @@ -197,12 +292,22 @@ ctypedef struct ncclDevResourceRequirements_t 'ncclDevResourceRequirements_t': int ginCounterCount ncclGinSignal_t* outGinSignalStart ncclGinCounter_t* outGinCounterStart +ctypedef ncclDevResourceRequirements ncclDevResourceRequirements_t -ctypedef struct ncclTeamRequirements_t 'ncclTeamRequirements_t': - void* next - ncclTeam_t team - uint8_t multimem - ncclMultimemHandle_t* outMultimemHandle +ctypedef struct ncclCollConfig_t 'ncclCollConfig_t': + size_t size + unsigned int magic + unsigned int version + ncclConfigExt_t* ext + int minCTAs + int maxCTAs + int nvlsCTAs + int cgaClusterSize + char* algSelection + int forceAlgSelection + int CTAPolicy + uint64_t userProfilerTag + cudaEvent_t launchCompletionEvent ctypedef struct ncclDevComm_t 'ncclDevComm_t': unsigned int magic @@ -216,24 +321,35 @@ ctypedef struct ncclDevComm_t 'ncclDevComm_t': ncclDevCommWindowTable_t windowTable ncclWindow_t resourceWindow ncclResourceWindow_vidmem_t resourceWindow_inlined - ncclGinBarrierHandle_t hybridWorldGinBarrier + ncclGinBarrierHandle_t hybridDenseGinBarrier ncclMultimemHandle_t lsaMultimem ncclLsaBarrierHandle_t lsaBarrier ncclGinBarrierHandle_t railGinBarrier uint8_t ginConnectionCount + uint8_t backendIndex uint8_t ginNetDeviceTypes[4] void* ginHandles[4] int ginSignalCount int ginCounterCount uint64_t* ginSignalShadows uint32_t ginContextCount - uint8_t ginConnectionsRailed - uint8_t ginStrongLegacySignals - uint8_t ginContextsRailed + int ginConnectionStride + int ginContextStride + _cyb_bool ginStrongLegacySignals uint32_t* abortFlag ncclLsaBarrierHandle_t hybridLsaBarrier ncclGinBarrierHandle_t hybridRailGinBarrier ncclGinBarrierHandle_t worldGinBarrier + uint32_t ginConnectionStride_rcp32 + int cftRank + int cftSize + int cftMultimemRank + int cftMultimemSize + uint32_t cftMultimemSize_rcp32 + ncclCftLeId ucLeId + ncclCftLeId mcLeId + ncclCftBarrierHandle_t cftBarrier + ncclCftBarrierHandle_t cftMultimemBarrier ctypedef struct ncclDevCommRequirements_t 'ncclDevCommRequirements_t': size_t size @@ -241,23 +357,28 @@ ctypedef struct ncclDevCommRequirements_t 'ncclDevCommRequirements_t': unsigned int version ncclDevResourceRequirements_t* resourceRequirementsList ncclTeamRequirements_t* teamRequirementsList - uint8_t lsaMultimem + _cyb_bool lsaMultimem int barrierCount int lsaBarrierCount int railGinBarrierCount int lsaLLA2ABlockCount int lsaLLA2ASlotCount - uint8_t ginForceEnable + _cyb_bool ginForceEnable int ginContextCount int ginSignalCount int ginCounterCount ncclGinConnectionType_t ginConnectionType - uint8_t ginExclusiveContexts + _cyb_bool ginExclusiveContexts int ginQueueDepth int ginTrafficClass int worldGinBarrierCount - uint8_t ginStrongSignalsRequired - uint8_t ginVaSignalsRequired + _cyb_bool ginStrongSignalsRequired + _cyb_bool ginVaSignalsRequired + int ginCustomStride + ncclGinType_t ginType + _cyb_bool useRuntimeVersion + int cftCaps + int cftBarrierCount ############################################################################### @@ -305,6 +426,14 @@ cdef ncclResult_t ncclAllGather(const void* sendbuff, void* recvbuff, size_t sen cdef ncclResult_t ncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGather(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclScatter(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclAllReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclBroadcastConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclAllGatherConfig(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclReduceScatterConfig(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclAlltoAllConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclGatherConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclScatterConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclPutSignal(const void* localbuff, size_t count, ncclDataType_t datatype, int peer, ncclWindow_t peerWin, size_t peerWinOffset, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil @@ -313,6 +442,7 @@ cdef ncclResult_t ncclWaitSignal(int nDesc, ncclWaitSignalDesc_t* signalDescs, n cdef ncclResult_t ncclGroupStart() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGroupEnd() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclSetEncryption(const ncclEncryptionConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclParamBind(ncclParamHandle_t* out, const char* key) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclParamGetI8(ncclParamHandle_t h, int8_t* out) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclParamGetI16(ncclParamHandle_t h, int16_t* out) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil @@ -331,15 +461,20 @@ cdef ncclResult_t ncclCommQueryProperties(ncclComm_t comm, ncclCommProperties_t* cdef ncclResult_t ncclDevCommCreate(ncclComm_t comm, const ncclDevCommRequirements_t* reqs, ncclDevComm_t* outDevComm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclDevCommDestroy(ncclComm_t comm, const ncclDevComm_t* devComm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGetLsaMultimemDevicePointer(ncclWindow_t window, size_t offset, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil -cdef ncclResult_t ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGetMultimemDevicePointer(ncclWindow_t window, size_t offset, ncclMultimemHandle_t multimem, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGetPeerDevicePointer(ncclWindow_t window, size_t offset, int peer, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclGetMultimemDeviceLeInfo(ncclWindow_t window, size_t offset, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclGetCftDeviceLeInfo(ncclWindow_t window, size_t offset, int peerCft, ncclTeam_t cftTeam, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil +cdef ncclResult_t ncclGetPeerDeviceLeInfo(ncclWindow_t window, size_t offset, int peerWorld, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclTeam_t ncclTeamWorld(ncclComm_t comm) except* nogil cdef ncclTeam_t ncclTeamLsa(ncclComm_t comm) except* nogil -cdef ncclTeam_t ncclTeamRail(ncclComm_t comm) except* nogil +cdef ncclTeam_t ncclTeamCft(ncclComm_t comm, ncclCftTeamMode_t mode) except* nogil +cdef ncclTeam_t ncclTeamCftMultimem(ncclComm_t comm) except* nogil cdef int ncclTeamRankToWorld(ncclComm_t comm, ncclTeam_t team, int rank) except?-42 nogil cdef int ncclTeamRankToLsa(ncclComm_t comm, ncclTeam_t team, int rank) except?-42 nogil +cdef ncclTeam_t ncclTeamRail(ncclComm_t comm) except* nogil cdef ncclResult_t ncclLsaBarrierCreateRequirement(ncclTeam_t team, int nBarriers, ncclLsaBarrierHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGinBarrierCreateRequirement(ncclComm_t comm, ncclTeam_t team, int nBarriers, ncclGinBarrierHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil -cdef ncclResult_t ncclLLA2ACreateRequirement(int nBlocks, int nSlots, ncclLLA2AHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef int ncclLLA2ACalcSlots(int maxElts, int maxEltSize) except?-42 nogil +cdef ncclResult_t ncclLLA2ACreateRequirement(int nBlocks, int nSlots, ncclLLA2AHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/cynccl.pyx b/projects/rccl/bindings/nccl4py/nccl/bindings/cynccl.pyx index bb0dbaeb07f1..20beae96bf70 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/cynccl.pyx +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/cynccl.pyx @@ -2,7 +2,25 @@ # # SPDX-License-Identifier: Apache-2.0 # -# This code was automatically generated with version 2.30.7. Do not modify it directly. +# This code was automatically generated with version 2.32.3. Do not modify it directly. + + +# <<<< PREAMBLE CONTENT >>>> + +cimport cython as _cyb_cython +from libc.stdint cimport ( + int16_t, + int32_t, + int64_t, + int8_t, + uint16_t, + uint32_t, + uint64_t, + uint8_t, +) + + +# <<<< END OF PREAMBLE CONTENT >>>> from ._internal cimport nccl as _nccl @@ -175,6 +193,38 @@ cdef ncclResult_t ncclScatter(const void* sendbuff, void* recvbuff, size_t count return _nccl._ncclScatter(sendbuff, recvbuff, count, datatype, root, comm, stream) +cdef ncclResult_t ncclAllReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclAllReduceConfig(sendbuff, recvbuff, count, datatype, op, comm, stream, config) + + +cdef ncclResult_t ncclBroadcastConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclBroadcastConfig(sendbuff, recvbuff, count, datatype, root, comm, stream, config) + + +cdef ncclResult_t ncclReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclReduceConfig(sendbuff, recvbuff, count, datatype, op, root, comm, stream, config) + + +cdef ncclResult_t ncclAllGatherConfig(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclAllGatherConfig(sendbuff, recvbuff, sendcount, datatype, comm, stream, config) + + +cdef ncclResult_t ncclReduceScatterConfig(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclReduceScatterConfig(sendbuff, recvbuff, recvcount, datatype, op, comm, stream, config) + + +cdef ncclResult_t ncclAlltoAllConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclAlltoAllConfig(sendbuff, recvbuff, count, datatype, comm, stream, config) + + +cdef ncclResult_t ncclGatherConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclGatherConfig(sendbuff, recvbuff, count, datatype, root, comm, stream, config) + + +cdef ncclResult_t ncclScatterConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclScatterConfig(sendbuff, recvbuff, count, datatype, root, comm, stream, config) + + cdef ncclResult_t ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclSend(sendbuff, count, datatype, peer, comm, stream) @@ -207,6 +257,10 @@ cdef ncclResult_t ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) except?_NCCLRESUL return _nccl._ncclGroupSimulateEnd(simInfo) +cdef ncclResult_t ncclSetEncryption(const ncclEncryptionConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclSetEncryption(config) + + cdef ncclResult_t ncclParamBind(ncclParamHandle_t* out, const char* key) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclParamBind(out, key) @@ -259,6 +313,7 @@ cdef ncclResult_t ncclParamGetAllParameterKeys(const char*** table, int* tableLe return _nccl._ncclParamGetAllParameterKeys(table, tableLen) +@_cyb_cython.show_performance_hints(False) cdef void ncclParamDumpAll() except* nogil: _nccl._ncclParamDumpAll() @@ -279,18 +334,30 @@ cdef ncclResult_t ncclGetLsaMultimemDevicePointer(ncclWindow_t window, size_t of return _nccl._ncclGetLsaMultimemDevicePointer(window, offset, outPtr) -cdef ncclResult_t ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: - return _nccl._ncclGetLsaDevicePointer(window, offset, lsaRank, outPtr) - - cdef ncclResult_t ncclGetMultimemDevicePointer(ncclWindow_t window, size_t offset, ncclMultimemHandle_t multimem, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclGetMultimemDevicePointer(window, offset, multimem, outPtr) +cdef ncclResult_t ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclGetLsaDevicePointer(window, offset, lsaRank, outPtr) + + cdef ncclResult_t ncclGetPeerDevicePointer(ncclWindow_t window, size_t offset, int peer, void** outPtr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclGetPeerDevicePointer(window, offset, peer, outPtr) +cdef ncclResult_t ncclGetMultimemDeviceLeInfo(ncclWindow_t window, size_t offset, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclGetMultimemDeviceLeInfo(window, offset, leId, leOffset) + + +cdef ncclResult_t ncclGetCftDeviceLeInfo(ncclWindow_t window, size_t offset, int peerCft, ncclTeam_t cftTeam, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclGetCftDeviceLeInfo(window, offset, peerCft, cftTeam, leId, leOffset) + + +cdef ncclResult_t ncclGetPeerDeviceLeInfo(ncclWindow_t window, size_t offset, int peerWorld, ncclCftLeId* leId, size_t* leOffset) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclGetPeerDeviceLeInfo(window, offset, peerWorld, leId, leOffset) + + cdef ncclTeam_t ncclTeamWorld(ncclComm_t comm) except* nogil: return _nccl._ncclTeamWorld(comm) @@ -299,8 +366,12 @@ cdef ncclTeam_t ncclTeamLsa(ncclComm_t comm) except* nogil: return _nccl._ncclTeamLsa(comm) -cdef ncclTeam_t ncclTeamRail(ncclComm_t comm) except* nogil: - return _nccl._ncclTeamRail(comm) +cdef ncclTeam_t ncclTeamCft(ncclComm_t comm, ncclCftTeamMode_t mode) except* nogil: + return _nccl._ncclTeamCft(comm, mode) + + +cdef ncclTeam_t ncclTeamCftMultimem(ncclComm_t comm) except* nogil: + return _nccl._ncclTeamCftMultimem(comm) cdef int ncclTeamRankToWorld(ncclComm_t comm, ncclTeam_t team, int rank) except?-42 nogil: @@ -311,6 +382,10 @@ cdef int ncclTeamRankToLsa(ncclComm_t comm, ncclTeam_t team, int rank) except?-4 return _nccl._ncclTeamRankToLsa(comm, team, rank) +cdef ncclTeam_t ncclTeamRail(ncclComm_t comm) except* nogil: + return _nccl._ncclTeamRail(comm) + + cdef ncclResult_t ncclLsaBarrierCreateRequirement(ncclTeam_t team, int nBarriers, ncclLsaBarrierHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclLsaBarrierCreateRequirement(team, nBarriers, outHandle, outReq) @@ -319,9 +394,9 @@ cdef ncclResult_t ncclGinBarrierCreateRequirement(ncclComm_t comm, ncclTeam_t te return _nccl._ncclGinBarrierCreateRequirement(comm, team, nBarriers, outHandle, outReq) -cdef ncclResult_t ncclLLA2ACreateRequirement(int nBlocks, int nSlots, ncclLLA2AHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: - return _nccl._ncclLLA2ACreateRequirement(nBlocks, nSlots, outHandle, outReq) - - cdef int ncclLLA2ACalcSlots(int maxElts, int maxEltSize) except?-42 nogil: return _nccl._ncclLLA2ACalcSlots(maxElts, maxEltSize) + + +cdef ncclResult_t ncclLLA2ACreateRequirement(int nBlocks, int nSlots, ncclLLA2AHandle_t* outHandle, ncclDevResourceRequirements_t* outReq) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: + return _nccl._ncclLLA2ACreateRequirement(nBlocks, nSlots, outHandle, outReq) diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/nccl.pxd b/projects/rccl/bindings/nccl4py/nccl/bindings/nccl.pxd index 65a9ffa940e9..acca3cb82e20 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/nccl.pxd +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/nccl.pxd @@ -2,9 +2,19 @@ # # SPDX-License-Identifier: Apache-2.0 # -# This code was automatically generated with version 2.30.7. Do not modify it directly. +# This code was automatically generated with version 2.32.3. Do not modify it directly. -from libc.stdint cimport intptr_t + + +# <<<< PREAMBLE CONTENT >>>> + +from libc.stdint cimport ( + intptr_t, + uint64_t, +) + + +# <<<< END OF PREAMBLE CONTENT >>>> from .cynccl cimport * @@ -13,29 +23,30 @@ from .cynccl cimport * # Types ############################################################################### -ctypedef ncclComm_t Comm -ctypedef ncclWindow_t Window ctypedef ncclParamHandle_t ParamHandle ctypedef ncclDevCommWindowTable_t DevCommWindowTable ctypedef cudaStream_t Stream -cdef class PointerBox: - cdef public intptr_t ptr - ############################################################################### # Enum ############################################################################### ctypedef ncclResult_t _Result +ctypedef ncclHostCftMode_t _HostCftMode +ctypedef ncclNvlsHostMode_t _NvlsHostMode ctypedef ncclCommMemStat_t _CommMemStat -ctypedef ncclRedOp_dummy_t _RedOp_dummy +ctypedef ncclRedOp_dummy_t _RedOpDummy ctypedef ncclRedOp_t _RedOp ctypedef ncclDataType_t _DataType ctypedef ncclScalarResidence_t _ScalarResidence -ctypedef ncclGinType_t _GinType +ctypedef ncclEncryptionMode_t _EncryptionMode ctypedef ncclGinConnectionType_t _GinConnectionType +ctypedef ncclGinType_t _GinType +ctypedef ncclCftTeamMode_t _CftTeamMode +ctypedef ncclCftCap_t _CftCap +ctypedef ncclGinSignalOp_t _GinSignalOp ############################################################################### @@ -46,70 +57,88 @@ cpdef intptr_t mem_alloc(size_t size) except? 0 cpdef mem_free(intptr_t ptr) cpdef int get_version() except? -1 cpdef object get_unique_id() -cpdef int comm_init_rank_config(intptr_t comm, int nranks, comm_id, int rank, intptr_t config) except? -1 -cpdef int comm_init_rank(intptr_t comm, int nranks, comm_id, int rank) except? -1 +cpdef object comm_init_rank_config(int nranks, comm_id, int rank, config) +cpdef object comm_init_rank(int nranks, comm_id, int rank) cpdef object comm_init_all(int ndev, devlist) -cpdef comm_finalize(intptr_t comm) -cpdef comm_destroy(intptr_t comm) -cpdef comm_abort(intptr_t comm) -cpdef comm_revoke(intptr_t comm, int revoke_flags) -cpdef int comm_split(intptr_t comm, int color, int key, intptr_t newcomm, intptr_t config) except? -1 -cpdef int comm_shrink(intptr_t comm, exclude_ranks_list, int exclude_ranks_count, intptr_t newcomm, intptr_t config, int shrink_flags) except? -1 -cpdef object comm_get_unique_id(intptr_t comm) -cpdef int comm_grow(intptr_t comm, int n_ranks, intptr_t unique_id, int rank, intptr_t newcomm, intptr_t config) except? -1 -cpdef int comm_init_rank_scalable(intptr_t newcomm, int nranks, int myrank, int n_id, comm_ids, intptr_t config) except? -1 +cpdef comm_finalize(object comm) +cpdef comm_destroy(object comm) +cpdef comm_abort(object comm) +cpdef comm_revoke(object comm, int revoke_flags) +cpdef object comm_split(object comm, int color, int key, config) +cpdef object comm_shrink(object comm, exclude_ranks_list, int exclude_ranks_count, config, int shrink_flags) +cpdef object comm_get_unique_id(object comm) +cpdef object comm_grow(object comm, int n_ranks, unique_id, int rank, config) +cpdef object comm_init_rank_scalable(int nranks, int myrank, int n_id, comm_ids, config) cpdef str get_error_string(int result) -cpdef str get_last_error(intptr_t comm) -cpdef int comm_get_async_error(intptr_t comm) except? -1 -cpdef int comm_count(intptr_t comm) except? -1 -cpdef int comm_cu_device(intptr_t comm) except? -1 -cpdef int comm_user_rank(intptr_t comm) except? -1 -cpdef intptr_t comm_register(intptr_t comm, intptr_t buff, size_t size) except? 0 -cpdef comm_deregister(intptr_t comm, intptr_t handle) -cpdef comm_suspend(intptr_t comm, int flags) -cpdef comm_resume(intptr_t comm) -cpdef uint64_t comm_mem_stats(intptr_t comm, int stat) except? -1 -cpdef int comm_window_register(intptr_t comm, intptr_t buff, size_t size, intptr_t win, int win_flags) except? -1 -cpdef comm_window_deregister(intptr_t comm, intptr_t win) -cpdef intptr_t win_get_user_ptr(intptr_t comm, intptr_t win) except? 0 -cpdef int red_op_create_pre_mul_sum(intptr_t scalar, int datatype, int residence, intptr_t comm) except? -1 -cpdef red_op_destroy(int op, intptr_t comm) -cpdef reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, int root, intptr_t comm, intptr_t stream) -cpdef bcast(intptr_t buff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream) -cpdef broadcast(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream) -cpdef all_reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, intptr_t comm, intptr_t stream) -cpdef reduce_scatter(intptr_t sendbuff, intptr_t recvbuff, size_t recvcount, int datatype, int op, intptr_t comm, intptr_t stream) -cpdef all_gather(intptr_t sendbuff, intptr_t recvbuff, size_t sendcount, int datatype, intptr_t comm, intptr_t stream) -cpdef allto_all(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, intptr_t comm, intptr_t stream) -cpdef gather(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream) -cpdef scatter(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream) -cpdef send(intptr_t sendbuff, size_t count, int datatype, int peer, intptr_t comm, intptr_t stream) -cpdef recv(intptr_t recvbuff, size_t count, int datatype, int peer, intptr_t comm, intptr_t stream) -cpdef put_signal(intptr_t localbuff, size_t count, int datatype, int peer, intptr_t peer_win, size_t peer_win_offset, int sig_idx, int ctx, unsigned int flags, intptr_t comm, intptr_t stream) -cpdef signal(int peer, int sig_idx, int ctx, unsigned int flags, intptr_t comm, intptr_t stream) -cpdef wait_signal(int n_desc, signal_descs, intptr_t comm, intptr_t stream) +cpdef str get_last_error(object comm) +cpdef int comm_get_async_error(object comm) except? -1 +cpdef int comm_count(object comm) except? -1 +cpdef int comm_cu_device(object comm) except? -1 +cpdef int comm_user_rank(object comm) except? -1 +cpdef intptr_t comm_register(object comm, intptr_t buff, size_t size) except? 0 +cpdef comm_deregister(object comm, intptr_t handle) +cpdef comm_suspend(object comm, int flags) +cpdef comm_resume(object comm) +cpdef uint64_t comm_mem_stats(object comm, int stat) except? -1 +cpdef object comm_window_register(object comm, intptr_t buff, size_t size, int win_flags) +cpdef comm_window_deregister(object comm, object win) +cpdef intptr_t win_get_user_ptr(object comm, object win) except? 0 +cpdef int red_op_create_pre_mul_sum(intptr_t scalar, int datatype, int residence, object comm) except? -1 +cpdef red_op_destroy(int op, object comm) +cpdef reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, int root, object comm, intptr_t stream) +cpdef bcast(intptr_t buff, size_t count, int datatype, int root, object comm, intptr_t stream) +cpdef broadcast(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream) +cpdef all_reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, object comm, intptr_t stream) +cpdef reduce_scatter(intptr_t sendbuff, intptr_t recvbuff, size_t recvcount, int datatype, int op, object comm, intptr_t stream) +cpdef all_gather(intptr_t sendbuff, intptr_t recvbuff, size_t sendcount, int datatype, object comm, intptr_t stream) +cpdef allto_all(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, object comm, intptr_t stream) +cpdef gather(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream) +cpdef scatter(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream) +cpdef all_reduce_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, object comm, intptr_t stream, config) +cpdef broadcast_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream, config) +cpdef reduce_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, int root, object comm, intptr_t stream, config) +cpdef all_gather_config(intptr_t sendbuff, intptr_t recvbuff, size_t sendcount, int datatype, object comm, intptr_t stream, config) +cpdef reduce_scatter_config(intptr_t sendbuff, intptr_t recvbuff, size_t recvcount, int datatype, int op, object comm, intptr_t stream, config) +cpdef allto_all_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, object comm, intptr_t stream, config) +cpdef gather_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream, config) +cpdef scatter_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream, config) +cpdef send(intptr_t sendbuff, size_t count, int datatype, int peer, object comm, intptr_t stream) +cpdef recv(intptr_t recvbuff, size_t count, int datatype, int peer, object comm, intptr_t stream) +cpdef put_signal(intptr_t localbuff, size_t count, int datatype, int peer, object peer_win, size_t peer_win_offset, int sig_idx, int ctx, unsigned int flags, object comm, intptr_t stream) +cpdef signal(int peer, int sig_idx, int ctx, unsigned int flags, object comm, intptr_t stream) +cpdef wait_signal(int n_desc, signal_descs, object comm, intptr_t stream) cpdef group_start() cpdef group_end() cpdef object group_simulate_end() -cpdef object comm_query_properties(intptr_t comm) -cpdef object dev_comm_create(intptr_t comm, intptr_t reqs) -cpdef dev_comm_destroy(intptr_t comm, intptr_t dev_comm) -cpdef intptr_t get_lsa_multimem_device_pointer(intptr_t window, size_t offset) except? 0 -cpdef intptr_t get_lsa_device_pointer(intptr_t window, size_t offset, int lsa_rank) except? 0 -cpdef intptr_t get_peer_device_pointer(intptr_t window, size_t offset, int peer) except? 0 -cpdef ll_a2a_create_requirement(int n_blocks, int n_slots, intptr_t out_handle, intptr_t out_req) - -# Hand-written: cybind cannot emit by-value struct returns. -cpdef object team_world(intptr_t comm) -cpdef object team_lsa(intptr_t comm) -cpdef object team_rail(intptr_t comm) -cpdef int team_rank_to_world(intptr_t comm, intptr_t team, int rank) -cpdef int team_rank_to_lsa(intptr_t comm, intptr_t team, int rank) -cpdef lsa_barrier_create_requirement(intptr_t team, int n_barriers, intptr_t out_handle, intptr_t out_req) -cpdef gin_barrier_create_requirement(intptr_t comm, intptr_t team, int n_barriers, intptr_t out_handle, intptr_t out_req) -cpdef object get_library_path() +cpdef set_encryption(config) +cpdef void param_dump_all() except* +cpdef object comm_query_properties(object comm) +cpdef object dev_comm_create(object comm, reqs) +cpdef dev_comm_destroy(object comm, dev_comm) +cpdef intptr_t get_lsa_multimem_device_pointer(object window, size_t offset) except? 0 +cpdef intptr_t get_multimem_device_pointer(object window, size_t offset, multimem) except? 0 +cpdef intptr_t get_lsa_device_pointer(object window, size_t offset, int lsa_rank) except? 0 +cpdef intptr_t get_peer_device_pointer(object window, size_t offset, int peer) except? 0 +cpdef tuple get_multimem_device_le_info(object window, size_t offset) +cpdef tuple get_cft_device_le_info(object window, size_t offset, int peer_cft, cft_team) +cpdef tuple get_peer_device_le_info(object window, size_t offset, int peer_world) +cpdef int team_rank_to_world(object comm, team, int rank) except? -1 +cpdef int team_rank_to_lsa(object comm, team, int rank) except? -1 +cpdef lsa_barrier_create_requirement(team, int n_barriers, out_handle, out_req) +cpdef gin_barrier_create_requirement(object comm, team, int n_barriers, out_handle, out_req) +cpdef int ll_a2a_calc_slots(int max_elts, int max_elt_size) except? -1 +cpdef ll_a2a_create_requirement(int n_blocks, int n_slots, out_handle, out_req) + +# Hand-written: the team getters return ncclTeam_t by value, which cybind cannot emit. +cpdef object team_world(object comm) +cpdef object team_lsa(object comm) +cpdef object team_rail(object comm) +cpdef object team_cft(object comm, int mode) +cpdef object team_cft_multimem(object comm) # Hand-written: Param API (SKIP_LOWPP in nccl.cybind.yaml). cpdef str param_get_parameter(str key) cpdef list param_get_all_keys() -cpdef param_dump_all() + +# Hand-written: not an NCCL entry point; reports the path of the loaded DSO. +cpdef object get_library_path() diff --git a/projects/rccl/bindings/nccl4py/nccl/bindings/nccl.pyx b/projects/rccl/bindings/nccl4py/nccl/bindings/nccl.pyx index 42ad1811f0eb..b722c2b6c7bb 100644 --- a/projects/rccl/bindings/nccl4py/nccl/bindings/nccl.pyx +++ b/projects/rccl/bindings/nccl4py/nccl/bindings/nccl.pyx @@ -2,57 +2,36 @@ # # SPDX-License-Identifier: Apache-2.0 # -# This code was automatically generated with version 2.30.7. Do not modify it directly. +# This code was automatically generated with version 2.32.3. Do not modify it directly. + + +# <<<< PREAMBLE CONTENT >>>> + +cimport cpython as _cyb_cpython +cimport cpython.buffer as _cyb_cpython_buffer +from cython cimport view as _cyb_view +from libc.stdint cimport ( + intptr_t, + uint32_t, + uint64_t, + uint8_t, +) +from libc.stdlib cimport ( + calloc as _cyb_calloc, + free as _cyb_free, + malloc as _cyb_malloc, +) +from libc.string cimport ( + memcmp as _cyb_memcmp, + memcpy as _cyb_memcpy, +) +from libcpp cimport bool as _cyb_bool + +from enum import IntEnum as _cyb_IntEnum -cimport cython # NOQA -from libcpp.vector cimport vector - -from ._internal.utils cimport (nested_resource, nullable_unique_ptr, get_buffer_pointer, - get_resource_ptr, get_nested_resource_ptr) - -from enum import IntEnum as _IntEnum - -_version_span = "with version 2.30.7" -__version__ = _version_span.split(", generator version")[0].split()[-1] - - -from libc.stdlib cimport calloc, free, malloc -from cython cimport view -cimport cpython.buffer -cimport cpython.memoryview -cimport cpython -from libc.string cimport memcmp, memcpy import numpy as _numpy - -cdef __from_data(data, dtype_name, expected_dtype, lowpp_type): - # _numpy.recarray is a subclass of _numpy.ndarray, so implicitly handled here. - if isinstance(data, lowpp_type): - return data - if not isinstance(data, _numpy.ndarray): - raise TypeError("data argument must be a NumPy ndarray") - if data.size != 1: - raise ValueError("data array must have a size of 1") - if data.dtype != expected_dtype: - raise ValueError(f"data array must be of dtype {dtype_name}") - return lowpp_type.from_ptr(data.ctypes.data, not data.flags.writeable, data) - - -cdef __from_buffer(buffer, size, lowpp_type): - cdef Py_buffer view - if cpython.PyObject_GetBuffer(buffer, &view, cpython.PyBUF_SIMPLE) != 0: - raise TypeError("buffer argument does not support the buffer protocol") - try: - if view.itemsize != 1: - raise ValueError("buffer itemsize must be 1 byte") - if view.len != size: - raise ValueError(f"buffer length must be {size} bytes") - return lowpp_type.from_ptr(view.buf, not view.readonly, buffer) - finally: - cpython.PyBuffer_Release(&view) - - -cdef __getbuffer(object self, cpython.Py_buffer *buffer, void *ptr, int size, bint readonly): +cdef _cyb___getbuffer(object self, _cyb_cpython.Py_buffer *buffer, void *ptr, int size, bint readonly): buffer.buf = ptr buffer.format = 'b' buffer.internal = NULL @@ -65,39 +44,87 @@ cdef __getbuffer(object self, cpython.Py_buffer *buffer, void *ptr, int size, bi buffer.strides = &buffer.itemsize buffer.suboffsets = NULL +cdef _cyb_from_buffer(buffer, size, lowpp_type): + cdef _cyb_cpython.Py_buffer view + if _cyb_cpython.PyObject_GetBuffer(buffer, &view, _cyb_cpython_buffer.PyBUF_SIMPLE) != 0: + raise TypeError("buffer argument does not support the buffer protocol") + try: + if view.itemsize != 1: + raise ValueError("buffer itemsize must be 1 byte") + if view.len != size: + raise ValueError(f"buffer length must be {size} bytes") + return lowpp_type.from_ptr(view.buf, not view.readonly, buffer) + finally: + _cyb_cpython.PyBuffer_Release(&view) -############################################################################### -# POD -############################################################################### +cdef _cyb_from_data(data, dtype_name, expected_dtype, lowpp_type): + # _numpy.recarray is a subclass of _numpy.ndarray, so implicitly handled here. + if isinstance(data, lowpp_type): + return data + if not isinstance(data, _numpy.ndarray): + raise TypeError("data argument must be a NumPy ndarray") + if data.size != 1: + raise ValueError("data array must have a size of 1") + if data.dtype != expected_dtype: + raise ValueError(f"data array must be of dtype {dtype_name}") + return lowpp_type.from_ptr(data.ctypes.data, not data.flags.writeable, data) -cdef class PointerBox: - """Stable storage for NCCL APIs that fill pointer outputs asynchronously.""" +cdef intptr_t _cyb_get_buffer_pointer(buf, Py_ssize_t size, readonly=True) except?-1: + cdef intptr_t ptr + cdef int flags = _cyb_cpython.PyBUF_ANY_CONTIGUOUS + if not readonly: + flags |= _cyb_cpython.PyBUF_WRITABLE + cdef int status = -1 + cdef _cyb_cpython.Py_buffer view + if buf is None: + ptr = 0 + elif isinstance(buf, int): + ptr = buf + else: + try: + status = _cyb_cpython.PyObject_GetBuffer(buf, &view, flags) + if size != -1: + assert view.len == size + assert view.ndim == 1 + except Exception as e: + adj = "writable " if not readonly else "" + raise ValueError( + "buf must be None, a Python int representing the pointer " + f"address to a valid buffer, or a 1D contiguous {adj}" + f"buffer, of size {size}" + ) from e + else: + ptr = view.buf + finally: + if status == 0: + _cyb_cpython.PyBuffer_Release(&view) + return ptr - def __init__(self, intptr_t ptr=0): - self.ptr = ptr - def __int__(self): - return self.ptr +# <<<< END OF PREAMBLE CONTENT >>>> - def __index__(self): - return self.ptr +cimport cython # NOQA +from libcpp.vector cimport vector - def __bool__(self): - return self.ptr != 0 +from ._internal.utils cimport (nested_resource, nullable_unique_ptr, + get_resource_ptr, get_nested_resource_ptr) - @property - def address(self): - return &self.ptr +_version_span = "with version 2.32.3" +__version__ = _version_span.split()[-1] - def __repr__(self): - return f"" +# NCCL_VERSION(X,Y,Z) = X*10000 + Y*100 + Z (NCCL >= 2.9). +_version_parts = __version__.split(".") +__version_code__ = ( + int(_version_parts[0]) * 10000 + int(_version_parts[1]) * 100 + int(_version_parts[2]) +) - def __format__(self, format_spec): - return format(self.ptr, format_spec) +############################################################################### +# POD +############################################################################### cdef _get_unique_id_dtype_offsets(): - cdef ncclUniqueId pod = ncclUniqueId() + cdef ncclUniqueId pod return _numpy.dtype({ 'names': ['internal'], 'formats': [(_numpy.int8, 128)], @@ -122,7 +149,7 @@ cdef class UniqueId: bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclUniqueId)) + self._ptr = _cyb_calloc(1, sizeof(ncclUniqueId)) if self._ptr == NULL: raise MemoryError("Error allocating UniqueId") self._owner = None @@ -134,7 +161,7 @@ cdef class UniqueId: if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): return f"<{__name__}.UniqueId object at {hex(id(self))}>" @@ -155,20 +182,20 @@ cdef class UniqueId: if not isinstance(other, UniqueId): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclUniqueId)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclUniqueId)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclUniqueId), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclUniqueId), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclUniqueId)) + self._ptr = _cyb_malloc(sizeof(ncclUniqueId)) if self._ptr == NULL: raise MemoryError("Error allocating UniqueId") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclUniqueId)) + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclUniqueId)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -178,7 +205,7 @@ cdef class UniqueId: @staticmethod def from_buffer(buffer): """Create an UniqueId instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclUniqueId), UniqueId) + return _cyb_from_buffer(buffer, sizeof(ncclUniqueId), UniqueId) @staticmethod def from_data(data): @@ -187,7 +214,7 @@ cdef class UniqueId: Args: data (_numpy.ndarray): a single-element array of dtype `unique_id_dtype` holding the data. """ - return __from_data(data, "unique_id_dtype", unique_id_dtype, UniqueId) + return _cyb_from_data(data, "unique_id_dtype", unique_id_dtype, UniqueId) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): @@ -202,10 +229,10 @@ cdef class UniqueId: raise ValueError("ptr must not be null (0)") cdef UniqueId obj = UniqueId.__new__(UniqueId) if owner is None: - obj._ptr = malloc(sizeof(ncclUniqueId)) + obj._ptr = _cyb_malloc(sizeof(ncclUniqueId)) if obj._ptr == NULL: raise MemoryError("Error allocating UniqueId") - memcpy((obj._ptr), ptr, sizeof(ncclUniqueId)) + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclUniqueId)) obj._owner = None obj._owned = True else: @@ -217,10 +244,10 @@ cdef class UniqueId: cdef _get_config_dtype_offsets(): - cdef ncclConfig_t pod = ncclConfig_t() + cdef ncclConfig_t pod return _numpy.dtype({ - 'names': ['size_', 'magic', 'version', 'blocking', 'cga_cluster_size', 'min_ctas', 'max_ctas', 'net_name', 'split_share', 'traffic_class', 'comm_name', 'collnet_enable', 'cta_policy', 'shrink_share', 'nvls_ctas', 'n_channels_per_net_peer', 'nvlink_centric_sched', 'graph_usage_mode', 'num_rma_ctx', 'max_p2p_peers', 'graph_stream_ordering'], - 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32], + 'names': ['size_', 'magic', 'version', 'blocking', 'cga_cluster_size', 'min_ctas', 'max_ctas', 'net_name', 'split_share', 'traffic_class', 'comm_name', 'collnet_enable', 'cta_policy', 'shrink_share', 'nvls_ctas', 'n_channels_per_net_peer', 'nvlink_centric_sched', 'graph_usage_mode', 'num_rma_ctx', 'max_p2p_peers', 'graph_stream_ordering', 'launch_order_implicit', 'num_rma_sig', 'rma_eager_init', 'host_cft_mode', 'nvls_host_mode'], + 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32], 'offsets': [ (&(pod.size)) - (&pod), (&(pod.magic)) - (&pod), @@ -243,6 +270,11 @@ cdef _get_config_dtype_offsets(): (&(pod.numRmaCtx)) - (&pod), (&(pod.maxP2pPeers)) - (&pod), (&(pod.graphStreamOrdering)) - (&pod), + (&(pod.launchOrderImplicit)) - (&pod), + (&(pod.numRmaSig)) - (&pod), + (&(pod.rmaEagerInit)) - (&pod), + (&(pod.hostCftMode)) - (&pod), + (&(pod.nvlsHostMode)) - (&pod), ], 'itemsize': sizeof(ncclConfig_t), }) @@ -263,7 +295,7 @@ cdef class Config: dict _refs def __init__(self): - self._ptr = calloc(1, sizeof(ncclConfig_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclConfig_t)) if self._ptr == NULL: raise MemoryError("Error allocating Config") self._owner = None @@ -273,13 +305,15 @@ cdef class Config: self._ptr[0].size = sizeof(ncclConfig_t) self._ptr[0].magic = 0xcafebeef - self._ptr[0].version = 23007 + self._ptr[0].version = __version_code__ self._ptr[0].blocking = -2147483648 self._ptr[0].cgaClusterSize = -2147483648 self._ptr[0].minCTAs = -2147483648 self._ptr[0].maxCTAs = -2147483648 + self._ptr[0].netName = NULL self._ptr[0].splitShare = -2147483648 self._ptr[0].trafficClass = -2147483648 + self._ptr[0].commName = NULL self._ptr[0].collnetEnable = -2147483648 self._ptr[0].CTAPolicy = -2147483648 self._ptr[0].shrinkShare = -2147483648 @@ -290,13 +324,18 @@ cdef class Config: self._ptr[0].numRmaCtx = -2147483648 self._ptr[0].maxP2pPeers = -2147483648 self._ptr[0].graphStreamOrdering = -2147483648 + self._ptr[0].launchOrderImplicit = -2147483648 + self._ptr[0].numRmaSig = -2147483648 + self._ptr[0].rmaEagerInit = -2147483648 + self._ptr[0].hostCftMode = -2147483648 + self._ptr[0].nvlsHostMode = -2147483648 def __dealloc__(self): cdef ncclConfig_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): return f"<{__name__}.Config object at {hex(id(self))}>" @@ -317,20 +356,20 @@ cdef class Config: if not isinstance(other, Config): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclConfig_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclConfig_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclConfig_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclConfig_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclConfig_t)) + self._ptr = _cyb_malloc(sizeof(ncclConfig_t)) if self._ptr == NULL: raise MemoryError("Error allocating Config") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclConfig_t)) + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclConfig_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -419,7 +458,7 @@ cdef class Config: """str: """ cdef char* ptr = self._ptr[0].netName if ptr: - return cpython.PyUnicode_FromString(ptr) + return _cyb_cpython.PyUnicode_FromString(ptr) return "" @net_name.setter @@ -429,7 +468,7 @@ cdef class Config: cdef bytes buf = val.encode() cdef char *ptr = buf self._refs["net_name"] = buf - self._ptr.netName = ptr + self._ptr[0].netName = ptr @property def split_share(self): @@ -458,7 +497,7 @@ cdef class Config: """str: """ cdef char* ptr = self._ptr[0].commName if ptr: - return cpython.PyUnicode_FromString(ptr) + return _cyb_cpython.PyUnicode_FromString(ptr) return "" @comm_name.setter @@ -468,7 +507,7 @@ cdef class Config: cdef bytes buf = val.encode() cdef char *ptr = buf self._refs["comm_name"] = buf - self._ptr.commName = ptr + self._ptr[0].commName = ptr @property def collnet_enable(self): @@ -580,10 +619,65 @@ cdef class Config: raise ValueError("This Config instance is read-only") self._ptr[0].graphStreamOrdering = val + @property + def launch_order_implicit(self): + """int: """ + return self._ptr[0].launchOrderImplicit + + @launch_order_implicit.setter + def launch_order_implicit(self, val): + if self._readonly: + raise ValueError("This Config instance is read-only") + self._ptr[0].launchOrderImplicit = val + + @property + def num_rma_sig(self): + """int: """ + return self._ptr[0].numRmaSig + + @num_rma_sig.setter + def num_rma_sig(self, val): + if self._readonly: + raise ValueError("This Config instance is read-only") + self._ptr[0].numRmaSig = val + + @property + def rma_eager_init(self): + """int: """ + return self._ptr[0].rmaEagerInit + + @rma_eager_init.setter + def rma_eager_init(self, val): + if self._readonly: + raise ValueError("This Config instance is read-only") + self._ptr[0].rmaEagerInit = val + + @property + def host_cft_mode(self): + """int: """ + return self._ptr[0].hostCftMode + + @host_cft_mode.setter + def host_cft_mode(self, val): + if self._readonly: + raise ValueError("This Config instance is read-only") + self._ptr[0].hostCftMode = val + + @property + def nvls_host_mode(self): + """int: """ + return self._ptr[0].nvlsHostMode + + @nvls_host_mode.setter + def nvls_host_mode(self, val): + if self._readonly: + raise ValueError("This Config instance is read-only") + self._ptr[0].nvlsHostMode = val + @staticmethod def from_buffer(buffer): """Create an Config instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclConfig_t), Config) + return _cyb_from_buffer(buffer, sizeof(ncclConfig_t), Config) @staticmethod def from_data(data): @@ -592,7 +686,7 @@ cdef class Config: Args: data (_numpy.ndarray): a single-element array of dtype `config_dtype` holding the data. """ - return __from_data(data, "config_dtype", config_dtype, Config) + return _cyb_from_data(data, "config_dtype", config_dtype, Config) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): @@ -607,10 +701,10 @@ cdef class Config: raise ValueError("ptr must not be null (0)") cdef Config obj = Config.__new__(Config) if owner is None: - obj._ptr = malloc(sizeof(ncclConfig_t)) + obj._ptr = _cyb_malloc(sizeof(ncclConfig_t)) if obj._ptr == NULL: raise MemoryError("Error allocating Config") - memcpy((obj._ptr), ptr, sizeof(ncclConfig_t)) + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclConfig_t)) obj._owner = None obj._owned = True else: @@ -622,56 +716,49 @@ cdef class Config: return obj -cdef _get_sim_info_dtype_offsets(): - cdef ncclSimInfo_t pod = ncclSimInfo_t() +cdef _get__py_anon_pod0_dtype_offsets(): + cdef nccl_bindings_nccl__anon_pod0 pod return _numpy.dtype({ - 'names': ['size_', 'magic', 'version', 'estimated_time'], - 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.float32], + 'names': ['vendor_id', 'option_id'], + 'formats': [_numpy.int32, _numpy.int32], 'offsets': [ - (&(pod.size)) - (&pod), - (&(pod.magic)) - (&pod), - (&(pod.version)) - (&pod), - (&(pod.estimatedTime)) - (&pod), + (&(pod.vendorId)) - (&pod), + (&(pod.optionId)) - (&pod), ], - 'itemsize': sizeof(ncclSimInfo_t), + 'itemsize': sizeof(nccl_bindings_nccl__anon_pod0), }) -sim_info_dtype = _get_sim_info_dtype_offsets() +_py_anon_pod0_dtype = _get__py_anon_pod0_dtype_offsets() -cdef class SimInfo: - """Initialize an instance of `ncclSimInfo_t` using configured defaults. +cdef class _py_anon_pod0: + """Empty-initialize an instance of `nccl_bindings_nccl__anon_pod0`. - .. seealso:: `ncclSimInfo_t` + .. seealso:: `nccl_bindings_nccl__anon_pod0` """ cdef: - ncclSimInfo_t *_ptr + nccl_bindings_nccl__anon_pod0 *_ptr object _owner bint _owned bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclSimInfo_t)) + self._ptr = _cyb_calloc(1, sizeof(nccl_bindings_nccl__anon_pod0)) if self._ptr == NULL: - raise MemoryError("Error allocating SimInfo") + raise MemoryError("Error allocating _py_anon_pod0") self._owner = None self._owned = True self._readonly = False - self._ptr[0].size = sizeof(ncclSimInfo_t) - self._ptr[0].magic = 0x74685283 - self._ptr[0].version = 23007 - self._ptr[0].estimatedTime = -1.0 - def __dealloc__(self): - cdef ncclSimInfo_t *ptr + cdef nccl_bindings_nccl__anon_pod0 *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.SimInfo object at {hex(id(self))}>" + return f"<{__name__}._py_anon_pod0 object at {hex(id(self))}>" @property def ptr(self): @@ -685,24 +772,24 @@ cdef class SimInfo: return (self._ptr) def __eq__(self, other): - cdef SimInfo other_ - if not isinstance(other, SimInfo): + cdef _py_anon_pod0 other_ + if not isinstance(other, _py_anon_pod0): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclSimInfo_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(nccl_bindings_nccl__anon_pod0)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclSimInfo_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(nccl_bindings_nccl__anon_pod0), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclSimInfo_t)) + self._ptr = _cyb_malloc(sizeof(nccl_bindings_nccl__anon_pod0)) if self._ptr == NULL: - raise MemoryError("Error allocating SimInfo") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclSimInfo_t)) + raise MemoryError("Error allocating _py_anon_pod0") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(nccl_bindings_nccl__anon_pod0)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -710,66 +797,44 @@ cdef class SimInfo: setattr(self, key, val) @property - def size_(self): - """int: """ - return self._ptr[0].size - - @size_.setter - def size_(self, val): - if self._readonly: - raise ValueError("This SimInfo instance is read-only") - self._ptr[0].size = val - - @property - def magic(self): + def vendor_id(self): """int: """ - return self._ptr[0].magic + return self._ptr[0].vendorId - @magic.setter - def magic(self, val): + @vendor_id.setter + def vendor_id(self, val): if self._readonly: - raise ValueError("This SimInfo instance is read-only") - self._ptr[0].magic = val + raise ValueError("This _py_anon_pod0 instance is read-only") + self._ptr[0].vendorId = val @property - def version(self): + def option_id(self): """int: """ - return self._ptr[0].version - - @version.setter - def version(self, val): - if self._readonly: - raise ValueError("This SimInfo instance is read-only") - self._ptr[0].version = val - - @property - def estimated_time(self): - """float: """ - return self._ptr[0].estimatedTime + return self._ptr[0].optionId - @estimated_time.setter - def estimated_time(self, val): + @option_id.setter + def option_id(self, val): if self._readonly: - raise ValueError("This SimInfo instance is read-only") - self._ptr[0].estimatedTime = val + raise ValueError("This _py_anon_pod0 instance is read-only") + self._ptr[0].optionId = val @staticmethod def from_buffer(buffer): - """Create an SimInfo instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclSimInfo_t), SimInfo) + """Create an _py_anon_pod0 instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(nccl_bindings_nccl__anon_pod0), _py_anon_pod0) @staticmethod def from_data(data): - """Create an SimInfo instance wrapping the given NumPy array. + """Create an _py_anon_pod0 instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `sim_info_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `_py_anon_pod0_dtype` holding the data. """ - return __from_data(data, "sim_info_dtype", sim_info_dtype, SimInfo) + return _cyb_from_data(data, "_py_anon_pod0_dtype", _py_anon_pod0_dtype, _py_anon_pod0) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an SimInfo instance wrapping the given pointer. + """Create an _py_anon_pod0 instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -778,67 +843,68 @@ cdef class SimInfo: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef SimInfo obj = SimInfo.__new__(SimInfo) + cdef _py_anon_pod0 obj = _py_anon_pod0.__new__(_py_anon_pod0) if owner is None: - obj._ptr = malloc(sizeof(ncclSimInfo_t)) + obj._ptr = _cyb_malloc(sizeof(nccl_bindings_nccl__anon_pod0)) if obj._ptr == NULL: - raise MemoryError("Error allocating SimInfo") - memcpy((obj._ptr), ptr, sizeof(ncclSimInfo_t)) + raise MemoryError("Error allocating _py_anon_pod0") + _cyb_memcpy((obj._ptr), ptr, sizeof(nccl_bindings_nccl__anon_pod0)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly return obj -cdef _get_wait_signal_desc_dtype_offsets(): - cdef ncclWaitSignalDesc_t pod = ncclWaitSignalDesc_t() +cdef _get__py_anon_pod1_dtype_offsets(): + cdef nccl_bindings_nccl__anon_pod1 pod return _numpy.dtype({ - 'names': ['op_cnt', 'peer', 'sig_idx', 'ctx'], - 'formats': [_numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32], + 'names': ['i', 's', 'raw'], + 'formats': [_numpy.int32, _numpy.intp, _numpy.intp], 'offsets': [ - (&(pod.opCnt)) - (&pod), - (&(pod.peer)) - (&pod), - (&(pod.sigIdx)) - (&pod), - (&(pod.ctx)) - (&pod), + (&(pod.i)) - (&pod), + (&(pod.s)) - (&pod), + (&(pod.raw)) - (&pod), ], - 'itemsize': sizeof(ncclWaitSignalDesc_t), + 'itemsize': sizeof(nccl_bindings_nccl__anon_pod1), }) -wait_signal_desc_dtype = _get_wait_signal_desc_dtype_offsets() +_py_anon_pod1_dtype = _get__py_anon_pod1_dtype_offsets() -cdef class WaitSignalDesc: - """Empty-initialize an instance of `ncclWaitSignalDesc_t`. +cdef class _py_anon_pod1: + """Empty-initialize an instance of `nccl_bindings_nccl__anon_pod1`. - .. seealso:: `ncclWaitSignalDesc_t` + .. seealso:: `nccl_bindings_nccl__anon_pod1` """ cdef: - ncclWaitSignalDesc_t *_ptr + nccl_bindings_nccl__anon_pod1 *_ptr object _owner bint _owned bint _readonly + dict _refs def __init__(self): - self._ptr = calloc(1, sizeof(ncclWaitSignalDesc_t)) + self._ptr = _cyb_calloc(1, sizeof(nccl_bindings_nccl__anon_pod1)) if self._ptr == NULL: - raise MemoryError("Error allocating WaitSignalDesc") + raise MemoryError("Error allocating _py_anon_pod1") self._owner = None self._owned = True self._readonly = False + self._refs = {} def __dealloc__(self): - cdef ncclWaitSignalDesc_t *ptr + cdef nccl_bindings_nccl__anon_pod1 *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.WaitSignalDesc object at {hex(id(self))}>" + return f"<{__name__}._py_anon_pod1 object at {hex(id(self))}>" @property def ptr(self): @@ -852,24 +918,24 @@ cdef class WaitSignalDesc: return (self._ptr) def __eq__(self, other): - cdef WaitSignalDesc other_ - if not isinstance(other, WaitSignalDesc): + cdef _py_anon_pod1 other_ + if not isinstance(other, _py_anon_pod1): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclWaitSignalDesc_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(nccl_bindings_nccl__anon_pod1)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclWaitSignalDesc_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(nccl_bindings_nccl__anon_pod1), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclWaitSignalDesc_t)) + self._ptr = _cyb_malloc(sizeof(nccl_bindings_nccl__anon_pod1)) if self._ptr == NULL: - raise MemoryError("Error allocating WaitSignalDesc") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclWaitSignalDesc_t)) + raise MemoryError("Error allocating _py_anon_pod1") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(nccl_bindings_nccl__anon_pod1)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -877,66 +943,61 @@ cdef class WaitSignalDesc: setattr(self, key, val) @property - def op_cnt(self): - """int: """ - return self._ptr[0].opCnt - - @op_cnt.setter - def op_cnt(self, val): - if self._readonly: - raise ValueError("This WaitSignalDesc instance is read-only") - self._ptr[0].opCnt = val - - @property - def peer(self): + def i(self): """int: """ - return self._ptr[0].peer + return self._ptr[0].i - @peer.setter - def peer(self, val): + @i.setter + def i(self, val): if self._readonly: - raise ValueError("This WaitSignalDesc instance is read-only") - self._ptr[0].peer = val + raise ValueError("This _py_anon_pod1 instance is read-only") + self._ptr[0].i = val @property - def sig_idx(self): - """int: """ - return self._ptr[0].sigIdx + def s(self): + """str: """ + cdef char* ptr = self._ptr[0].s + if ptr: + return _cyb_cpython.PyUnicode_FromString(ptr) + return "" - @sig_idx.setter - def sig_idx(self, val): + @s.setter + def s(self, val): if self._readonly: - raise ValueError("This WaitSignalDesc instance is read-only") - self._ptr[0].sigIdx = val + raise ValueError("This _py_anon_pod1 instance is read-only") + cdef bytes buf = val.encode() + cdef char *ptr = buf + self._refs["s"] = buf + self._ptr[0].s = ptr @property - def ctx(self): + def raw(self): """int: """ - return self._ptr[0].ctx + return (self._ptr[0].raw) - @ctx.setter - def ctx(self, val): + @raw.setter + def raw(self, val): if self._readonly: - raise ValueError("This WaitSignalDesc instance is read-only") - self._ptr[0].ctx = val + raise ValueError("This _py_anon_pod1 instance is read-only") + self._ptr[0].raw = val @staticmethod def from_buffer(buffer): - """Create an WaitSignalDesc instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclWaitSignalDesc_t), WaitSignalDesc) + """Create an _py_anon_pod1 instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(nccl_bindings_nccl__anon_pod1), _py_anon_pod1) @staticmethod def from_data(data): - """Create an WaitSignalDesc instance wrapping the given NumPy array. + """Create an _py_anon_pod1 instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `wait_signal_desc_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `_py_anon_pod1_dtype` holding the data. """ - return __from_data(data, "wait_signal_desc_dtype", wait_signal_desc_dtype, WaitSignalDesc) + return _cyb_from_data(data, "_py_anon_pod1_dtype", _py_anon_pod1_dtype, _py_anon_pod1) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an WaitSignalDesc instance wrapping the given pointer. + """Create an _py_anon_pod1 instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -945,85 +1006,73 @@ cdef class WaitSignalDesc: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef WaitSignalDesc obj = WaitSignalDesc.__new__(WaitSignalDesc) + cdef _py_anon_pod1 obj = _py_anon_pod1.__new__(_py_anon_pod1) if owner is None: - obj._ptr = malloc(sizeof(ncclWaitSignalDesc_t)) + obj._ptr = _cyb_malloc(sizeof(nccl_bindings_nccl__anon_pod1)) if obj._ptr == NULL: - raise MemoryError("Error allocating WaitSignalDesc") - memcpy((obj._ptr), ptr, sizeof(ncclWaitSignalDesc_t)) + raise MemoryError("Error allocating _py_anon_pod1") + _cyb_memcpy((obj._ptr), ptr, sizeof(nccl_bindings_nccl__anon_pod1)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly + obj._refs = {} return obj -cdef _get_comm_properties_dtype_offsets(): - cdef ncclCommProperties_t pod = ncclCommProperties_t() +cdef _get_sim_info_dtype_offsets(): + cdef ncclSimInfo_t pod return _numpy.dtype({ - 'names': ['size_', 'magic', 'version', 'rank', 'n_ranks', 'cuda_dev', 'nvml_dev', 'device_api_support', 'multimem_support', 'gin_type', 'n_lsa_teams', 'host_rma_support', 'railed_gin_type', 'comm_hash', 'gin_min_stride', 'gin_connection_type', 'gin_support', 'dev_comm_runtime_version_size'], - 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.uint8, _numpy.uint8, _numpy.int32, _numpy.int32, _numpy.uint8, _numpy.int32, _numpy.uint64, _numpy.int32, _numpy.int32, (_numpy.uint8, 64), _numpy.uint64], + 'names': ['size_', 'magic', 'version', 'estimated_time'], + 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.float32], 'offsets': [ (&(pod.size)) - (&pod), (&(pod.magic)) - (&pod), (&(pod.version)) - (&pod), - (&(pod.rank)) - (&pod), - (&(pod.nRanks)) - (&pod), - (&(pod.cudaDev)) - (&pod), - (&(pod.nvmlDev)) - (&pod), - (&(pod.deviceApiSupport)) - (&pod), - (&(pod.multimemSupport)) - (&pod), - (&(pod.ginType)) - (&pod), - (&(pod.nLsaTeams)) - (&pod), - (&(pod.hostRmaSupport)) - (&pod), - (&(pod.railedGinType)) - (&pod), - (&(pod.commHash)) - (&pod), - (&(pod.ginMinStride)) - (&pod), - (&(pod.ginConnectionType)) - (&pod), - (&(pod.ginSupport)) - (&pod), - (&(pod.devCommRuntimeVersionSize)) - (&pod), + (&(pod.estimatedTime)) - (&pod), ], - 'itemsize': sizeof(ncclCommProperties_t), + 'itemsize': sizeof(ncclSimInfo_t), }) -comm_properties_dtype = _get_comm_properties_dtype_offsets() +sim_info_dtype = _get_sim_info_dtype_offsets() -cdef class CommProperties: - """Initialize an instance of `ncclCommProperties_t` using configured defaults. +cdef class SimInfo: + """Initialize an instance of `ncclSimInfo_t` using configured defaults. - .. seealso:: `ncclCommProperties_t` + .. seealso:: `ncclSimInfo_t` """ cdef: - ncclCommProperties_t *_ptr + ncclSimInfo_t *_ptr object _owner bint _owned bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclCommProperties_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclSimInfo_t)) if self._ptr == NULL: - raise MemoryError("Error allocating CommProperties") + raise MemoryError("Error allocating SimInfo") self._owner = None self._owned = True self._readonly = False - self._ptr[0].size = sizeof(ncclCommProperties_t) - self._ptr[0].magic = 0xcafebeef - self._ptr[0].version = 23007 + self._ptr[0].size = sizeof(ncclSimInfo_t) + self._ptr[0].magic = 0x74685283 + self._ptr[0].version = __version_code__ + self._ptr[0].estimatedTime = -1.0 def __dealloc__(self): - cdef ncclCommProperties_t *ptr + cdef ncclSimInfo_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.CommProperties object at {hex(id(self))}>" + return f"<{__name__}.SimInfo object at {hex(id(self))}>" @property def ptr(self): @@ -1037,24 +1086,24 @@ cdef class CommProperties: return (self._ptr) def __eq__(self, other): - cdef CommProperties other_ - if not isinstance(other, CommProperties): + cdef SimInfo other_ + if not isinstance(other, SimInfo): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclCommProperties_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclSimInfo_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclCommProperties_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclSimInfo_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclCommProperties_t)) + self._ptr = _cyb_malloc(sizeof(ncclSimInfo_t)) if self._ptr == NULL: - raise MemoryError("Error allocating CommProperties") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclCommProperties_t)) + raise MemoryError("Error allocating SimInfo") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclSimInfo_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -1069,7 +1118,7 @@ cdef class CommProperties: @size_.setter def size_(self, val): if self._readonly: - raise ValueError("This CommProperties instance is read-only") + raise ValueError("This SimInfo instance is read-only") self._ptr[0].size = val @property @@ -1080,7 +1129,7 @@ cdef class CommProperties: @magic.setter def magic(self, val): if self._readonly: - raise ValueError("This CommProperties instance is read-only") + raise ValueError("This SimInfo instance is read-only") self._ptr[0].magic = val @property @@ -1091,196 +1140,37 @@ cdef class CommProperties: @version.setter def version(self, val): if self._readonly: - raise ValueError("This CommProperties instance is read-only") + raise ValueError("This SimInfo instance is read-only") self._ptr[0].version = val @property - def rank(self): - """int: """ - return self._ptr[0].rank - - @rank.setter - def rank(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].rank = val - - @property - def n_ranks(self): - """int: """ - return self._ptr[0].nRanks - - @n_ranks.setter - def n_ranks(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].nRanks = val - - @property - def cuda_dev(self): - """int: """ - return self._ptr[0].cudaDev - - @cuda_dev.setter - def cuda_dev(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].cudaDev = val - - @property - def nvml_dev(self): - """int: """ - return self._ptr[0].nvmlDev - - @nvml_dev.setter - def nvml_dev(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].nvmlDev = val - - @property - def device_api_support(self): - """int: """ - return self._ptr[0].deviceApiSupport - - @device_api_support.setter - def device_api_support(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].deviceApiSupport = val - - @property - def multimem_support(self): - """int: """ - return self._ptr[0].multimemSupport - - @multimem_support.setter - def multimem_support(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].multimemSupport = val - - @property - def gin_type(self): - """int: """ - return (self._ptr[0].ginType) - - @gin_type.setter - def gin_type(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].ginType = val - - @property - def n_lsa_teams(self): - """int: """ - return self._ptr[0].nLsaTeams - - @n_lsa_teams.setter - def n_lsa_teams(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].nLsaTeams = val - - @property - def host_rma_support(self): - """int: """ - return self._ptr[0].hostRmaSupport - - @host_rma_support.setter - def host_rma_support(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].hostRmaSupport = val - - @property - def railed_gin_type(self): - """int: """ - return (self._ptr[0].railedGinType) - - @railed_gin_type.setter - def railed_gin_type(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].railedGinType = val - - @property - def comm_hash(self): - """int: """ - return self._ptr[0].commHash - - @comm_hash.setter - def comm_hash(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].commHash = val - - @property - def gin_min_stride(self): - """int: """ - return self._ptr[0].ginMinStride - - @gin_min_stride.setter - def gin_min_stride(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].ginMinStride = val - - @property - def gin_connection_type(self): - """int: """ - return (self._ptr[0].ginConnectionType) - - @gin_connection_type.setter - def gin_connection_type(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].ginConnectionType = val - - @property - def gin_support(self): - """bytes: """ - cdef bytes out = bytes(64) - memcpy(out, self._ptr[0].ginSupport, 64) - return out - - @gin_support.setter - def gin_support(self, val): - if self._readonly: - raise ValueError("This CommProperties instance is read-only") - cdef bytes b = val - if len(b) != 64: - raise ValueError("gin_support must be exactly 64 bytes") - memcpy(self._ptr[0].ginSupport, b, 64) - - @property - def dev_comm_runtime_version_size(self): - """int: """ - return self._ptr[0].devCommRuntimeVersionSize + def estimated_time(self): + """float: """ + return self._ptr[0].estimatedTime - @dev_comm_runtime_version_size.setter - def dev_comm_runtime_version_size(self, val): + @estimated_time.setter + def estimated_time(self, val): if self._readonly: - raise ValueError("This CommProperties instance is read-only") - self._ptr[0].devCommRuntimeVersionSize = val + raise ValueError("This SimInfo instance is read-only") + self._ptr[0].estimatedTime = val @staticmethod def from_buffer(buffer): - """Create an CommProperties instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclCommProperties_t), CommProperties) + """Create an SimInfo instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclSimInfo_t), SimInfo) @staticmethod def from_data(data): - """Create an CommProperties instance wrapping the given NumPy array. + """Create an SimInfo instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `comm_properties_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `sim_info_dtype` holding the data. """ - return __from_data(data, "comm_properties_dtype", comm_properties_dtype, CommProperties) + return _cyb_from_data(data, "sim_info_dtype", sim_info_dtype, SimInfo) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an CommProperties instance wrapping the given pointer. + """Create an SimInfo instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -1289,71 +1179,67 @@ cdef class CommProperties: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef CommProperties obj = CommProperties.__new__(CommProperties) + cdef SimInfo obj = SimInfo.__new__(SimInfo) if owner is None: - obj._ptr = malloc(sizeof(ncclCommProperties_t)) + obj._ptr = _cyb_malloc(sizeof(ncclSimInfo_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating CommProperties") - memcpy((obj._ptr), ptr, sizeof(ncclCommProperties_t)) + raise MemoryError("Error allocating SimInfo") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclSimInfo_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly return obj -cdef _get_dev_resource_requirements_dtype_offsets(): - cdef ncclDevResourceRequirements_t pod = ncclDevResourceRequirements_t() +cdef _get_wait_signal_desc_dtype_offsets(): + cdef ncclWaitSignalDesc_t pod return _numpy.dtype({ - 'names': ['next', 'buffer_size', 'buffer_align', 'out_buffer_handle', 'gin_signal_count', 'gin_counter_count', 'out_gin_signal_start', 'out_gin_counter_start'], - 'formats': [_numpy.intp, _numpy.uint64, _numpy.uint64, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.intp], + 'names': ['op_cnt', 'peer', 'sig_idx', 'ctx'], + 'formats': [_numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32], 'offsets': [ - (&(pod.next)) - (&pod), - (&(pod.bufferSize)) - (&pod), - (&(pod.bufferAlign)) - (&pod), - (&(pod.outBufferHandle)) - (&pod), - (&(pod.ginSignalCount)) - (&pod), - (&(pod.ginCounterCount)) - (&pod), - (&(pod.outGinSignalStart)) - (&pod), - (&(pod.outGinCounterStart)) - (&pod), + (&(pod.opCnt)) - (&pod), + (&(pod.peer)) - (&pod), + (&(pod.sigIdx)) - (&pod), + (&(pod.ctx)) - (&pod), ], - 'itemsize': sizeof(ncclDevResourceRequirements_t), + 'itemsize': sizeof(ncclWaitSignalDesc_t), }) -dev_resource_requirements_dtype = _get_dev_resource_requirements_dtype_offsets() +wait_signal_desc_dtype = _get_wait_signal_desc_dtype_offsets() -cdef class DevResourceRequirements: - """Empty-initialize an instance of `ncclDevResourceRequirements_t`. +cdef class WaitSignalDesc: + """Empty-initialize an instance of `ncclWaitSignalDesc_t`. - .. seealso:: `ncclDevResourceRequirements_t` + .. seealso:: `ncclWaitSignalDesc_t` """ cdef: - ncclDevResourceRequirements_t *_ptr + ncclWaitSignalDesc_t *_ptr object _owner bint _owned bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclDevResourceRequirements_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclWaitSignalDesc_t)) if self._ptr == NULL: - raise MemoryError("Error allocating DevResourceRequirements") + raise MemoryError("Error allocating WaitSignalDesc") self._owner = None self._owned = True self._readonly = False def __dealloc__(self): - cdef ncclDevResourceRequirements_t *ptr + cdef ncclWaitSignalDesc_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.DevResourceRequirements object at {hex(id(self))}>" + return f"<{__name__}.WaitSignalDesc object at {hex(id(self))}>" @property def ptr(self): @@ -1367,24 +1253,24 @@ cdef class DevResourceRequirements: return (self._ptr) def __eq__(self, other): - cdef DevResourceRequirements other_ - if not isinstance(other, DevResourceRequirements): + cdef WaitSignalDesc other_ + if not isinstance(other, WaitSignalDesc): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclDevResourceRequirements_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclWaitSignalDesc_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclDevResourceRequirements_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclWaitSignalDesc_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclDevResourceRequirements_t)) + self._ptr = _cyb_malloc(sizeof(ncclWaitSignalDesc_t)) if self._ptr == NULL: - raise MemoryError("Error allocating DevResourceRequirements") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclDevResourceRequirements_t)) + raise MemoryError("Error allocating WaitSignalDesc") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclWaitSignalDesc_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -1392,110 +1278,66 @@ cdef class DevResourceRequirements: setattr(self, key, val) @property - def next(self): + def op_cnt(self): """int: """ - return (self._ptr[0].next) + return self._ptr[0].opCnt - @next.setter - def next(self, val): + @op_cnt.setter + def op_cnt(self, val): if self._readonly: - raise ValueError("This DevResourceRequirements instance is read-only") - self._ptr[0].next = val + raise ValueError("This WaitSignalDesc instance is read-only") + self._ptr[0].opCnt = val @property - def buffer_size(self): + def peer(self): """int: """ - return self._ptr[0].bufferSize + return self._ptr[0].peer - @buffer_size.setter - def buffer_size(self, val): + @peer.setter + def peer(self, val): if self._readonly: - raise ValueError("This DevResourceRequirements instance is read-only") - self._ptr[0].bufferSize = val - - @property - def buffer_align(self): - """int: """ - return self._ptr[0].bufferAlign - - @buffer_align.setter - def buffer_align(self, val): - if self._readonly: - raise ValueError("This DevResourceRequirements instance is read-only") - self._ptr[0].bufferAlign = val - - @property - def out_buffer_handle(self): - """int: """ - return (self._ptr[0].outBufferHandle) - - @out_buffer_handle.setter - def out_buffer_handle(self, val): - if self._readonly: - raise ValueError("This DevResourceRequirements instance is read-only") - self._ptr[0].outBufferHandle = val - - @property - def gin_signal_count(self): - """int: """ - return self._ptr[0].ginSignalCount - - @gin_signal_count.setter - def gin_signal_count(self, val): - if self._readonly: - raise ValueError("This DevResourceRequirements instance is read-only") - self._ptr[0].ginSignalCount = val - - @property - def gin_counter_count(self): - """int: """ - return self._ptr[0].ginCounterCount - - @gin_counter_count.setter - def gin_counter_count(self, val): - if self._readonly: - raise ValueError("This DevResourceRequirements instance is read-only") - self._ptr[0].ginCounterCount = val + raise ValueError("This WaitSignalDesc instance is read-only") + self._ptr[0].peer = val @property - def out_gin_signal_start(self): + def sig_idx(self): """int: """ - return (self._ptr[0].outGinSignalStart) + return self._ptr[0].sigIdx - @out_gin_signal_start.setter - def out_gin_signal_start(self, val): + @sig_idx.setter + def sig_idx(self, val): if self._readonly: - raise ValueError("This DevResourceRequirements instance is read-only") - self._ptr[0].outGinSignalStart = val + raise ValueError("This WaitSignalDesc instance is read-only") + self._ptr[0].sigIdx = val @property - def out_gin_counter_start(self): + def ctx(self): """int: """ - return (self._ptr[0].outGinCounterStart) + return self._ptr[0].ctx - @out_gin_counter_start.setter - def out_gin_counter_start(self, val): + @ctx.setter + def ctx(self, val): if self._readonly: - raise ValueError("This DevResourceRequirements instance is read-only") - self._ptr[0].outGinCounterStart = val + raise ValueError("This WaitSignalDesc instance is read-only") + self._ptr[0].ctx = val @staticmethod def from_buffer(buffer): - """Create an DevResourceRequirements instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclDevResourceRequirements_t), DevResourceRequirements) + """Create an WaitSignalDesc instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclWaitSignalDesc_t), WaitSignalDesc) @staticmethod def from_data(data): - """Create an DevResourceRequirements instance wrapping the given NumPy array. + """Create an WaitSignalDesc instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `dev_resource_requirements_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `wait_signal_desc_dtype` holding the data. """ - return __from_data(data, "dev_resource_requirements_dtype", dev_resource_requirements_dtype, DevResourceRequirements) + return _cyb_from_data(data, "wait_signal_desc_dtype", wait_signal_desc_dtype, WaitSignalDesc) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an DevResourceRequirements instance wrapping the given pointer. + """Create an WaitSignalDesc instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -1504,66 +1346,76 @@ cdef class DevResourceRequirements: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef DevResourceRequirements obj = DevResourceRequirements.__new__(DevResourceRequirements) + cdef WaitSignalDesc obj = WaitSignalDesc.__new__(WaitSignalDesc) if owner is None: - obj._ptr = malloc(sizeof(ncclDevResourceRequirements_t)) + obj._ptr = _cyb_malloc(sizeof(ncclWaitSignalDesc_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating DevResourceRequirements") - memcpy((obj._ptr), ptr, sizeof(ncclDevResourceRequirements_t)) + raise MemoryError("Error allocating WaitSignalDesc") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclWaitSignalDesc_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly return obj -cdef _get_team_dtype_offsets(): - cdef ncclTeam_t pod = ncclTeam_t() +cdef _get_encryption_config_dtype_offsets(): + cdef ncclEncryptionConfig_t pod return _numpy.dtype({ - 'names': ['n_ranks', 'rank', 'stride'], - 'formats': [_numpy.int32, _numpy.int32, _numpy.int32], + 'names': ['size_', 'magic', 'version', 'mode', 'psk'], + 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.int32, _numpy.intp], 'offsets': [ - (&(pod.nRanks)) - (&pod), - (&(pod.rank)) - (&pod), - (&(pod.stride)) - (&pod), + (&(pod.size)) - (&pod), + (&(pod.magic)) - (&pod), + (&(pod.version)) - (&pod), + (&(pod.mode)) - (&pod), + (&(pod.psk)) - (&pod), ], - 'itemsize': sizeof(ncclTeam_t), + 'itemsize': sizeof(ncclEncryptionConfig_t), }) -team_dtype = _get_team_dtype_offsets() +encryption_config_dtype = _get_encryption_config_dtype_offsets() -cdef class Team: - """Empty-initialize an instance of `ncclTeam_t`. +cdef class EncryptionConfig: + """Initialize an instance of `ncclEncryptionConfig_t` using configured defaults. - .. seealso:: `ncclTeam_t` + .. seealso:: `ncclEncryptionConfig_t` """ cdef: - ncclTeam_t *_ptr + ncclEncryptionConfig_t *_ptr object _owner bint _owned bint _readonly + dict _refs def __init__(self): - self._ptr = calloc(1, sizeof(ncclTeam_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclEncryptionConfig_t)) if self._ptr == NULL: - raise MemoryError("Error allocating Team") + raise MemoryError("Error allocating EncryptionConfig") self._owner = None self._owned = True self._readonly = False + self._refs = {} + + self._ptr[0].size = sizeof(ncclEncryptionConfig_t) + self._ptr[0].magic = 0xcafebeef + self._ptr[0].version = __version_code__ + self._ptr[0].mode = NCCL_ENCRYPTION_MODE_NONE + self._ptr[0].psk = NULL def __dealloc__(self): - cdef ncclTeam_t *ptr + cdef ncclEncryptionConfig_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.Team object at {hex(id(self))}>" + return f"<{__name__}.EncryptionConfig object at {hex(id(self))}>" @property def ptr(self): @@ -1577,24 +1429,24 @@ cdef class Team: return (self._ptr) def __eq__(self, other): - cdef Team other_ - if not isinstance(other, Team): + cdef EncryptionConfig other_ + if not isinstance(other, EncryptionConfig): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclTeam_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclEncryptionConfig_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclTeam_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclEncryptionConfig_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclTeam_t)) + self._ptr = _cyb_malloc(sizeof(ncclEncryptionConfig_t)) if self._ptr == NULL: - raise MemoryError("Error allocating Team") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclTeam_t)) + raise MemoryError("Error allocating EncryptionConfig") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclEncryptionConfig_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -1602,55 +1454,83 @@ cdef class Team: setattr(self, key, val) @property - def n_ranks(self): + def size_(self): """int: """ - return self._ptr[0].nRanks + return self._ptr[0].size - @n_ranks.setter - def n_ranks(self, val): + @size_.setter + def size_(self, val): if self._readonly: - raise ValueError("This Team instance is read-only") - self._ptr[0].nRanks = val + raise ValueError("This EncryptionConfig instance is read-only") + self._ptr[0].size = val @property - def rank(self): + def magic(self): """int: """ - return self._ptr[0].rank + return self._ptr[0].magic - @rank.setter - def rank(self, val): + @magic.setter + def magic(self, val): if self._readonly: - raise ValueError("This Team instance is read-only") - self._ptr[0].rank = val + raise ValueError("This EncryptionConfig instance is read-only") + self._ptr[0].magic = val @property - def stride(self): + def version(self): """int: """ - return self._ptr[0].stride + return self._ptr[0].version - @stride.setter - def stride(self, val): + @version.setter + def version(self, val): if self._readonly: - raise ValueError("This Team instance is read-only") - self._ptr[0].stride = val + raise ValueError("This EncryptionConfig instance is read-only") + self._ptr[0].version = val + + @property + def mode(self): + """int: """ + return self._ptr[0].mode + + @mode.setter + def mode(self, val): + if self._readonly: + raise ValueError("This EncryptionConfig instance is read-only") + self._ptr[0].mode = val + + @property + def psk(self): + """str: """ + cdef char* ptr = self._ptr[0].psk + if ptr: + return _cyb_cpython.PyUnicode_FromString(ptr) + return "" + + @psk.setter + def psk(self, val): + if self._readonly: + raise ValueError("This EncryptionConfig instance is read-only") + cdef bytes buf = val.encode() + cdef char *ptr = buf + self._refs["psk"] = buf + self._ptr[0].psk = ptr @staticmethod def from_buffer(buffer): - """Create an Team instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclTeam_t), Team) + """Create an EncryptionConfig instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclEncryptionConfig_t), EncryptionConfig) @staticmethod def from_data(data): - """Create an Team instance wrapping the given NumPy array. + """Create an EncryptionConfig instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `team_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `encryption_config_dtype` holding the data. """ - return __from_data(data, "team_dtype", team_dtype, Team) + return _cyb_from_data(data, "encryption_config_dtype", encryption_config_dtype, EncryptionConfig) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an Team instance wrapping the given pointer. + """Create an EncryptionConfig instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -1659,64 +1539,69 @@ cdef class Team: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef Team obj = Team.__new__(Team) + cdef EncryptionConfig obj = EncryptionConfig.__new__(EncryptionConfig) if owner is None: - obj._ptr = malloc(sizeof(ncclTeam_t)) + obj._ptr = _cyb_malloc(sizeof(ncclEncryptionConfig_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating Team") - memcpy((obj._ptr), ptr, sizeof(ncclTeam_t)) + raise MemoryError("Error allocating EncryptionConfig") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclEncryptionConfig_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly + obj._refs = {} return obj -cdef _get_multimem_handle_dtype_offsets(): - cdef ncclMultimemHandle_t pod = ncclMultimemHandle_t() +cdef _get_resource_window_vidmem_dtype_offsets(): + cdef ncclResourceWindow_vidmem_t pod return _numpy.dtype({ - 'names': ['mc_base_ptr'], - 'formats': [_numpy.intp], + 'names': ['lsa_flat_base', 'stride4g', 'mc_offset4k'], + 'formats': [_numpy.intp, _numpy.uint32, _numpy.uint32], 'offsets': [ - (&(pod.mcBasePtr)) - (&pod), + (&(pod.lsaFlatBase)) - (&pod), + (&(pod.stride4G)) - (&pod), + (&(pod.mcOffset4K)) - (&pod), ], - 'itemsize': sizeof(ncclMultimemHandle_t), + 'itemsize': sizeof(ncclResourceWindow_vidmem_t), }) -multimem_handle_dtype = _get_multimem_handle_dtype_offsets() +resource_window_vidmem_dtype = _get_resource_window_vidmem_dtype_offsets() -cdef class MultimemHandle: - """Empty-initialize an instance of `ncclMultimemHandle_t`. +cdef class ResourceWindowVidmem: + """Empty-initialize an instance of `ncclResourceWindow_vidmem_t`. - .. seealso:: `ncclMultimemHandle_t` + .. seealso:: `ncclResourceWindow_vidmem_t` """ cdef: - ncclMultimemHandle_t *_ptr + ncclResourceWindow_vidmem_t *_ptr object _owner bint _owned bint _readonly + dict _refs def __init__(self): - self._ptr = calloc(1, sizeof(ncclMultimemHandle_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclResourceWindow_vidmem_t)) if self._ptr == NULL: - raise MemoryError("Error allocating MultimemHandle") + raise MemoryError("Error allocating ResourceWindowVidmem") self._owner = None self._owned = True self._readonly = False + self._refs = {} def __dealloc__(self): - cdef ncclMultimemHandle_t *ptr + cdef ncclResourceWindow_vidmem_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.MultimemHandle object at {hex(id(self))}>" + return f"<{__name__}.ResourceWindowVidmem object at {hex(id(self))}>" @property def ptr(self): @@ -1730,24 +1615,24 @@ cdef class MultimemHandle: return (self._ptr) def __eq__(self, other): - cdef MultimemHandle other_ - if not isinstance(other, MultimemHandle): + cdef ResourceWindowVidmem other_ + if not isinstance(other, ResourceWindowVidmem): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclMultimemHandle_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclResourceWindow_vidmem_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclMultimemHandle_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclResourceWindow_vidmem_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclMultimemHandle_t)) + self._ptr = _cyb_malloc(sizeof(ncclResourceWindow_vidmem_t)) if self._ptr == NULL: - raise MemoryError("Error allocating MultimemHandle") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclMultimemHandle_t)) + raise MemoryError("Error allocating ResourceWindowVidmem") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclResourceWindow_vidmem_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -1755,33 +1640,61 @@ cdef class MultimemHandle: setattr(self, key, val) @property - def mc_base_ptr(self): + def lsa_flat_base(self): + """str: """ + cdef char* ptr = self._ptr[0].lsaFlatBase + if ptr: + return _cyb_cpython.PyUnicode_FromString(ptr) + return "" + + @lsa_flat_base.setter + def lsa_flat_base(self, val): + if self._readonly: + raise ValueError("This ResourceWindowVidmem instance is read-only") + cdef bytes buf = val.encode() + cdef char *ptr = buf + self._refs["lsa_flat_base"] = buf + self._ptr[0].lsaFlatBase = ptr + + @property + def stride4g(self): """int: """ - return (self._ptr[0].mcBasePtr) + return self._ptr[0].stride4G - @mc_base_ptr.setter - def mc_base_ptr(self, val): + @stride4g.setter + def stride4g(self, val): if self._readonly: - raise ValueError("This MultimemHandle instance is read-only") - self._ptr[0].mcBasePtr = val + raise ValueError("This ResourceWindowVidmem instance is read-only") + self._ptr[0].stride4G = val + + @property + def mc_offset4k(self): + """int: """ + return self._ptr[0].mcOffset4K + + @mc_offset4k.setter + def mc_offset4k(self, val): + if self._readonly: + raise ValueError("This ResourceWindowVidmem instance is read-only") + self._ptr[0].mcOffset4K = val @staticmethod def from_buffer(buffer): - """Create an MultimemHandle instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclMultimemHandle_t), MultimemHandle) + """Create an ResourceWindowVidmem instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclResourceWindow_vidmem_t), ResourceWindowVidmem) @staticmethod def from_data(data): - """Create an MultimemHandle instance wrapping the given NumPy array. + """Create an ResourceWindowVidmem instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `multimem_handle_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `resource_window_vidmem_dtype` holding the data. """ - return __from_data(data, "multimem_handle_dtype", multimem_handle_dtype, MultimemHandle) + return _cyb_from_data(data, "resource_window_vidmem_dtype", resource_window_vidmem_dtype, ResourceWindowVidmem) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an MultimemHandle instance wrapping the given pointer. + """Create an ResourceWindowVidmem instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -1790,71 +1703,66 @@ cdef class MultimemHandle: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef MultimemHandle obj = MultimemHandle.__new__(MultimemHandle) + cdef ResourceWindowVidmem obj = ResourceWindowVidmem.__new__(ResourceWindowVidmem) if owner is None: - obj._ptr = malloc(sizeof(ncclMultimemHandle_t)) + obj._ptr = _cyb_malloc(sizeof(ncclResourceWindow_vidmem_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating MultimemHandle") - memcpy((obj._ptr), ptr, sizeof(ncclMultimemHandle_t)) + raise MemoryError("Error allocating ResourceWindowVidmem") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclResourceWindow_vidmem_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly + obj._refs = {} return obj -cdef _get_resource_window_vidmem_dtype_offsets(): - cdef ncclResourceWindow_vidmem_t pod = ncclResourceWindow_vidmem_t() +cdef _get_gin_barrier_handle_dtype_offsets(): + cdef ncclGinBarrierHandle_t pod return _numpy.dtype({ - 'names': ['reserved1', 'lsa_flat_base', 'reserved2', 'stride4g', 'mc_offset4k', 'reserved3'], - 'formats': [(_numpy.int8, 8), _numpy.intp, (_numpy.int8, 8), _numpy.uint32, _numpy.uint32, (_numpy.int8, 32)], + 'names': ['signal0', 'unused'], + 'formats': [_numpy.uint32, _numpy.uint32], 'offsets': [ - (&(pod.reserved1)) - (&pod), - (&(pod.lsaFlatBase)) - (&pod), - (&(pod.reserved2)) - (&pod), - (&(pod.stride4G)) - (&pod), - (&(pod.mcOffset4K)) - (&pod), - (&(pod.reserved3)) - (&pod), + (&(pod.signal0)) - (&pod), + (&(pod.unused)) - (&pod), ], - 'itemsize': sizeof(ncclResourceWindow_vidmem_t), + 'itemsize': sizeof(ncclGinBarrierHandle_t), }) -resource_window_vidmem_dtype = _get_resource_window_vidmem_dtype_offsets() +gin_barrier_handle_dtype = _get_gin_barrier_handle_dtype_offsets() -cdef class ResourceWindow_vidmem: - """Empty-initialize an instance of `ncclResourceWindow_vidmem_t`. +cdef class GinBarrierHandle: + """Empty-initialize an instance of `ncclGinBarrierHandle_t`. - .. seealso:: `ncclResourceWindow_vidmem_t` + .. seealso:: `ncclGinBarrierHandle_t` """ cdef: - ncclResourceWindow_vidmem_t *_ptr + ncclGinBarrierHandle_t *_ptr object _owner bint _owned bint _readonly - dict _refs def __init__(self): - self._ptr = calloc(1, sizeof(ncclResourceWindow_vidmem_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclGinBarrierHandle_t)) if self._ptr == NULL: - raise MemoryError("Error allocating ResourceWindow_vidmem") + raise MemoryError("Error allocating GinBarrierHandle") self._owner = None self._owned = True self._readonly = False - self._refs = {} def __dealloc__(self): - cdef ncclResourceWindow_vidmem_t *ptr + cdef ncclGinBarrierHandle_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.ResourceWindow_vidmem object at {hex(id(self))}>" + return f"<{__name__}.GinBarrierHandle object at {hex(id(self))}>" @property def ptr(self): @@ -1868,24 +1776,24 @@ cdef class ResourceWindow_vidmem: return (self._ptr) def __eq__(self, other): - cdef ResourceWindow_vidmem other_ - if not isinstance(other, ResourceWindow_vidmem): + cdef GinBarrierHandle other_ + if not isinstance(other, GinBarrierHandle): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclResourceWindow_vidmem_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclGinBarrierHandle_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclResourceWindow_vidmem_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclGinBarrierHandle_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclResourceWindow_vidmem_t)) + self._ptr = _cyb_malloc(sizeof(ncclGinBarrierHandle_t)) if self._ptr == NULL: - raise MemoryError("Error allocating ResourceWindow_vidmem") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclResourceWindow_vidmem_t)) + raise MemoryError("Error allocating GinBarrierHandle") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclGinBarrierHandle_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -1893,61 +1801,33 @@ cdef class ResourceWindow_vidmem: setattr(self, key, val) @property - def lsa_flat_base(self): - """str: """ - cdef char* ptr = self._ptr[0].lsaFlatBase - if ptr: - return cpython.PyUnicode_FromString(ptr) - return "" - - @lsa_flat_base.setter - def lsa_flat_base(self, val): - if self._readonly: - raise ValueError("This ResourceWindow_vidmem instance is read-only") - cdef bytes buf = val.encode() - cdef char *ptr = buf - self._refs["lsa_flat_base"] = buf - self._ptr.lsaFlatBase = ptr - - @property - def stride4g(self): - """int: """ - return self._ptr[0].stride4G - - @stride4g.setter - def stride4g(self, val): - if self._readonly: - raise ValueError("This ResourceWindow_vidmem instance is read-only") - self._ptr[0].stride4G = val - - @property - def mc_offset4k(self): + def signal0(self): """int: """ - return self._ptr[0].mcOffset4K + return (self._ptr[0].signal0) - @mc_offset4k.setter - def mc_offset4k(self, val): + @signal0.setter + def signal0(self, val): if self._readonly: - raise ValueError("This ResourceWindow_vidmem instance is read-only") - self._ptr[0].mcOffset4K = val + raise ValueError("This GinBarrierHandle instance is read-only") + self._ptr[0].signal0 = val @staticmethod def from_buffer(buffer): - """Create an ResourceWindow_vidmem instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclResourceWindow_vidmem_t), ResourceWindow_vidmem) + """Create an GinBarrierHandle instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclGinBarrierHandle_t), GinBarrierHandle) @staticmethod def from_data(data): - """Create an ResourceWindow_vidmem instance wrapping the given NumPy array. + """Create an GinBarrierHandle instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `resource_window_vidmem_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `gin_barrier_handle_dtype` holding the data. """ - return __from_data(data, "resource_window_vidmem_dtype", resource_window_vidmem_dtype, ResourceWindow_vidmem) + return _cyb_from_data(data, "gin_barrier_handle_dtype", gin_barrier_handle_dtype, GinBarrierHandle) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an ResourceWindow_vidmem instance wrapping the given pointer. + """Create an GinBarrierHandle instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -1956,66 +1836,64 @@ cdef class ResourceWindow_vidmem: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef ResourceWindow_vidmem obj = ResourceWindow_vidmem.__new__(ResourceWindow_vidmem) + cdef GinBarrierHandle obj = GinBarrierHandle.__new__(GinBarrierHandle) if owner is None: - obj._ptr = malloc(sizeof(ncclResourceWindow_vidmem_t)) + obj._ptr = _cyb_malloc(sizeof(ncclGinBarrierHandle_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating ResourceWindow_vidmem") - memcpy((obj._ptr), ptr, sizeof(ncclResourceWindow_vidmem_t)) + raise MemoryError("Error allocating GinBarrierHandle") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclGinBarrierHandle_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly - obj._refs = {} return obj -cdef _get_gin_barrier_handle_dtype_offsets(): - cdef ncclGinBarrierHandle_t pod = ncclGinBarrierHandle_t() +cdef _get_multimem_handle_dtype_offsets(): + cdef ncclMultimemHandle_t pod return _numpy.dtype({ - 'names': ['signal0', 'unused'], - 'formats': [_numpy.uint32, _numpy.uint32], + 'names': ['mc_base_ptr'], + 'formats': [_numpy.intp], 'offsets': [ - (&(pod.signal0)) - (&pod), - (&(pod.unused)) - (&pod), + (&(pod.mcBasePtr)) - (&pod), ], - 'itemsize': sizeof(ncclGinBarrierHandle_t), + 'itemsize': sizeof(ncclMultimemHandle_t), }) -gin_barrier_handle_dtype = _get_gin_barrier_handle_dtype_offsets() +multimem_handle_dtype = _get_multimem_handle_dtype_offsets() -cdef class GinBarrierHandle: - """Empty-initialize an instance of `ncclGinBarrierHandle_t`. +cdef class MultimemHandle: + """Empty-initialize an instance of `ncclMultimemHandle_t`. - .. seealso:: `ncclGinBarrierHandle_t` + .. seealso:: `ncclMultimemHandle_t` """ cdef: - ncclGinBarrierHandle_t *_ptr + ncclMultimemHandle_t *_ptr object _owner bint _owned bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclGinBarrierHandle_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclMultimemHandle_t)) if self._ptr == NULL: - raise MemoryError("Error allocating GinBarrierHandle") + raise MemoryError("Error allocating MultimemHandle") self._owner = None self._owned = True self._readonly = False def __dealloc__(self): - cdef ncclGinBarrierHandle_t *ptr + cdef ncclMultimemHandle_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.GinBarrierHandle object at {hex(id(self))}>" + return f"<{__name__}.MultimemHandle object at {hex(id(self))}>" @property def ptr(self): @@ -2029,24 +1907,24 @@ cdef class GinBarrierHandle: return (self._ptr) def __eq__(self, other): - cdef GinBarrierHandle other_ - if not isinstance(other, GinBarrierHandle): + cdef MultimemHandle other_ + if not isinstance(other, MultimemHandle): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclGinBarrierHandle_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclMultimemHandle_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclGinBarrierHandle_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclMultimemHandle_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclGinBarrierHandle_t)) + self._ptr = _cyb_malloc(sizeof(ncclMultimemHandle_t)) if self._ptr == NULL: - raise MemoryError("Error allocating GinBarrierHandle") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclGinBarrierHandle_t)) + raise MemoryError("Error allocating MultimemHandle") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclMultimemHandle_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -2054,44 +1932,33 @@ cdef class GinBarrierHandle: setattr(self, key, val) @property - def signal0(self): - """int: """ - return (self._ptr[0].signal0) - - @signal0.setter - def signal0(self, val): - if self._readonly: - raise ValueError("This GinBarrierHandle instance is read-only") - self._ptr[0].signal0 = val - - @property - def unused(self): + def mc_base_ptr(self): """int: """ - return (self._ptr[0].unused) + return (self._ptr[0].mcBasePtr) - @unused.setter - def unused(self, val): + @mc_base_ptr.setter + def mc_base_ptr(self, val): if self._readonly: - raise ValueError("This GinBarrierHandle instance is read-only") - self._ptr[0].unused = val + raise ValueError("This MultimemHandle instance is read-only") + self._ptr[0].mcBasePtr = val @staticmethod def from_buffer(buffer): - """Create an GinBarrierHandle instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclGinBarrierHandle_t), GinBarrierHandle) + """Create an MultimemHandle instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclMultimemHandle_t), MultimemHandle) @staticmethod def from_data(data): - """Create an GinBarrierHandle instance wrapping the given NumPy array. + """Create an MultimemHandle instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `gin_barrier_handle_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `multimem_handle_dtype` holding the data. """ - return __from_data(data, "gin_barrier_handle_dtype", gin_barrier_handle_dtype, GinBarrierHandle) + return _cyb_from_data(data, "multimem_handle_dtype", multimem_handle_dtype, MultimemHandle) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an GinBarrierHandle instance wrapping the given pointer. + """Create an MultimemHandle instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -2100,16 +1967,16 @@ cdef class GinBarrierHandle: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef GinBarrierHandle obj = GinBarrierHandle.__new__(GinBarrierHandle) + cdef MultimemHandle obj = MultimemHandle.__new__(MultimemHandle) if owner is None: - obj._ptr = malloc(sizeof(ncclGinBarrierHandle_t)) + obj._ptr = _cyb_malloc(sizeof(ncclMultimemHandle_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating GinBarrierHandle") - memcpy((obj._ptr), ptr, sizeof(ncclGinBarrierHandle_t)) + raise MemoryError("Error allocating MultimemHandle") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclMultimemHandle_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly @@ -2117,7 +1984,7 @@ cdef class GinBarrierHandle: cdef _get_lsa_barrier_handle_dtype_offsets(): - cdef ncclLsaBarrierHandle_t pod = ncclLsaBarrierHandle_t() + cdef ncclLsaBarrierHandle_t pod return _numpy.dtype({ 'names': ['buf_handle', 'n_barriers'], 'formats': [_numpy.uint32, _numpy.int32], @@ -2143,7 +2010,7 @@ cdef class LsaBarrierHandle: bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclLsaBarrierHandle_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclLsaBarrierHandle_t)) if self._ptr == NULL: raise MemoryError("Error allocating LsaBarrierHandle") self._owner = None @@ -2155,7 +2022,7 @@ cdef class LsaBarrierHandle: if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): return f"<{__name__}.LsaBarrierHandle object at {hex(id(self))}>" @@ -2176,20 +2043,20 @@ cdef class LsaBarrierHandle: if not isinstance(other, LsaBarrierHandle): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclLsaBarrierHandle_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclLsaBarrierHandle_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclLsaBarrierHandle_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclLsaBarrierHandle_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclLsaBarrierHandle_t)) + self._ptr = _cyb_malloc(sizeof(ncclLsaBarrierHandle_t)) if self._ptr == NULL: raise MemoryError("Error allocating LsaBarrierHandle") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclLsaBarrierHandle_t)) + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclLsaBarrierHandle_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -2221,7 +2088,7 @@ cdef class LsaBarrierHandle: @staticmethod def from_buffer(buffer): """Create an LsaBarrierHandle instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclLsaBarrierHandle_t), LsaBarrierHandle) + return _cyb_from_buffer(buffer, sizeof(ncclLsaBarrierHandle_t), LsaBarrierHandle) @staticmethod def from_data(data): @@ -2230,7 +2097,7 @@ cdef class LsaBarrierHandle: Args: data (_numpy.ndarray): a single-element array of dtype `lsa_barrier_handle_dtype` holding the data. """ - return __from_data(data, "lsa_barrier_handle_dtype", lsa_barrier_handle_dtype, LsaBarrierHandle) + return _cyb_from_data(data, "lsa_barrier_handle_dtype", lsa_barrier_handle_dtype, LsaBarrierHandle) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): @@ -2245,10 +2112,10 @@ cdef class LsaBarrierHandle: raise ValueError("ptr must not be null (0)") cdef LsaBarrierHandle obj = LsaBarrierHandle.__new__(LsaBarrierHandle) if owner is None: - obj._ptr = malloc(sizeof(ncclLsaBarrierHandle_t)) + obj._ptr = _cyb_malloc(sizeof(ncclLsaBarrierHandle_t)) if obj._ptr == NULL: raise MemoryError("Error allocating LsaBarrierHandle") - memcpy((obj._ptr), ptr, sizeof(ncclLsaBarrierHandle_t)) + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclLsaBarrierHandle_t)) obj._owner = None obj._owned = True else: @@ -2259,49 +2126,49 @@ cdef class LsaBarrierHandle: return obj -cdef _get_ll_a2a_handle_dtype_offsets(): - cdef ncclLLA2AHandle_t pod = ncclLLA2AHandle_t() +cdef _get_cft_barrier_handle_dtype_offsets(): + cdef ncclCftBarrierHandle_t pod return _numpy.dtype({ - 'names': ['buf_handle', 'n_slots'], - 'formats': [_numpy.uint32, _numpy.uint32], + 'names': ['buf_handle', 'n_barriers'], + 'formats': [_numpy.uint32, _numpy.int32], 'offsets': [ (&(pod.bufHandle)) - (&pod), - (&(pod.nSlots)) - (&pod), + (&(pod.nBarriers)) - (&pod), ], - 'itemsize': sizeof(ncclLLA2AHandle_t), + 'itemsize': sizeof(ncclCftBarrierHandle_t), }) -ll_a2a_handle_dtype = _get_ll_a2a_handle_dtype_offsets() +cft_barrier_handle_dtype = _get_cft_barrier_handle_dtype_offsets() -cdef class LLA2AHandle: - """Empty-initialize an instance of `ncclLLA2AHandle_t`. +cdef class CftBarrierHandle: + """Empty-initialize an instance of `ncclCftBarrierHandle_t`. - .. seealso:: `ncclLLA2AHandle_t` + .. seealso:: `ncclCftBarrierHandle_t` """ cdef: - ncclLLA2AHandle_t *_ptr + ncclCftBarrierHandle_t *_ptr object _owner bint _owned bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclLLA2AHandle_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclCftBarrierHandle_t)) if self._ptr == NULL: - raise MemoryError("Error allocating LLA2AHandle") + raise MemoryError("Error allocating CftBarrierHandle") self._owner = None self._owned = True self._readonly = False def __dealloc__(self): - cdef ncclLLA2AHandle_t *ptr + cdef ncclCftBarrierHandle_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.LLA2AHandle object at {hex(id(self))}>" + return f"<{__name__}.CftBarrierHandle object at {hex(id(self))}>" @property def ptr(self): @@ -2315,24 +2182,24 @@ cdef class LLA2AHandle: return (self._ptr) def __eq__(self, other): - cdef LLA2AHandle other_ - if not isinstance(other, LLA2AHandle): + cdef CftBarrierHandle other_ + if not isinstance(other, CftBarrierHandle): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclLLA2AHandle_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclCftBarrierHandle_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclLLA2AHandle_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclCftBarrierHandle_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclLLA2AHandle_t)) + self._ptr = _cyb_malloc(sizeof(ncclCftBarrierHandle_t)) if self._ptr == NULL: - raise MemoryError("Error allocating LLA2AHandle") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclLLA2AHandle_t)) + raise MemoryError("Error allocating CftBarrierHandle") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclCftBarrierHandle_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -2347,37 +2214,37 @@ cdef class LLA2AHandle: @buf_handle.setter def buf_handle(self, val): if self._readonly: - raise ValueError("This LLA2AHandle instance is read-only") + raise ValueError("This CftBarrierHandle instance is read-only") self._ptr[0].bufHandle = val @property - def n_slots(self): + def n_barriers(self): """int: """ - return self._ptr[0].nSlots + return self._ptr[0].nBarriers - @n_slots.setter - def n_slots(self, val): + @n_barriers.setter + def n_barriers(self, val): if self._readonly: - raise ValueError("This LLA2AHandle instance is read-only") - self._ptr[0].nSlots = val + raise ValueError("This CftBarrierHandle instance is read-only") + self._ptr[0].nBarriers = val @staticmethod def from_buffer(buffer): - """Create an LLA2AHandle instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclLLA2AHandle_t), LLA2AHandle) + """Create an CftBarrierHandle instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclCftBarrierHandle_t), CftBarrierHandle) @staticmethod def from_data(data): - """Create an LLA2AHandle instance wrapping the given NumPy array. + """Create an CftBarrierHandle instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `ll_a2a_handle_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `cft_barrier_handle_dtype` holding the data. """ - return __from_data(data, "ll_a2a_handle_dtype", ll_a2a_handle_dtype, LLA2AHandle) + return _cyb_from_data(data, "cft_barrier_handle_dtype", cft_barrier_handle_dtype, CftBarrierHandle) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an LLA2AHandle instance wrapping the given pointer. + """Create an CftBarrierHandle instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -2386,67 +2253,66 @@ cdef class LLA2AHandle: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef LLA2AHandle obj = LLA2AHandle.__new__(LLA2AHandle) + cdef CftBarrierHandle obj = CftBarrierHandle.__new__(CftBarrierHandle) if owner is None: - obj._ptr = malloc(sizeof(ncclLLA2AHandle_t)) + obj._ptr = _cyb_malloc(sizeof(ncclCftBarrierHandle_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating LLA2AHandle") - memcpy((obj._ptr), ptr, sizeof(ncclLLA2AHandle_t)) + raise MemoryError("Error allocating CftBarrierHandle") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclCftBarrierHandle_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly return obj -cdef _get_team_requirements_dtype_offsets(): - cdef ncclTeamRequirements_t pod = ncclTeamRequirements_t() +cdef _get_team_dtype_offsets(): + cdef ncclTeam_t pod return _numpy.dtype({ - 'names': ['next', 'team', 'multimem', 'out_multimem_handle'], - 'formats': [_numpy.intp, team_dtype, _numpy.uint8, _numpy.intp], + 'names': ['n_ranks', 'rank', 'stride'], + 'formats': [_numpy.int32, _numpy.int32, _numpy.int32], 'offsets': [ - (&(pod.next)) - (&pod), - (&(pod.team)) - (&pod), - (&(pod.multimem)) - (&pod), - (&(pod.outMultimemHandle)) - (&pod), + (&(pod.nRanks)) - (&pod), + (&(pod.rank)) - (&pod), + (&(pod.stride)) - (&pod), ], - 'itemsize': sizeof(ncclTeamRequirements_t), + 'itemsize': sizeof(ncclTeam_t), }) -team_requirements_dtype = _get_team_requirements_dtype_offsets() +team_dtype = _get_team_dtype_offsets() -cdef class TeamRequirements: - """Empty-initialize an instance of `ncclTeamRequirements_t`. +cdef class Team: + """Empty-initialize an instance of `ncclTeam_t`. - .. seealso:: `ncclTeamRequirements_t` + .. seealso:: `ncclTeam_t` """ cdef: - ncclTeamRequirements_t *_ptr + ncclTeam_t *_ptr object _owner bint _owned bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclTeamRequirements_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclTeam_t)) if self._ptr == NULL: - raise MemoryError("Error allocating TeamRequirements") + raise MemoryError("Error allocating Team") self._owner = None self._owned = True self._readonly = False def __dealloc__(self): - cdef ncclTeamRequirements_t *ptr + cdef ncclTeam_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.TeamRequirements object at {hex(id(self))}>" + return f"<{__name__}.Team object at {hex(id(self))}>" @property def ptr(self): @@ -2460,24 +2326,24 @@ cdef class TeamRequirements: return (self._ptr) def __eq__(self, other): - cdef TeamRequirements other_ - if not isinstance(other, TeamRequirements): + cdef Team other_ + if not isinstance(other, Team): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclTeamRequirements_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclTeam_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclTeamRequirements_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclTeam_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclTeamRequirements_t)) + self._ptr = _cyb_malloc(sizeof(ncclTeam_t)) if self._ptr == NULL: - raise MemoryError("Error allocating TeamRequirements") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclTeamRequirements_t)) + raise MemoryError("Error allocating Team") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclTeam_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -2485,67 +2351,55 @@ cdef class TeamRequirements: setattr(self, key, val) @property - def team(self): - """Team: """ - return Team.from_ptr(&(self._ptr[0].team), self._readonly, self) - - @team.setter - def team(self, val): - if self._readonly: - raise ValueError("This TeamRequirements instance is read-only") - cdef Team val_ = val - memcpy(&(self._ptr[0].team), (val_._get_ptr()), sizeof(ncclTeam_t) * 1) - - @property - def next(self): + def n_ranks(self): """int: """ - return (self._ptr[0].next) + return self._ptr[0].nRanks - @next.setter - def next(self, val): + @n_ranks.setter + def n_ranks(self, val): if self._readonly: - raise ValueError("This TeamRequirements instance is read-only") - self._ptr[0].next = val + raise ValueError("This Team instance is read-only") + self._ptr[0].nRanks = val @property - def multimem(self): + def rank(self): """int: """ - return self._ptr[0].multimem + return self._ptr[0].rank - @multimem.setter - def multimem(self, val): + @rank.setter + def rank(self, val): if self._readonly: - raise ValueError("This TeamRequirements instance is read-only") - self._ptr[0].multimem = val + raise ValueError("This Team instance is read-only") + self._ptr[0].rank = val @property - def out_multimem_handle(self): + def stride(self): """int: """ - return (self._ptr[0].outMultimemHandle) + return self._ptr[0].stride - @out_multimem_handle.setter - def out_multimem_handle(self, val): + @stride.setter + def stride(self, val): if self._readonly: - raise ValueError("This TeamRequirements instance is read-only") - self._ptr[0].outMultimemHandle = val + raise ValueError("This Team instance is read-only") + self._ptr[0].stride = val @staticmethod def from_buffer(buffer): - """Create an TeamRequirements instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclTeamRequirements_t), TeamRequirements) + """Create an Team instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclTeam_t), Team) @staticmethod def from_data(data): - """Create an TeamRequirements instance wrapping the given NumPy array. + """Create an Team instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `team_requirements_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `team_dtype` holding the data. """ - return __from_data(data, "team_requirements_dtype", team_requirements_dtype, TeamRequirements) + return _cyb_from_data(data, "team_dtype", team_dtype, Team) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an TeamRequirements instance wrapping the given pointer. + """Create an Team instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -2554,92 +2408,1895 @@ cdef class TeamRequirements: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef TeamRequirements obj = TeamRequirements.__new__(TeamRequirements) + cdef Team obj = Team.__new__(Team) if owner is None: - obj._ptr = malloc(sizeof(ncclTeamRequirements_t)) + obj._ptr = _cyb_malloc(sizeof(ncclTeam_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating TeamRequirements") - memcpy((obj._ptr), ptr, sizeof(ncclTeamRequirements_t)) + raise MemoryError("Error allocating Team") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclTeam_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly return obj -cdef _get_dev_comm_dtype_offsets(): - cdef ncclDevComm_t pod = ncclDevComm_t() +cdef _get_ll_a2a_handle_dtype_offsets(): + cdef ncclLLA2AHandle_t pod return _numpy.dtype({ - 'names': ['magic', 'version', 'rank', 'n_ranks', 'n_ranks_rcp32', 'lsa_rank', 'lsa_size', 'lsa_size_rcp32', 'window_table', 'resource_window', 'resource_window_inlined', 'hybrid_world_gin_barrier', 'lsa_multimem', 'lsa_barrier', 'rail_gin_barrier', 'gin_connection_count', 'gin_net_device_types', 'gin_handles', 'gin_signal_count', 'gin_counter_count', 'gin_signal_shadows', 'gin_context_count', 'gin_connections_railed', 'gin_strong_legacy_signals', 'gin_contexts_railed', 'abort_flag', 'hybrid_lsa_barrier', 'hybrid_rail_gin_barrier', 'world_gin_barrier'], - 'formats': [_numpy.uint32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.uint32, _numpy.intp, _numpy.intp, resource_window_vidmem_dtype, gin_barrier_handle_dtype, multimem_handle_dtype, lsa_barrier_handle_dtype, gin_barrier_handle_dtype, _numpy.uint8, (_numpy.uint8, 4), (_numpy.int64, 4), _numpy.int32, _numpy.int32, _numpy.intp, _numpy.uint32, _numpy.uint8, _numpy.uint8, _numpy.uint8, _numpy.intp, lsa_barrier_handle_dtype, gin_barrier_handle_dtype, gin_barrier_handle_dtype], + 'names': ['buf_handle', 'n_slots'], + 'formats': [_numpy.uint32, _numpy.uint32], 'offsets': [ - (&(pod.magic)) - (&pod), - (&(pod.version)) - (&pod), - (&(pod.rank)) - (&pod), - (&(pod.nRanks)) - (&pod), - (&(pod.nRanks_rcp32)) - (&pod), - (&(pod.lsaRank)) - (&pod), - (&(pod.lsaSize)) - (&pod), - (&(pod.lsaSize_rcp32)) - (&pod), - (&(pod.windowTable)) - (&pod), - (&(pod.resourceWindow)) - (&pod), - (&(pod.resourceWindow_inlined)) - (&pod), - (&(pod.hybridWorldGinBarrier)) - (&pod), - (&(pod.lsaMultimem)) - (&pod), - (&(pod.lsaBarrier)) - (&pod), - (&(pod.railGinBarrier)) - (&pod), - (&(pod.ginConnectionCount)) - (&pod), - (&(pod.ginNetDeviceTypes)) - (&pod), - (&(pod.ginHandles)) - (&pod), + (&(pod.bufHandle)) - (&pod), + (&(pod.nSlots)) - (&pod), + ], + 'itemsize': sizeof(ncclLLA2AHandle_t), + }) + +ll_a2a_handle_dtype = _get_ll_a2a_handle_dtype_offsets() + +cdef class LLA2AHandle: + """Empty-initialize an instance of `ncclLLA2AHandle_t`. + + + .. seealso:: `ncclLLA2AHandle_t` + """ + cdef: + ncclLLA2AHandle_t *_ptr + object _owner + bint _owned + bint _readonly + + def __init__(self): + self._ptr = _cyb_calloc(1, sizeof(ncclLLA2AHandle_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating LLA2AHandle") + self._owner = None + self._owned = True + self._readonly = False + + def __dealloc__(self): + cdef ncclLLA2AHandle_t *ptr + if self._owned and self._ptr != NULL: + ptr = self._ptr + self._ptr = NULL + _cyb_free(ptr) + + def __repr__(self): + return f"<{__name__}.LLA2AHandle object at {hex(id(self))}>" + + @property + def ptr(self): + """Get the pointer address to the data as Python :class:`int`.""" + return (self._ptr) + + cdef intptr_t _get_ptr(self): + return (self._ptr) + + def __int__(self): + return (self._ptr) + + def __eq__(self, other): + cdef LLA2AHandle other_ + if not isinstance(other, LLA2AHandle): + return False + other_ = other + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclLLA2AHandle_t)) == 0) + + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclLLA2AHandle_t), self._readonly) + + def __releasebuffer__(self, Py_buffer *buffer): + pass + + def __setitem__(self, key, val): + if key == 0 and isinstance(val, _numpy.ndarray): + self._ptr = _cyb_malloc(sizeof(ncclLLA2AHandle_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating LLA2AHandle") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclLLA2AHandle_t)) + self._owner = None + self._owned = True + self._readonly = not val.flags.writeable + else: + setattr(self, key, val) + + @property + def buf_handle(self): + """int: """ + return (self._ptr[0].bufHandle) + + @buf_handle.setter + def buf_handle(self, val): + if self._readonly: + raise ValueError("This LLA2AHandle instance is read-only") + self._ptr[0].bufHandle = val + + @property + def n_slots(self): + """int: """ + return self._ptr[0].nSlots + + @n_slots.setter + def n_slots(self, val): + if self._readonly: + raise ValueError("This LLA2AHandle instance is read-only") + self._ptr[0].nSlots = val + + @staticmethod + def from_buffer(buffer): + """Create an LLA2AHandle instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclLLA2AHandle_t), LLA2AHandle) + + @staticmethod + def from_data(data): + """Create an LLA2AHandle instance wrapping the given NumPy array. + + Args: + data (_numpy.ndarray): a single-element array of dtype `ll_a2a_handle_dtype` holding the data. + """ + return _cyb_from_data(data, "ll_a2a_handle_dtype", ll_a2a_handle_dtype, LLA2AHandle) + + @staticmethod + def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): + """Create an LLA2AHandle instance wrapping the given pointer. + + Args: + ptr (intptr_t): pointer address as Python :class:`int` to the data. + owner (object): The Python object that owns the pointer. If not provided, data will be copied. + readonly (bool): whether the data is read-only (to the user). default is `False`. + """ + if ptr == 0: + raise ValueError("ptr must not be null (0)") + cdef LLA2AHandle obj = LLA2AHandle.__new__(LLA2AHandle) + if owner is None: + obj._ptr = _cyb_malloc(sizeof(ncclLLA2AHandle_t)) + if obj._ptr == NULL: + raise MemoryError("Error allocating LLA2AHandle") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclLLA2AHandle_t)) + obj._owner = None + obj._owned = True + else: + obj._ptr = ptr + obj._owner = owner + obj._owned = False + obj._readonly = readonly + return obj + + +cdef _get_dev_resource_requirements_dtype_offsets(): + cdef ncclDevResourceRequirements_t pod + return _numpy.dtype({ + 'names': ['next', 'buffer_size', 'buffer_align', 'out_buffer_handle', 'gin_signal_count', 'gin_counter_count', 'out_gin_signal_start', 'out_gin_counter_start'], + 'formats': [_numpy.intp, _numpy.uint64, _numpy.uint64, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.intp], + 'offsets': [ + (&(pod.next)) - (&pod), + (&(pod.bufferSize)) - (&pod), + (&(pod.bufferAlign)) - (&pod), + (&(pod.outBufferHandle)) - (&pod), (&(pod.ginSignalCount)) - (&pod), (&(pod.ginCounterCount)) - (&pod), - (&(pod.ginSignalShadows)) - (&pod), - (&(pod.ginContextCount)) - (&pod), - (&(pod.ginConnectionsRailed)) - (&pod), - (&(pod.ginStrongLegacySignals)) - (&pod), - (&(pod.ginContextsRailed)) - (&pod), - (&(pod.abortFlag)) - (&pod), - (&(pod.hybridLsaBarrier)) - (&pod), - (&(pod.hybridRailGinBarrier)) - (&pod), - (&(pod.worldGinBarrier)) - (&pod), + (&(pod.outGinSignalStart)) - (&pod), + (&(pod.outGinCounterStart)) - (&pod), ], - 'itemsize': sizeof(ncclDevComm_t), + 'itemsize': sizeof(ncclDevResourceRequirements_t), }) -dev_comm_dtype = _get_dev_comm_dtype_offsets() +dev_resource_requirements_dtype = _get_dev_resource_requirements_dtype_offsets() -cdef class DevComm: - """Empty-initialize an instance of `ncclDevComm_t`. +cdef class DevResourceRequirements: + """Empty-initialize an instance of `ncclDevResourceRequirements_t`. - .. seealso:: `ncclDevComm_t` + .. seealso:: `ncclDevResourceRequirements_t` """ cdef: - ncclDevComm_t *_ptr + ncclDevResourceRequirements_t *_ptr object _owner bint _owned bint _readonly + dict _refs def __init__(self): - self._ptr = calloc(1, sizeof(ncclDevComm_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclDevResourceRequirements_t)) if self._ptr == NULL: - raise MemoryError("Error allocating DevComm") + raise MemoryError("Error allocating DevResourceRequirements") self._owner = None self._owned = True self._readonly = False + self._refs = {} def __dealloc__(self): - cdef ncclDevComm_t *ptr + cdef ncclDevResourceRequirements_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): - return f"<{__name__}.DevComm object at {hex(id(self))}>" + return f"<{__name__}.DevResourceRequirements object at {hex(id(self))}>" + + @property + def ptr(self): + """Get the pointer address to the data as Python :class:`int`.""" + return (self._ptr) + + cdef intptr_t _get_ptr(self): + return (self._ptr) + + def __int__(self): + return (self._ptr) + + def __eq__(self, other): + cdef DevResourceRequirements other_ + if not isinstance(other, DevResourceRequirements): + return False + other_ = other + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclDevResourceRequirements_t)) == 0) + + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclDevResourceRequirements_t), self._readonly) + + def __releasebuffer__(self, Py_buffer *buffer): + pass + + def __setitem__(self, key, val): + if key == 0 and isinstance(val, _numpy.ndarray): + self._ptr = _cyb_malloc(sizeof(ncclDevResourceRequirements_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating DevResourceRequirements") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclDevResourceRequirements_t)) + self._owner = None + self._owned = True + self._readonly = not val.flags.writeable + else: + setattr(self, key, val) + + @property + def next(self): + """DevResourceRequirements: """ + if self._ptr[0].next == NULL: + return None + ref = self._refs.get("next") + if ( + ref is not None + and (ref)._get_ptr() == (self._ptr[0].next) + ): + return ref + return DevResourceRequirements.from_ptr( + (self._ptr[0].next), + readonly=self._readonly, + owner=self, + ) + + @next.setter + def next(self, val): + if self._readonly: + raise ValueError("This DevResourceRequirements instance is read-only") + if val is None: + self._ptr[0].next = NULL + self._refs.pop("next", None) + return + if not isinstance(val, DevResourceRequirements): + raise TypeError("next must be DevResourceRequirements or None") + self._ptr[0].next = (val)._get_ptr() + self._refs["next"] = val + + @property + def buffer_size(self): + """int: """ + return self._ptr[0].bufferSize + + @buffer_size.setter + def buffer_size(self, val): + if self._readonly: + raise ValueError("This DevResourceRequirements instance is read-only") + self._ptr[0].bufferSize = val + + @property + def buffer_align(self): + """int: """ + return self._ptr[0].bufferAlign + + @buffer_align.setter + def buffer_align(self, val): + if self._readonly: + raise ValueError("This DevResourceRequirements instance is read-only") + self._ptr[0].bufferAlign = val + + @property + def out_buffer_handle(self): + """int: """ + return (self._ptr[0].outBufferHandle) + + @out_buffer_handle.setter + def out_buffer_handle(self, val): + if self._readonly: + raise ValueError("This DevResourceRequirements instance is read-only") + self._ptr[0].outBufferHandle = val + + @property + def gin_signal_count(self): + """int: """ + return self._ptr[0].ginSignalCount + + @gin_signal_count.setter + def gin_signal_count(self, val): + if self._readonly: + raise ValueError("This DevResourceRequirements instance is read-only") + self._ptr[0].ginSignalCount = val + + @property + def gin_counter_count(self): + """int: """ + return self._ptr[0].ginCounterCount + + @gin_counter_count.setter + def gin_counter_count(self, val): + if self._readonly: + raise ValueError("This DevResourceRequirements instance is read-only") + self._ptr[0].ginCounterCount = val + + @property + def out_gin_signal_start(self): + """int: """ + return (self._ptr[0].outGinSignalStart) + + @out_gin_signal_start.setter + def out_gin_signal_start(self, val): + if self._readonly: + raise ValueError("This DevResourceRequirements instance is read-only") + self._ptr[0].outGinSignalStart = val + + @property + def out_gin_counter_start(self): + """int: """ + return (self._ptr[0].outGinCounterStart) + + @out_gin_counter_start.setter + def out_gin_counter_start(self, val): + if self._readonly: + raise ValueError("This DevResourceRequirements instance is read-only") + self._ptr[0].outGinCounterStart = val + + @staticmethod + def from_buffer(buffer): + """Create an DevResourceRequirements instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclDevResourceRequirements_t), DevResourceRequirements) + + @staticmethod + def from_data(data): + """Create an DevResourceRequirements instance wrapping the given NumPy array. + + Args: + data (_numpy.ndarray): a single-element array of dtype `dev_resource_requirements_dtype` holding the data. + """ + return _cyb_from_data(data, "dev_resource_requirements_dtype", dev_resource_requirements_dtype, DevResourceRequirements) + + @staticmethod + def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): + """Create an DevResourceRequirements instance wrapping the given pointer. + + Args: + ptr (intptr_t): pointer address as Python :class:`int` to the data. + owner (object): The Python object that owns the pointer. If not provided, data will be copied. + readonly (bool): whether the data is read-only (to the user). default is `False`. + """ + if ptr == 0: + raise ValueError("ptr must not be null (0)") + cdef DevResourceRequirements obj = DevResourceRequirements.__new__(DevResourceRequirements) + if owner is None: + obj._ptr = _cyb_malloc(sizeof(ncclDevResourceRequirements_t)) + if obj._ptr == NULL: + raise MemoryError("Error allocating DevResourceRequirements") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclDevResourceRequirements_t)) + obj._owner = None + obj._owned = True + else: + obj._ptr = ptr + obj._owner = owner + obj._owned = False + obj._readonly = readonly + obj._refs = {} + return obj + + +cdef _get_comm_properties_dtype_offsets(): + cdef ncclCommProperties_t pod + return _numpy.dtype({ + 'names': ['size_', 'magic', 'version', 'rank', 'n_ranks', 'cuda_dev', 'nvml_dev', 'device_api_support', 'multimem_support', 'gin_type', 'n_lsa_teams', 'host_rma_support', 'railed_gin_type', 'comm_hash', 'gin_min_stride', 'gin_connection_type', 'gin_support', 'dev_comm_runtime_version_size', 'cft_support', 'cft_multicast_support', 'cft_counted_support'], + 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.bool_, _numpy.bool_, _numpy.int32, _numpy.int32, _numpy.bool_, _numpy.int32, _numpy.uint64, _numpy.int32, _numpy.int32, (_numpy.bool_, 64), _numpy.uint64, _numpy.bool_, _numpy.bool_, _numpy.bool_], + 'offsets': [ + (&(pod.size)) - (&pod), + (&(pod.magic)) - (&pod), + (&(pod.version)) - (&pod), + (&(pod.rank)) - (&pod), + (&(pod.nRanks)) - (&pod), + (&(pod.cudaDev)) - (&pod), + (&(pod.nvmlDev)) - (&pod), + (&(pod.deviceApiSupport)) - (&pod), + (&(pod.multimemSupport)) - (&pod), + (&(pod.ginType)) - (&pod), + (&(pod.nLsaTeams)) - (&pod), + (&(pod.hostRmaSupport)) - (&pod), + (&(pod.railedGinType)) - (&pod), + (&(pod.commHash)) - (&pod), + (&(pod.ginMinStride)) - (&pod), + (&(pod.ginConnectionType)) - (&pod), + (&(pod.ginSupport)) - (&pod), + (&(pod.devCommRuntimeVersionSize)) - (&pod), + (&(pod.cftSupport)) - (&pod), + (&(pod.cftMulticastSupport)) - (&pod), + (&(pod.cftCountedSupport)) - (&pod), + ], + 'itemsize': sizeof(ncclCommProperties_t), + }) + +comm_properties_dtype = _get_comm_properties_dtype_offsets() + +cdef class CommProperties: + """Initialize an instance of `ncclCommProperties_t` using configured defaults. + + + .. seealso:: `ncclCommProperties_t` + """ + cdef: + ncclCommProperties_t *_ptr + object _owner + bint _owned + bint _readonly + + def __init__(self): + self._ptr = _cyb_calloc(1, sizeof(ncclCommProperties_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating CommProperties") + self._owner = None + self._owned = True + self._readonly = False + + self._ptr[0].size = sizeof(ncclCommProperties_t) + self._ptr[0].magic = 0xcafebeef + self._ptr[0].version = __version_code__ + + def __dealloc__(self): + cdef ncclCommProperties_t *ptr + if self._owned and self._ptr != NULL: + ptr = self._ptr + self._ptr = NULL + _cyb_free(ptr) + + def __repr__(self): + return f"<{__name__}.CommProperties object at {hex(id(self))}>" + + @property + def ptr(self): + """Get the pointer address to the data as Python :class:`int`.""" + return (self._ptr) + + cdef intptr_t _get_ptr(self): + return (self._ptr) + + def __int__(self): + return (self._ptr) + + def __eq__(self, other): + cdef CommProperties other_ + if not isinstance(other, CommProperties): + return False + other_ = other + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclCommProperties_t)) == 0) + + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclCommProperties_t), self._readonly) + + def __releasebuffer__(self, Py_buffer *buffer): + pass + + def __setitem__(self, key, val): + if key == 0 and isinstance(val, _numpy.ndarray): + self._ptr = _cyb_malloc(sizeof(ncclCommProperties_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating CommProperties") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclCommProperties_t)) + self._owner = None + self._owned = True + self._readonly = not val.flags.writeable + else: + setattr(self, key, val) + + @property + def size_(self): + """int: """ + return self._ptr[0].size + + @size_.setter + def size_(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].size = val + + @property + def magic(self): + """int: """ + return self._ptr[0].magic + + @magic.setter + def magic(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].magic = val + + @property + def version(self): + """int: """ + return self._ptr[0].version + + @version.setter + def version(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].version = val + + @property + def rank(self): + """int: """ + return self._ptr[0].rank + + @rank.setter + def rank(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].rank = val + + @property + def n_ranks(self): + """int: """ + return self._ptr[0].nRanks + + @n_ranks.setter + def n_ranks(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].nRanks = val + + @property + def cuda_dev(self): + """int: """ + return self._ptr[0].cudaDev + + @cuda_dev.setter + def cuda_dev(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].cudaDev = val + + @property + def nvml_dev(self): + """int: """ + return self._ptr[0].nvmlDev + + @nvml_dev.setter + def nvml_dev(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].nvmlDev = val + + @property + def device_api_support(self): + """: """ + return (self._ptr[0].deviceApiSupport) + + @device_api_support.setter + def device_api_support(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].deviceApiSupport = <_cyb_bool>val + + @property + def multimem_support(self): + """: """ + return (self._ptr[0].multimemSupport) + + @multimem_support.setter + def multimem_support(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].multimemSupport = <_cyb_bool>val + + @property + def gin_type(self): + """int: """ + return (self._ptr[0].ginType) + + @gin_type.setter + def gin_type(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].ginType = val + + @property + def n_lsa_teams(self): + """int: """ + return self._ptr[0].nLsaTeams + + @n_lsa_teams.setter + def n_lsa_teams(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].nLsaTeams = val + + @property + def host_rma_support(self): + """: """ + return (self._ptr[0].hostRmaSupport) + + @host_rma_support.setter + def host_rma_support(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].hostRmaSupport = <_cyb_bool>val + + @property + def railed_gin_type(self): + """int: """ + return (self._ptr[0].railedGinType) + + @railed_gin_type.setter + def railed_gin_type(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].railedGinType = val + + @property + def comm_hash(self): + """int: """ + return self._ptr[0].commHash + + @comm_hash.setter + def comm_hash(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].commHash = val + + @property + def gin_min_stride(self): + """int: """ + return self._ptr[0].ginMinStride + + @gin_min_stride.setter + def gin_min_stride(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].ginMinStride = val + + @property + def gin_connection_type(self): + """int: """ + return (self._ptr[0].ginConnectionType) + + @gin_connection_type.setter + def gin_connection_type(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].ginConnectionType = val + + @property + def gin_support(self): + """~_numpy.bool_: (array of length 64).""" + cdef _cyb_view.array arr = _cyb_view.array(shape=(64,), itemsize=sizeof(_cyb_bool), format="?", mode="c", allocate_buffer=False) + arr.data = (&(self._ptr[0].ginSupport)) + return _numpy.asarray(arr) + + @gin_support.setter + def gin_support(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + if len(val) != 64: + raise ValueError(f"Expected length { 64 } for field gin_support, got {len(val)}") + cdef _cyb_view.array arr = _cyb_view.array(shape=(64,), itemsize=sizeof(_cyb_bool), format="?", mode="c") + arr[:] = _numpy.asarray(val, dtype=_numpy.bool_) + _cyb_memcpy((&(self._ptr[0].ginSupport)), (arr.data), sizeof(_cyb_bool) * len(val)) + + @property + def dev_comm_runtime_version_size(self): + """int: """ + return self._ptr[0].devCommRuntimeVersionSize + + @dev_comm_runtime_version_size.setter + def dev_comm_runtime_version_size(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].devCommRuntimeVersionSize = val + + @property + def cft_support(self): + """: """ + return (self._ptr[0].cftSupport) + + @cft_support.setter + def cft_support(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].cftSupport = <_cyb_bool>val + + @property + def cft_multicast_support(self): + """: """ + return (self._ptr[0].cftMulticastSupport) + + @cft_multicast_support.setter + def cft_multicast_support(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].cftMulticastSupport = <_cyb_bool>val + + @property + def cft_counted_support(self): + """: """ + return (self._ptr[0].cftCountedSupport) + + @cft_counted_support.setter + def cft_counted_support(self, val): + if self._readonly: + raise ValueError("This CommProperties instance is read-only") + self._ptr[0].cftCountedSupport = <_cyb_bool>val + + @staticmethod + def from_buffer(buffer): + """Create an CommProperties instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclCommProperties_t), CommProperties) + + @staticmethod + def from_data(data): + """Create an CommProperties instance wrapping the given NumPy array. + + Args: + data (_numpy.ndarray): a single-element array of dtype `comm_properties_dtype` holding the data. + """ + return _cyb_from_data(data, "comm_properties_dtype", comm_properties_dtype, CommProperties) + + @staticmethod + def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): + """Create an CommProperties instance wrapping the given pointer. + + Args: + ptr (intptr_t): pointer address as Python :class:`int` to the data. + owner (object): The Python object that owns the pointer. If not provided, data will be copied. + readonly (bool): whether the data is read-only (to the user). default is `False`. + """ + if ptr == 0: + raise ValueError("ptr must not be null (0)") + cdef CommProperties obj = CommProperties.__new__(CommProperties) + if owner is None: + obj._ptr = _cyb_malloc(sizeof(ncclCommProperties_t)) + if obj._ptr == NULL: + raise MemoryError("Error allocating CommProperties") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclCommProperties_t)) + obj._owner = None + obj._owned = True + else: + obj._ptr = ptr + obj._owner = owner + obj._owned = False + obj._readonly = readonly + return obj + + +cdef _get_config_ext_dtype_offsets(): + cdef ncclConfigExt_t pod + return _numpy.dtype({ + 'names': ['next', 'key', 'val'], + 'formats': [_numpy.intp, _py_anon_pod0_dtype, _py_anon_pod1_dtype], + 'offsets': [ + (&(pod.next)) - (&pod), + (&(pod.key)) - (&pod), + (&(pod.val)) - (&pod), + ], + 'itemsize': sizeof(ncclConfigExt_t), + }) + +config_ext_dtype = _get_config_ext_dtype_offsets() + +cdef class ConfigExt: + """Empty-initialize an instance of `ncclConfigExt_t`. + + + .. seealso:: `ncclConfigExt_t` + """ + cdef: + ncclConfigExt_t *_ptr + object _owner + bint _owned + bint _readonly + dict _refs + + def __init__(self): + self._ptr = _cyb_calloc(1, sizeof(ncclConfigExt_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating ConfigExt") + self._owner = None + self._owned = True + self._readonly = False + self._refs = {} + + def __dealloc__(self): + cdef ncclConfigExt_t *ptr + if self._owned and self._ptr != NULL: + ptr = self._ptr + self._ptr = NULL + _cyb_free(ptr) + + def __repr__(self): + return f"<{__name__}.ConfigExt object at {hex(id(self))}>" + + @property + def ptr(self): + """Get the pointer address to the data as Python :class:`int`.""" + return (self._ptr) + + cdef intptr_t _get_ptr(self): + return (self._ptr) + + def __int__(self): + return (self._ptr) + + def __eq__(self, other): + cdef ConfigExt other_ + if not isinstance(other, ConfigExt): + return False + other_ = other + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclConfigExt_t)) == 0) + + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclConfigExt_t), self._readonly) + + def __releasebuffer__(self, Py_buffer *buffer): + pass + + def __setitem__(self, key, val): + if key == 0 and isinstance(val, _numpy.ndarray): + self._ptr = _cyb_malloc(sizeof(ncclConfigExt_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating ConfigExt") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclConfigExt_t)) + self._owner = None + self._owned = True + self._readonly = not val.flags.writeable + else: + setattr(self, key, val) + + @property + def next(self): + """ConfigExt: """ + if self._ptr[0].next == NULL: + return None + ref = self._refs.get("next") + if ( + ref is not None + and (ref)._get_ptr() == (self._ptr[0].next) + ): + return ref + return ConfigExt.from_ptr( + (self._ptr[0].next), + readonly=self._readonly, + owner=self, + ) + + @next.setter + def next(self, val): + if self._readonly: + raise ValueError("This ConfigExt instance is read-only") + if val is None: + self._ptr[0].next = NULL + self._refs.pop("next", None) + return + if not isinstance(val, ConfigExt): + raise TypeError("next must be ConfigExt or None") + self._ptr[0].next = (val)._get_ptr() + self._refs["next"] = val + + @property + def key(self): + """_py_anon_pod0: """ + return _py_anon_pod0.from_ptr( + &(self._ptr[0].key), + readonly=self._readonly, + owner=self, + ) + + @key.setter + def key(self, val): + if self._readonly: + raise ValueError("This ConfigExt instance is read-only") + cdef _py_anon_pod0 val_ = val + _cyb_memcpy(&(self._ptr[0].key), (val_._get_ptr()), sizeof(nccl_bindings_nccl__anon_pod0) * 1) + + @property + def val(self): + """_py_anon_pod1: """ + return _py_anon_pod1.from_ptr( + &(self._ptr[0].val), + readonly=self._readonly, + owner=self, + ) + + @val.setter + def val(self, val): + if self._readonly: + raise ValueError("This ConfigExt instance is read-only") + cdef _py_anon_pod1 val_ = val + _cyb_memcpy(&(self._ptr[0].val), (val_._get_ptr()), sizeof(nccl_bindings_nccl__anon_pod1) * 1) + + @staticmethod + def from_buffer(buffer): + """Create an ConfigExt instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclConfigExt_t), ConfigExt) + + @staticmethod + def from_data(data): + """Create an ConfigExt instance wrapping the given NumPy array. + + Args: + data (_numpy.ndarray): a single-element array of dtype `config_ext_dtype` holding the data. + """ + return _cyb_from_data(data, "config_ext_dtype", config_ext_dtype, ConfigExt) + + @staticmethod + def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): + """Create an ConfigExt instance wrapping the given pointer. + + Args: + ptr (intptr_t): pointer address as Python :class:`int` to the data. + owner (object): The Python object that owns the pointer. If not provided, data will be copied. + readonly (bool): whether the data is read-only (to the user). default is `False`. + """ + if ptr == 0: + raise ValueError("ptr must not be null (0)") + cdef ConfigExt obj = ConfigExt.__new__(ConfigExt) + if owner is None: + obj._ptr = _cyb_malloc(sizeof(ncclConfigExt_t)) + if obj._ptr == NULL: + raise MemoryError("Error allocating ConfigExt") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclConfigExt_t)) + obj._owner = None + obj._owned = True + else: + obj._ptr = ptr + obj._owner = owner + obj._owned = False + obj._readonly = readonly + obj._refs = {} + return obj + + +cdef _get_dev_comm_dtype_offsets(): + cdef ncclDevComm_t pod + return _numpy.dtype({ + 'names': ['magic', 'version', 'rank', 'n_ranks', 'n_ranks_rcp32', 'lsa_rank', 'lsa_size', 'lsa_size_rcp32', 'window_table', 'resource_window', 'resource_window_inlined', 'hybrid_dense_gin_barrier', 'lsa_multimem', 'lsa_barrier', 'rail_gin_barrier', 'gin_connection_count', 'backend_index', 'gin_net_device_types', 'gin_handles', 'gin_signal_count', 'gin_counter_count', 'gin_signal_shadows', 'gin_context_count', 'gin_connection_stride', 'gin_context_stride', 'gin_strong_legacy_signals', 'abort_flag', 'hybrid_lsa_barrier', 'hybrid_rail_gin_barrier', 'world_gin_barrier', 'gin_connection_stride_rcp32', 'cft_rank', 'cft_size', 'cft_multimem_rank', 'cft_multimem_size', 'cft_multimem_size_rcp32', 'uc_le_id', 'mc_le_id', 'cft_barrier', 'cft_multimem_barrier'], + 'formats': [_numpy.uint32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.uint32, _numpy.intp, _numpy.intp, resource_window_vidmem_dtype, gin_barrier_handle_dtype, multimem_handle_dtype, lsa_barrier_handle_dtype, gin_barrier_handle_dtype, _numpy.uint8, _numpy.uint8, (_numpy.uint8, 4), (_numpy.int64, 4), _numpy.int32, _numpy.int32, _numpy.intp, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.bool_, _numpy.intp, lsa_barrier_handle_dtype, gin_barrier_handle_dtype, gin_barrier_handle_dtype, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.uint32, _numpy.uint32, _numpy.uint32, cft_barrier_handle_dtype, cft_barrier_handle_dtype], + 'offsets': [ + (&(pod.magic)) - (&pod), + (&(pod.version)) - (&pod), + (&(pod.rank)) - (&pod), + (&(pod.nRanks)) - (&pod), + (&(pod.nRanks_rcp32)) - (&pod), + (&(pod.lsaRank)) - (&pod), + (&(pod.lsaSize)) - (&pod), + (&(pod.lsaSize_rcp32)) - (&pod), + (&(pod.windowTable)) - (&pod), + (&(pod.resourceWindow)) - (&pod), + (&(pod.resourceWindow_inlined)) - (&pod), + (&(pod.hybridDenseGinBarrier)) - (&pod), + (&(pod.lsaMultimem)) - (&pod), + (&(pod.lsaBarrier)) - (&pod), + (&(pod.railGinBarrier)) - (&pod), + (&(pod.ginConnectionCount)) - (&pod), + (&(pod.backendIndex)) - (&pod), + (&(pod.ginNetDeviceTypes)) - (&pod), + (&(pod.ginHandles)) - (&pod), + (&(pod.ginSignalCount)) - (&pod), + (&(pod.ginCounterCount)) - (&pod), + (&(pod.ginSignalShadows)) - (&pod), + (&(pod.ginContextCount)) - (&pod), + (&(pod.ginConnectionStride)) - (&pod), + (&(pod.ginContextStride)) - (&pod), + (&(pod.ginStrongLegacySignals)) - (&pod), + (&(pod.abortFlag)) - (&pod), + (&(pod.hybridLsaBarrier)) - (&pod), + (&(pod.hybridRailGinBarrier)) - (&pod), + (&(pod.worldGinBarrier)) - (&pod), + (&(pod.ginConnectionStride_rcp32)) - (&pod), + (&(pod.cftRank)) - (&pod), + (&(pod.cftSize)) - (&pod), + (&(pod.cftMultimemRank)) - (&pod), + (&(pod.cftMultimemSize)) - (&pod), + (&(pod.cftMultimemSize_rcp32)) - (&pod), + (&(pod.ucLeId)) - (&pod), + (&(pod.mcLeId)) - (&pod), + (&(pod.cftBarrier)) - (&pod), + (&(pod.cftMultimemBarrier)) - (&pod), + ], + 'itemsize': sizeof(ncclDevComm_t), + }) + +dev_comm_dtype = _get_dev_comm_dtype_offsets() + +cdef class DevComm: + """Empty-initialize an instance of `ncclDevComm_t`. + + + .. seealso:: `ncclDevComm_t` + """ + cdef: + ncclDevComm_t *_ptr + object _owner + bint _owned + bint _readonly + + def __init__(self): + self._ptr = _cyb_calloc(1, sizeof(ncclDevComm_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating DevComm") + self._owner = None + self._owned = True + self._readonly = False + + def __dealloc__(self): + cdef ncclDevComm_t *ptr + if self._owned and self._ptr != NULL: + ptr = self._ptr + self._ptr = NULL + _cyb_free(ptr) + + def __repr__(self): + return f"<{__name__}.DevComm object at {hex(id(self))}>" + + @property + def ptr(self): + """Get the pointer address to the data as Python :class:`int`.""" + return (self._ptr) + + cdef intptr_t _get_ptr(self): + return (self._ptr) + + def __int__(self): + return (self._ptr) + + def __eq__(self, other): + cdef DevComm other_ + if not isinstance(other, DevComm): + return False + other_ = other + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclDevComm_t)) == 0) + + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclDevComm_t), self._readonly) + + def __releasebuffer__(self, Py_buffer *buffer): + pass + + def __setitem__(self, key, val): + if key == 0 and isinstance(val, _numpy.ndarray): + self._ptr = _cyb_malloc(sizeof(ncclDevComm_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating DevComm") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclDevComm_t)) + self._owner = None + self._owned = True + self._readonly = not val.flags.writeable + else: + setattr(self, key, val) + + @property + def resource_window_inlined(self): + """ResourceWindowVidmem: """ + return ResourceWindowVidmem.from_ptr( + &(self._ptr[0].resourceWindow_inlined), + readonly=self._readonly, + owner=self, + ) + + @resource_window_inlined.setter + def resource_window_inlined(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef ResourceWindowVidmem val_ = val + _cyb_memcpy(&(self._ptr[0].resourceWindow_inlined), (val_._get_ptr()), sizeof(ncclResourceWindow_vidmem_t) * 1) + + @property + def hybrid_dense_gin_barrier(self): + """GinBarrierHandle: """ + return GinBarrierHandle.from_ptr( + &(self._ptr[0].hybridDenseGinBarrier), + readonly=self._readonly, + owner=self, + ) + + @hybrid_dense_gin_barrier.setter + def hybrid_dense_gin_barrier(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef GinBarrierHandle val_ = val + _cyb_memcpy(&(self._ptr[0].hybridDenseGinBarrier), (val_._get_ptr()), sizeof(ncclGinBarrierHandle_t) * 1) + + @property + def lsa_multimem(self): + """MultimemHandle: """ + return MultimemHandle.from_ptr( + &(self._ptr[0].lsaMultimem), + readonly=self._readonly, + owner=self, + ) + + @lsa_multimem.setter + def lsa_multimem(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef MultimemHandle val_ = val + _cyb_memcpy(&(self._ptr[0].lsaMultimem), (val_._get_ptr()), sizeof(ncclMultimemHandle_t) * 1) + + @property + def lsa_barrier(self): + """LsaBarrierHandle: """ + return LsaBarrierHandle.from_ptr( + &(self._ptr[0].lsaBarrier), + readonly=self._readonly, + owner=self, + ) + + @lsa_barrier.setter + def lsa_barrier(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef LsaBarrierHandle val_ = val + _cyb_memcpy(&(self._ptr[0].lsaBarrier), (val_._get_ptr()), sizeof(ncclLsaBarrierHandle_t) * 1) + + @property + def rail_gin_barrier(self): + """GinBarrierHandle: """ + return GinBarrierHandle.from_ptr( + &(self._ptr[0].railGinBarrier), + readonly=self._readonly, + owner=self, + ) + + @rail_gin_barrier.setter + def rail_gin_barrier(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef GinBarrierHandle val_ = val + _cyb_memcpy(&(self._ptr[0].railGinBarrier), (val_._get_ptr()), sizeof(ncclGinBarrierHandle_t) * 1) + + @property + def hybrid_lsa_barrier(self): + """LsaBarrierHandle: """ + return LsaBarrierHandle.from_ptr( + &(self._ptr[0].hybridLsaBarrier), + readonly=self._readonly, + owner=self, + ) + + @hybrid_lsa_barrier.setter + def hybrid_lsa_barrier(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef LsaBarrierHandle val_ = val + _cyb_memcpy(&(self._ptr[0].hybridLsaBarrier), (val_._get_ptr()), sizeof(ncclLsaBarrierHandle_t) * 1) + + @property + def hybrid_rail_gin_barrier(self): + """GinBarrierHandle: """ + return GinBarrierHandle.from_ptr( + &(self._ptr[0].hybridRailGinBarrier), + readonly=self._readonly, + owner=self, + ) + + @hybrid_rail_gin_barrier.setter + def hybrid_rail_gin_barrier(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef GinBarrierHandle val_ = val + _cyb_memcpy(&(self._ptr[0].hybridRailGinBarrier), (val_._get_ptr()), sizeof(ncclGinBarrierHandle_t) * 1) + + @property + def world_gin_barrier(self): + """GinBarrierHandle: """ + return GinBarrierHandle.from_ptr( + &(self._ptr[0].worldGinBarrier), + readonly=self._readonly, + owner=self, + ) + + @world_gin_barrier.setter + def world_gin_barrier(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef GinBarrierHandle val_ = val + _cyb_memcpy(&(self._ptr[0].worldGinBarrier), (val_._get_ptr()), sizeof(ncclGinBarrierHandle_t) * 1) + + @property + def cft_barrier(self): + """CftBarrierHandle: """ + return CftBarrierHandle.from_ptr( + &(self._ptr[0].cftBarrier), + readonly=self._readonly, + owner=self, + ) + + @cft_barrier.setter + def cft_barrier(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef CftBarrierHandle val_ = val + _cyb_memcpy(&(self._ptr[0].cftBarrier), (val_._get_ptr()), sizeof(ncclCftBarrierHandle_t) * 1) + + @property + def cft_multimem_barrier(self): + """CftBarrierHandle: """ + return CftBarrierHandle.from_ptr( + &(self._ptr[0].cftMultimemBarrier), + readonly=self._readonly, + owner=self, + ) + + @cft_multimem_barrier.setter + def cft_multimem_barrier(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + cdef CftBarrierHandle val_ = val + _cyb_memcpy(&(self._ptr[0].cftMultimemBarrier), (val_._get_ptr()), sizeof(ncclCftBarrierHandle_t) * 1) + + @property + def magic(self): + """int: """ + return self._ptr[0].magic + + @magic.setter + def magic(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].magic = val + + @property + def version(self): + """int: """ + return self._ptr[0].version + + @version.setter + def version(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].version = val + + @property + def rank(self): + """int: """ + return self._ptr[0].rank + + @rank.setter + def rank(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].rank = val + + @property + def n_ranks(self): + """int: """ + return self._ptr[0].nRanks + + @n_ranks.setter + def n_ranks(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].nRanks = val + + @property + def n_ranks_rcp32(self): + """int: """ + return self._ptr[0].nRanks_rcp32 + + @n_ranks_rcp32.setter + def n_ranks_rcp32(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].nRanks_rcp32 = val + + @property + def lsa_rank(self): + """int: """ + return self._ptr[0].lsaRank + + @lsa_rank.setter + def lsa_rank(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].lsaRank = val + + @property + def lsa_size(self): + """int: """ + return self._ptr[0].lsaSize + + @lsa_size.setter + def lsa_size(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].lsaSize = val + + @property + def lsa_size_rcp32(self): + """int: """ + return self._ptr[0].lsaSize_rcp32 + + @lsa_size_rcp32.setter + def lsa_size_rcp32(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].lsaSize_rcp32 = val + + @property + def window_table(self): + """int: """ + return (self._ptr[0].windowTable) + + @window_table.setter + def window_table(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].windowTable = val + + @property + def resource_window(self): + """int: """ + return (self._ptr[0].resourceWindow) + + @resource_window.setter + def resource_window(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].resourceWindow = val + + @property + def gin_connection_count(self): + """int: """ + return self._ptr[0].ginConnectionCount + + @gin_connection_count.setter + def gin_connection_count(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginConnectionCount = val + + @property + def backend_index(self): + """int: """ + return self._ptr[0].backendIndex + + @backend_index.setter + def backend_index(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].backendIndex = val + + @property + def gin_net_device_types(self): + """~_numpy.uint8: (array of length 4).""" + cdef _cyb_view.array arr = _cyb_view.array(shape=(4,), itemsize=sizeof(uint8_t), format="B", mode="c", allocate_buffer=False) + arr.data = (&(self._ptr[0].ginNetDeviceTypes)) + return _numpy.asarray(arr) + + @gin_net_device_types.setter + def gin_net_device_types(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + if len(val) != 4: + raise ValueError(f"Expected length { 4 } for field gin_net_device_types, got {len(val)}") + cdef _cyb_view.array arr = _cyb_view.array(shape=(4,), itemsize=sizeof(uint8_t), format="B", mode="c") + arr[:] = _numpy.asarray(val, dtype=_numpy.uint8) + _cyb_memcpy((&(self._ptr[0].ginNetDeviceTypes)), (arr.data), sizeof(uint8_t) * len(val)) + + @property + def gin_handles(self): + """~_numpy.int64: (array of length 4).""" + cdef _cyb_view.array arr = _cyb_view.array(shape=(4,), itemsize=sizeof(intptr_t), format="q", mode="c", allocate_buffer=False) + arr.data = (&(self._ptr[0].ginHandles)) + return _numpy.asarray(arr) + + @gin_handles.setter + def gin_handles(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + if len(val) != 4: + raise ValueError(f"Expected length { 4 } for field gin_handles, got {len(val)}") + cdef _cyb_view.array arr = _cyb_view.array(shape=(4,), itemsize=sizeof(intptr_t), format="q", mode="c") + arr[:] = _numpy.asarray(val, dtype=_numpy.intp) + _cyb_memcpy((&(self._ptr[0].ginHandles)), (arr.data), sizeof(intptr_t) * len(val)) + + @property + def gin_signal_count(self): + """int: """ + return self._ptr[0].ginSignalCount + + @gin_signal_count.setter + def gin_signal_count(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginSignalCount = val + + @property + def gin_counter_count(self): + """int: """ + return self._ptr[0].ginCounterCount + + @gin_counter_count.setter + def gin_counter_count(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginCounterCount = val + + @property + def gin_signal_shadows(self): + """int: """ + return (self._ptr[0].ginSignalShadows) + + @gin_signal_shadows.setter + def gin_signal_shadows(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginSignalShadows = val + + @property + def gin_context_count(self): + """int: """ + return self._ptr[0].ginContextCount + + @gin_context_count.setter + def gin_context_count(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginContextCount = val + + @property + def gin_connection_stride(self): + """int: """ + return self._ptr[0].ginConnectionStride + + @gin_connection_stride.setter + def gin_connection_stride(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginConnectionStride = val + + @property + def gin_context_stride(self): + """int: """ + return self._ptr[0].ginContextStride + + @gin_context_stride.setter + def gin_context_stride(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginContextStride = val + + @property + def gin_strong_legacy_signals(self): + """: """ + return (self._ptr[0].ginStrongLegacySignals) + + @gin_strong_legacy_signals.setter + def gin_strong_legacy_signals(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginStrongLegacySignals = <_cyb_bool>val + + @property + def abort_flag(self): + """int: """ + return (self._ptr[0].abortFlag) + + @abort_flag.setter + def abort_flag(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].abortFlag = val + + @property + def gin_connection_stride_rcp32(self): + """int: """ + return self._ptr[0].ginConnectionStride_rcp32 + + @gin_connection_stride_rcp32.setter + def gin_connection_stride_rcp32(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ginConnectionStride_rcp32 = val + + @property + def cft_rank(self): + """int: """ + return self._ptr[0].cftRank + + @cft_rank.setter + def cft_rank(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].cftRank = val + + @property + def cft_size(self): + """int: """ + return self._ptr[0].cftSize + + @cft_size.setter + def cft_size(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].cftSize = val + + @property + def cft_multimem_rank(self): + """int: """ + return self._ptr[0].cftMultimemRank + + @cft_multimem_rank.setter + def cft_multimem_rank(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].cftMultimemRank = val + + @property + def cft_multimem_size(self): + """int: """ + return self._ptr[0].cftMultimemSize + + @cft_multimem_size.setter + def cft_multimem_size(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].cftMultimemSize = val + + @property + def cft_multimem_size_rcp32(self): + """int: """ + return self._ptr[0].cftMultimemSize_rcp32 + + @cft_multimem_size_rcp32.setter + def cft_multimem_size_rcp32(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].cftMultimemSize_rcp32 = val + + @property + def uc_le_id(self): + """int: """ + return (self._ptr[0].ucLeId) + + @uc_le_id.setter + def uc_le_id(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].ucLeId = val + + @property + def mc_le_id(self): + """int: """ + return (self._ptr[0].mcLeId) + + @mc_le_id.setter + def mc_le_id(self, val): + if self._readonly: + raise ValueError("This DevComm instance is read-only") + self._ptr[0].mcLeId = val + + @staticmethod + def from_buffer(buffer): + """Create an DevComm instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclDevComm_t), DevComm) + + @staticmethod + def from_data(data): + """Create an DevComm instance wrapping the given NumPy array. + + Args: + data (_numpy.ndarray): a single-element array of dtype `dev_comm_dtype` holding the data. + """ + return _cyb_from_data(data, "dev_comm_dtype", dev_comm_dtype, DevComm) + + @staticmethod + def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): + """Create an DevComm instance wrapping the given pointer. + + Args: + ptr (intptr_t): pointer address as Python :class:`int` to the data. + owner (object): The Python object that owns the pointer. If not provided, data will be copied. + readonly (bool): whether the data is read-only (to the user). default is `False`. + """ + if ptr == 0: + raise ValueError("ptr must not be null (0)") + cdef DevComm obj = DevComm.__new__(DevComm) + if owner is None: + obj._ptr = _cyb_malloc(sizeof(ncclDevComm_t)) + if obj._ptr == NULL: + raise MemoryError("Error allocating DevComm") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclDevComm_t)) + obj._owner = None + obj._owned = True + else: + obj._ptr = ptr + obj._owner = owner + obj._owned = False + obj._readonly = readonly + return obj + + +cdef _get_team_requirements_dtype_offsets(): + cdef ncclTeamRequirements_t pod + return _numpy.dtype({ + 'names': ['next', 'team', 'multimem', 'out_multimem_handle'], + 'formats': [_numpy.intp, team_dtype, _numpy.bool_, _numpy.intp], + 'offsets': [ + (&(pod.next)) - (&pod), + (&(pod.team)) - (&pod), + (&(pod.multimem)) - (&pod), + (&(pod.outMultimemHandle)) - (&pod), + ], + 'itemsize': sizeof(ncclTeamRequirements_t), + }) + +team_requirements_dtype = _get_team_requirements_dtype_offsets() + +cdef class TeamRequirements: + """Empty-initialize an instance of `ncclTeamRequirements_t`. + + + .. seealso:: `ncclTeamRequirements_t` + """ + cdef: + ncclTeamRequirements_t *_ptr + object _owner + bint _owned + bint _readonly + dict _refs + + def __init__(self): + self._ptr = _cyb_calloc(1, sizeof(ncclTeamRequirements_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating TeamRequirements") + self._owner = None + self._owned = True + self._readonly = False + self._refs = {} + + def __dealloc__(self): + cdef ncclTeamRequirements_t *ptr + if self._owned and self._ptr != NULL: + ptr = self._ptr + self._ptr = NULL + _cyb_free(ptr) + + def __repr__(self): + return f"<{__name__}.TeamRequirements object at {hex(id(self))}>" + + @property + def ptr(self): + """Get the pointer address to the data as Python :class:`int`.""" + return (self._ptr) + + cdef intptr_t _get_ptr(self): + return (self._ptr) + + def __int__(self): + return (self._ptr) + + def __eq__(self, other): + cdef TeamRequirements other_ + if not isinstance(other, TeamRequirements): + return False + other_ = other + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclTeamRequirements_t)) == 0) + + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclTeamRequirements_t), self._readonly) + + def __releasebuffer__(self, Py_buffer *buffer): + pass + + def __setitem__(self, key, val): + if key == 0 and isinstance(val, _numpy.ndarray): + self._ptr = _cyb_malloc(sizeof(ncclTeamRequirements_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating TeamRequirements") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclTeamRequirements_t)) + self._owner = None + self._owned = True + self._readonly = not val.flags.writeable + else: + setattr(self, key, val) + + @property + def next(self): + """TeamRequirements: """ + if self._ptr[0].next == NULL: + return None + ref = self._refs.get("next") + if ( + ref is not None + and (ref)._get_ptr() == (self._ptr[0].next) + ): + return ref + return TeamRequirements.from_ptr( + (self._ptr[0].next), + readonly=self._readonly, + owner=self, + ) + + @next.setter + def next(self, val): + if self._readonly: + raise ValueError("This TeamRequirements instance is read-only") + if val is None: + self._ptr[0].next = NULL + self._refs.pop("next", None) + return + if not isinstance(val, TeamRequirements): + raise TypeError("next must be TeamRequirements or None") + self._ptr[0].next = (val)._get_ptr() + self._refs["next"] = val + + @property + def team(self): + """Team: """ + return Team.from_ptr( + &(self._ptr[0].team), + readonly=self._readonly, + owner=self, + ) + + @team.setter + def team(self, val): + if self._readonly: + raise ValueError("This TeamRequirements instance is read-only") + cdef Team val_ = val + _cyb_memcpy(&(self._ptr[0].team), (val_._get_ptr()), sizeof(ncclTeam_t) * 1) + + @property + def multimem(self): + """: """ + return (self._ptr[0].multimem) + + @multimem.setter + def multimem(self, val): + if self._readonly: + raise ValueError("This TeamRequirements instance is read-only") + self._ptr[0].multimem = <_cyb_bool>val + + @property + def out_multimem_handle(self): + """int: """ + return (self._ptr[0].outMultimemHandle) + + @out_multimem_handle.setter + def out_multimem_handle(self, val): + if self._readonly: + raise ValueError("This TeamRequirements instance is read-only") + self._ptr[0].outMultimemHandle = val + + @staticmethod + def from_buffer(buffer): + """Create an TeamRequirements instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclTeamRequirements_t), TeamRequirements) + + @staticmethod + def from_data(data): + """Create an TeamRequirements instance wrapping the given NumPy array. + + Args: + data (_numpy.ndarray): a single-element array of dtype `team_requirements_dtype` holding the data. + """ + return _cyb_from_data(data, "team_requirements_dtype", team_requirements_dtype, TeamRequirements) + + @staticmethod + def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): + """Create an TeamRequirements instance wrapping the given pointer. + + Args: + ptr (intptr_t): pointer address as Python :class:`int` to the data. + owner (object): The Python object that owns the pointer. If not provided, data will be copied. + readonly (bool): whether the data is read-only (to the user). default is `False`. + """ + if ptr == 0: + raise ValueError("ptr must not be null (0)") + cdef TeamRequirements obj = TeamRequirements.__new__(TeamRequirements) + if owner is None: + obj._ptr = _cyb_malloc(sizeof(ncclTeamRequirements_t)) + if obj._ptr == NULL: + raise MemoryError("Error allocating TeamRequirements") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclTeamRequirements_t)) + obj._owner = None + obj._owned = True + else: + obj._ptr = ptr + obj._owner = owner + obj._owned = False + obj._readonly = readonly + obj._refs = {} + return obj + + +cdef _get_coll_config_dtype_offsets(): + cdef ncclCollConfig_t pod + return _numpy.dtype({ + 'names': ['size_', 'magic', 'version', 'ext', 'min_ctas', 'max_ctas', 'nvls_ctas', 'cga_cluster_size', 'alg_selection', 'force_alg_selection', 'cta_policy', 'user_profiler_tag', 'launch_completion_event_handle'], + 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.uint64, _numpy.intp], + 'offsets': [ + (&(pod.size)) - (&pod), + (&(pod.magic)) - (&pod), + (&(pod.version)) - (&pod), + (&(pod.ext)) - (&pod), + (&(pod.minCTAs)) - (&pod), + (&(pod.maxCTAs)) - (&pod), + (&(pod.nvlsCTAs)) - (&pod), + (&(pod.cgaClusterSize)) - (&pod), + (&(pod.algSelection)) - (&pod), + (&(pod.forceAlgSelection)) - (&pod), + (&(pod.CTAPolicy)) - (&pod), + (&(pod.userProfilerTag)) - (&pod), + (&(pod.launchCompletionEvent)) - (&pod), + ], + 'itemsize': sizeof(ncclCollConfig_t), + }) + +coll_config_dtype = _get_coll_config_dtype_offsets() + +cdef class CollConfig: + """Initialize an instance of `ncclCollConfig_t` using configured defaults. + + + .. seealso:: `ncclCollConfig_t` + """ + cdef: + ncclCollConfig_t *_ptr + object _owner + bint _owned + bint _readonly + dict _refs + + def __init__(self): + self._ptr = _cyb_calloc(1, sizeof(ncclCollConfig_t)) + if self._ptr == NULL: + raise MemoryError("Error allocating CollConfig") + self._owner = None + self._owned = True + self._readonly = False + self._refs = {} + + self._ptr[0].size = sizeof(ncclCollConfig_t) + self._ptr[0].magic = 0xcafebeef + self._ptr[0].version = __version_code__ + self._ptr[0].ext = NULL + self._ptr[0].minCTAs = -2147483648 + self._ptr[0].maxCTAs = -2147483648 + self._ptr[0].nvlsCTAs = -2147483648 + self._ptr[0].cgaClusterSize = -2147483648 + self._ptr[0].algSelection = NULL + self._ptr[0].forceAlgSelection = 1 + self._ptr[0].CTAPolicy = -2147483648 + self._ptr[0].userProfilerTag = 0 + self._ptr[0].launchCompletionEvent = NULL + + def __dealloc__(self): + cdef ncclCollConfig_t *ptr + if self._owned and self._ptr != NULL: + ptr = self._ptr + self._ptr = NULL + _cyb_free(ptr) + + def __repr__(self): + return f"<{__name__}.CollConfig object at {hex(id(self))}>" @property def ptr(self): @@ -2653,24 +4310,24 @@ cdef class DevComm: return (self._ptr) def __eq__(self, other): - cdef DevComm other_ - if not isinstance(other, DevComm): + cdef CollConfig other_ + if not isinstance(other, CollConfig): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclDevComm_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclCollConfig_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclDevComm_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclCollConfig_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclDevComm_t)) + self._ptr = _cyb_malloc(sizeof(ncclCollConfig_t)) if self._ptr == NULL: - raise MemoryError("Error allocating DevComm") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclDevComm_t)) + raise MemoryError("Error allocating CollConfig") + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclCollConfig_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -2678,100 +4335,15 @@ cdef class DevComm: setattr(self, key, val) @property - def resource_window_inlined(self): - """ResourceWindow_vidmem: """ - return ResourceWindow_vidmem.from_ptr(&(self._ptr[0].resourceWindow_inlined), self._readonly, self) - - @resource_window_inlined.setter - def resource_window_inlined(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - cdef ResourceWindow_vidmem val_ = val - memcpy(&(self._ptr[0].resourceWindow_inlined), (val_._get_ptr()), sizeof(ncclResourceWindow_vidmem_t) * 1) - - @property - def hybrid_world_gin_barrier(self): - """GinBarrierHandle: """ - return GinBarrierHandle.from_ptr(&(self._ptr[0].hybridWorldGinBarrier), self._readonly, self) - - @hybrid_world_gin_barrier.setter - def hybrid_world_gin_barrier(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - cdef GinBarrierHandle val_ = val - memcpy(&(self._ptr[0].hybridWorldGinBarrier), (val_._get_ptr()), sizeof(ncclGinBarrierHandle_t) * 1) - - @property - def lsa_multimem(self): - """MultimemHandle: """ - return MultimemHandle.from_ptr(&(self._ptr[0].lsaMultimem), self._readonly, self) - - @lsa_multimem.setter - def lsa_multimem(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - cdef MultimemHandle val_ = val - memcpy(&(self._ptr[0].lsaMultimem), (val_._get_ptr()), sizeof(ncclMultimemHandle_t) * 1) - - @property - def lsa_barrier(self): - """LsaBarrierHandle: """ - return LsaBarrierHandle.from_ptr(&(self._ptr[0].lsaBarrier), self._readonly, self) - - @lsa_barrier.setter - def lsa_barrier(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - cdef LsaBarrierHandle val_ = val - memcpy(&(self._ptr[0].lsaBarrier), (val_._get_ptr()), sizeof(ncclLsaBarrierHandle_t) * 1) - - @property - def rail_gin_barrier(self): - """GinBarrierHandle: """ - return GinBarrierHandle.from_ptr(&(self._ptr[0].railGinBarrier), self._readonly, self) - - @rail_gin_barrier.setter - def rail_gin_barrier(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - cdef GinBarrierHandle val_ = val - memcpy(&(self._ptr[0].railGinBarrier), (val_._get_ptr()), sizeof(ncclGinBarrierHandle_t) * 1) - - @property - def hybrid_lsa_barrier(self): - """LsaBarrierHandle: """ - return LsaBarrierHandle.from_ptr(&(self._ptr[0].hybridLsaBarrier), self._readonly, self) - - @hybrid_lsa_barrier.setter - def hybrid_lsa_barrier(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - cdef LsaBarrierHandle val_ = val - memcpy(&(self._ptr[0].hybridLsaBarrier), (val_._get_ptr()), sizeof(ncclLsaBarrierHandle_t) * 1) - - @property - def hybrid_rail_gin_barrier(self): - """GinBarrierHandle: """ - return GinBarrierHandle.from_ptr(&(self._ptr[0].hybridRailGinBarrier), self._readonly, self) - - @hybrid_rail_gin_barrier.setter - def hybrid_rail_gin_barrier(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - cdef GinBarrierHandle val_ = val - memcpy(&(self._ptr[0].hybridRailGinBarrier), (val_._get_ptr()), sizeof(ncclGinBarrierHandle_t) * 1) - - @property - def world_gin_barrier(self): - """GinBarrierHandle: """ - return GinBarrierHandle.from_ptr(&(self._ptr[0].worldGinBarrier), self._readonly, self) + def size_(self): + """int: """ + return self._ptr[0].size - @world_gin_barrier.setter - def world_gin_barrier(self, val): + @size_.setter + def size_(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - cdef GinBarrierHandle val_ = val - memcpy(&(self._ptr[0].worldGinBarrier), (val_._get_ptr()), sizeof(ncclGinBarrierHandle_t) * 1) + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].size = val @property def magic(self): @@ -2781,7 +4353,7 @@ cdef class DevComm: @magic.setter def magic(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") + raise ValueError("This CollConfig instance is read-only") self._ptr[0].magic = val @property @@ -2792,247 +4364,142 @@ cdef class DevComm: @version.setter def version(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") + raise ValueError("This CollConfig instance is read-only") self._ptr[0].version = val @property - def rank(self): - """int: """ - return self._ptr[0].rank - - @rank.setter - def rank(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].rank = val - - @property - def n_ranks(self): - """int: """ - return self._ptr[0].nRanks - - @n_ranks.setter - def n_ranks(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].nRanks = val - - @property - def n_ranks_rcp32(self): - """int: """ - return self._ptr[0].nRanks_rcp32 - - @n_ranks_rcp32.setter - def n_ranks_rcp32(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].nRanks_rcp32 = val - - @property - def lsa_rank(self): - """int: """ - return self._ptr[0].lsaRank - - @lsa_rank.setter - def lsa_rank(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].lsaRank = val - - @property - def lsa_size(self): - """int: """ - return self._ptr[0].lsaSize - - @lsa_size.setter - def lsa_size(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].lsaSize = val - - @property - def lsa_size_rcp32(self): - """int: """ - return self._ptr[0].lsaSize_rcp32 - - @lsa_size_rcp32.setter - def lsa_size_rcp32(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].lsaSize_rcp32 = val - - @property - def window_table(self): - """int: """ - return (self._ptr[0].windowTable) - - @window_table.setter - def window_table(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].windowTable = val - - @property - def resource_window(self): + def ext(self): """int: """ - return (self._ptr[0].resourceWindow) + return (self._ptr[0].ext) - @resource_window.setter - def resource_window(self, val): + @ext.setter + def ext(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].resourceWindow = val + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].ext = val @property - def gin_connection_count(self): + def min_ctas(self): """int: """ - return self._ptr[0].ginConnectionCount - - @gin_connection_count.setter - def gin_connection_count(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].ginConnectionCount = val - - @property - def gin_net_device_types(self): - """~_numpy.uint8: (array of length 4).""" - cdef view.array arr = view.array(shape=(4,), itemsize=sizeof(uint8_t), format="B", mode="c", allocate_buffer=False) - arr.data = (&(self._ptr[0].ginNetDeviceTypes)) - return _numpy.asarray(arr) - - @gin_net_device_types.setter - def gin_net_device_types(self, val): - if self._readonly: - raise ValueError("This DevComm instance is read-only") - if len(val) != 4: - raise ValueError(f"Expected length { 4 } for field gin_net_device_types, got {len(val)}") - cdef view.array arr = view.array(shape=(4,), itemsize=sizeof(uint8_t), format="B", mode="c") - arr[:] = _numpy.asarray(val, dtype=_numpy.uint8) - memcpy((&(self._ptr[0].ginNetDeviceTypes)), (arr.data), sizeof(uint8_t) * len(val)) - - @property - def gin_handles(self): - """~_numpy.int64: (array of length 4).""" - cdef view.array arr = view.array(shape=(4,), itemsize=sizeof(intptr_t), format="q", mode="c", allocate_buffer=False) - arr.data = (&(self._ptr[0].ginHandles)) - return _numpy.asarray(arr) + return self._ptr[0].minCTAs - @gin_handles.setter - def gin_handles(self, val): + @min_ctas.setter + def min_ctas(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - if len(val) != 4: - raise ValueError(f"Expected length { 4 } for field gin_handles, got {len(val)}") - cdef view.array arr = view.array(shape=(4,), itemsize=sizeof(intptr_t), format="q", mode="c") - arr[:] = _numpy.asarray(val, dtype=_numpy.intp) - memcpy((&(self._ptr[0].ginHandles)), (arr.data), sizeof(intptr_t) * len(val)) + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].minCTAs = val @property - def gin_signal_count(self): + def max_ctas(self): """int: """ - return self._ptr[0].ginSignalCount + return self._ptr[0].maxCTAs - @gin_signal_count.setter - def gin_signal_count(self, val): + @max_ctas.setter + def max_ctas(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].ginSignalCount = val + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].maxCTAs = val @property - def gin_counter_count(self): + def nvls_ctas(self): """int: """ - return self._ptr[0].ginCounterCount + return self._ptr[0].nvlsCTAs - @gin_counter_count.setter - def gin_counter_count(self, val): + @nvls_ctas.setter + def nvls_ctas(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].ginCounterCount = val + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].nvlsCTAs = val @property - def gin_signal_shadows(self): + def cga_cluster_size(self): """int: """ - return (self._ptr[0].ginSignalShadows) + return self._ptr[0].cgaClusterSize - @gin_signal_shadows.setter - def gin_signal_shadows(self, val): + @cga_cluster_size.setter + def cga_cluster_size(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].ginSignalShadows = val + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].cgaClusterSize = val @property - def gin_context_count(self): - """int: """ - return self._ptr[0].ginContextCount + def alg_selection(self): + """str: """ + cdef char* ptr = self._ptr[0].algSelection + if ptr: + return _cyb_cpython.PyUnicode_FromString(ptr) + return "" - @gin_context_count.setter - def gin_context_count(self, val): + @alg_selection.setter + def alg_selection(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].ginContextCount = val + raise ValueError("This CollConfig instance is read-only") + cdef bytes buf = val.encode() + cdef char *ptr = buf + self._refs["alg_selection"] = buf + self._ptr[0].algSelection = ptr @property - def gin_connections_railed(self): + def force_alg_selection(self): """int: """ - return self._ptr[0].ginConnectionsRailed + return self._ptr[0].forceAlgSelection - @gin_connections_railed.setter - def gin_connections_railed(self, val): + @force_alg_selection.setter + def force_alg_selection(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].ginConnectionsRailed = val + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].forceAlgSelection = val @property - def gin_strong_legacy_signals(self): + def cta_policy(self): """int: """ - return self._ptr[0].ginStrongLegacySignals - - @gin_strong_legacy_signals.setter - def gin_strong_legacy_signals(self, val): + return self._ptr[0].CTAPolicy + + @cta_policy.setter + def cta_policy(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].ginStrongLegacySignals = val + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].CTAPolicy = val @property - def gin_contexts_railed(self): + def user_profiler_tag(self): """int: """ - return self._ptr[0].ginContextsRailed + return self._ptr[0].userProfilerTag - @gin_contexts_railed.setter - def gin_contexts_railed(self, val): + @user_profiler_tag.setter + def user_profiler_tag(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].ginContextsRailed = val + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].userProfilerTag = val @property - def abort_flag(self): + def launch_completion_event_handle(self): """int: """ - return (self._ptr[0].abortFlag) + return (self._ptr[0].launchCompletionEvent) - @abort_flag.setter - def abort_flag(self, val): + @launch_completion_event_handle.setter + def launch_completion_event_handle(self, val): if self._readonly: - raise ValueError("This DevComm instance is read-only") - self._ptr[0].abortFlag = val + raise ValueError("This CollConfig instance is read-only") + self._ptr[0].launchCompletionEvent = val @staticmethod def from_buffer(buffer): - """Create an DevComm instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclDevComm_t), DevComm) + """Create an CollConfig instance with the memory from the given buffer.""" + return _cyb_from_buffer(buffer, sizeof(ncclCollConfig_t), CollConfig) @staticmethod def from_data(data): - """Create an DevComm instance wrapping the given NumPy array. + """Create an CollConfig instance wrapping the given NumPy array. Args: - data (_numpy.ndarray): a single-element array of dtype `dev_comm_dtype` holding the data. + data (_numpy.ndarray): a single-element array of dtype `coll_config_dtype` holding the data. """ - return __from_data(data, "dev_comm_dtype", dev_comm_dtype, DevComm) + return _cyb_from_data(data, "coll_config_dtype", coll_config_dtype, CollConfig) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): - """Create an DevComm instance wrapping the given pointer. + """Create an CollConfig instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. @@ -3041,27 +4508,28 @@ cdef class DevComm: """ if ptr == 0: raise ValueError("ptr must not be null (0)") - cdef DevComm obj = DevComm.__new__(DevComm) + cdef CollConfig obj = CollConfig.__new__(CollConfig) if owner is None: - obj._ptr = malloc(sizeof(ncclDevComm_t)) + obj._ptr = _cyb_malloc(sizeof(ncclCollConfig_t)) if obj._ptr == NULL: - raise MemoryError("Error allocating DevComm") - memcpy((obj._ptr), ptr, sizeof(ncclDevComm_t)) + raise MemoryError("Error allocating CollConfig") + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclCollConfig_t)) obj._owner = None obj._owned = True else: - obj._ptr = ptr + obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly + obj._refs = {} return obj cdef _get_dev_comm_requirements_dtype_offsets(): - cdef ncclDevCommRequirements_t pod = ncclDevCommRequirements_t() + cdef ncclDevCommRequirements_t pod return _numpy.dtype({ - 'names': ['size_', 'magic', 'version', 'resource_requirements_list', 'team_requirements_list', 'lsa_multimem', 'barrier_count', 'lsa_barrier_count', 'rail_gin_barrier_count', 'lsa_ll_a2a_block_count', 'lsa_ll_a2a_slot_count', 'gin_force_enable', 'gin_context_count', 'gin_signal_count', 'gin_counter_count', 'gin_connection_type', 'gin_exclusive_contexts', 'gin_queue_depth', 'gin_traffic_class', 'world_gin_barrier_count', 'gin_strong_signals_required', 'gin_va_signals_required'], - 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.intp, _numpy.intp, _numpy.uint8, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.uint8, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.uint8, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.uint8, _numpy.uint8], + 'names': ['size_', 'magic', 'version', 'resource_requirements_list', 'team_requirements_list', 'lsa_multimem', 'barrier_count', 'lsa_barrier_count', 'rail_gin_barrier_count', 'lsa_ll_a2a_block_count', 'lsa_ll_a2a_slot_count', 'gin_force_enable', 'gin_context_count', 'gin_signal_count', 'gin_counter_count', 'gin_connection_type', 'gin_exclusive_contexts', 'gin_queue_depth', 'gin_traffic_class', 'world_gin_barrier_count', 'gin_strong_signals_required', 'gin_va_signals_required', 'gin_custom_stride', 'gin_type', 'use_runtime_version', 'cft_caps', 'cft_barrier_count'], + 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.intp, _numpy.intp, _numpy.bool_, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.bool_, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.bool_, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.bool_, _numpy.bool_, _numpy.int32, _numpy.int32, _numpy.bool_, _numpy.int32, _numpy.int32], 'offsets': [ (&(pod.size)) - (&pod), (&(pod.magic)) - (&pod), @@ -3085,6 +4553,11 @@ cdef _get_dev_comm_requirements_dtype_offsets(): (&(pod.worldGinBarrierCount)) - (&pod), (&(pod.ginStrongSignalsRequired)) - (&pod), (&(pod.ginVaSignalsRequired)) - (&pod), + (&(pod.ginCustomStride)) - (&pod), + (&(pod.ginType)) - (&pod), + (&(pod.useRuntimeVersion)) - (&pod), + (&(pod.cftCaps)) - (&pod), + (&(pod.cftBarrierCount)) - (&pod), ], 'itemsize': sizeof(ncclDevCommRequirements_t), }) @@ -3104,7 +4577,7 @@ cdef class DevCommRequirements: bint _readonly def __init__(self): - self._ptr = calloc(1, sizeof(ncclDevCommRequirements_t)) + self._ptr = _cyb_calloc(1, sizeof(ncclDevCommRequirements_t)) if self._ptr == NULL: raise MemoryError("Error allocating DevCommRequirements") self._owner = None @@ -3113,19 +4586,38 @@ cdef class DevCommRequirements: self._ptr[0].size = sizeof(ncclDevCommRequirements_t) self._ptr[0].magic = 0xcafebeef - self._ptr[0].version = 23007 + self._ptr[0].version = __version_code__ + self._ptr[0].resourceRequirementsList = NULL + self._ptr[0].teamRequirementsList = NULL + self._ptr[0].lsaMultimem = 0 + self._ptr[0].barrierCount = 0 + self._ptr[0].lsaBarrierCount = 0 + self._ptr[0].railGinBarrierCount = 0 + self._ptr[0].lsaLLA2ABlockCount = 0 + self._ptr[0].lsaLLA2ASlotCount = 0 + self._ptr[0].ginForceEnable = 0 self._ptr[0].ginContextCount = 4 + self._ptr[0].ginSignalCount = 0 + self._ptr[0].ginCounterCount = 0 self._ptr[0].ginConnectionType = NCCL_GIN_CONNECTION_NONE + self._ptr[0].ginExclusiveContexts = 0 + self._ptr[0].ginQueueDepth = 0 self._ptr[0].ginTrafficClass = -2147483648 + self._ptr[0].worldGinBarrierCount = 0 self._ptr[0].ginStrongSignalsRequired = 1 self._ptr[0].ginVaSignalsRequired = 1 + self._ptr[0].ginCustomStride = 1 + self._ptr[0].ginType = NCCL_GIN_TYPE_NONE + self._ptr[0].useRuntimeVersion = 0 + self._ptr[0].cftCaps = 0 + self._ptr[0].cftBarrierCount = 0 def __dealloc__(self): cdef ncclDevCommRequirements_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL - free(ptr) + _cyb_free(ptr) def __repr__(self): return f"<{__name__}.DevCommRequirements object at {hex(id(self))}>" @@ -3146,20 +4638,20 @@ cdef class DevCommRequirements: if not isinstance(other, DevCommRequirements): return False other_ = other - return (memcmp((self._ptr), (other_._ptr), sizeof(ncclDevCommRequirements_t)) == 0) + return (_cyb_memcmp((self._ptr), (other_._ptr), sizeof(ncclDevCommRequirements_t)) == 0) - def __getbuffer__(self, Py_buffer *buffer, int flags): - __getbuffer(self, buffer, self._ptr, sizeof(ncclDevCommRequirements_t), self._readonly) + def __getbuffer__(self, _cyb_cpython.Py_buffer *buffer, int flags): + _cyb___getbuffer(self, buffer, self._ptr, sizeof(ncclDevCommRequirements_t), self._readonly) def __releasebuffer__(self, Py_buffer *buffer): pass def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): - self._ptr = malloc(sizeof(ncclDevCommRequirements_t)) + self._ptr = _cyb_malloc(sizeof(ncclDevCommRequirements_t)) if self._ptr == NULL: raise MemoryError("Error allocating DevCommRequirements") - memcpy(self._ptr, val.ctypes.data, sizeof(ncclDevCommRequirements_t)) + _cyb_memcpy(self._ptr, val.ctypes.data, sizeof(ncclDevCommRequirements_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable @@ -3223,14 +4715,14 @@ cdef class DevCommRequirements: @property def lsa_multimem(self): - """int: """ - return self._ptr[0].lsaMultimem + """: """ + return (self._ptr[0].lsaMultimem) @lsa_multimem.setter def lsa_multimem(self, val): if self._readonly: raise ValueError("This DevCommRequirements instance is read-only") - self._ptr[0].lsaMultimem = val + self._ptr[0].lsaMultimem = <_cyb_bool>val @property def barrier_count(self): @@ -3289,14 +4781,14 @@ cdef class DevCommRequirements: @property def gin_force_enable(self): - """int: """ - return self._ptr[0].ginForceEnable + """: """ + return (self._ptr[0].ginForceEnable) @gin_force_enable.setter def gin_force_enable(self, val): if self._readonly: raise ValueError("This DevCommRequirements instance is read-only") - self._ptr[0].ginForceEnable = val + self._ptr[0].ginForceEnable = <_cyb_bool>val @property def gin_context_count(self): @@ -3344,14 +4836,14 @@ cdef class DevCommRequirements: @property def gin_exclusive_contexts(self): - """int: """ - return self._ptr[0].ginExclusiveContexts + """: """ + return (self._ptr[0].ginExclusiveContexts) @gin_exclusive_contexts.setter def gin_exclusive_contexts(self, val): if self._readonly: raise ValueError("This DevCommRequirements instance is read-only") - self._ptr[0].ginExclusiveContexts = val + self._ptr[0].ginExclusiveContexts = <_cyb_bool>val @property def gin_queue_depth(self): @@ -3388,30 +4880,85 @@ cdef class DevCommRequirements: @property def gin_strong_signals_required(self): - """int: """ - return self._ptr[0].ginStrongSignalsRequired + """: """ + return (self._ptr[0].ginStrongSignalsRequired) @gin_strong_signals_required.setter def gin_strong_signals_required(self, val): if self._readonly: raise ValueError("This DevCommRequirements instance is read-only") - self._ptr[0].ginStrongSignalsRequired = val + self._ptr[0].ginStrongSignalsRequired = <_cyb_bool>val @property def gin_va_signals_required(self): - """int: """ - return self._ptr[0].ginVaSignalsRequired + """: """ + return (self._ptr[0].ginVaSignalsRequired) @gin_va_signals_required.setter def gin_va_signals_required(self, val): if self._readonly: raise ValueError("This DevCommRequirements instance is read-only") - self._ptr[0].ginVaSignalsRequired = val + self._ptr[0].ginVaSignalsRequired = <_cyb_bool>val + + @property + def gin_custom_stride(self): + """int: """ + return self._ptr[0].ginCustomStride + + @gin_custom_stride.setter + def gin_custom_stride(self, val): + if self._readonly: + raise ValueError("This DevCommRequirements instance is read-only") + self._ptr[0].ginCustomStride = val + + @property + def gin_type(self): + """int: """ + return (self._ptr[0].ginType) + + @gin_type.setter + def gin_type(self, val): + if self._readonly: + raise ValueError("This DevCommRequirements instance is read-only") + self._ptr[0].ginType = val + + @property + def use_runtime_version(self): + """: """ + return (self._ptr[0].useRuntimeVersion) + + @use_runtime_version.setter + def use_runtime_version(self, val): + if self._readonly: + raise ValueError("This DevCommRequirements instance is read-only") + self._ptr[0].useRuntimeVersion = <_cyb_bool>val + + @property + def cft_caps(self): + """int: """ + return self._ptr[0].cftCaps + + @cft_caps.setter + def cft_caps(self, val): + if self._readonly: + raise ValueError("This DevCommRequirements instance is read-only") + self._ptr[0].cftCaps = val + + @property + def cft_barrier_count(self): + """int: """ + return self._ptr[0].cftBarrierCount + + @cft_barrier_count.setter + def cft_barrier_count(self, val): + if self._readonly: + raise ValueError("This DevCommRequirements instance is read-only") + self._ptr[0].cftBarrierCount = val @staticmethod def from_buffer(buffer): """Create an DevCommRequirements instance with the memory from the given buffer.""" - return __from_buffer(buffer, sizeof(ncclDevCommRequirements_t), DevCommRequirements) + return _cyb_from_buffer(buffer, sizeof(ncclDevCommRequirements_t), DevCommRequirements) @staticmethod def from_data(data): @@ -3420,7 +4967,7 @@ cdef class DevCommRequirements: Args: data (_numpy.ndarray): a single-element array of dtype `dev_comm_requirements_dtype` holding the data. """ - return __from_data(data, "dev_comm_requirements_dtype", dev_comm_requirements_dtype, DevCommRequirements) + return _cyb_from_data(data, "dev_comm_requirements_dtype", dev_comm_requirements_dtype, DevCommRequirements) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): @@ -3435,10 +4982,10 @@ cdef class DevCommRequirements: raise ValueError("ptr must not be null (0)") cdef DevCommRequirements obj = DevCommRequirements.__new__(DevCommRequirements) if owner is None: - obj._ptr = malloc(sizeof(ncclDevCommRequirements_t)) + obj._ptr = _cyb_malloc(sizeof(ncclDevCommRequirements_t)) if obj._ptr == NULL: raise MemoryError("Error allocating DevCommRequirements") - memcpy((obj._ptr), ptr, sizeof(ncclDevCommRequirements_t)) + _cyb_memcpy((obj._ptr), ptr, sizeof(ncclDevCommRequirements_t)) obj._owner = None obj._owned = True else: @@ -3449,11 +4996,109 @@ cdef class DevCommRequirements: return obj +cdef class Comm: + cdef ncclComm_t _handle + cdef ncclComm_t* _handle_ptr + cdef object _owner + + def __cinit__(self, *args, **kwargs): + self._handle_ptr = &self._handle + + def __init__(self, intptr_t handle=0): + self._handle = handle + self._handle_ptr = &self._handle + self._owner = None + + @property + def handle(self): + """Get the current handle value as a Python :class:`int`.""" + return self._handle_ptr[0] + + @property + def handle_ptr(self): + """Get the handle slot address as a Python :class:`int`.""" + return self._handle_ptr + + cdef ncclComm_t _get_handle(self): + return self._handle_ptr[0] + + cdef ncclComm_t* _get_handle_ptr(self): + return self._handle_ptr + + @staticmethod + cdef Comm _from_handle_ptr(ncclComm_t* handle_ptr, object owner): + cdef Comm obj = Comm.__new__(Comm) + obj._handle_ptr = handle_ptr + obj._owner = owner + return obj + + def __int__(self): + return self._handle_ptr[0] + + def __index__(self): + return self._handle_ptr[0] + + def __bool__(self): + return self._handle_ptr[0] != NULL + + def __repr__(self): + return f"<{__name__}.Comm handle={self.handle:#x} object at {hex(id(self))}>" + + +cdef class Window: + cdef ncclWindow_t _handle + cdef ncclWindow_t* _handle_ptr + cdef object _owner + + def __cinit__(self, *args, **kwargs): + self._handle_ptr = &self._handle + + def __init__(self, intptr_t handle=0): + self._handle = handle + self._handle_ptr = &self._handle + self._owner = None + + @property + def handle(self): + """Get the current handle value as a Python :class:`int`.""" + return self._handle_ptr[0] + + @property + def handle_ptr(self): + """Get the handle slot address as a Python :class:`int`.""" + return self._handle_ptr + + cdef ncclWindow_t _get_handle(self): + return self._handle_ptr[0] + + cdef ncclWindow_t* _get_handle_ptr(self): + return self._handle_ptr + + @staticmethod + cdef Window _from_handle_ptr(ncclWindow_t* handle_ptr, object owner): + cdef Window obj = Window.__new__(Window) + obj._handle_ptr = handle_ptr + obj._owner = owner + return obj + + def __int__(self): + return self._handle_ptr[0] + + def __index__(self): + return self._handle_ptr[0] + + def __bool__(self): + return self._handle_ptr[0] != NULL + + def __repr__(self): + return f"<{__name__}.Window handle={self.handle:#x} object at {hex(id(self))}>" + + ############################################################################### # Enum ############################################################################### -class Result(_IntEnum): +class Result(_cyb_IntEnum): """ See `ncclResult_t`. """ @@ -3468,7 +5113,26 @@ class Result(_IntEnum): Timeout = ncclTimeout NumResults = ncclNumResults -class CommMemStat(_IntEnum): +class HostCftMode(_cyb_IntEnum): + """ + See `ncclHostCftMode_t`. + """ + Default = ncclHostCftDefault + Enable = ncclHostCftEnable + Disable = ncclHostCftDisable + Fallback = ncclHostCftFallback + +class NvlsHostMode(_cyb_IntEnum): + """ + See `ncclNvlsHostMode_t`. + """ + Default = ncclNvlsHostModeDefault + Enable = ncclNvlsHostModeEnable + DisableTransport = ncclNvlsHostModeDisableTransport + DisableSymmetricMultimem = ncclNvlsHostModeDisableSymmetricMultimem + Disable = ncclNvlsHostModeDisable + +class CommMemStat(_cyb_IntEnum): """ See `ncclCommMemStat_t`. """ @@ -3477,13 +5141,13 @@ class CommMemStat(_IntEnum): GpuMemPersist = ncclStatGpuMemPersist GpuMemTotal = ncclStatGpuMemTotal -class RedOp_dummy(_IntEnum): +class RedOpDummy(_cyb_IntEnum): """ See `ncclRedOp_dummy_t`. """ NumOps_dummy = ncclNumOps_dummy -class RedOp(_IntEnum): +class RedOp(_cyb_IntEnum): """ See `ncclRedOp_t`. """ @@ -3495,7 +5159,7 @@ class RedOp(_IntEnum): NumOps = ncclNumOps MaxRedOp = ncclMaxRedOp -class DataType(_IntEnum): +class DataType(_cyb_IntEnum): """ See `ncclDataType_t`. """ @@ -3518,14 +5182,30 @@ class DataType(_IntEnum): Float8e5m2 = ncclFloat8e5m2 NumTypes = ncclNumTypes -class ScalarResidence(_IntEnum): +class ScalarResidence(_cyb_IntEnum): """ See `ncclScalarResidence_t`. """ Device = ncclScalarDevice HostImmediate = ncclScalarHostImmediate -class GinType(_IntEnum): +class EncryptionMode(_cyb_IntEnum): + """ + See `ncclEncryptionMode_t`. + """ + NONE = NCCL_ENCRYPTION_MODE_NONE + PSK = NCCL_ENCRYPTION_MODE_PSK + +class GinConnectionType(_cyb_IntEnum): + """ + See `ncclGinConnectionType_t`. + """ + NONE = NCCL_GIN_CONNECTION_NONE + FULL = NCCL_GIN_CONNECTION_FULL + RAIL = NCCL_GIN_CONNECTION_RAIL + CUSTOM_STRIDE = NCCL_GIN_CONNECTION_CUSTOM_STRIDE + +class GinType(_cyb_IntEnum): """ See `ncclGinType_t`. """ @@ -3533,14 +5213,31 @@ class GinType(_IntEnum): PROXY = NCCL_GIN_TYPE_PROXY GDAKI = NCCL_GIN_TYPE_GDAKI GPI = NCCL_GIN_TYPE_GPI + EFA_GDA = NCCL_GIN_TYPE_EFA_GDA + GIN_MAX_TYPES = NCCL_GIN_MAX_TYPES -class GinConnectionType(_IntEnum): +class CftTeamMode(_cyb_IntEnum): """ - See `ncclGinConnectionType_t`. + See `ncclCftTeamMode_t`. """ - NONE = NCCL_GIN_CONNECTION_NONE - FULL = NCCL_GIN_CONNECTION_FULL - RAIL = NCCL_GIN_CONNECTION_RAIL + FLAT = NCCL_CFT_TEAM_FLAT + HIER_MULTIMEM = NCCL_CFT_TEAM_HIER_MULTIMEM + HIER_LSA = NCCL_CFT_TEAM_HIER_LSA + +class CftCap(_cyb_IntEnum): + """ + See `ncclCftCap_t`. + """ + NONE = NCCL_CFT_NONE + CFT = NCCL_CFT + MULTIMEM = NCCL_CFT_MULTIMEM + +class GinSignalOp(_cyb_IntEnum): + """ + See `ncclGinSignalOp_t`. + """ + Inc = ncclGinSignalInc + Add = ncclGinSignalAdd ############################################################################### @@ -3561,7 +5258,7 @@ class NCCLError(Exception): @cython.profile(False) cpdef inline check_status(int status): - if status != Result.Success and status != Result.InProgress: + if status != ncclSuccess and status != ncclInProgress: raise NCCLError(status) @@ -3600,301 +5297,433 @@ cpdef object get_unique_id(): return unique_id_py -cpdef int comm_init_rank_config(intptr_t comm, int nranks, comm_id, int rank, intptr_t config) except? -1: - cdef void* _comm_id_ = get_buffer_pointer(comm_id, -1, readonly=False) - cdef int ret +cpdef object comm_init_rank_config(int nranks, comm_id, int rank, config): + cdef void* _comm_id_ = _cyb_get_buffer_pointer(comm_id, -1, readonly=False) + cdef intptr_t _config_ptr_ = int(config) + cdef Comm comm_py = Comm() + cdef ncclComm_t *comm = comm_py._get_handle_ptr() with nogil: - ret = ncclCommInitRankConfig(comm, nranks, ((_comm_id_))[0], rank, config) - check_status(ret) - return ret + __status__ = ncclCommInitRankConfig(comm, nranks, ((_comm_id_))[0], rank, _config_ptr_) + check_status(__status__) + return comm_py -cpdef int comm_init_rank(intptr_t comm, int nranks, comm_id, int rank) except? -1: - cdef void* _comm_id_ = get_buffer_pointer(comm_id, -1, readonly=False) - cdef int ret +cpdef object comm_init_rank(int nranks, comm_id, int rank): + cdef void* _comm_id_ = _cyb_get_buffer_pointer(comm_id, -1, readonly=False) + cdef Comm comm_py = Comm() + cdef ncclComm_t *comm = comm_py._get_handle_ptr() with nogil: - ret = ncclCommInitRank(comm, nranks, ((_comm_id_))[0], rank) - check_status(ret) - return ret + __status__ = ncclCommInitRank(comm, nranks, ((_comm_id_))[0], rank) + check_status(__status__) + return comm_py cpdef object comm_init_all(int ndev, devlist): cdef nullable_unique_ptr[ vector[int] ] _devlist_ get_resource_ptr[int](_devlist_, devlist, NULL) if ndev == 0: - return view.array(shape=(1,), itemsize=sizeof(intptr_t), format="q", mode="c")[:0] - cdef view.array comm = view.array(shape=(ndev,), itemsize=sizeof(intptr_t), format="q", mode="c") + return [] + cdef _cyb_view.array comm = _cyb_view.array(shape=(ndev,), itemsize=sizeof(intptr_t), format="q", mode="c") cdef intptr_t *comm_ptr = (comm.data) + cdef list comm_wrappers = [] + cdef Py_ssize_t comm_index + for comm_index in range(ndev): + comm_wrappers.append( + Comm._from_handle_ptr( + (comm_ptr + comm_index), + comm, + ) + ) with nogil: __status__ = ncclCommInitAll(comm_ptr, ndev, (_devlist_.data())) check_status(__status__) - return comm + return comm_wrappers -cpdef comm_finalize(intptr_t comm): +cpdef comm_finalize(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclCommFinalize(comm) + __status__ = ncclCommFinalize(_comm_handle) check_status(__status__) -cpdef comm_destroy(intptr_t comm): +cpdef comm_destroy(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclCommDestroy(comm) + __status__ = ncclCommDestroy(_comm_handle) check_status(__status__) -cpdef comm_abort(intptr_t comm): +cpdef comm_abort(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclCommAbort(comm) + __status__ = ncclCommAbort(_comm_handle) check_status(__status__) -cpdef comm_revoke(intptr_t comm, int revoke_flags): +cpdef comm_revoke(object comm, int revoke_flags): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclCommRevoke(comm, revoke_flags) + __status__ = ncclCommRevoke(_comm_handle, revoke_flags) check_status(__status__) -cpdef int comm_split(intptr_t comm, int color, int key, intptr_t newcomm, intptr_t config) except? -1: - cdef int ret +cpdef object comm_split(object comm, int color, int key, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) + cdef Comm newcomm_py = Comm() + cdef ncclComm_t *newcomm = newcomm_py._get_handle_ptr() with nogil: - ret = ncclCommSplit(comm, color, key, newcomm, config) - check_status(ret) - return ret + __status__ = ncclCommSplit(_comm_handle, color, key, newcomm, _config_ptr_) + check_status(__status__) + return newcomm_py -cpdef int comm_shrink(intptr_t comm, exclude_ranks_list, int exclude_ranks_count, intptr_t newcomm, intptr_t config, int shrink_flags) except? -1: +cpdef object comm_shrink(object comm, exclude_ranks_list, int exclude_ranks_count, config, int shrink_flags): + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef nullable_unique_ptr[ vector[int] ] _exclude_ranks_list_ get_resource_ptr[int](_exclude_ranks_list_, exclude_ranks_list, NULL) - cdef int ret + cdef intptr_t _config_ptr_ = int(config) + cdef Comm newcomm_py = Comm() + cdef ncclComm_t *newcomm = newcomm_py._get_handle_ptr() with nogil: - ret = ncclCommShrink(comm, (_exclude_ranks_list_.data()), exclude_ranks_count, newcomm, config, shrink_flags) - check_status(ret) - return ret + __status__ = ncclCommShrink(_comm_handle, (_exclude_ranks_list_.data()), exclude_ranks_count, newcomm, _config_ptr_, shrink_flags) + check_status(__status__) + return newcomm_py -cpdef object comm_get_unique_id(intptr_t comm): +cpdef object comm_get_unique_id(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef UniqueId unique_id_py = UniqueId() cdef ncclUniqueId *unique_id = (unique_id_py._get_ptr()) with nogil: - __status__ = ncclCommGetUniqueId(comm, unique_id) + __status__ = ncclCommGetUniqueId(_comm_handle, unique_id) check_status(__status__) return unique_id_py -cpdef int comm_grow(intptr_t comm, int n_ranks, intptr_t unique_id, int rank, intptr_t newcomm, intptr_t config) except? -1: - cdef int ret +cpdef object comm_grow(object comm, int n_ranks, unique_id, int rank, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _unique_id_ptr_ = int(unique_id) + cdef intptr_t _config_ptr_ = int(config) + cdef Comm newcomm_py = Comm() + cdef ncclComm_t *newcomm = newcomm_py._get_handle_ptr() with nogil: - ret = ncclCommGrow(comm, n_ranks, unique_id, rank, newcomm, config) - check_status(ret) - return ret + __status__ = ncclCommGrow(_comm_handle, n_ranks, _unique_id_ptr_, rank, newcomm, _config_ptr_) + check_status(__status__) + return newcomm_py -cpdef int comm_init_rank_scalable(intptr_t newcomm, int nranks, int myrank, int n_id, comm_ids, intptr_t config) except? -1: - cdef void* _comm_ids_ = get_buffer_pointer(comm_ids, -1, readonly=False) - cdef int ret +cpdef object comm_init_rank_scalable(int nranks, int myrank, int n_id, comm_ids, config): + cdef void* _comm_ids_ = _cyb_get_buffer_pointer(comm_ids, -1, readonly=False) + cdef intptr_t _config_ptr_ = int(config) + cdef Comm newcomm_py = Comm() + cdef ncclComm_t *newcomm = newcomm_py._get_handle_ptr() with nogil: - ret = ncclCommInitRankScalable(newcomm, nranks, myrank, n_id, _comm_ids_, config) - check_status(ret) - return ret + __status__ = ncclCommInitRankScalable(newcomm, nranks, myrank, n_id, _comm_ids_, _config_ptr_) + check_status(__status__) + return newcomm_py cpdef str get_error_string(int result): + cdef const char *_output_cstr_ cdef bytes _output_ - _output_ = ncclGetErrorString(<_Result>result) + with nogil: + _output_cstr_ = ncclGetErrorString(<_Result>result) + _output_ = _output_cstr_ return _output_.decode() -cpdef str get_last_error(intptr_t comm): +cpdef str get_last_error(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef const char *_output_cstr_ cdef bytes _output_ - _output_ = ncclGetLastError(comm) + with nogil: + _output_cstr_ = ncclGetLastError(_comm_handle) + _output_ = _output_cstr_ return _output_.decode() -cpdef int comm_get_async_error(intptr_t comm) except? -1: +cpdef int comm_get_async_error(object comm) except? -1: + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef _Result async_error with nogil: - __status__ = ncclCommGetAsyncError(comm, &async_error) + __status__ = ncclCommGetAsyncError(_comm_handle, &async_error) check_status(__status__) return async_error -cpdef int comm_count(intptr_t comm) except? -1: +cpdef int comm_count(object comm) except? -1: + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef int count with nogil: - __status__ = ncclCommCount(comm, &count) + __status__ = ncclCommCount(_comm_handle, &count) check_status(__status__) return count -cpdef int comm_cu_device(intptr_t comm) except? -1: +cpdef int comm_cu_device(object comm) except? -1: + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef int device with nogil: - __status__ = ncclCommCuDevice(comm, &device) + __status__ = ncclCommCuDevice(_comm_handle, &device) check_status(__status__) return device -cpdef int comm_user_rank(intptr_t comm) except? -1: +cpdef int comm_user_rank(object comm) except? -1: + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef int rank with nogil: - __status__ = ncclCommUserRank(comm, &rank) + __status__ = ncclCommUserRank(_comm_handle, &rank) check_status(__status__) return rank -cpdef intptr_t comm_register(intptr_t comm, intptr_t buff, size_t size) except? 0: +cpdef intptr_t comm_register(object comm, intptr_t buff, size_t size) except? 0: + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef void* handle with nogil: - __status__ = ncclCommRegister(comm, buff, size, &handle) + __status__ = ncclCommRegister(_comm_handle, buff, size, &handle) check_status(__status__) return handle -cpdef comm_deregister(intptr_t comm, intptr_t handle): +cpdef comm_deregister(object comm, intptr_t handle): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclCommDeregister(comm, handle) + __status__ = ncclCommDeregister(_comm_handle, handle) check_status(__status__) -cpdef comm_suspend(intptr_t comm, int flags): +cpdef comm_suspend(object comm, int flags): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclCommSuspend(comm, flags) + __status__ = ncclCommSuspend(_comm_handle, flags) check_status(__status__) -cpdef comm_resume(intptr_t comm): +cpdef comm_resume(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclCommResume(comm) + __status__ = ncclCommResume(_comm_handle) check_status(__status__) -cpdef uint64_t comm_mem_stats(intptr_t comm, int stat) except? -1: +cpdef uint64_t comm_mem_stats(object comm, int stat) except? -1: + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef uint64_t value with nogil: - __status__ = ncclCommMemStats(comm, <_CommMemStat>stat, &value) + __status__ = ncclCommMemStats(_comm_handle, <_CommMemStat>stat, &value) check_status(__status__) return value -cpdef int comm_window_register(intptr_t comm, intptr_t buff, size_t size, intptr_t win, int win_flags) except? -1: - cdef int ret +cpdef object comm_window_register(object comm, intptr_t buff, size_t size, int win_flags): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef Window win_py = Window() + cdef ncclWindow_t *win = win_py._get_handle_ptr() with nogil: - ret = ncclCommWindowRegister(comm, buff, size, win, win_flags) - check_status(ret) - return ret + __status__ = ncclCommWindowRegister(_comm_handle, buff, size, win, win_flags) + check_status(__status__) + return win_py -cpdef comm_window_deregister(intptr_t comm, intptr_t win): +cpdef comm_window_deregister(object comm, object win): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef ncclWindow_t _win_handle = (win)._get_handle() with nogil: - __status__ = ncclCommWindowDeregister(comm, win) + __status__ = ncclCommWindowDeregister(_comm_handle, _win_handle) check_status(__status__) -cpdef intptr_t win_get_user_ptr(intptr_t comm, intptr_t win) except? 0: +cpdef intptr_t win_get_user_ptr(object comm, object win) except? 0: + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef ncclWindow_t _win_handle = (win)._get_handle() cdef void* out_user_ptr with nogil: - __status__ = ncclWinGetUserPtr(comm, win, &out_user_ptr) + __status__ = ncclWinGetUserPtr(_comm_handle, _win_handle, &out_user_ptr) check_status(__status__) return out_user_ptr -cpdef int red_op_create_pre_mul_sum(intptr_t scalar, int datatype, int residence, intptr_t comm) except? -1: +cpdef int red_op_create_pre_mul_sum(intptr_t scalar, int datatype, int residence, object comm) except? -1: + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef _RedOp op with nogil: - __status__ = ncclRedOpCreatePreMulSum(&op, scalar, <_DataType>datatype, <_ScalarResidence>residence, comm) + __status__ = ncclRedOpCreatePreMulSum(&op, scalar, <_DataType>datatype, <_ScalarResidence>residence, _comm_handle) check_status(__status__) return op -cpdef red_op_destroy(int op, intptr_t comm): +cpdef red_op_destroy(int op, object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + with nogil: + __status__ = ncclRedOpDestroy(<_RedOp>op, _comm_handle) + check_status(__status__) + + +cpdef reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, int root, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + with nogil: + __status__ = ncclReduce(sendbuff, recvbuff, count, <_DataType>datatype, <_RedOp>op, root, _comm_handle, stream) + check_status(__status__) + + +cpdef bcast(intptr_t buff, size_t count, int datatype, int root, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + with nogil: + __status__ = ncclBcast(buff, count, <_DataType>datatype, root, _comm_handle, stream) + check_status(__status__) + + +cpdef broadcast(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + with nogil: + __status__ = ncclBroadcast(sendbuff, recvbuff, count, <_DataType>datatype, root, _comm_handle, stream) + check_status(__status__) + + +cpdef all_reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + with nogil: + __status__ = ncclAllReduce(sendbuff, recvbuff, count, <_DataType>datatype, <_RedOp>op, _comm_handle, stream) + check_status(__status__) + + +cpdef reduce_scatter(intptr_t sendbuff, intptr_t recvbuff, size_t recvcount, int datatype, int op, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + with nogil: + __status__ = ncclReduceScatter(sendbuff, recvbuff, recvcount, <_DataType>datatype, <_RedOp>op, _comm_handle, stream) + check_status(__status__) + + +cpdef all_gather(intptr_t sendbuff, intptr_t recvbuff, size_t sendcount, int datatype, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + with nogil: + __status__ = ncclAllGather(sendbuff, recvbuff, sendcount, <_DataType>datatype, _comm_handle, stream) + check_status(__status__) + + +cpdef allto_all(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + with nogil: + __status__ = ncclAlltoAll(sendbuff, recvbuff, count, <_DataType>datatype, _comm_handle, stream) + check_status(__status__) + + +cpdef gather(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclRedOpDestroy(<_RedOp>op, comm) + __status__ = ncclGather(sendbuff, recvbuff, count, <_DataType>datatype, root, _comm_handle, stream) check_status(__status__) -cpdef reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, int root, intptr_t comm, intptr_t stream): +cpdef scatter(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclReduce(sendbuff, recvbuff, count, <_DataType>datatype, <_RedOp>op, root, comm, stream) + __status__ = ncclScatter(sendbuff, recvbuff, count, <_DataType>datatype, root, _comm_handle, stream) check_status(__status__) -cpdef bcast(intptr_t buff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream): +cpdef all_reduce_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, object comm, intptr_t stream, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) with nogil: - __status__ = ncclBcast(buff, count, <_DataType>datatype, root, comm, stream) + __status__ = ncclAllReduceConfig(sendbuff, recvbuff, count, <_DataType>datatype, <_RedOp>op, _comm_handle, stream, _config_ptr_) check_status(__status__) -cpdef broadcast(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream): +cpdef broadcast_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) with nogil: - __status__ = ncclBroadcast(sendbuff, recvbuff, count, <_DataType>datatype, root, comm, stream) + __status__ = ncclBroadcastConfig(sendbuff, recvbuff, count, <_DataType>datatype, root, _comm_handle, stream, _config_ptr_) check_status(__status__) -cpdef all_reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, intptr_t comm, intptr_t stream): +cpdef reduce_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, int root, object comm, intptr_t stream, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) with nogil: - __status__ = ncclAllReduce(sendbuff, recvbuff, count, <_DataType>datatype, <_RedOp>op, comm, stream) + __status__ = ncclReduceConfig(sendbuff, recvbuff, count, <_DataType>datatype, <_RedOp>op, root, _comm_handle, stream, _config_ptr_) check_status(__status__) -cpdef reduce_scatter(intptr_t sendbuff, intptr_t recvbuff, size_t recvcount, int datatype, int op, intptr_t comm, intptr_t stream): +cpdef all_gather_config(intptr_t sendbuff, intptr_t recvbuff, size_t sendcount, int datatype, object comm, intptr_t stream, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) with nogil: - __status__ = ncclReduceScatter(sendbuff, recvbuff, recvcount, <_DataType>datatype, <_RedOp>op, comm, stream) + __status__ = ncclAllGatherConfig(sendbuff, recvbuff, sendcount, <_DataType>datatype, _comm_handle, stream, _config_ptr_) check_status(__status__) -cpdef all_gather(intptr_t sendbuff, intptr_t recvbuff, size_t sendcount, int datatype, intptr_t comm, intptr_t stream): +cpdef reduce_scatter_config(intptr_t sendbuff, intptr_t recvbuff, size_t recvcount, int datatype, int op, object comm, intptr_t stream, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) with nogil: - __status__ = ncclAllGather(sendbuff, recvbuff, sendcount, <_DataType>datatype, comm, stream) + __status__ = ncclReduceScatterConfig(sendbuff, recvbuff, recvcount, <_DataType>datatype, <_RedOp>op, _comm_handle, stream, _config_ptr_) check_status(__status__) -cpdef allto_all(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, intptr_t comm, intptr_t stream): +cpdef allto_all_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, object comm, intptr_t stream, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) with nogil: - __status__ = ncclAlltoAll(sendbuff, recvbuff, count, <_DataType>datatype, comm, stream) + __status__ = ncclAlltoAllConfig(sendbuff, recvbuff, count, <_DataType>datatype, _comm_handle, stream, _config_ptr_) check_status(__status__) -cpdef gather(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream): +cpdef gather_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) with nogil: - __status__ = ncclGather(sendbuff, recvbuff, count, <_DataType>datatype, root, comm, stream) + __status__ = ncclGatherConfig(sendbuff, recvbuff, count, <_DataType>datatype, root, _comm_handle, stream, _config_ptr_) check_status(__status__) -cpdef scatter(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream): +cpdef scatter_config(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, object comm, intptr_t stream, config): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _config_ptr_ = int(config) with nogil: - __status__ = ncclScatter(sendbuff, recvbuff, count, <_DataType>datatype, root, comm, stream) + __status__ = ncclScatterConfig(sendbuff, recvbuff, count, <_DataType>datatype, root, _comm_handle, stream, _config_ptr_) check_status(__status__) -cpdef send(intptr_t sendbuff, size_t count, int datatype, int peer, intptr_t comm, intptr_t stream): +cpdef send(intptr_t sendbuff, size_t count, int datatype, int peer, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclSend(sendbuff, count, <_DataType>datatype, peer, comm, stream) + __status__ = ncclSend(sendbuff, count, <_DataType>datatype, peer, _comm_handle, stream) check_status(__status__) -cpdef recv(intptr_t recvbuff, size_t count, int datatype, int peer, intptr_t comm, intptr_t stream): +cpdef recv(intptr_t recvbuff, size_t count, int datatype, int peer, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclRecv(recvbuff, count, <_DataType>datatype, peer, comm, stream) + __status__ = ncclRecv(recvbuff, count, <_DataType>datatype, peer, _comm_handle, stream) check_status(__status__) -cpdef put_signal(intptr_t localbuff, size_t count, int datatype, int peer, intptr_t peer_win, size_t peer_win_offset, int sig_idx, int ctx, unsigned int flags, intptr_t comm, intptr_t stream): +cpdef put_signal(intptr_t localbuff, size_t count, int datatype, int peer, object peer_win, size_t peer_win_offset, int sig_idx, int ctx, unsigned int flags, object comm, intptr_t stream): + cdef ncclWindow_t _peer_win_handle = (peer_win)._get_handle() + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclPutSignal(localbuff, count, <_DataType>datatype, peer, peer_win, peer_win_offset, sig_idx, ctx, flags, comm, stream) + __status__ = ncclPutSignal(localbuff, count, <_DataType>datatype, peer, _peer_win_handle, peer_win_offset, sig_idx, ctx, flags, _comm_handle, stream) check_status(__status__) -cpdef signal(int peer, int sig_idx, int ctx, unsigned int flags, intptr_t comm, intptr_t stream): +cpdef signal(int peer, int sig_idx, int ctx, unsigned int flags, object comm, intptr_t stream): + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclSignal(peer, sig_idx, ctx, flags, comm, stream) + __status__ = ncclSignal(peer, sig_idx, ctx, flags, _comm_handle, stream) check_status(__status__) -cpdef wait_signal(int n_desc, signal_descs, intptr_t comm, intptr_t stream): - cdef void* _signal_descs_ = get_buffer_pointer(signal_descs, -1, readonly=False) +cpdef wait_signal(int n_desc, signal_descs, object comm, intptr_t stream): + cdef void* _signal_descs_ = _cyb_get_buffer_pointer(signal_descs, -1, readonly=False) + cdef ncclComm_t _comm_handle = (comm)._get_handle() with nogil: - __status__ = ncclWaitSignal(n_desc, _signal_descs_, comm, stream) + __status__ = ncclWaitSignal(n_desc, _signal_descs_, _comm_handle, stream) check_status(__status__) @@ -3919,151 +5748,214 @@ cpdef object group_simulate_end(): return sim_info_py -cpdef object comm_query_properties(intptr_t comm): +cpdef set_encryption(config): + cdef intptr_t _config_ptr_ = int(config) + with nogil: + __status__ = ncclSetEncryption(_config_ptr_) + check_status(__status__) + + +cpdef void param_dump_all() except*: + with nogil: + ncclParamDumpAll() + + +cpdef object comm_query_properties(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() cdef CommProperties props_py = CommProperties() cdef ncclCommProperties_t *props = (props_py._get_ptr()) with nogil: - __status__ = ncclCommQueryProperties(comm, props) + __status__ = ncclCommQueryProperties(_comm_handle, props) check_status(__status__) return props_py -cpdef object dev_comm_create(intptr_t comm, intptr_t reqs): +cpdef object dev_comm_create(object comm, reqs): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _reqs_ptr_ = int(reqs) cdef DevComm out_dev_comm_py = DevComm() cdef ncclDevComm_t *out_dev_comm = (out_dev_comm_py._get_ptr()) with nogil: - __status__ = ncclDevCommCreate(comm, reqs, out_dev_comm) + __status__ = ncclDevCommCreate(_comm_handle, _reqs_ptr_, out_dev_comm) check_status(__status__) return out_dev_comm_py -cpdef dev_comm_destroy(intptr_t comm, intptr_t dev_comm): +cpdef dev_comm_destroy(object comm, dev_comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _dev_comm_ptr_ = int(dev_comm) with nogil: - __status__ = ncclDevCommDestroy(comm, dev_comm) + __status__ = ncclDevCommDestroy(_comm_handle, _dev_comm_ptr_) check_status(__status__) -cpdef intptr_t get_lsa_multimem_device_pointer(intptr_t window, size_t offset) except? 0: +cpdef intptr_t get_lsa_multimem_device_pointer(object window, size_t offset) except? 0: + cdef ncclWindow_t _window_handle = (window)._get_handle() cdef void* out_ptr with nogil: - __status__ = ncclGetLsaMultimemDevicePointer(window, offset, &out_ptr) + __status__ = ncclGetLsaMultimemDevicePointer(_window_handle, offset, &out_ptr) check_status(__status__) return out_ptr -cpdef intptr_t get_lsa_device_pointer(intptr_t window, size_t offset, int lsa_rank) except? 0: +cpdef intptr_t get_multimem_device_pointer(object window, size_t offset, multimem) except? 0: + cdef ncclWindow_t _window_handle = (window)._get_handle() + cdef intptr_t _multimem_ptr_ = (multimem)._get_ptr() cdef void* out_ptr with nogil: - __status__ = ncclGetLsaDevicePointer(window, offset, lsa_rank, &out_ptr) + __status__ = ncclGetMultimemDevicePointer(_window_handle, offset, (_multimem_ptr_)[0], &out_ptr) check_status(__status__) return out_ptr -cpdef intptr_t get_peer_device_pointer(intptr_t window, size_t offset, int peer) except? 0: +cpdef intptr_t get_lsa_device_pointer(object window, size_t offset, int lsa_rank) except? 0: + cdef ncclWindow_t _window_handle = (window)._get_handle() cdef void* out_ptr with nogil: - __status__ = ncclGetPeerDevicePointer(window, offset, peer, &out_ptr) + __status__ = ncclGetLsaDevicePointer(_window_handle, offset, lsa_rank, &out_ptr) check_status(__status__) return out_ptr -cpdef ll_a2a_create_requirement(int n_blocks, int n_slots, intptr_t out_handle, intptr_t out_req): +cpdef intptr_t get_peer_device_pointer(object window, size_t offset, int peer) except? 0: + cdef ncclWindow_t _window_handle = (window)._get_handle() + cdef void* out_ptr with nogil: - __status__ = ncclLLA2ACreateRequirement(n_blocks, n_slots, out_handle, out_req) + __status__ = ncclGetPeerDevicePointer(_window_handle, offset, peer, &out_ptr) check_status(__status__) + return out_ptr +cpdef tuple get_multimem_device_le_info(object window, size_t offset): + cdef ncclWindow_t _window_handle = (window)._get_handle() + cdef ncclCftLeId le_id + cdef size_t le_offset + with nogil: + __status__ = ncclGetMultimemDeviceLeInfo(_window_handle, offset, &le_id, &le_offset) + check_status(__status__) + return (le_id, le_offset) -# Hand-written: cybind cannot emit by-value struct returns (ncclTeam_t). -cpdef object team_world(intptr_t comm): - cdef Team team_py = Team() - cdef ncclTeam_t *team = (team_py._get_ptr()) +cpdef tuple get_cft_device_le_info(object window, size_t offset, int peer_cft, cft_team): + cdef ncclWindow_t _window_handle = (window)._get_handle() + cdef intptr_t _cft_team_ptr_ = (cft_team)._get_ptr() + cdef ncclCftLeId le_id + cdef size_t le_offset with nogil: - team[0] = ncclTeamWorld(comm) - return team_py + __status__ = ncclGetCftDeviceLeInfo(_window_handle, offset, peer_cft, (_cft_team_ptr_)[0], &le_id, &le_offset) + check_status(__status__) + return (le_id, le_offset) -cpdef object team_lsa(intptr_t comm): - cdef Team team_py = Team() - cdef ncclTeam_t *team = (team_py._get_ptr()) +cpdef tuple get_peer_device_le_info(object window, size_t offset, int peer_world): + cdef ncclWindow_t _window_handle = (window)._get_handle() + cdef ncclCftLeId le_id + cdef size_t le_offset with nogil: - team[0] = ncclTeamLsa(comm) - return team_py + __status__ = ncclGetPeerDeviceLeInfo(_window_handle, offset, peer_world, &le_id, &le_offset) + check_status(__status__) + return (le_id, le_offset) -cpdef object team_rail(intptr_t comm): - cdef Team team_py = Team() - cdef ncclTeam_t *team = (team_py._get_ptr()) +cpdef int team_rank_to_world(object comm, team, int rank) except? -1: + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _team_ptr_ = (team)._get_ptr() + cdef int ret with nogil: - team[0] = ncclTeamRail(comm) - return team_py + ret = ncclTeamRankToWorld(_comm_handle, (_team_ptr_)[0], rank) + return ret + +cpdef int team_rank_to_lsa(object comm, team, int rank) except? -1: + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _team_ptr_ = (team)._get_ptr() + cdef int ret + with nogil: + ret = ncclTeamRankToLsa(_comm_handle, (_team_ptr_)[0], rank) + return ret -# Hand-written: the team rank mappers take ncclTeam_t by value and return int -# (not ncclResult_t), so there is no status to check. -cpdef int team_rank_to_world(intptr_t comm, intptr_t team, int rank): - cdef int result +cpdef lsa_barrier_create_requirement(team, int n_barriers, out_handle, out_req): + cdef intptr_t _team_ptr_ = (team)._get_ptr() + cdef intptr_t _out_handle_ptr_ = int(out_handle) + cdef intptr_t _out_req_ptr_ = int(out_req) with nogil: - result = ncclTeamRankToWorld(comm, (team)[0], rank) - return result + __status__ = ncclLsaBarrierCreateRequirement((_team_ptr_)[0], n_barriers, _out_handle_ptr_, _out_req_ptr_) + check_status(__status__) -cpdef int team_rank_to_lsa(intptr_t comm, intptr_t team, int rank): - cdef int result +cpdef gin_barrier_create_requirement(object comm, team, int n_barriers, out_handle, out_req): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef intptr_t _team_ptr_ = (team)._get_ptr() + cdef intptr_t _out_handle_ptr_ = int(out_handle) + cdef intptr_t _out_req_ptr_ = int(out_req) with nogil: - result = ncclTeamRankToLsa(comm, (team)[0], rank) - return result + __status__ = ncclGinBarrierCreateRequirement(_comm_handle, (_team_ptr_)[0], n_barriers, _out_handle_ptr_, _out_req_ptr_) + check_status(__status__) + +cpdef int ll_a2a_calc_slots(int max_elts, int max_elt_size) except? -1: + cdef int ret + with nogil: + ret = ncclLLA2ACalcSlots(max_elts, max_elt_size) + return ret -# Hand-written: LSA/GIN barrier requirement creators take ncclTeam_t by value -# (SKIP_LOWPP in nccl.cybind.yaml; cybind cannot emit by-value structs). Each -# fills the out-req node and wires it to write into the out-handle during -# ncclDevCommCreate. ncclLLA2ACreateRequirement has no by-value struct and is -# generated by cybind. -cpdef lsa_barrier_create_requirement(intptr_t team, int n_barriers, intptr_t out_handle, intptr_t out_req): +cpdef ll_a2a_create_requirement(int n_blocks, int n_slots, out_handle, out_req): + cdef intptr_t _out_handle_ptr_ = int(out_handle) + cdef intptr_t _out_req_ptr_ = int(out_req) with nogil: - __status__ = ncclLsaBarrierCreateRequirement( - (team)[0], n_barriers, - out_handle, out_req) + __status__ = ncclLLA2ACreateRequirement(n_blocks, n_slots, _out_handle_ptr_, _out_req_ptr_) check_status(__status__) +# Hand-written: cybind cannot emit by-value struct returns (ncclTeam_t). + +cpdef object team_world(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef Team team_py = Team() + cdef ncclTeam_t *team = (team_py._get_ptr()) + with nogil: + team[0] = ncclTeamWorld(_comm_handle) + return team_py + -cpdef gin_barrier_create_requirement(intptr_t comm, intptr_t team, int n_barriers, intptr_t out_handle, intptr_t out_req): +cpdef object team_lsa(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef Team team_py = Team() + cdef ncclTeam_t *team = (team_py._get_ptr()) with nogil: - __status__ = ncclGinBarrierCreateRequirement( - comm, (team)[0], n_barriers, - out_handle, out_req) - check_status(__status__) + team[0] = ncclTeamLsa(_comm_handle) + return team_py -# Hand-written: ncclLLA2ACalcSlots returns int (not ncclResult_t). -cpdef int lla2a_calc_slots(int max_elts, int max_elt_size): - cdef int result + +cpdef object team_rail(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef Team team_py = Team() + cdef ncclTeam_t *team = (team_py._get_ptr()) with nogil: - result = ncclLLA2ACalcSlots(max_elts, max_elt_size) - return result + team[0] = ncclTeamRail(_comm_handle) + return team_py -# Hand-written: ncclGetMultimemDevicePointer takes ncclMultimemHandle_t by value -# (SKIP_LOWPP in nccl.cybind.yaml). ``multimem`` is a pointer to a MultimemHandle -# lowpp, dereferenced here. -cpdef intptr_t get_multimem_device_pointer(intptr_t window, size_t offset, intptr_t multimem) except? 0: - cdef void* out_ptr + +cpdef object team_cft(object comm, int mode): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef Team team_py = Team() + cdef ncclTeam_t *team = (team_py._get_ptr()) with nogil: - __status__ = ncclGetMultimemDevicePointer( - window, offset, (multimem)[0], &out_ptr) - check_status(__status__) - return out_ptr + team[0] = ncclTeamCft(_comm_handle, mode) + return team_py -cpdef object get_library_path(): - from ._internal.nccl import _inspect_loaded_library_path - return _inspect_loaded_library_path() + +cpdef object team_cft_multimem(object comm): + cdef ncclComm_t _comm_handle = (comm)._get_handle() + cdef Team team_py = Team() + cdef ncclTeam_t *team = (team_py._get_ptr()) + with nogil: + team[0] = ncclTeamCftMultimem(_comm_handle) + return team_py # Hand-written: Param API (SKIP_LOWPP in nccl.cybind.yaml). -# param_get_parameter raises KeyError on unknown key (not NCCLError). -# ncclParamDumpAll returns void — auto-gen wraps only ncclResult_t. cpdef str param_get_parameter(str key): cdef const char* value @@ -4087,6 +5979,8 @@ cpdef list param_get_all_keys(): return [table[i].decode() for i in range(table_len)] -cpdef param_dump_all(): - with nogil: - ncclParamDumpAll() +# Hand-written: not an NCCL entry point; reports the path of the loaded DSO. +cpdef object get_library_path(): + from ._internal.nccl import _inspect_loaded_library_path + return _inspect_loaded_library_path() +del _cyb_IntEnum diff --git a/projects/rccl/bindings/nccl4py/nccl/core/__init__.py b/projects/rccl/bindings/nccl4py/nccl/core/__init__.py index 61bdf26628a8..a869eec1ad9b 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/__init__.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/__init__.py @@ -46,11 +46,16 @@ "NcclRedOp", "NcclGinType", "NcclGinConnectionType", + "NcclHostCftMode", + "NcclNvlsHostMode", + "NcclCftTeamMode", + "NcclCftCap", "NcclCommMemStat", "NcclBufferSpec", "NcclScalarSpec", "NcclDeviceSpec", "NcclStreamSpec", + "NcclEventSpec", # Exceptions "NcclInvalid", # Data type constants @@ -84,6 +89,9 @@ "WindowFlag", # Communicator "NCCLConfig", + "NCCLCollConfig", + "VendorOption", + "NCCLCommProperties", "WaitSignalDesc", "TeamRequirement", "LsaBarrierRequirement", @@ -100,6 +108,7 @@ "LLA2AHandle", "RegisteredBufferHandle", "RegisteredWindowHandle", + "CftLeInfo", "CustomRedOp", "DevCommResource", # Group diff --git a/projects/rccl/bindings/nccl4py/nccl/core/_binding_helpers.py b/projects/rccl/bindings/nccl4py/nccl/core/_binding_helpers.py index 07def793abbd..7bbacdc8ad64 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/_binding_helpers.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/_binding_helpers.py @@ -9,10 +9,10 @@ (e.g. ``nccl.bindings.nccl.Config``); it holds a ``.ptr`` to the struct and exposes its members as properties. Two roles sit above it: -- :class:`LowppSpec` — a user-facing dataclass whose fields are the source of +- :class:`LowppSpec` -- a user-facing dataclass whose fields are the source of truth. ``obj._to_lowpp()`` builds a fresh lowpp struct from the current field values, to pass *into* a binding wrapper function. -- :class:`LowppView` — a read-only, live view over a lowpp struct *returned from* +- :class:`LowppView` -- a read-only, live view over a lowpp struct *returned from* a binding wrapper. Each field reads the lowpp directly, so a value NCCL writes later (e.g. non-blocking completion) is always current. The wrapped struct is available as ``obj._lowpp``. @@ -38,7 +38,7 @@ class PassBy(Enum): """How a field's value is written into the lowpp struct. ``VALUE`` assigns the value (or a nested facade's lowpp) by value. - ``POINTER`` assigns ``int(value.ptr)`` — a borrowed pointer. The referent + ``POINTER`` assigns ``int(value.ptr)`` -- a borrowed pointer. The referent must outlive any native use of the materialization. """ @@ -116,7 +116,7 @@ class WaitSignalDesc(LowppSpec, lowpp_cls=_bindings.WaitSignalDesc): class, re-running ``__init_subclass__`` without the ``lowpp_cls`` keyword. ``_to_lowpp()`` builds a fresh ``lowpp_cls`` from the current field values. - A field whose value is ``None`` — or that has been ``del``-eted — is left at + A field whose value is ``None`` -- or that has been ``del``-eted -- is left at the lowpp default, so a facade may mirror a subset of the lowpp members and add Python-only fields: an implicitly-named field not naming a writable lowpp member is Python-only, diff --git a/projects/rccl/bindings/nccl4py/nccl/core/_version.py b/projects/rccl/bindings/nccl4py/nccl/core/_version.py index 124de736f424..d286cfe08f3a 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/_version.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/_version.py @@ -3,4 +3,12 @@ # # See LICENSE.txt for more license information -__version__ = "0.4.1" +"""NCCL4Py version from installed distribution metadata.""" + +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _distribution_version + +try: + __version__ = _distribution_version("nccl4py") or "0+unknown" +except PackageNotFoundError: + __version__ = "0+unknown" diff --git a/projects/rccl/bindings/nccl4py/nccl/core/communicator.py b/projects/rccl/bindings/nccl4py/nccl/core/communicator.py index 123012ca31bd..e7dc5c0fc72a 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/communicator.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/communicator.py @@ -31,7 +31,7 @@ CommSuspendFlag, WindowFlag, ) -from nccl.core.cuda import get_stream_ptr +from nccl.core.cuda import get_event_ptr, get_stream_ptr from nccl.core.team import NCCLTeam from nccl.core.resources import ( CommResource, @@ -46,19 +46,26 @@ NcclRedOp, NcclGinType, NcclGinConnectionType, + NcclHostCftMode, + NcclNvlsHostMode, + NcclCftTeamMode, + NcclCftCap, NcclStreamSpec, + NcclEventSpec, NcclScalarSpec, NcclInvalid, NcclCommMemStat, ) from nccl.core.utils import UniqueId -_PointerBox = _nccl_bindings.PointerBox _Result = _nccl_bindings.Result __all__ = [ "NCCLConfig", + "NCCLCollConfig", + "VendorOption", + "NCCLCommProperties", "WaitSignalDesc", "TeamRequirement", "LsaBarrierRequirement", @@ -83,58 +90,428 @@ class NCCLConfig(LowppSpec, lowpp_cls=_nccl_bindings.Config): """ blocking: bool | None = None - """Blocking (True) or non-blocking (False) communicator behavior. If unset, NCCL uses True.""" + """Blocking (True) or non-blocking (False) communicator behavior. If unset, + NCCL uses True. + + Available since NCCL 2.14.0. + """ cga_cluster_size: int | None = None - """Cooperative Group Array (CGA) size for kernels (0-8). If unset, NCCL uses 4 for sm90+, 0 otherwise.""" + """Cooperative Group Array (CGA) size for kernels (0-8). If unset, NCCL uses + 4 for sm90+, 0 otherwise. + + Available since NCCL 2.17.0. + """ min_ctas: int | None = None - """Minimum number of CTAs per kernel; positive integer up to 32. If unset, NCCL uses 1.""" + """Minimum number of CTAs per kernel; positive integer up to 32. If unset, + NCCL uses 1. + + Available since NCCL 2.17.0. + """ max_ctas: int | None = None - """Maximum number of CTAs per kernel; positive integer up to 32. If unset, NCCL uses 32.""" + """Maximum number of CTAs per kernel; positive integer up to 32. If unset, + NCCL uses 32. + + Available since NCCL 2.17.0. + """ net_name: str | None = None - """Network module name (e.g. 'IB', 'Socket'). Case-insensitive. If unset, NCCL auto-selects.""" + """Network module name (e.g. 'IB', 'Socket'). Case-insensitive. If unset, + NCCL auto-selects. + + Available since NCCL 2.17.0. + """ split_share: bool | None = None - """Share resources with the child communicator during split. If unset, NCCL uses False.""" + """Share resources with the child communicator during split. If unset, + NCCL uses False. + + Available since NCCL 2.18.0. + """ traffic_class: int | None = None - """Traffic class (TC) for network operations (>= 0). Network-specific meaning.""" + """Traffic class (TC) for network operations (>= 0). Network-specific meaning. + + Available since NCCL 2.26.0. + """ comm_name: str | None = None - """User-defined communicator name for logging and profiling.""" + """User-defined communicator name for logging and profiling. + + Available since NCCL 2.27.0. + """ collnet_enable: bool | None = None - """Enable (True) or disable (False) IB SHARP. If unset, NCCL uses False.""" + """Enable (True) or disable (False) IB SHARP. If unset, NCCL uses False. + + Available since NCCL 2.27.0. + """ cta_policy: CTAPolicy | None = None - """CTA scheduling policy. If unset, NCCL uses CTAPolicy.DEFAULT.""" + """CTA scheduling policy. If unset, NCCL uses CTAPolicy.DEFAULT. + + Available since NCCL 2.27.0. + """ shrink_share: bool | None = None - """Share resources with the child communicator during shrink. If unset, NCCL uses False.""" + """Share resources with the child communicator during shrink. If unset, + NCCL uses False. + + Available since NCCL 2.27.0. + """ nvls_ctas: int | None = None - """Total number of CTAs for NVLS kernels (positive integer). If unset, NCCL auto-determines.""" + """Total number of CTAs for NVLS kernels (positive integer). If unset, + NCCL auto-determines. + + Available since NCCL 2.27.1. + """ n_channels_per_net_peer: int | None = None - """Number of network channels for pairwise communication. Positive integer, rounded up to power of 2. If unset, NCCL uses an AlltoAll-optimized value.""" + """Number of network channels for pairwise communication. Positive integer, + rounded up to power of 2. If unset, NCCL uses an AlltoAll-optimized value. + + Available since NCCL 2.28.0. + """ nvlink_centric_sched: bool | None = None - """Enable NVLink-centric scheduling. If unset, NCCL uses False.""" + """Enable NVLink-centric scheduling. If unset, NCCL uses False. + + Available since NCCL 2.28.2. + """ graph_usage_mode: int | None = None - """Graph usage mode (NCCL 2.29+). Supported values are 0 (no graphs), 1 (one graph), 2 (multiple graphs or mix of graph and non-graph). If unset, NCCL uses 2.""" + """Graph usage mode. Supported values are 0 (no graphs), 1 (one graph), and 2 + (multiple graphs or a mix of graph and non-graph). If unset, NCCL uses 2. + + Available since NCCL 2.29.0. + """ num_rma_ctx: int | None = None - """Number of RMA contexts (NCCL 2.29+). Positive integer. If unset, NCCL uses 1.""" + """Number of RMA contexts. Positive integer. If unset, NCCL uses 1. + + Available since NCCL 2.29.0. + """ max_p2p_peers: int | None = None - """Maximum number of peers any rank will concurrently communicate with using P2P (NCCL 2.30+). Positive integer. If unset, NCCL uses the communicator size.""" + """Maximum number of peers any rank will concurrently communicate with using + P2P. Positive integer. If unset, NCCL uses the communicator size. + + Available since NCCL 2.30.0. + """ graph_stream_ordering: int | None = None - """Whether NCCL preserves stream-ordering semantics for collectives captured into CUDA graphs. Supported values are 0 (disabled) or 1 (enabled). The value 0 cannot be combined with ``graph_usage_mode=2``. Also controllable via the ``NCCL_GRAPH_STREAM_ORDERING`` environment variable. If unset, NCCL uses 1.""" + """Whether NCCL preserves stream-ordering semantics for collectives captured + into CUDA graphs. Supported values are 0 (disabled) or 1 (enabled). The value 0 + cannot be combined with ``graph_usage_mode=2``. Also controllable via the + ``NCCL_GRAPH_STREAM_ORDERING`` environment variable. If unset, NCCL uses 1. + + Available since NCCL 2.30.5. + """ + + launch_order_implicit: bool | None = None + """Whether this communicator takes part in implicit launch ordering. Within one + CUDA context, operations on communicators that enable it must not overlap with + operations on communicators that do not. Also controllable via the + ``NCCL_LAUNCH_ORDER_IMPLICIT`` environment variable, which takes precedence. + If unset, NCCL uses False. + + Available since NCCL 2.31.0. + """ + + num_rma_sig: int | None = None + """Number of one-sided RMA signal indexes available per context. Non-negative + integer; bounds the ``signal_index`` accepted by the signal and wait-signal + operations. If unset, NCCL uses 1. + + Available since NCCL 2.31.0. + """ + + rma_eager_init: bool | None = None + """Whether the collective one-sided RMA signal setup is initialized at + communicator creation rather than at the first window registration. True is + required if the communicator issues signal or wait-signal operations without + first registering a symmetric window. Also controllable via the + ``NCCL_RMA_EAGER_INIT`` environment variable, which takes precedence. If unset, + NCCL uses False. + + Available since NCCL 2.31.0. + """ + + host_cft_mode: NcclHostCftMode | None = None + """Host-side Compute Fabric Transport mode. Controls whether the communicator + creates the CUDA fabric logical endpoints backing the host-side CFT queries. If + unset, NCCL uses :py:attr:`NcclHostCftMode.DEFAULT`. + + Available since NCCL 2.31.1. + """ + + nvls_host_mode: NcclNvlsHostMode | None = None + """Host-side NVLS mode. Selects which host NVLS components + the communicator uses. If unset, NCCL uses its library-defined default, + which is currently equivalent to :py:attr:`NcclNvlsHostMode.ENABLE`. + + Available since NCCL 2.32.0. + """ + + +@dataclass(frozen=True) +class VendorOption: + """A single vendor-specific option attached to an :py:class:`NCCLCollConfig`. + + Mirrors one :c:type:`ncclConfigExt_t` node. Options are identified by the + ``(vendor_id, option_id)`` pair; the official NCCL library ignores every + extension, so an option only has an effect on a vendor library that + recognizes its ``vendor_id``. Vendors pick a non-zero ``vendor_id`` + less than 2**24 that is unlikely to collide. + + Exactly one of the three value fields must be set. + + See Also: + :c:type:`ncclConfigExt_t` + """ + + vendor_id: int + """Vendor-chosen identifier, unique across vendor libraries.""" + + option_id: int + """Vendor-defined identifier distinguishing options within a vendor.""" + + int_value: int | None = None + """Integer value (``val.i``).""" + + str_value: str | None = None + """String value (``val.s``), encoded to UTF-8.""" + + raw_value: int | None = None + """Value of any other type (``val.raw``), as an integer. If it is an + address, the referent must stay valid for the duration of the call.""" + + def __post_init__(self): + set_fields = [ + name for name in ("int_value", "str_value", "raw_value") + if getattr(self, name) is not None + ] + if len(set_fields) != 1: + raise NcclInvalid( + "VendorOption requires exactly one of int_value, str_value, " + f"raw_value to be set, got {set_fields or 'none'}" + ) + + +def _validate_vendor_options(options: Sequence[Any]) -> None: + if isinstance(options, (VendorOption, str, bytes)) or not isinstance(options, Sequence): + raise NcclInvalid( + "vendor_options must be a sequence of VendorOption, got " + f"{type(options).__name__}" + ) + seen = set() + for opt in options: + if not isinstance(opt, VendorOption): + raise NcclInvalid( + "vendor_options must contain VendorOption instances, got " + f"{type(opt).__name__}" + ) + key = (opt.vendor_id, opt.option_id) + if key in seen: + raise NcclInvalid( + f"Duplicate vendor option key (vendor_id={opt.vendor_id}, " + f"option_id={opt.option_id}); keys must be unique" + ) + seen.add(key) + + +@dataclass(kw_only=True) +class NCCLCollConfig(LowppSpec, lowpp_cls=_nccl_bindings.CollConfig): + """Per-call configuration for a single collective. + + Accepted as the ``config`` argument of every collective on + :py:class:`Communicator`, tuning that one call. The same configuration must + be set on every rank; NCCL validates it only locally, when the call is + issued. + + See Also: + :c:type:`ncclCollConfig_t` + """ + + min_ctas: int | None = None + """Lower bound on channels/CTAs for this call. Also set by + ``NCCL_MIN_CTAS``, which takes precedence. If unset, inherits + :py:attr:`NCCLConfig.min_ctas`. + + Available since NCCL 2.31.0. + """ + + max_ctas: int | None = None + """Upper bound on channels/CTAs for this call, clamped to the + communicator's ``max_ctas``. Also set by ``NCCL_MAX_CTAS``, which takes + precedence. If unset, inherits :py:attr:`NCCLConfig.max_ctas`. + + Available since NCCL 2.31.0. + """ + + nvls_ctas: int | None = None + """NVLS-pool-specific channel cap for this call. Also set by + ``NCCL_NVLS_NCHANNELS``, which takes precedence. If unset, inherits + :py:attr:`NCCLConfig.nvls_ctas`. + + Available since NCCL 2.31.0. + """ + + cga_cluster_size: int | None = None + """CUDA thread-block-cluster size (0-8, Hopper+). Inconsistent values + within one group are undefined behavior. Also set by + ``NCCL_CGA_CLUSTER_SIZE``, which takes precedence. If unset, inherits + :py:attr:`NCCLConfig.cga_cluster_size`. + + Available since NCCL 2.31.0. + """ + + alg_selection: str | None = None + """Selection string filtering which algorithms this call may use, e.g. + ``"ring"``, ``"tree,ring"``, ``"^symk"``. If unset or empty, NCCL selects + automatically. + + Available since NCCL 2.31.0. + """ + + force_alg_selection: bool | None = None + """Whether an unsatisfiable :py:attr:`alg_selection` is an error rather + than a fallback to automatic selection. If unset, NCCL uses True. + + Available since NCCL 2.31.0. + """ + + cta_policy: CTAPolicy | None = None + """CTA scheduling policy for this call. Also set by ``NCCL_CTA_POLICY``, + which takes precedence. If unset, inherits + :py:attr:`NCCLConfig.cta_policy`. + + Available since NCCL 2.31.0. + """ + + user_profiler_tag: int | None = None + """Opaque value delivered verbatim to profiler plugins with this call's + profiler events; does not affect execution. Values with the + most-significant bit set are reserved by NCCL. If unset, NCCL uses 0. + + Available since NCCL 2.31.0. + """ + + launch_completion_event: NcclEventSpec | None = None + """Caller-owned, rank-local CUDA event recorded at collective kernel launch + completion. With CUDA versions earlier than 12.3, NCCL records the event + before the kernel launch instead. A ``cuda.core.Event`` created by + ``Device().create_event()`` has the required timing-disabled configuration; + events passed as integer handles must likewise have timing disabled. + Interprocess and interop events are unsupported. Either every rank passes + an event or none does, and at most one per communicator in a group. Keep it + alive through all queued waits and captured-graph executions. If unset, + NCCL records no event. + + See Also: + :c:member:`ncclCollConfig_t.launchCompletionEvent` + + Available since NCCL 2.32.0. + """ + + vendor_options: tuple[VendorOption, ...] = () + """Vendor-specific options; ``(vendor_id, option_id)`` keys must be + unique. + + Available since NCCL 2.31.0. + """ + + def __post_init__(self): + _validate_vendor_options(self.vendor_options) + + +@dataclass(frozen=True, kw_only=True) +class NCCLCommProperties: + """The properties NCCL reports for a communicator. + + Returned by :py:attr:`Communicator.properties`. These values are fixed + for the lifetime of the communicator. Version-marked fields are + ``None`` when nccl4py was built against an older NCCL. + + See Also: + :c:type:`ncclCommProperties` for the description of each field. + """ + + rank: int + """This caller's rank within the communicator.""" + + n_ranks: int + """Number of ranks in the communicator.""" + + cuda_dev: int + """CUDA device ID associated with the communicator.""" + + nvml_dev: int + """NVML device ID for the GPU. Uses the NVML indexing space, which may + differ from CUDA indexing.""" + + device_api_support: bool + """Whether device-side NCCL operations are supported.""" + + multimem_support: bool + """Whether ranks in the same LSA team can communicate using multimem.""" + + gin_type: NcclGinType + """GIN transport reaching every rank. ``NONE`` unless + :py:attr:`gin_connection_type` is ``FULL``, even when a rail-restricted + transport is available.""" + + n_lsa_teams: int + """Number of LSA teams.""" + + host_rma_support: bool + """Whether host RMA is supported.""" + + railed_gin_type: NcclGinType + """GIN transport reaching ranks within a rail. ``NONE`` only when no GIN + transport is available at all.""" + + comm_hash: int | None = None + """Hash identifying the communicator, shared by all its ranks (NCCL 2.31+).""" + + gin_min_stride: int | None = None + """Granularity of the GIN rank stride this communicator supports. A stride + passed as :py:attr:`NCCLDevCommRequirements.gin_custom_stride` must be a + multiple of this value, and no larger than the rail team's stride. It is 1 + when :py:attr:`gin_connection_type` is ``FULL`` (NCCL 2.31+).""" + + gin_connection_type: NcclGinConnectionType | None = None + """Widest GIN connection topology this communicator supports: ``NONE``, + ``RAIL`` or ``FULL``. A device communicator may request this topology or a + narrower one via + :py:attr:`NCCLDevCommRequirements.gin_connection_type` (NCCL 2.31+).""" + + available_gin_types: frozenset[NcclGinType] | None = None + """The GIN transports this communicator can use, e.g. + ``NcclGinType.GDAKI in props.available_gin_types``. Empty when GIN is + unavailable (NCCL 2.31+).""" + + dev_comm_runtime_version_size: int | None = None + """Size, in bytes, of the device communicator structure in the running NCCL + library (NCCL 2.31+).""" + + cft_support: bool | None = None + """Whether every rank in the communicator supports CFT unicast logical + endpoints, which requires CUDA and driver 13.3+ on each. NCCL reduces this + across ranks, so ``False`` does not mean the local GPU lacks support + (NCCL 2.32+).""" + + cft_multicast_support: bool | None = None + """Whether every rank in the communicator supports multicast CFT logical + endpoints. Independent of :py:attr:`cft_support`; a GPU may support + multicast endpoints without unicast ones (NCCL 2.32+).""" + + cft_counted_support: bool | None = None + """Whether every rank in the communicator supports counted CFT + logical-endpoint operations (NCCL 2.32+).""" @dataclass(frozen=True) @@ -154,10 +531,12 @@ class WaitSignalDesc(LowppSpec, lowpp_cls=_nccl_bindings.WaitSignalDesc): """Number of signal operations to wait for from the peer. Defaults to 1.""" signal_index: int = Field(default=0, lowpp_name="sig_idx") - """Signal index identifier. Currently must be 0. Defaults to 0.""" + """Signal index identifier. Must lie in ``[0, num_rma_sig)``; see + :py:attr:`NCCLConfig.num_rma_sig`. Defaults to 0.""" context: int = Field(default=0, lowpp_name="ctx") - """Context identifier. Currently must be 0. Defaults to 0.""" + """Context identifier. Must lie in ``[0, num_rma_ctx)``; see + :py:attr:`NCCLConfig.num_rma_ctx`. Defaults to 0.""" @dataclass(frozen=True) @@ -289,6 +668,26 @@ class NCCLDevCommRequirements(LowppSpec, lowpp_cls=_nccl_bindings.DevCommRequire """Whether GIN VA signals are required by kernels using this devComm. When False, using GIN VA signals results in undefined behavior. If unset, NCCL uses True.""" + gin_custom_stride: int | None = None + """Stride of ranks to connect for GIN. Only consulted when + :py:attr:`gin_connection_type` is + :py:attr:`NcclGinConnectionType.CUSTOM_STRIDE`, and must be a multiple of + :py:attr:`NCCLCommProperties.gin_min_stride`. If unset, NCCL uses 1.""" + + gin_type: NcclGinType | None = None + """GIN transport to require. If unset, NCCL uses + :py:attr:`NcclGinType.NONE`, accepting any available transport.""" + + cft_caps: NcclCftCap | None = None + """Compute Fabric Transport capabilities to request, as a bitmask of + :py:class:`NcclCftCap` values. Creation fails if CFT resources are + requested on a communicator where not all ranks support CFT. If unset, + NCCL uses :py:attr:`NcclCftCap.NONE`.""" + + cft_barrier_count: int | None = None + """Number of CFT barriers to allocate, one per independently addressed + barrier slot the kernel uses (commonly one per CTA). If unset, NCCL uses 0.""" + teams: tuple[TeamRequirement, ...] = () """Per-team requirements. Entries for the same team (by value) are merged, keeping first-appearance order; multimem is requested for a team if any of @@ -303,6 +702,58 @@ class NCCLDevCommRequirements(LowppSpec, lowpp_cls=_nccl_bindings.DevCommRequire kept as-is (not merged): each is a distinct resource.""" +def _materialize_coll_config( + config: NCCLCollConfig | None, +) -> tuple[Any | None, list[Any]]: + """Builds the lowpp ``CollConfig`` and its vendor-option chain. + + Returns ``(lowpp, keepalive)``. ``keepalive`` holds the ``ConfigExt`` + nodes and the ``val`` views that own encoded string buffers; the caller + must keep it referenced until the NCCL call returns, since the config + stores borrowed pointers into them. The call is the whole window: NCCL + resolves the config -- parsing ``alg_selection`` into an algorithm mask + -- before the entry point returns, even inside a group. A path that + defers that resolution must copy the config rather than borrow it. + + Returns ``(None, [])`` when ``config`` is ``None``. + """ + if config is None: + return None, [] + + # NCCLCollConfig is mutable, so revalidate Python-only extensions at the + # point of use rather than relying solely on construction-time validation. + _validate_vendor_options(config.vendor_options) + cfg = config._to_lowpp() + cfg.launch_completion_event_handle = get_event_ptr(config.launch_completion_event) + keepalive: list[Any] = [] + + nodes = [] + for opt in config.vendor_options: + node = _nccl_bindings.ConfigExt() + node.key.vendor_id = int(opt.vendor_id) + node.key.option_id = int(opt.option_id) + # `node.val` is a fresh view each access, and the string setter parks + # the encoded bytes on the view it was called on -- hold that same + # view so the char* it wrote stays alive. + val = node.val + if opt.int_value is not None: + val.i = int(opt.int_value) + elif opt.str_value is not None: + val.s = opt.str_value + else: + val.raw = int(opt.raw_value) + nodes.append(node) + keepalive += [node, val] + + for node, nxt in zip(nodes, nodes[1:]): + node.next = nxt + if nodes: + # A borrowed pointer: the `ext` setter stores no reference of its own. + cfg.ext = nodes[0].ptr + + return cfg, keepalive + + def _materialize_team_requirements( teams: tuple[TeamRequirement, ...], ) -> tuple[ @@ -340,7 +791,7 @@ def _materialize_team_requirements( multimem_handles[team] = handle if previous is not None: - previous.next = node.ptr + previous.next = node team_nodes.append(node) previous = node @@ -349,7 +800,7 @@ def _materialize_team_requirements( def _materialize_resource_requirements( - comm_ptr: int, + comm: _nccl_bindings.Comm, resources: tuple[LsaBarrierRequirement | GinBarrierRequirement | LLA2ARequirement, ...], ) -> tuple[ tuple[_nccl_bindings.DevResourceRequirements, ...], @@ -381,17 +832,17 @@ def _materialize_resource_requirements( handle = _nccl_bindings.LsaBarrierHandle() team_lowpp = resource.team._to_lowpp() _nccl_bindings.lsa_barrier_create_requirement( - team_lowpp.ptr, resource.n_barriers, handle.ptr, node.ptr + team_lowpp, resource.n_barriers, handle.ptr, node.ptr ) elif isinstance(resource, GinBarrierRequirement): handle = _nccl_bindings.GinBarrierHandle() team_lowpp = resource.team._to_lowpp() _nccl_bindings.gin_barrier_create_requirement( - comm_ptr, team_lowpp.ptr, resource.n_barriers, handle.ptr, node.ptr + comm, team_lowpp, resource.n_barriers, handle.ptr, node.ptr ) elif isinstance(resource, LLA2ARequirement): handle = _nccl_bindings.LLA2AHandle() - n_slots = _nccl_bindings.lla2a_calc_slots( + n_slots = _nccl_bindings.ll_a2a_calc_slots( resource.max_elements, resource.max_element_size ) _nccl_bindings.ll_a2a_create_requirement( @@ -401,7 +852,7 @@ def _materialize_resource_requirements( raise TypeError(f"unknown resource requirement: {type(resource).__name__}") if previous is not None: - previous.next = node.ptr + previous.next = node resource_nodes.append(node) resource_handles.append(handle) @@ -413,53 +864,72 @@ def _materialize_resource_requirements( class Communicator: """NCCL communicator for collective and point-to-point operations. - A communicator represents a group of ranks that can perform collective - operations (e.g. allreduce, broadcast) and point-to-point operations - (send/recv). Each rank has a unique ID in ``[0, nranks)``. + A communicator represents a group of participants that perform NCCL + operations. Each participant is assigned an integer rank in + ``[0, nranks)``. + + Most users should create communicators with :py:meth:`init` or + :py:meth:`init_all`. The constructor is a low-level interoperability + entry point for wrapping an existing NCCL communicator pointer or creating + a null communicator for later initialization. - Communicator instances expose a number of properties for inspection - (``ptr``, ``nranks``, ``device``, ``rank``, plus device-API related - properties like ``cuda_dev``, ``nvml_dev``, ``device_api_support``, - ``multimem_support``, ``gin_type``, ``n_lsa_teams``, - ``host_rma_support``, ``railed_gin_type``); see the per-property - documentation for details. + A communicator instance provides collective and point-to-point operations, + lifecycle and resource management, and properties describing its rank, + device, topology, and capabilities. """ - def __init__(self, ptr: int = 0) -> None: - """Initializes a communicator with a raw NCCL pointer. + _comm: _nccl_bindings.Comm + _resources: list[CommResource] + _nranks: int | None + _device: Device | None + _rank: int | None + _comm_properties: NCCLCommProperties | None + _children_in_progress: list[Communicator] - Unlike the :py:meth:`init` classmethod, this constructor allows - ``ptr=0`` for creating null communicators (e.g. when :py:meth:`split` - excludes a rank). A null communicator can later be initialized via - :py:meth:`initialize`, or used as the caller for :py:meth:`grow` to - join an existing communicator. + def __init__(self, ptr: int | None = None) -> None: + """Wraps an existing NCCL communicator pointer. + + This is a low-level interoperability entry point for an NCCL + communicator pointer obtained from another library or framework. Most + users should create communicators with :py:meth:`init` or + :py:meth:`init_all` instead. + + Omitting ``ptr`` or passing 0 creates a null communicator. A null + communicator can later be initialized with :py:meth:`initialize`, or + used with :py:meth:`grow` to join an existing communicator. Args: - ptr: Integer representing an NCCL communicator pointer (0 for a - null communicator). Defaults to 0. + ptr: Address of an existing NCCL communicator, represented as a + Python integer. ``None`` and 0 create a null communicator. + Defaults to ``None``. """ - if isinstance(ptr, _PointerBox): - self._comm_box = ptr - else: - self._comm_box = _PointerBox(ptr) - self._resources: list[CommResource] = [] - self._nranks: int | None = None - self._device: Device | None = None - self._rank: int | None = None - self._comm_properties: _nccl_bindings.CommProperties | None = None + self._comm = _nccl_bindings.Comm(0 if ptr is None else ptr) + self._resources = [] + self._nranks = None + self._device = None + self._rank = None + self._comm_properties = None self._children_in_progress = [] + @classmethod + def _from_comm(cls, comm: _nccl_bindings.Comm) -> Communicator: + """Constructs a communicator from an internal binding wrapper.""" + obj = cls() + obj._comm = comm + return obj + def _check_valid(self, operation: str) -> None: - """Raises if the communicator is not initialized. + """Validates that this object references an active communicator. Args: operation: Name of the operation being attempted (for the error message). Raises: - NcclInvalid: If the communicator is not initialized (ptr == 0). + NcclInvalid: If this object does not reference an active NCCL + communicator. """ - if self._comm == 0: + if not self._comm: raise NcclInvalid(f"Cannot {operation}: Communicator not initialized") def _validate_buffer_device(self, buffer: NcclBuffer, buffer_name: str = "buffer") -> None: @@ -480,28 +950,14 @@ def _validate_buffer_device(self, buffer: NcclBuffer, buffer_name: str = "buffer f"device as the communicator." ) - def _get_comm_properties(self) -> _nccl_bindings.CommProperties: - """Queries and caches communicator properties. - - Returns: - Cached CommProperties object. - - Raises: - NcclInvalid: If the communicator is not initialized. - """ - self._check_valid("query properties") - if self._comm_properties is None: - self._comm_properties = _nccl_bindings.comm_query_properties(self._comm) - return self._comm_properties - def __repr__(self) -> str: - if self._comm == 0: + if not self._comm: return "" try: - return f"" + return f"" except RuntimeError: # If we can't get properties, just show the pointer - return f"" + return f"" @classmethod def init( @@ -589,10 +1045,9 @@ def init_all( # Call NCCL binding to initialize all communicators # Note: ncclCommInitAll preserves the current device internally - # The binding returns a Cython array containing communicator pointers - comm_array = _nccl_bindings.comm_init_all(ndev, devlist) + comm_handles = _nccl_bindings.comm_init_all(ndev, devlist) - return [cls(int(comm_ptr)) for comm_ptr in comm_array] + return [cls._from_comm(comm) for comm in comm_handles] def initialize( self, @@ -618,7 +1073,7 @@ def initialize( NcclInvalid: If ``unique_id`` has an invalid type or this communicator is already initialized. """ - if self._comm != 0: + if self._comm: raise NcclInvalid("Communicator is already initialized") config_lowpp = None if config is None else config._to_lowpp() @@ -629,8 +1084,8 @@ def initialize( raise NcclInvalid("unique_id must be a UniqueId or a sequence of UniqueIds") commIds = bytearray().join(bytes(uid) for uid in unique_id) - _nccl_bindings.comm_init_rank_scalable( - self._comm_box.address, int(nranks), int(rank), int(len(unique_id)), commIds, cfg_ptr + self._comm = _nccl_bindings.comm_init_rank_scalable( + int(nranks), int(rank), int(len(unique_id)), commIds, cfg_ptr ) self._resources = [] @@ -682,11 +1137,10 @@ def split( color = -1 # NCCL_SPLIT_NOCOLOR from nccl.h config_lowpp = None if config is None else config._to_lowpp() cfg_ptr = 0 if config_lowpp is None else config_lowpp.ptr - newcomm = type(self)() - self._children_in_progress.append(newcomm) - _nccl_bindings.comm_split( - self._comm, int(color), int(key), newcomm._comm_box.address, cfg_ptr + newcomm = type(self)._from_comm( + _nccl_bindings.comm_split(self._comm, int(color), int(key), cfg_ptr) ) + self._children_in_progress.append(newcomm) return newcomm @@ -735,14 +1189,14 @@ def shrink( ranks_to_exclude = list(exclude_ranks) if exclude_ranks is not None else [] config_lowpp = None if config is None else config._to_lowpp() cfg_ptr = 0 if config_lowpp is None else config_lowpp.ptr - newcomm = type(self)() - _nccl_bindings.comm_shrink( - self._comm, - ranks_to_exclude, - len(ranks_to_exclude), - newcomm._comm_box.address, - cfg_ptr, - int(flag), + newcomm = type(self)._from_comm( + _nccl_bindings.comm_shrink( + self._comm, + ranks_to_exclude, + len(ranks_to_exclude), + cfg_ptr, + int(flag), + ) ) return newcomm @@ -818,18 +1272,17 @@ def grow( or an existing rank is given a null communicator. """ is_new_rank = rank is not None - if is_new_rank and self._comm != 0: + if is_new_rank and self._comm: raise NcclInvalid("New ranks must use a null communicator (Communicator())") - if not is_new_rank and self._comm == 0: + if not is_new_rank and not self._comm: raise NcclInvalid("Existing ranks must use an initialized communicator") uid_ptr = 0 if unique_id is None else unique_id.ptr rank_val = -1 if rank is None else int(rank) config_lowpp = None if config is None else config._to_lowpp() cfg_ptr = 0 if config_lowpp is None else config_lowpp.ptr - newcomm = type(self)() - _nccl_bindings.comm_grow( - self._comm, int(nranks), uid_ptr, rank_val, newcomm._comm_box.address, cfg_ptr + newcomm = type(self)._from_comm( + _nccl_bindings.comm_grow(self._comm, int(nranks), uid_ptr, rank_val, cfg_ptr) ) return newcomm @@ -857,11 +1310,11 @@ def destroy(self) -> None: # Close all resources first (best-effort, ignore errors) self.close_all_resources() - if self._comm == 0: + if not self._comm: return _nccl_bindings.comm_destroy(self._comm) - self._comm_box.ptr = 0 + self._comm = _nccl_bindings.Comm() def abort(self) -> None: """Aborts the communicator and frees resources, terminating in-flight operations. @@ -882,11 +1335,11 @@ def abort(self) -> None: # Close all resources first (best-effort, ignore errors) self.close_all_resources() - if self._comm == 0: + if not self._comm: return _nccl_bindings.comm_abort(self._comm) - self._comm_box.ptr = 0 + self._comm = _nccl_bindings.Comm() def finalize(self) -> None: """Finalizes the communicator, flushing uncompleted operations and network resources. @@ -907,7 +1360,7 @@ def finalize(self) -> None: See Also: :c:func:`ncclCommFinalize` """ - if self._comm == 0: + if not self._comm: return _nccl_bindings.comm_finalize(self._comm) @@ -964,21 +1417,15 @@ def resume(self) -> None: _nccl_bindings.comm_resume(self._comm) # --- Properties --- - @property - def _comm(self) -> int: - return int(self._comm_box.ptr) - @property def ptr(self) -> int: - """Raw pointer to the underlying :c:type:`ncclComm_t` structure - (0 if destroyed or null). - """ - return self._comm + """Integer value of the underlying :c:type:`ncclComm_t` (0 if destroyed or null).""" + return self._comm.handle @property def is_valid(self) -> bool: """Whether the communicator is valid (not destroyed or null).""" - return self._comm != 0 + return bool(self._comm) @property def nranks(self) -> int: @@ -1020,6 +1467,67 @@ def rank(self) -> int: self._rank = int(_nccl_bindings.comm_user_rank(self._comm)) return self._rank + @property + def properties(self) -> NCCLCommProperties: + """All properties NCCL reports for this communicator. + + Use this to read several properties at once, or to reach the fields + that have no dedicated accessor. + + Returns: + An :py:class:`NCCLCommProperties` holding every field NCCL reports + for this communicator. + + Raises: + NcclInvalid: If the communicator is not initialized. + """ + self._check_valid("get properties") + if self._comm_properties is None: + pod = _nccl_bindings.comm_query_properties(self._comm) + gin_support = getattr(pod, "gin_support", None) + gin_connection_type = getattr(pod, "gin_connection_type", None) + cft_support = getattr(pod, "cft_support", None) + cft_multicast_support = getattr(pod, "cft_multicast_support", None) + cft_counted_support = getattr(pod, "cft_counted_support", None) + self._comm_properties = NCCLCommProperties( + rank=pod.rank, + n_ranks=pod.n_ranks, + cuda_dev=pod.cuda_dev, + nvml_dev=pod.nvml_dev, + device_api_support=bool(pod.device_api_support), + multimem_support=bool(pod.multimem_support), + gin_type=NcclGinType(pod.gin_type), + n_lsa_teams=pod.n_lsa_teams, + host_rma_support=bool(pod.host_rma_support), + railed_gin_type=NcclGinType(pod.railed_gin_type), + comm_hash=getattr(pod, "comm_hash", None), + gin_min_stride=getattr(pod, "gin_min_stride", None), + gin_connection_type=( + None + if gin_connection_type is None + else NcclGinConnectionType(gin_connection_type) + ), + # Copied out: the lowpp accessor aliases the CommProperties + # buffer. Iterating the enum skips the array's reserved tail; + # NONE is not a transport. + available_gin_types=( + None + if gin_support is None + else frozenset( + t for t in NcclGinType if t is not NcclGinType.NONE and gin_support[t] + ) + ), + dev_comm_runtime_version_size=getattr(pod, "dev_comm_runtime_version_size", None), + cft_support=None if cft_support is None else bool(cft_support), + cft_multicast_support=( + None if cft_multicast_support is None else bool(cft_multicast_support) + ), + cft_counted_support=( + None if cft_counted_support is None else bool(cft_counted_support) + ), + ) + return self._comm_properties + @property def cuda_dev(self) -> int: """CUDA device ID associated with this communicator. @@ -1029,7 +1537,7 @@ def cuda_dev(self) -> int: """ self._check_valid("get cuda_dev") - return self._get_comm_properties().cuda_dev + return self.properties.cuda_dev @property def nvml_dev(self) -> int: @@ -1042,7 +1550,7 @@ def nvml_dev(self) -> int: """ self._check_valid("get nvml_dev") - return self._get_comm_properties().nvml_dev + return self.properties.nvml_dev @property def device_api_support(self) -> bool: @@ -1055,7 +1563,7 @@ def device_api_support(self) -> bool: """ self._check_valid("get device_api_support") - return bool(self._get_comm_properties().device_api_support) + return self.properties.device_api_support @property def multimem_support(self) -> bool: @@ -1069,21 +1577,23 @@ def multimem_support(self) -> bool: """ self._check_valid("get multimem_support") - return bool(self._get_comm_properties().multimem_support) + return self.properties.multimem_support @property def gin_type(self) -> NcclGinType: - """GPU-Initiated Networking (GIN) type. + """GPU-Initiated Networking (GIN) type reaching every rank. - If equal to NcclGinType.NONE, a device communicator cannot be - created with GIN resources. + If equal to :py:attr:`NcclGinType.NONE`, a device communicator + cannot be created with GIN connection type + :py:attr:`NcclGinConnectionType.FULL`. A rail-restricted transport + may still be available; see :py:attr:`railed_gin_type`. Raises: NcclInvalid: If the communicator is not initialized. """ self._check_valid("get gin_type") - return NcclGinType(self._get_comm_properties().gin_type) + return self.properties.gin_type @property def n_lsa_teams(self) -> int: @@ -1094,7 +1604,7 @@ def n_lsa_teams(self) -> int: """ self._check_valid("get n_lsa_teams") - return self._get_comm_properties().n_lsa_teams + return self.properties.n_lsa_teams @property def team_world(self) -> NCCLTeam: @@ -1104,7 +1614,7 @@ def team_world(self) -> NCCLTeam: NcclInvalid: If the communicator is not initialized. """ self._check_valid("get team_world") - pod = _nccl_bindings.team_world(self.ptr) + pod = _nccl_bindings.team_world(self._comm) return NCCLTeam(pod.n_ranks, pod.rank, pod.stride) @property @@ -1115,7 +1625,7 @@ def team_lsa(self) -> NCCLTeam: NcclInvalid: If the communicator is not initialized. """ self._check_valid("get team_lsa") - pod = _nccl_bindings.team_lsa(self.ptr) + pod = _nccl_bindings.team_lsa(self._comm) return NCCLTeam(pod.n_ranks, pod.rank, pod.stride) @property @@ -1126,7 +1636,38 @@ def team_rail(self) -> NCCLTeam: NcclInvalid: If the communicator is not initialized. """ self._check_valid("get team_rail") - pod = _nccl_bindings.team_rail(self.ptr) + pod = _nccl_bindings.team_rail(self._comm) + return NCCLTeam(pod.n_ranks, pod.rank, pod.stride) + + def team_cft(self, mode: NcclCftTeamMode = NcclCftTeamMode.FLAT) -> NCCLTeam: + """The CFT team for this communicator, in the requested layout. + + Args: + mode: Team layout. Defaults to + :py:attr:`NcclCftTeamMode.FLAT`, matching the C default. + + Raises: + NcclInvalid: If the communicator is not initialized. + + See Also: + :c:func:`ncclTeamCft` + """ + self._check_valid("get team_cft") + pod = _nccl_bindings.team_cft(self._comm, int(mode)) + return NCCLTeam(pod.n_ranks, pod.rank, pod.stride) + + @property + def team_cft_multimem(self) -> NCCLTeam: + """The CFT multimem team for this communicator. + + Raises: + NcclInvalid: If the communicator is not initialized. + + See Also: + :c:func:`ncclTeamCftMultimem` + """ + self._check_valid("get team_cft_multimem") + pod = _nccl_bindings.team_cft_multimem(self._comm) return NCCLTeam(pod.n_ranks, pod.rank, pod.stride) @property @@ -1138,7 +1679,7 @@ def host_rma_support(self) -> bool: """ self._check_valid("get host_rma_support") - return bool(self._get_comm_properties().host_rma_support) + return self.properties.host_rma_support @property def railed_gin_type(self) -> NcclGinType: @@ -1153,7 +1694,7 @@ def railed_gin_type(self) -> NcclGinType: """ self._check_valid("get railed_gin_type") - return NcclGinType(self._get_comm_properties().railed_gin_type) + return self.properties.railed_gin_type def team_rank_to_world(self, team: NCCLTeam, team_rank: int) -> int: """Maps a rank within ``team`` to its rank in this communicator. @@ -1175,7 +1716,7 @@ def team_rank_to_world(self, team: NCCLTeam, team_rank: int) -> int: """ self._check_valid("team_rank_to_world") team_lowpp = team._to_lowpp() - return _nccl_bindings.team_rank_to_world(self.ptr, team_lowpp.ptr, team_rank) + return _nccl_bindings.team_rank_to_world(self._comm, team_lowpp, team_rank) def team_rank_to_lsa(self, team: NCCLTeam, team_rank: int) -> int: """Maps a rank within ``team`` to its rank in the LSA team. @@ -1200,7 +1741,7 @@ def team_rank_to_lsa(self, team: NCCLTeam, team_rank: int) -> int: """ self._check_valid("team_rank_to_lsa") team_lowpp = team._to_lowpp() - return _nccl_bindings.team_rank_to_lsa(self.ptr, team_lowpp.ptr, team_rank) + return _nccl_bindings.team_rank_to_lsa(self._comm, team_lowpp, team_rank) # --- Point-to-Point Communication --- def send( @@ -1227,7 +1768,7 @@ def send( self._validate_buffer_device(s, "sendbuf") _nccl_bindings.send( - s.ptr, s.count, int(s.dtype), int(peer), int(self._comm), get_stream_ptr(stream) + s.ptr, s.count, int(s.dtype), int(peer), self._comm, get_stream_ptr(stream) ) def recv( @@ -1254,7 +1795,7 @@ def recv( self._validate_buffer_device(r, "recvbuf") _nccl_bindings.recv( - r.ptr, r.count, int(r.dtype), int(peer), int(self._comm), get_stream_ptr(stream) + r.ptr, r.count, int(r.dtype), int(peer), self._comm, get_stream_ptr(stream) ) def wait_signal( @@ -1290,7 +1831,7 @@ def wait_signal( lowpp_descs = [d._to_lowpp() for d in descs] buf = bytearray().join(bytes(d) for d in lowpp_descs) - _nccl_bindings.wait_signal(len(descs), buf, int(self._comm), get_stream_ptr(stream)) + _nccl_bindings.wait_signal(len(descs), buf, self._comm, get_stream_ptr(stream)) def signal( self, @@ -1309,8 +1850,10 @@ def signal( Args: peer: Target rank to send the signal to. - signal_index: Signal index identifier. Currently must be 0. - context: Context identifier. Currently must be 0. + signal_index: Signal index identifier. Must lie in + ``[0, num_rma_sig)``; see :py:attr:`NCCLConfig.num_rma_sig`. + context: Context identifier. Must lie in ``[0, num_rma_ctx)``; + see :py:attr:`NCCLConfig.num_rma_ctx`. flags: Reserved for future use. Currently must be 0. stream: CUDA stream to enqueue the signal operation on. Defaults to ``None`` (the default stream). @@ -1357,8 +1900,10 @@ def put_signal( (from :py:meth:`register_window`). peer_window_offset: Offset in the peer's window in elements. Defaults to 0. - signal_index: Signal index identifier. Currently must be 0. - context: Context identifier. Currently must be 0. + signal_index: Signal index identifier. Must lie in + ``[0, num_rma_sig)``; see :py:attr:`NCCLConfig.num_rma_sig`. + context: Context identifier. Must lie in ``[0, num_rma_ctx)``; + see :py:attr:`NCCLConfig.num_rma_ctx`. flags: Reserved for future use. Currently must be 0. stream: CUDA stream to enqueue the put_signal operation on. Defaults to ``None`` (the default stream). @@ -1381,7 +1926,7 @@ def put_signal( buffer.count, int(buffer.dtype), peer, - peer_window.handle, + peer_window._window, int(peer_window_offset) * buffer.dtype.itemsize, signal_index, context, @@ -1399,19 +1944,21 @@ def allreduce( op: NcclRedOp | CustomRedOp, *, stream: NcclStreamSpec | None = None, + config: NCCLCollConfig | None = None, ) -> None: """All-reduce variant of :py:meth:`reduce`. Equivalent to ``reduce(sendbuf, recvbuf, op, root=None, stream=stream)``: reduces data across all ranks and stores identical copies in each - rank's recvbuf. See :py:meth:`reduce` for argument semantics. + rank's recvbuf. ``config`` is forwarded unchanged. See + :py:meth:`reduce` for argument semantics. See Also: :py:meth:`reduce`, :c:func:`ncclAllReduce` """ self._check_valid("allreduce") - self.reduce(sendbuf, recvbuf, op, stream=stream) + self.reduce(sendbuf, recvbuf, op, stream=stream, config=config) def broadcast( self, @@ -1420,6 +1967,7 @@ def broadcast( root: int, *, stream: NcclStreamSpec | None = None, + config: NCCLCollConfig | None = None, ) -> None: """Copies data from ``sendbuf`` on the root rank to all ranks' ``recvbuf``. @@ -1438,6 +1986,8 @@ def broadcast( root: Root rank that broadcasts the data (0 to ``nranks - 1``). stream: CUDA stream for the operation. Defaults to ``None`` (the default stream). + config: Per-call :py:class:`NCCLCollConfig`. Must be identical on + every rank. Defaults to ``None`` (NCCL's own defaults). Raises: NcclInvalid: If send and receive buffers have mismatched dtypes, @@ -1471,9 +2021,16 @@ def broadcast( count = r.count dtype = r.dtype - _nccl_bindings.broadcast( - s_ptr, r_ptr, count, int(dtype), int(root), int(self._comm), get_stream_ptr(stream) - ) + cfg, _cfg_keepalive = _materialize_coll_config(config) + if cfg is None: + _nccl_bindings.broadcast( + s_ptr, r_ptr, count, int(dtype), int(root), self._comm, get_stream_ptr(stream) + ) + else: + _nccl_bindings.broadcast_config( + s_ptr, r_ptr, count, int(dtype), int(root), self._comm, + get_stream_ptr(stream), cfg.ptr, + ) def reduce( self, @@ -1483,6 +2040,7 @@ def reduce( root: int | None = None, *, stream: NcclStreamSpec | None = None, + config: NCCLCollConfig | None = None, ) -> None: """Reduces data from all ranks using the specified operation. @@ -1512,6 +2070,8 @@ def reduce( Defaults to ``None``. stream: CUDA stream for the operation. Defaults to ``None`` (the default stream). + config: Per-call :py:class:`NCCLCollConfig`. Must be identical on + every rank. Defaults to ``None`` (NCCL's own defaults). Raises: NcclInvalid: If send and receive buffers have mismatched dtypes, @@ -1544,21 +2104,26 @@ def reduce( count = s.count dtype = s.dtype - if root is None: + cfg, _cfg_keepalive = _materialize_coll_config(config) + if root is None and cfg is None: _nccl_bindings.all_reduce( - s_ptr, r_ptr, count, int(dtype), int(op), int(self._comm), get_stream_ptr(stream) + s_ptr, r_ptr, count, int(dtype), int(op), self._comm, get_stream_ptr(stream) ) - else: + elif root is None: + _nccl_bindings.all_reduce_config( + s_ptr, r_ptr, count, int(dtype), int(op), self._comm, + get_stream_ptr(stream), cfg.ptr, + ) + elif cfg is None: _nccl_bindings.reduce( - s_ptr, - r_ptr, - count, - int(dtype), - int(op), - int(root), - int(self._comm), + s_ptr, r_ptr, count, int(dtype), int(op), int(root), self._comm, get_stream_ptr(stream), ) + else: + _nccl_bindings.reduce_config( + s_ptr, r_ptr, count, int(dtype), int(op), int(root), self._comm, + get_stream_ptr(stream), cfg.ptr, + ) def allgather( self, @@ -1566,20 +2131,22 @@ def allgather( recvbuf: NcclBufferSpec, *, stream: NcclStreamSpec | None = None, + config: NCCLCollConfig | None = None, ) -> None: """All-gather variant of :py:meth:`gather`. Equivalent to ``gather(sendbuf, recvbuf, root=None, stream=stream)``: gathers ``sendcount`` values from each rank and places identical - copies of the concatenated result in every rank's recvbuf. See - :py:meth:`gather` for argument semantics. + copies of the concatenated result in every rank's recvbuf. + ``config`` is forwarded unchanged. See :py:meth:`gather` for + argument semantics. See Also: :py:meth:`gather`, :c:func:`ncclAllGather` """ self._check_valid("allgather") - self.gather(sendbuf, recvbuf, stream=stream) + self.gather(sendbuf, recvbuf, stream=stream, config=config) def reduce_scatter( self, @@ -1588,6 +2155,7 @@ def reduce_scatter( op: NcclRedOp | CustomRedOp, *, stream: NcclStreamSpec | None = None, + config: NCCLCollConfig | None = None, ) -> None: """Reduces data from all ranks and scatters the result across ranks. @@ -1609,6 +2177,8 @@ def reduce_scatter( :py:class:`~nccl.core.CustomRedOp`). stream: CUDA stream for the operation. Defaults to ``None`` (the default stream). + config: Per-call :py:class:`NCCLCollConfig`. Must be identical on + every rank. Defaults to ``None`` (NCCL's own defaults). Raises: NcclInvalid: If send and receive buffers have mismatched dtypes, @@ -1644,9 +2214,16 @@ def reduce_scatter( count = per_rank_count dtype = s.dtype - _nccl_bindings.reduce_scatter( - s_ptr, r_ptr, count, int(dtype), int(op), int(self._comm), get_stream_ptr(stream) - ) + cfg, _cfg_keepalive = _materialize_coll_config(config) + if cfg is None: + _nccl_bindings.reduce_scatter( + s_ptr, r_ptr, count, int(dtype), int(op), self._comm, get_stream_ptr(stream) + ) + else: + _nccl_bindings.reduce_scatter_config( + s_ptr, r_ptr, count, int(dtype), int(op), self._comm, + get_stream_ptr(stream), cfg.ptr, + ) def alltoall( self, @@ -1654,6 +2231,7 @@ def alltoall( recvbuf: NcclBufferSpec, *, stream: NcclStreamSpec | None = None, + config: NCCLCollConfig | None = None, ) -> None: """Each rank sends and receives ``count`` values to and from every other rank. @@ -1671,6 +2249,8 @@ def alltoall( recvbuf: Destination buffer (size ``>= nranks * count`` elements). stream: CUDA stream for the operation. Defaults to ``None`` (the default stream). + config: Per-call :py:class:`NCCLCollConfig`. Must be identical on + every rank. Defaults to ``None`` (NCCL's own defaults). Raises: NcclInvalid: If send and receive buffers have mismatched dtypes, @@ -1706,9 +2286,16 @@ def alltoall( count = per_rank_count dtype = s.dtype - _nccl_bindings.allto_all( - s_ptr, r_ptr, count, int(dtype), int(self._comm), get_stream_ptr(stream) - ) + cfg, _cfg_keepalive = _materialize_coll_config(config) + if cfg is None: + _nccl_bindings.allto_all( + s_ptr, r_ptr, count, int(dtype), self._comm, get_stream_ptr(stream) + ) + else: + _nccl_bindings.allto_all_config( + s_ptr, r_ptr, count, int(dtype), self._comm, + get_stream_ptr(stream), cfg.ptr, + ) def gather( self, @@ -1717,6 +2304,7 @@ def gather( root: int | None = None, *, stream: NcclStreamSpec | None = None, + config: NCCLCollConfig | None = None, ) -> None: """Gathers ``sendcount`` values from all ranks. @@ -1745,6 +2333,8 @@ def gather( Defaults to ``None``. stream: CUDA stream for the operation. Defaults to ``None`` (the default stream). + config: Per-call :py:class:`NCCLCollConfig`. Must be identical on + every rank. Defaults to ``None`` (NCCL's own defaults). Raises: NcclInvalid: If send and receive buffers have mismatched dtypes, @@ -1779,13 +2369,24 @@ def gather( count = s.count dtype = s.dtype - if root is None: + cfg, _cfg_keepalive = _materialize_coll_config(config) + if root is None and cfg is None: _nccl_bindings.all_gather( - s_ptr, r_ptr, count, int(dtype), int(self._comm), get_stream_ptr(stream) + s_ptr, r_ptr, count, int(dtype), self._comm, get_stream_ptr(stream) ) - else: + elif root is None: + _nccl_bindings.all_gather_config( + s_ptr, r_ptr, count, int(dtype), self._comm, + get_stream_ptr(stream), cfg.ptr, + ) + elif cfg is None: _nccl_bindings.gather( - s_ptr, r_ptr, count, int(dtype), int(root), int(self._comm), get_stream_ptr(stream) + s_ptr, r_ptr, count, int(dtype), int(root), self._comm, get_stream_ptr(stream) + ) + else: + _nccl_bindings.gather_config( + s_ptr, r_ptr, count, int(dtype), int(root), self._comm, + get_stream_ptr(stream), cfg.ptr, ) def scatter( @@ -1795,6 +2396,7 @@ def scatter( root: int, *, stream: NcclStreamSpec | None = None, + config: NCCLCollConfig | None = None, ) -> None: """Scatters data from the root rank to all ranks. @@ -1815,6 +2417,8 @@ def scatter( root: Root rank that scatters the data (0 to ``nranks - 1``). stream: CUDA stream for the operation. Defaults to ``None`` (the default stream). + config: Per-call :py:class:`NCCLCollConfig`. Must be identical on + every rank. Defaults to ``None`` (NCCL's own defaults). Raises: NcclInvalid: If send and receive buffers have mismatched dtypes, @@ -1854,9 +2458,16 @@ def scatter( count = r.count dtype = r.dtype - _nccl_bindings.scatter( - s_ptr, r_ptr, count, int(dtype), int(root), int(self._comm), get_stream_ptr(stream) - ) + cfg, _cfg_keepalive = _materialize_coll_config(config) + if cfg is None: + _nccl_bindings.scatter( + s_ptr, r_ptr, count, int(dtype), int(root), self._comm, get_stream_ptr(stream) + ) + else: + _nccl_bindings.scatter_config( + s_ptr, r_ptr, count, int(dtype), int(root), self._comm, + get_stream_ptr(stream), cfg.ptr, + ) # --- Registration --- def register_buffer(self, buffer: NcclBufferSpec) -> RegisteredBufferHandle: @@ -1897,12 +2508,13 @@ def register_buffer(self, buffer: NcclBufferSpec) -> RegisteredBufferHandle: def register_window( self, buffer: NcclBufferSpec, flags: WindowFlag | None = None - ) -> RegisteredWindowHandle | None: + ) -> RegisteredWindowHandle: """Collectively registers a local buffer into an NCCL window. This is a collective call: every rank in the communicator must - participate, and buffer size must be equal among ranks by default. - Buffer size is automatically derived from buffer count and dtype. + participate. Buffer size is automatically derived from buffer count + and dtype. + If called within a group, the handle value may not be filled until ``ncclGroupEnd`` completes. For non-blocking communicators, the handle may remain ``0`` until :py:meth:`get_async_error` reports success. @@ -1919,8 +2531,8 @@ def register_window( Returns: :py:class:`~nccl.core.RegisteredWindowHandle` for the registered - window, or ``None`` if NCCL returns a NULL handle (e.g. windows - are unsupported on this platform). + window. Its handle remains ``0`` while registration is pending, + or when windows are unsupported on the platform. Raises: NcclInvalid: If the buffer is on the wrong device or the @@ -1936,17 +2548,10 @@ def register_window( buffer_ptr = nccl_buf.ptr size = nccl_buf.count * nccl_buf.dtype.itemsize + # NCCL itself gates registration on platform support. Always retain and + # return its handle storage because a null handle can also mean that a + # grouped or non-blocking registration has not filled it yet. resource = RegisteredWindowHandle(self._comm, buffer_ptr, size, flags) - if self.get_async_error() == int(_Result.Success) and resource.handle == 0: - # A non-blocking communicator only initializes the handle value - # after InProgress has changed to Success. Until then the handle - # will remain NULL, but we cannot assume its invalid. - # - # But if status is Success and handle is still null, then we can - # simply return None. - resource.close() - return None - self._resources.append(resource) return resource diff --git a/projects/rccl/bindings/nccl4py/nccl/core/constants.py b/projects/rccl/bindings/nccl4py/nccl/core/constants.py index 6bcc60adc55a..0d0162ae9d8c 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/constants.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/constants.py @@ -23,7 +23,7 @@ # CTA (Cooperative Thread Array) Policy flags class CTAPolicy(IntFlag): """NCCL performance policy for CTA scheduling, used by - :py:attr:`NCCLConfig.cta_policy`. + :py:attr:`NCCLConfig.cta_policy` and :py:attr:`NCCLCollConfig.cta_policy`. """ DEFAULT = 0x00 @@ -79,6 +79,11 @@ class WindowFlag(IntFlag): """Collective symmetric window registration.""" STRICT_ORDERING = 0x02 """Strict ordering for window operations.""" + GIN_ONLY = 0x04 + """Register the window for GIN access only (NCCL 2.32+).""" + CFT_COUNTED = 0x08 + """Register the window against counted CFT logical endpoints instead of + the default non-counted ones (NCCL 2.32+).""" # Backward-compat aliases. Default = 0x00 diff --git a/projects/rccl/bindings/nccl4py/nccl/core/cuda.py b/projects/rccl/bindings/nccl4py/nccl/core/cuda.py index fabced0fe691..d51070e8d29d 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/cuda.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/cuda.py @@ -3,19 +3,20 @@ # # See LICENSE.txt for more license information -"""CUDA device and stream utilities for NCCL operations. +"""CUDA device, stream, and event utilities for NCCL operations. This module provides context managers and helper functions for working with -CUDA devices and streams in NCCL operations, including device context management -and stream resolution. These helpers are used internally to translate user -device/stream specifications into the concrete forms NCCL expects. +CUDA devices, streams, and events in NCCL operations, including device context +management and stream/event handle resolution. These helpers are used +internally to translate user specifications into the concrete forms NCCL +expects. """ from __future__ import annotations -from cuda.core import Device, Stream +from cuda.core import Device, Event, Stream -from nccl.core.typing import NcclDeviceSpec, NcclStreamSpec +from nccl.core.typing import NcclDeviceSpec, NcclEventSpec, NcclStreamSpec class CudaDeviceContext: @@ -90,35 +91,78 @@ def get_cuda_stream( Returns: Resolved cuda.core.Stream. + + Raises: + TypeError: If ``stream`` is a boolean. """ + if isinstance(stream, bool): + raise TypeError("stream must not be a bool") if stream is None: device = get_cuda_device(device) return device.default_stream - elif isinstance(stream, Stream): + if isinstance(stream, Stream): return stream - elif isinstance(stream, int): + if isinstance(stream, int): return Stream.from_handle(handle=stream) - else: - device = get_cuda_device(device) - return device.create_stream(stream) + device = get_cuda_device(device) + return device.create_stream(stream) def get_stream_ptr(stream: NcclStreamSpec | None = None) -> int: - """Resolves a stream specification to its raw stream handle (as int). + """Resolves a CUDA stream specification to an integer ``cudaStream_t`` handle. Args: - stream: A Stream instance, an integer stream handle, an object - implementing __cuda_stream__, or ``None`` to use the default - stream (handle 0). + stream: A ``cuda.core.Stream``, an object implementing + ``__cuda_stream__``, an integer ``cudaStream_t`` handle, or + ``None`` for the default stream (handle 0). Returns: - Raw CUDA stream handle as int. Returns 0 for the default stream. + The CUDA stream handle as an integer, or 0 for the default stream. + + Raises: + TypeError: If ``stream`` is a boolean. """ + if isinstance(stream, bool): + raise TypeError("stream must not be a bool") if stream is None: return 0 - elif isinstance(stream, int): + if isinstance(stream, int): return stream - elif isinstance(stream, Stream): + if isinstance(stream, Stream): return int(stream.handle) - else: - return int(stream.__cuda_stream__()[1]) + return int(stream.__cuda_stream__()[1]) + + +def get_event_ptr(event: NcclEventSpec | None = None) -> int: + """Resolves a CUDA event specification to an integer ``cudaEvent_t`` handle. + + Args: + event: A ``cuda.core.Event``, a nonzero integer ``cudaEvent_t`` handle, + or ``None`` for no event. Convert other integer-convertible handle + objects explicitly with ``int()`` before passing them. + + Returns: + The CUDA event handle as an integer, or 0 when no event is given. + + Raises: + TypeError: If ``event`` is a boolean or is not a ``cuda.core.Event`` + or ``int``. + RuntimeError: If ``event`` is a closed ``cuda.core.Event``. + ValueError: If ``event`` is an integer with value 0. + """ + if event is None: + return 0 + if isinstance(event, Event): + handle = int(event.handle) + if handle == 0: + raise RuntimeError("Event has been closed") + return handle + if isinstance(event, bool) or not isinstance(event, int): + raise TypeError( + "event must be a cuda.core.Event or a nonzero integer cudaEvent_t handle" + ) + if event == 0: + raise ValueError( + "CUDA event has a null handle; initialize it before passing it to NCCL" + ) + return event diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/__init__.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/__init__.py index 8481e02ea76a..ce85a6908cca 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/__init__.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/__init__.py @@ -14,7 +14,7 @@ check is performed; a mismatch can cause link errors or incorrect kernel behavior. -See ``examples/cute/main.py`` for a complete, runnable example. +See ``examples/cute/00_basic.py`` for a complete, runnable example. """ try: @@ -27,7 +27,7 @@ " pip install 'nccl4py[cu13]' # for CUDA 13" ) from e -from . import types, coop, handles, comm, window, gin, barrier +from . import types, coop, handles, comm, window, gin, barrier, reduce_copy, runtime from .types import * # MemoryOrder, ThreadScope, GinFenceLevel, GinBackendMask, GinResourceSharingMode from .coop import * # Coop, cta, warp, thread, lanes, warp_span from .handles import * # MultimemHandle, LsaBarrierHandle, GinBarrierHandle @@ -35,6 +35,7 @@ from .window import * # Window from .gin import * # Gin from .barrier import * # session classes + factories +from .reduce_copy import * # ReduceCopy functions __all__ = [ "types", @@ -44,6 +45,8 @@ "window", "gin", "barrier", + "reduce_copy", + "runtime", *types.__all__, *coop.__all__, *handles.__all__, @@ -51,4 +54,5 @@ *window.__all__, *gin.__all__, *barrier.__all__, + *reduce_copy.__all__, ] diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_bindings.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_bindings.py index da12b587c8f5..40f17da7023f 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_bindings.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_bindings.py @@ -1,18 +1,15 @@ -"""1:1 CuTeDSL ``@cute.extern`` prototypes for ``bindings/ir/nccl_device_wrapper.h``. +"""Low-level CuTeDSL bindings for ``bindings/ir/nccl_device_wrapper.h``. -Codegen target. Each entry is a typed ``@cute.extern`` stub bound to the -device bitcode (``source=_BC``): parameter types and return type are -derived from the stub signature, and the C symbol is named explicitly via -``name=`` so the Python stub can keep a PEP 8 ``snake_case`` name. The -stubs are the raw FFI surface — argument coercion (``_to_ptr`` / -``_to_value`` / cutlass numeric casts) and return-value wrapping are the -caller's job. +Fixed symbols use ``@cute.extern``. ReduceCopy symbols are selected by dtype +through overloaded ``@cute.extern`` declarations. Callers handle argument +conversion and result wrapping. Struct decls live in :mod:`_structs`; coercion helpers in :mod:`_helpers`. """ import os from pathlib import Path +from typing import TypeVar import cutlass import cutlass.cute as cute @@ -105,6 +102,16 @@ def nccl_team_lsa(dev_comm: _LLVMPtrType) -> ncclTeam: ... @cute.extern(name="ncclTeamRail", source=_BC) def nccl_team_rail(dev_comm: _LLVMPtrType) -> ncclTeam: ... +# `mode` is ncclCftTeamMode_t, i.e. nccl.bindings.nccl.CftTeamMode. The C +# declaration defaults it to FLAT; the symbol takes it explicitly. +@cute.extern(name="ncclTeamCft", source=_BC) +def nccl_team_cft( + dev_comm: _LLVMPtrType, mode: cutlass.Int32, +) -> ncclTeam: ... + +@cute.extern(name="ncclTeamCftMultimem", source=_BC) +def nccl_team_cft_multimem(dev_comm: _LLVMPtrType) -> ncclTeam: ... + @cute.extern(name="ncclTeamRankToWorld", source=_BC) def nccl_team_rank_to_world( dev_comm: _LLVMPtrType, team: ncclTeam, rank: cutlass.Int32, @@ -178,6 +185,227 @@ def nccl_get_resource_buffer_lsa_multimem_pointer( ) -> _LLVMPtrType: ... +# === CFT logical endpoints === +# +# `le_id` and `le_offset` are out-params: pointers to caller-owned +# ncclCftLeId (uint32) and size_t storage. + +@cute.extern(name="ncclGetCftLeInfo", source=_BC) +def nccl_get_cft_le_info( + window: _LLVMPtrType, offset: cutlass.Int64, peer_cft: cutlass.Int32, + cft_team: ncclTeam, dev_comm: _LLVMPtrType, le_id: _LLVMPtrType, + le_offset: _LLVMPtrType, +) -> None: ... + +@cute.extern(name="ncclGetPeerLeInfo", source=_BC) +def nccl_get_peer_le_info( + window: _LLVMPtrType, offset: cutlass.Int64, peer_world: cutlass.Int32, + dev_comm: _LLVMPtrType, le_id: _LLVMPtrType, le_offset: _LLVMPtrType, +) -> None: ... + +@cute.extern(name="ncclGetMultimemLeInfo", source=_BC) +def nccl_get_multimem_le_info( + window: _LLVMPtrType, offset: cutlass.Int64, dev_comm: _LLVMPtrType, + le_id: _LLVMPtrType, le_offset: _LLVMPtrType, +) -> None: ... + +@cute.extern(name="ncclGetResourceBufferCftLeInfo", source=_BC) +def nccl_get_resource_buffer_cft_le_info( + dev_comm: _LLVMPtrType, handle: cutlass.Uint32, peer_cft: cutlass.Int32, + le_id: _LLVMPtrType, le_offset: _LLVMPtrType, +) -> None: ... + +@cute.extern(name="ncclGetResourceBufferPeerLeInfo", source=_BC) +def nccl_get_resource_buffer_peer_le_info( + dev_comm: _LLVMPtrType, handle: cutlass.Uint32, peer_world: cutlass.Int32, + le_id: _LLVMPtrType, le_offset: _LLVMPtrType, +) -> None: ... + +@cute.extern(name="ncclGetResourceBufferMultimemLeInfo", source=_BC) +def nccl_get_resource_buffer_multimem_le_info( + dev_comm: _LLVMPtrType, handle: cutlass.Uint32, le_id: _LLVMPtrType, + le_offset: _LLVMPtrType, +) -> None: ... + + +_CUTE_DTYPE_SUFFIX_MAPPING = { + cutlass.Int8: "I8", + cutlass.Uint8: "U8", + cutlass.Int32: "I32", + cutlass.Uint32: "U32", + cutlass.Int64: "I64", + cutlass.Uint64: "U64", + cutlass.Float16: "F16", + cutlass.Float32: "F32", + cutlass.Float64: "F64", + cutlass.BFloat16: "BF16", + cutlass.Float8E4M3: "F8E4M3", + cutlass.Float8E5M2: "F8E5M2", +} + +_REDUCE_COPY_ALL_DTYPES = ( + cutlass.Int8, + cutlass.Uint8, + cutlass.Int32, + cutlass.Uint32, + cutlass.Int64, + cutlass.Uint64, + cutlass.Float16, + cutlass.Float32, + cutlass.Float64, + cutlass.BFloat16, + cutlass.Float8E4M3, + cutlass.Float8E5M2, +) + +_REDUCE_COPY_MULTIMEM_DTYPES = ( + cutlass.Int32, + cutlass.Uint32, + cutlass.Int64, + cutlass.Uint64, + cutlass.Float16, + cutlass.Float32, + cutlass.Float64, + cutlass.BFloat16, + cutlass.Float8E4M3, + cutlass.Float8E5M2, +) + +ReduceCopyAllDTypes = TypeVar( + "ReduceCopyAllDTypes", + *_REDUCE_COPY_ALL_DTYPES, +) +ReduceCopyMultiMemDTypes = TypeVar( + "ReduceCopyMultiMemDTypes", + *_REDUCE_COPY_MULTIMEM_DTYPES, +) + + +def _reduce_copy_name_mangler(ffi): + dtype_param = next( + (param for param in ffi.params_types if isinstance(param, cute.ConstValue)), + None, + ) + if dtype_param is None: + raise TypeError(f"{ffi.name} requires a constexpr dtype") + dtype = dtype_param.value + + if ffi.name.startswith("ncclMultimem") and dtype not in _REDUCE_COPY_MULTIMEM_DTYPES: + raise TypeError(f"{ffi.name} does not support {dtype}") + + suffix = _CUTE_DTYPE_SUFFIX_MAPPING[dtype] + return f"{ffi.name}_{suffix}" + + +@cute.extern( + name="ncclLsaReduceSum", + source=_BC, + name_mangler=_reduce_copy_name_mangler, +) +def nccl_lsa_reduce_sum( + coop: ncclCoopAny, + src_window: _LLVMPtrType, + src_offset: cutlass.Int64, + dst: _LLVMPtrType, + count: cutlass.Int64, + team: ncclTeam, + dtype: cutlass.Constexpr[ReduceCopyAllDTypes], +) -> None: ... + + +@cute.extern( + name="ncclMultimemReduceSum", + source=_BC, + name_mangler=_reduce_copy_name_mangler, +) +def nccl_multimem_reduce_sum( + coop: ncclCoopAny, + mc_src: _LLVMPtrType, + dst: _LLVMPtrType, + count: cutlass.Int64, + dtype: cutlass.Constexpr[ReduceCopyMultiMemDTypes], +) -> None: ... + + +@cute.extern( + name="ncclLsaCopy", + source=_BC, + name_mangler=_reduce_copy_name_mangler, +) +def nccl_lsa_copy( + coop: ncclCoopAny, + src: _LLVMPtrType, + dst_window: _LLVMPtrType, + dst_offset: cutlass.Int64, + count: cutlass.Int64, + team: ncclTeam, + dtype: cutlass.Constexpr[ReduceCopyAllDTypes], +) -> None: ... + + +@cute.extern( + name="ncclMultimemCopy", + source=_BC, + name_mangler=_reduce_copy_name_mangler, +) +def nccl_multimem_copy( + coop: ncclCoopAny, + src: _LLVMPtrType, + mc_dst: _LLVMPtrType, + count: cutlass.Int64, + dtype: cutlass.Constexpr[ReduceCopyMultiMemDTypes], +) -> None: ... + + +@cute.extern( + name="ncclLsaReduceSumCopy", + source=_BC, + name_mangler=_reduce_copy_name_mangler, +) +def nccl_lsa_reduce_sum_copy( + coop: ncclCoopAny, + src_window: _LLVMPtrType, + src_offset: cutlass.Int64, + dst_window: _LLVMPtrType, + dst_offset: cutlass.Int64, + count: cutlass.Int64, + team: ncclTeam, + dtype: cutlass.Constexpr[ReduceCopyAllDTypes], +) -> None: ... + + +@cute.extern( + name="ncclMultimemReduceSumCopy", + source=_BC, + name_mangler=_reduce_copy_name_mangler, +) +def nccl_multimem_reduce_sum_copy( + coop: ncclCoopAny, + mc_src: _LLVMPtrType, + mc_dst: _LLVMPtrType, + count: cutlass.Int64, + dtype: cutlass.Constexpr[ReduceCopyMultiMemDTypes], +) -> None: ... + + +@cute.extern( + name="ncclLocalReduceSumCopy", + source=_BC, + name_mangler=_reduce_copy_name_mangler, +) +def nccl_local_reduce_sum_copy( + coop: ncclCoopAny, + n_src: cutlass.Int32, + src_base: _LLVMPtrType, + src_displ: cutlass.Int64, + n_dst: cutlass.Int32, + dst_base: _LLVMPtrType, + dst_displ: cutlass.Int64, + count: cutlass.Int64, + dtype: cutlass.Constexpr[ReduceCopyAllDTypes], +) -> None: ... + + # === GIN API === @cute.extern(name="ncclGin_C_initWithResourceSharingMode", source=_BC) @@ -333,6 +561,9 @@ def nccl_lsa_barrier_session_sync( session: _LLVMPtrType, coop: ncclCoopAny, order: cutlass.Int32, ) -> None: ... +@cute.extern(name="ncclLsaBarrierSessionDestroy", source=_BC) +def nccl_lsa_barrier_session_destroy(session: _LLVMPtrType) -> None: ... + # === GIN Barrier Session API === @@ -354,6 +585,9 @@ def nccl_gin_barrier_session_sync( fence: cutlass.Int32, ) -> None: ... +@cute.extern(name="ncclGinBarrierSessionDestroy", source=_BC) +def nccl_gin_barrier_session_destroy(session: _LLVMPtrType) -> None: ... + # === Hybrid Barrier Session API === @@ -371,6 +605,9 @@ def nccl_barrier_session_sync( fence: cutlass.Int32, ) -> None: ... +@cute.extern(name="ncclBarrierSessionDestroy", source=_BC) +def nccl_barrier_session_destroy(session: _LLVMPtrType) -> None: ... + # === Session size getters === diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_helpers.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_helpers.py index 25d31b7ab9ff..7ca8c62d733a 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_helpers.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_helpers.py @@ -1,7 +1,7 @@ """Hand-written internal utilities for the CuTeDSL bindings. -Holds the ``_alloca_struct`` stack-storage helper and the ``_to_ptr`` / -``_to_coop_value`` / ``_to_value`` coercion helpers used by the callers. +Holds the ``_alloca_*`` stack-storage helpers, ``_load_cft_le_info``, and the +``_to_ptr`` / ``_to_coop_value`` / ``_to_value`` coercion helpers. Device bitcode discovery lives in :mod:`_bindings`, next to the ``@cute.extern`` stubs that link against it. """ @@ -12,6 +12,7 @@ from cutlass.cutlass_dsl import dsl_user_op from ._structs import ncclCoopAny +from .types import CftLeInfo @dsl_user_op @@ -35,6 +36,41 @@ def _alloca_struct(struct_cls, *, alignment=None, loc=None, ip=None) -> ir.Value ) +@dsl_user_op +def _alloca_cft_le_info(*, loc=None, ip=None) -> tuple[ir.Value, ir.Value]: + """Alloca the ``(ncclCftLeId, size_t)`` out-params of ``ncclGet*LeInfo``. + + Returns: + ``(le_id_ptr, le_offset_ptr)``, both ``!llvm.ptr`` ir.Values. + """ + ptr_type = ir.Type.parse("!llvm.ptr") + one = cutlass.Int32(1).ir_value() + return ( + llvm.alloca(res=ptr_type, array_size=one, + elem_type=ir.IntegerType.get_signless(32), loc=loc, ip=ip), + llvm.alloca(res=ptr_type, array_size=one, + elem_type=ir.IntegerType.get_signless(64), loc=loc, ip=ip), + ) + + +@dsl_user_op +def _load_cft_le_info(le_id_ptr, le_offset_ptr, *, loc=None, ip=None) -> CftLeInfo: + """Load the out-params filled by ``ncclGet*LeInfo``. + + Returns: + Logical endpoint information containing ``cutlass.Uint32`` and + ``cutlass.Uint64`` values. + """ + return CftLeInfo( + le_id=cutlass.Uint32( + llvm.load(ir.IntegerType.get_signless(32), le_id_ptr, loc=loc, ip=ip) + ), + le_offset=cutlass.Uint64( + llvm.load(ir.IntegerType.get_signless(64), le_offset_ptr, loc=loc, ip=ip) + ), + ) + + # === Coercion helpers === @dsl_user_op @@ -42,13 +78,13 @@ def _to_ptr(x, *, loc=None, ip=None): """Coerce an address ``x`` to an ``!llvm.ptr`` ir.Value. Callers pass materialized pointers (``.ptr``) to the bindings directly; - this only handles the arguments given as an address — a null + this only handles the arguments given as an address -- a null ``descriptor_ptr=0`` or an explicit descriptor pointer: - * ``!llvm.ptr`` ir.Value — passthrough. - * cutlass numeric (has ``.ir_value()``) — ``inttoptr``. - * integer ``ir.Value`` — ``inttoptr``. - * Python int — wrap in ``cutlass.Int64``, then ``inttoptr``. + * ``!llvm.ptr`` ir.Value -- passthrough. + * cutlass numeric (has ``.ir_value()``) -- ``inttoptr``. + * integer ``ir.Value`` -- ``inttoptr``. + * Python int -- wrap in ``cutlass.Int64``, then ``inttoptr``. Returns: ``!llvm.ptr`` ir.Value. @@ -76,11 +112,11 @@ def _to_coop_value(x, *, loc=None, ip=None): annotation. Args: - x: accepted forms — + x: accepted forms -- - * ``ncclCoopAny`` struct ir.Value — passthrough. + * ``ncclCoopAny`` struct ir.Value -- passthrough. * ``@cute.native_struct`` pointer-wrapper around alloca'd - ``ncclCoopAny`` storage (e.g. :class:`Coop`) — load it. + ``ncclCoopAny`` storage (e.g. :class:`Coop`) -- load it. Returns: ``ncclCoopAny`` struct ir.Value. @@ -97,7 +133,7 @@ def _to_value(x): ``@cute.extern`` matches a native struct only as a bare struct ir.Value (a value-mode wrapper is rejected), so teams and barrier handles passed - by value — which ``cute.ffi`` accepted as wrappers — are unwrapped here. + by value -- which ``cute.ffi`` accepted as wrappers -- are unwrapped here. Args: x: a struct ir.Value (passthrough) or a value-mode native-struct diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_structs.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_structs.py index 13bc01d135fe..41f018967360 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_structs.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/_structs.py @@ -1,7 +1,7 @@ """CuTeDSL bindings for C struct types declared in ``bindings/ir/nccl_device_wrapper.h`` and ``src/include/nccl_device/``. -Codegen target — each entry is 1:1 with a C struct definition. +Codegen target -- each entry is 1:1 with a C struct definition. ``_LLVMPtrType`` and ``_array_i8`` are MLIR-type adapters used as ``@cute.native_struct`` field annotations. """ @@ -29,8 +29,8 @@ def __get_mlir_types__(): def isinstance(cls, value): """Match an ``!llvm.ptr`` ir.Value during ``@cute.extern`` dispatch. - Callers pass pointers as bare ``!llvm.ptr`` ir.Values — a wrapper's - ``.ptr``, or ``_to_ptr`` for an integer address — so the extern + Callers pass pointers as bare ``!llvm.ptr`` ir.Values -- a wrapper's + ``.ptr``, or ``_to_ptr`` for an integer address -- so the extern overload matcher needs this hook to recognize them against a ``_LLVMPtrType`` annotation. """ @@ -89,6 +89,15 @@ class ncclGinBarrierHandle: unused: cutlass.Uint32 +@cute.native_struct +class ncclCftBarrierHandle: + """``struct ncclCftBarrierHandle { ncclDevResourceHandle_t bufHandle; int nBarriers; }`` + (src/include/nccl_device/impl/cft_barrier__types.h).""" + + bufHandle: cutlass.Uint32 + nBarriers: cutlass.Int32 + + @cute.native_struct class ncclMultimemHandle: """``struct ncclMultimemHandle { void* mcBasePtr; }`` @@ -101,21 +110,18 @@ class ncclMultimemHandle: class ncclResourceWindow_vidmem: """``ncclResourceWindow_vidmem_t`` from ``impl/core__types.h``.""" - reserved1: _array_i8(8) lsa_flat_base: _LLVMPtrType - reserved2: _array_i8(8) stride4g: cutlass.Uint32 mc_offset4k: cutlass.Uint32 - reserved3: _array_i8(32) @cute.native_struct class DevCommValue: """By-value ABI mirror of ``struct ncclDevComm``. - Field order and types must stay synchronized with - ``nccl_device_expanded.h`` used to generate the low-level bindings. The - unpacked LLVM struct supplies the same natural padding as the C structure. + Field order and types must stay synchronized with ``struct ncclDevComm`` + in ``nccl_device/impl/comm__types.h``. The unpacked LLVM struct supplies + the same natural padding as the C structure. """ magic: cutlass.Uint32 @@ -132,22 +138,23 @@ class DevCommValue: resource_window: _LLVMPtrType resource_window_inlined: ncclResourceWindow_vidmem - hybrid_world_gin_barrier: ncclGinBarrierHandle + hybrid_dense_gin_barrier: ncclGinBarrierHandle lsa_multimem: ncclMultimemHandle lsa_barrier: ncclLsaBarrierHandle rail_gin_barrier: ncclGinBarrierHandle gin_connection_count: cutlass.Uint8 + backend_index: cutlass.Uint8 gin_net_device_types: _array_i8(4) gin_handles: _array_ptr(4) gin_signal_count: cutlass.Int32 gin_counter_count: cutlass.Int32 gin_signal_shadows: _LLVMPtrType gin_context_count: cutlass.Uint32 - gin_connections_railed: cutlass.Uint8 + gin_connection_stride: cutlass.Int32 + gin_context_stride: cutlass.Int32 gin_strong_legacy_signals: cutlass.Uint8 - gin_contexts_railed: cutlass.Uint8 abort_flag: _LLVMPtrType @@ -155,6 +162,17 @@ class DevCommValue: hybrid_rail_gin_barrier: ncclGinBarrierHandle world_gin_barrier: ncclGinBarrierHandle + gin_connection_stride_rcp32: cutlass.Uint32 + + cft_rank: cutlass.Int32 + cft_size: cutlass.Int32 + cft_multimem_rank: cutlass.Int32 + cft_multimem_size: cutlass.Int32 + cft_multimem_size_rcp32: cutlass.Uint32 + uc_le_id: cutlass.Uint32 + mc_le_id: cutlass.Uint32 + cft_barrier: ncclCftBarrierHandle + cft_multimem_barrier: ncclCftBarrierHandle @cute.native_struct @@ -164,7 +182,7 @@ class _ncclCoopStorage: @cute.native_struct class ncclCoopAny: - """``struct ncclCoopAny`` — 16-byte aligned-to-ptr storage + vtable ptr + """``struct ncclCoopAny`` -- 16-byte aligned-to-ptr storage + vtable ptr (src/include/nccl_device/coop.h).""" storage: _ncclCoopStorage diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/barrier.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/barrier.py index 611ba9a27de1..71e5cfc5799e 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/barrier.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/barrier.py @@ -2,13 +2,13 @@ Three session types: - * :class:`LsaBarrierSession` — LSA-only (intra-node, NVLink/peer-access). - * :class:`GinBarrierSession` — GIN-only (inter-node, network). - * :class:`BarrierSession` — hybrid (LSA inner + GIN outer). + * :class:`LsaBarrierSession` -- LSA-only (intra-node, NVLink/peer-access). + * :class:`GinBarrierSession` -- GIN-only (inter-node, network). + * :class:`BarrierSession` -- hybrid (LSA inner + GIN outer). Construct via the module-level factories: - * Explicit — caller supplies the team and barrier handle:: + * Explicit -- caller supplies the team and barrier handle:: sess = barrier.lsa_session(coop, dev_comm, team, lsa_handle, index=0) sess = barrier.gin_session(coop, gin, dev_comm, team, gin_handle, @@ -16,7 +16,7 @@ sess = barrier.hybrid_session(coop, inner_team, outer_team, gin, lsa_handle, gin_handle, index=0) - * DevComm-derived — team and handle pulled from ``dev_comm``:: + * DevComm-derived -- team and handle pulled from ``dev_comm``:: sess = barrier.lsa_default(coop, dev_comm, index=0) sess = barrier.world_gin(coop, gin, dev_comm, index=0) @@ -28,6 +28,12 @@ sess = barrier.world_gin(coop, barrier.GIN_ALL_CONTEXTS, dev_comm, index=0) + +Call ``destroy()`` exactly once after the session's final operation. All +threads in the session's cooperative group must call it from uniform control +flow. For LSA and hybrid sessions, this makes the same handle and index safe to +reuse. Omitting it can cause a later barrier to stop synchronizing without +hanging. """ import cutlass @@ -54,7 +60,7 @@ from .types import MemoryOrder, GinFenceLevel -# Session storage alignment. Conservatively chosen — covers ncclCoopAny +# Session storage alignment. Conservatively chosen -- covers ncclCoopAny # (ptr-aligned) and any wider fields any future session struct might add. _SESSION_ALIGN = 16 @@ -94,7 +100,7 @@ def _zero_multimem_handle() -> ncclMultimemHandle: @cute.native_struct class LsaBarrierSession: - """LSA (Load/Store Accessible) barrier session — intra-node, peer-access + """LSA (Load/Store Accessible) barrier session -- intra-node, peer-access based. Constructed via :func:`lsa_session`.""" ptr: _LLVMPtrType @@ -126,6 +132,14 @@ def sync(self, coop: ncclCoopAny, order: MemoryOrder) -> None: """ _bindings.nccl_lsa_barrier_session_sync(self.ptr, _to_coop_value(coop), cutlass.Int32(int(order))) + def destroy(self) -> None: + """Finalize the session so its handle and index can be safely reused. + + All threads in the session's cooperative group must call this exactly + once. + """ + _bindings.nccl_lsa_barrier_session_destroy(self.ptr) + def lsa_session( coop: ncclCoopAny, @@ -164,7 +178,7 @@ def lsa_session( @cute.native_struct class GinBarrierSession: - """GIN (network) barrier session — inter-node. Constructed via + """GIN (network) barrier session -- inter-node. Constructed via :func:`gin_session`. Only ``sync`` is supported.""" ptr: _LLVMPtrType @@ -180,6 +194,14 @@ def sync(self, coop: ncclCoopAny, order: MemoryOrder, fence: GinFenceLevel) -> N _bindings.nccl_gin_barrier_session_sync( self.ptr, _to_coop_value(coop), cutlass.Int32(int(order)), cutlass.Int32(int(fence))) + def destroy(self) -> None: + """Finalize the session. + + All threads in the session's cooperative group must call this exactly + once. + """ + _bindings.nccl_gin_barrier_session_destroy(self.ptr) + class _GinAllContexts: """Type of :data:`GIN_ALL_CONTEXTS`.""" @@ -236,7 +258,7 @@ def gin_session( @cute.native_struct class BarrierSession: - """Hybrid barrier — LSA inner stage + GIN outer stage. Constructed + """Hybrid barrier -- LSA inner stage + GIN outer stage. Constructed via :func:`hybrid_session`. Only ``sync`` is supported.""" ptr: _LLVMPtrType @@ -253,6 +275,14 @@ def sync(self, coop: ncclCoopAny, order: MemoryOrder, fence: GinFenceLevel) -> N _bindings.nccl_barrier_session_sync( self.ptr, _to_coop_value(coop), cutlass.Int32(int(order)), cutlass.Int32(int(fence))) + def destroy(self) -> None: + """Finalize the session so its handles and index can be safely reused. + + All threads in the session's cooperative group must call this exactly + once. + """ + _bindings.nccl_barrier_session_destroy(self.ptr) + def hybrid_session( coop: ncclCoopAny, diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/comm.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/comm.py index 662f9272f4d2..99feb71200d9 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/comm.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/comm.py @@ -14,7 +14,7 @@ from ...resources import DevCommResource from . import _bindings -from ._helpers import _alloca_struct, _to_value +from ._helpers import _alloca_cft_le_info, _alloca_struct, _load_cft_le_info, _to_value from ._structs import ( DevCommValue, ncclGin_C, @@ -22,7 +22,7 @@ ) from .gin import Gin from .handles import GinBarrierHandle, LsaBarrierHandle, MultimemHandle -from .types import GinBackendMask, GinResourceSharingMode +from .types import CftLeInfo, CftTeamMode, GinBackendMask, GinResourceSharingMode def _device_function_entry_block(): @@ -53,7 +53,7 @@ def _materialize_dev_comm(value, *, loc=None, ip=None) -> ir.Value: # The store has to follow both operands. The struct is a block argument # while it stays within one block, and then it already dominates the # alloca. But a DevComm used on both sides of a conditional is threaded - # through the region by the DSL, making the value an scf.if result — so + # through the region by the DSL, making the value an scf.if result -- so # anchor the store to whatever defines it. owner = struct_value.owner after = ptr.owner if isinstance(owner, ir.Block) else owner @@ -183,6 +183,20 @@ def team_lsa(self) -> Team: def team_rail(self) -> Team: return Team(_bindings.nccl_team_rail(self.ptr)) + def team_cft(self, mode: CftTeamMode = CftTeamMode.FLAT) -> Team: + """The CFT team, in the requested layout. + + Args: + mode: Team layout. Defaults to :py:attr:`CftTeamMode.FLAT`, + matching the C default argument. + """ + return Team(_bindings.nccl_team_cft(self.ptr, cutlass.Int32(int(mode)))) + + @property + def team_cft_multimem(self) -> Team: + """The CFT multimem team.""" + return Team(_bindings.nccl_team_cft_multimem(self.ptr)) + def team_rank_to_world(self, team: Team, rank: int) -> cutlass.Int32: """Translate ``rank`` within ``team`` to its world rank. @@ -264,7 +278,7 @@ def resource_buffer_multimem_pointer( Args: handle: ``ncclDevResourceHandle`` from ``DevCommResource``. - mm_handle: Multimem handle covering the resource window — + mm_handle: Multimem handle covering the resource window -- :py:attr:`lsa_multimem`, or one passed in from the host for a non-LSA team. @@ -286,6 +300,65 @@ def resource_buffer_lsa_multimem_pointer(self, handle: int) -> ir.Value: return _bindings.nccl_get_resource_buffer_lsa_multimem_pointer( self.ptr, cutlass.Uint32(handle)) + # === Resource-buffer CFT logical endpoints === + # + # As Window.*_le_info, but addressing a resource buffer by handle. The CFT + # variant takes no team: it resolves against the flat CFT team. + + def resource_buffer_cft_le_info( + self, handle: int, peer_cft: int + ) -> CftLeInfo: + """Resolve the buffer to the logical endpoint of ``peer_cft``. + + Args: + handle: ``ncclDevResourceHandle`` from ``DevCommResource``. + peer_cft: Rank within the flat CFT team. + + Returns: + The logical endpoint, as a :class:`CftLeInfo`. + """ + le_id_ptr, le_offset_ptr = _alloca_cft_le_info() + _bindings.nccl_get_resource_buffer_cft_le_info( + self.ptr, cutlass.Uint32(handle), cutlass.Int32(peer_cft), + le_id_ptr, le_offset_ptr) + return _load_cft_le_info(le_id_ptr, le_offset_ptr) + + def resource_buffer_peer_le_info( + self, handle: int, peer: int + ) -> CftLeInfo: + """Resolve the buffer to the logical endpoint of ``peer``. + + Args: + handle: ``ncclDevResourceHandle`` from ``DevCommResource``. + peer: World rank of the peer. Must fall within the flat CFT team; + the device API does not check, and a peer outside it yields a + meaningless logical endpoint. + + Returns: + The logical endpoint, as a :class:`CftLeInfo`. + """ + le_id_ptr, le_offset_ptr = _alloca_cft_le_info() + _bindings.nccl_get_resource_buffer_peer_le_info( + self.ptr, cutlass.Uint32(handle), cutlass.Int32(peer), + le_id_ptr, le_offset_ptr) + return _load_cft_le_info(le_id_ptr, le_offset_ptr) + + def resource_buffer_multimem_le_info( + self, handle: int + ) -> CftLeInfo: + """Resolve the buffer to its multicast logical endpoint. + + Args: + handle: ``ncclDevResourceHandle`` from ``DevCommResource``. + + Returns: + The logical endpoint, as a :class:`CftLeInfo`. + """ + le_id_ptr, le_offset_ptr = _alloca_cft_le_info() + _bindings.nccl_get_resource_buffer_multimem_le_info( + self.ptr, cutlass.Uint32(handle), le_id_ptr, le_offset_ptr) + return _load_cft_le_info(le_id_ptr, le_offset_ptr) + # === Gin factory === def gin( diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/coop.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/coop.py index 05c0e9bba346..43a3cf28f33d 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/coop.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/coop.py @@ -33,7 +33,7 @@ class Coop: ptr: _LLVMPtrType - # === Read-only accessors (cheap — direct ptr dispatch) === + # === Read-only accessors (cheap -- direct ptr dispatch) === @property def thread_rank(self) -> cutlass.Int32: diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/reduce_copy.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/reduce_copy.py new file mode 100644 index 000000000000..306b4a4e1fbf --- /dev/null +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/reduce_copy.py @@ -0,0 +1,135 @@ +"""CuTeDSL wrappers for the NCCL ReduceCopy device APIs. + +Data operands are ``cute.Tensor`` instances. Their element type selects the +dtype-specialized device symbol, and LSA window offsets are derived from their +iterators. +""" + +import cutlass +import cutlass.cute as cute + +from . import _bindings +from ._helpers import _to_coop_value, _to_ptr, _to_value + + +def _tensor_ptr(tensor): + return _to_ptr(tensor.iterator.toint()) + + +def _window_offset(window, tensor): + base = cute.make_ptr(cutlass.Int8, window.local_pointer(0)) + return cutlass.Int64(tensor.iterator.toint() - base.toint()) + + +def lsa_reduce_sum( + coop, src_window, src, dst, count, *, team, +) -> None: + """Reduce an LSA window tensor into a rank-local destination tensor.""" + _bindings.nccl_lsa_reduce_sum( + _to_coop_value(coop), + src_window.ptr, + _window_offset(src_window, src), + _tensor_ptr(dst), + cutlass.Int64(count), + _to_value(team), + src.element_type, + ) + + +def multimem_reduce_sum(coop, mc_src, dst, count) -> None: + """Reduce from a multimem tensor into a local destination tensor.""" + _bindings.nccl_multimem_reduce_sum( + _to_coop_value(coop), + _tensor_ptr(mc_src), + _tensor_ptr(dst), + cutlass.Int64(count), + mc_src.element_type, + ) + + +def lsa_copy( + coop, src, dst_window, dst, count, *, team, +) -> None: + """Copy a rank-local source tensor into an LSA window tensor.""" + _bindings.nccl_lsa_copy( + _to_coop_value(coop), + _tensor_ptr(src), + dst_window.ptr, + _window_offset(dst_window, dst), + cutlass.Int64(count), + _to_value(team), + src.element_type, + ) + + +def multimem_copy(coop, src, mc_dst, count) -> None: + """Copy a local source tensor into a multimem destination tensor.""" + _bindings.nccl_multimem_copy( + _to_coop_value(coop), + _tensor_ptr(src), + _tensor_ptr(mc_dst), + cutlass.Int64(count), + src.element_type, + ) + + +def lsa_reduce_sum_copy( + coop, src_window, src, dst_window, dst, count, *, team, +) -> None: + """Reduce and copy between LSA window tensors in the same team.""" + _bindings.nccl_lsa_reduce_sum_copy( + _to_coop_value(coop), + src_window.ptr, + _window_offset(src_window, src), + dst_window.ptr, + _window_offset(dst_window, dst), + cutlass.Int64(count), + _to_value(team), + src.element_type, + ) + + +def multimem_reduce_sum_copy(coop, mc_src, mc_dst, count) -> None: + """Reduce from and copy to multimem tensors.""" + _bindings.nccl_multimem_reduce_sum_copy( + _to_coop_value(coop), + _tensor_ptr(mc_src), + _tensor_ptr(mc_dst), + cutlass.Int64(count), + mc_src.element_type, + ) + + +def local_reduce_sum_copy( + coop, + n_src, + src_base, + src_displ, + n_dst, + dst_base, + dst_displ, + count, +) -> None: + """Reduce from local source tensors and copy to destination tensors.""" + _bindings.nccl_local_reduce_sum_copy( + _to_coop_value(coop), + cutlass.Int32(n_src), + _tensor_ptr(src_base), + cutlass.Int64(src_displ), + cutlass.Int32(n_dst), + _tensor_ptr(dst_base), + cutlass.Int64(dst_displ), + cutlass.Int64(count), + src_base.element_type, + ) + + +__all__ = [ + "lsa_reduce_sum", + "multimem_reduce_sum", + "lsa_copy", + "multimem_copy", + "lsa_reduce_sum_copy", + "multimem_reduce_sum_copy", + "local_reduce_sum_copy", +] diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/runtime.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/runtime.py new file mode 100644 index 000000000000..d53f3ddf3b50 --- /dev/null +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/runtime.py @@ -0,0 +1,132 @@ +"""Compile-only representative arguments for CuTeDSL NCCL views. + +Each factory builds a placeholder that carries a ``@cute.jit`` argument's +type without owning an NCCL resource, so an application can ``cute.compile`` +before it creates a communicator or registers memory and invoke the compiled +callable with real resources later. The nccl4py analogue of +``cutlass.cute.runtime.make_fake_stream``. + +.. warning:: + + A placeholder is only valid as a ``cute.compile`` argument. Passing one + to the compiled callable contributes no execution pointer, which shifts + every following argument and launches with garbage instead of raising; + only CuTeDSL's debugging mode reports it. +""" + +from typing import ClassVar, cast + +from .comm import DevComm +from .handles import GinBarrierHandle, LsaBarrierHandle, MultimemHandle +from .window import Window + + +class _CompileOnlyJitArgument: + """Type-only JIT argument that deliberately has no ``__c_pointers__``. + + Omitting it is what marks the object compile-only: CuTeDSL takes a host + argument that yields MLIR signature types but no execution argument as a + placeholder rather than an unsupported type. + """ + + _real_cls: ClassVar[type] + + @property # type: ignore[misc] + def __class__(self) -> type: + """Make isinstance-based annotation checks see the real view class. + + Load-bearing: CuTeDSL validates an argument against its parameter + annotation before it looks for an adapter, so a placeholder passed to + a ``dev_comm: DevComm`` parameter is rejected without this. + """ + return self._real_cls + + def __get_mlir_types__(self): + """Return the MLIR argument types of the real view.""" + return self._real_cls.__get_mlir_types__() + + def __new_from_mlir_values__(self, values): + """Reconstruct the real view used while tracing the function body. + + Called unbound on a bare prototype, since the real implementations + populate an instance they derive from ``type(self)``. + """ + prototype = object.__new__(self._real_cls) + return self._real_cls.__new_from_mlir_values__(prototype, values) + + def __repr__(self) -> str: + return f"" + + +class _FakeDevComm(_CompileOnlyJitArgument): + _real_cls = DevComm + + +class _FakeWindow(_CompileOnlyJitArgument): + _real_cls = Window + + +class _FakeMultimemHandle(_CompileOnlyJitArgument): + _real_cls = MultimemHandle + + +class _FakeLsaBarrierHandle(_CompileOnlyJitArgument): + _real_cls = LsaBarrierHandle + + +class _FakeGinBarrierHandle(_CompileOnlyJitArgument): + _real_cls = GinBarrierHandle + + +def make_fake_dev_comm() -> DevComm: + """Create a type-only ``DevComm`` placeholder for ``cute.compile``. + + Valid only as a ``cute.compile`` argument; invoke the compiled callable + with a real ``DevCommResource``, or a :class:`DevComm` wrapping one. + """ + return cast(DevComm, _FakeDevComm()) + + +def make_fake_window() -> Window: + """Create a type-only ``Window`` placeholder for ``cute.compile``. + + Valid only as a ``cute.compile`` argument; invoke the compiled callable + with a real ``RegisteredWindowHandle``, or a :class:`Window` wrapping one. + """ + return cast(Window, _FakeWindow()) + + +def make_fake_multimem_handle() -> MultimemHandle: + """Create a type-only ``MultimemHandle`` placeholder for ``cute.compile``. + + Valid only as a ``cute.compile`` argument; invoke the compiled callable + with the real handle. + """ + return cast(MultimemHandle, _FakeMultimemHandle()) + + +def make_fake_lsa_barrier_handle() -> LsaBarrierHandle: + """Create a type-only ``LsaBarrierHandle`` placeholder for ``cute.compile``. + + Valid only as a ``cute.compile`` argument; invoke the compiled callable + with the real handle. + """ + return cast(LsaBarrierHandle, _FakeLsaBarrierHandle()) + + +def make_fake_gin_barrier_handle() -> GinBarrierHandle: + """Create a type-only ``GinBarrierHandle`` placeholder for ``cute.compile``. + + Valid only as a ``cute.compile`` argument; invoke the compiled callable + with the real handle. + """ + return cast(GinBarrierHandle, _FakeGinBarrierHandle()) + + +__all__ = [ + "make_fake_dev_comm", + "make_fake_gin_barrier_handle", + "make_fake_lsa_barrier_handle", + "make_fake_multimem_handle", + "make_fake_window", +] diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/types.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/types.py index a156df70f20e..4f4ec60dfa06 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/types.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/types.py @@ -1,14 +1,27 @@ -"""User-facing enums shared by the rest of the device API. - - - :class:`MemoryOrder` — argument to barrier ``arrive`` / ``wait`` / ``sync``. - - :class:`ThreadScope` — release-scope argument to GIN ``put`` / ``signal``. - - :class:`GinFenceLevel` — fence-level argument to GIN barrier ``sync``. - - :class:`GinBackendMask` — backend selection for :meth:`DevComm.gin`. - - :class:`GinResourceSharingMode` — resource-sharing mode for :meth:`DevComm.gin`. +"""User-facing types shared by the rest of the device API. + + - :class:`CftLeInfo` -- logical endpoint address returned by CFT queries. + - :class:`MemoryOrder` -- argument to barrier ``arrive`` / ``wait`` / ``sync``. + - :class:`ThreadScope` -- release-scope argument to GIN ``put`` / ``signal``. + - :class:`GinFenceLevel` -- fence-level argument to GIN barrier ``sync``. + - :class:`GinBackendMask` -- backend selection for :meth:`DevComm.gin`. + - :class:`CftTeamMode` -- team layout for :meth:`DevComm.team_cft`. + - :class:`GinResourceSharingMode` -- resource-sharing mode for :meth:`DevComm.gin`. """ +from dataclasses import dataclass from enum import IntEnum, IntFlag +import cutlass + + +@dataclass(frozen=True) +class CftLeInfo: + """CFT logical endpoint address produced while tracing device code.""" + + le_id: cutlass.Uint32 + le_offset: cutlass.Uint64 + class MemoryOrder(IntEnum): """Mirrors C++ ``cuda::memory_order`` from . @@ -39,7 +52,7 @@ class ThreadScope(IntEnum): SYSTEM = 0 DEVICE = 1 BLOCK = 2 - THREAD = 3 + THREAD = 10 # libcu++ __ATOMIC_THREAD deliberately leaves a gap. class GinFenceLevel(IntFlag): @@ -63,7 +76,18 @@ class GinBackendMask(IntFlag): PROXY = 1 << 2 # 4 GDAKI = 1 << 3 # 8 GPI = 1 << 4 # 16 - ALL = PROXY | GDAKI | GPI + EFA_GDA = 1 << 5 # 32 (NCCL 2.31+) + ALL = PROXY | GDAKI | GPI | EFA_GDA + + +class CftTeamMode(IntEnum): + """Mirrors ``enum ncclCftTeamMode_t`` -- the layout + :meth:`DevComm.team_cft` returns. + """ + + FLAT = 0 # every rank in the CFT unicast group + HIER_MULTIMEM = 1 # same index across multicast CFT groups + HIER_LSA = 2 # same index across LSA groups class GinResourceSharingMode(IntEnum): @@ -81,9 +105,11 @@ class GinResourceSharingMode(IntEnum): __all__ = [ + "CftLeInfo", "MemoryOrder", "ThreadScope", "GinFenceLevel", "GinBackendMask", "GinResourceSharingMode", + "CftTeamMode", ] diff --git a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/window.py b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/window.py index 9868508ebcf5..78fafefae250 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/device/cute/window.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/device/cute/window.py @@ -10,9 +10,10 @@ from ...resources import RegisteredWindowHandle from . import _bindings -from ._helpers import _to_value +from ._helpers import _alloca_cft_le_info, _load_cft_le_info, _to_value from ._structs import _LLVMPtrType, ncclTeam as Team from .handles import MultimemHandle +from .types import CftLeInfo if TYPE_CHECKING: from .comm import DevComm @@ -47,8 +48,10 @@ def __c_pointers__(self) -> list[int]: "Window.__c_pointers__ is only available on a host-mode Window" ) if not self._resource.is_valid: - raise RuntimeError("RegisteredWindowHandle has been closed") - return [self._resource._handle.address] + raise RuntimeError( + "RegisteredWindowHandle does not reference an active NCCL window" + ) + return [self._resource._window.handle_ptr] @staticmethod def __get_mlir_types__() -> list[ir.Type]: @@ -122,7 +125,7 @@ def multimem_pointer( Args: offset: Byte offset within the window. - mm_handle: Multimem handle covering this window — + mm_handle: Multimem handle covering this window -- :py:attr:`DevComm.lsa_multimem`, or one passed in from the host for a non-LSA team. @@ -145,6 +148,70 @@ def lsa_multimem_pointer(self, offset: int, dev_comm: DevComm) -> ir.Value: return _bindings.nccl_get_lsa_multimem_pointer( self.ptr, cutlass.Int64(offset), dev_comm.ptr) + # === CFT logical endpoints === + # + # CFT ops address memory by an (le_id, le_offset) pair rather than by + # pointer, so these mirror the *_pointer methods above. + + def cft_le_info( + self, offset: int, peer_cft: int, cft_team: Team, dev_comm: DevComm + ) -> CftLeInfo: + """Resolve ``offset`` to the logical endpoint of ``peer_cft``. + + Args: + offset: Byte offset within the window. + peer_cft: Rank within ``cft_team``. + cft_team: CFT team, from :meth:`DevComm.team_cft`. + dev_comm: Device communicator owning this window. + + Returns: + The logical endpoint, as a :class:`CftLeInfo`. + """ + le_id_ptr, le_offset_ptr = _alloca_cft_le_info() + _bindings.nccl_get_cft_le_info( + self.ptr, cutlass.Int64(offset), cutlass.Int32(peer_cft), + _to_value(cft_team), dev_comm.ptr, le_id_ptr, le_offset_ptr) + return _load_cft_le_info(le_id_ptr, le_offset_ptr) + + def peer_le_info( + self, offset: int, peer: int, dev_comm: DevComm + ) -> CftLeInfo: + """Resolve ``offset`` to the logical endpoint of ``peer``. + + Args: + offset: Byte offset within the window. + peer: World rank of the peer. Must fall within the flat CFT team; + the device API does not check, and a peer outside it yields a + meaningless logical endpoint. + dev_comm: Device communicator owning this window. + + Returns: + The logical endpoint, as a :class:`CftLeInfo`. + """ + le_id_ptr, le_offset_ptr = _alloca_cft_le_info() + _bindings.nccl_get_peer_le_info( + self.ptr, cutlass.Int64(offset), cutlass.Int32(peer), + dev_comm.ptr, le_id_ptr, le_offset_ptr) + return _load_cft_le_info(le_id_ptr, le_offset_ptr) + + def multimem_le_info( + self, offset: int, dev_comm: DevComm + ) -> CftLeInfo: + """Resolve ``offset`` to the multicast logical endpoint. + + Args: + offset: Byte offset within the window. + dev_comm: Device communicator owning this window. + + Returns: + The logical endpoint, as a :class:`CftLeInfo`. + """ + le_id_ptr, le_offset_ptr = _alloca_cft_le_info() + _bindings.nccl_get_multimem_le_info( + self.ptr, cutlass.Int64(offset), dev_comm.ptr, + le_id_ptr, le_offset_ptr) + return _load_cft_le_info(le_id_ptr, le_offset_ptr) + def tensor(self, dtype, layout, offset: int = 0): """Construct a ``cute.Tensor`` view over the registered buffer. diff --git a/projects/rccl/bindings/nccl4py/nccl/core/params.py b/projects/rccl/bindings/nccl4py/nccl/core/params.py index 74899d1df25c..f00a7e7153b0 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/params.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/params.py @@ -5,8 +5,8 @@ """NCCL parameter access. -:py:data:`params` — read-only mapping of NCCL parameter names to their values. -:py:func:`dump_params` — print parameters to stdout. +:py:data:`params` -- read-only mapping of NCCL parameter names to their values. +:py:func:`dump_params` -- print parameters to stdout. """ from __future__ import annotations diff --git a/projects/rccl/bindings/nccl4py/nccl/core/py.typed b/projects/rccl/bindings/nccl4py/nccl/core/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/projects/rccl/bindings/nccl4py/nccl/core/resources.py b/projects/rccl/bindings/nccl4py/nccl/core/resources.py index 2517bfee6e10..eb686eb8b942 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/resources.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/resources.py @@ -5,25 +5,22 @@ """Communicator-owned resource management. -This module provides resource classes for NCCL communicator-owned objects including -registered buffers for zero-copy communication, registered windows for RMA operations, -and custom reduction operators. All resources are automatically cleaned up when the -owning communicator is destroyed or aborted. +This module provides resource abstractions for objects owned by an NCCL +communicator. Resources created through a communicator are tracked and +automatically closed when the communicator is destroyed or aborted. """ from __future__ import annotations from abc import ABC, abstractmethod +from dataclasses import dataclass from nccl.bindings import nccl as _nccl_bindings - from nccl.core._binding_helpers import LowppView from nccl.core.constants import WindowFlag from nccl.core.team import NCCLTeam from nccl.core.typing import NcclDataType, NcclInvalid -_PointerBox = _nccl_bindings.PointerBox - __all__ = [ "MultimemHandle", "LsaBarrierHandle", @@ -31,11 +28,23 @@ "LLA2AHandle", "RegisteredBufferHandle", "RegisteredWindowHandle", + "CftLeInfo", "CustomRedOp", "DevCommResource", ] +@dataclass(frozen=True, slots=True) +class CftLeInfo: + """CFT logical endpoint address returned by window LE-info queries.""" + + le_id: int + """Logical endpoint identifier.""" + + le_offset: int + """Byte offset within the logical endpoint.""" + + class MultimemHandle(LowppView, lowpp_cls=_nccl_bindings.MultimemHandle): """Multimem handle, returned by :py:meth:`~nccl.core.DevCommResource.multimem_handle` for a team requested @@ -49,7 +58,7 @@ def mc_base_pointer(self) -> int: class LsaBarrierHandle(LowppView, lowpp_cls=_nccl_bindings.LsaBarrierHandle): - """LSA barrier handle, returned by + """Load/Store Accessible (LSA) barrier handle, returned by :py:attr:`~nccl.core.DevCommResource.resource_handles` for each :py:class:`~nccl.core.LsaBarrierRequirement`. Pass it to device-side barrier sessions. @@ -57,7 +66,7 @@ class LsaBarrierHandle(LowppView, lowpp_cls=_nccl_bindings.LsaBarrierHandle): @property def buf_handle(self) -> int: - """Buffer resource offset assigned by NCCL during device comm creation.""" + """Buffer resource offset assigned during device communicator creation.""" return self._lowpp.buf_handle @property @@ -67,7 +76,7 @@ def n_barriers(self) -> int: class GinBarrierHandle(LowppView, lowpp_cls=_nccl_bindings.GinBarrierHandle): - """GIN barrier handle, returned by + """GPU-Initiated Networking (GIN) barrier handle, returned by :py:attr:`~nccl.core.DevCommResource.resource_handles` for each :py:class:`~nccl.core.GinBarrierRequirement`. Pass it to device-side barrier sessions. @@ -75,12 +84,12 @@ class GinBarrierHandle(LowppView, lowpp_cls=_nccl_bindings.GinBarrierHandle): @property def signal0(self) -> int: - """Starting GIN signal index assigned by NCCL during device comm creation.""" + """Starting GIN signal index assigned during device communicator creation.""" return self._lowpp.signal0 class LLA2AHandle(LowppView, lowpp_cls=_nccl_bindings.LLA2AHandle): - """Low-latency all-to-all handle, returned by + """Low-latency all-to-all (LLA2A) handle, returned by :py:attr:`~nccl.core.DevCommResource.resource_handles` for each :py:class:`~nccl.core.LLA2ARequirement`. Pass it to device-side all-to-all sessions. @@ -88,7 +97,7 @@ class LLA2AHandle(LowppView, lowpp_cls=_nccl_bindings.LLA2AHandle): @property def buf_handle(self) -> int: - """Buffer resource offset assigned by NCCL during device comm creation.""" + """Buffer resource offset assigned during device communicator creation.""" return self._lowpp.buf_handle @property @@ -101,24 +110,24 @@ class CommResource(ABC): """Abstract base class for NCCL communicator-owned resources. Resources are tied to a specific communicator. They can be released - explicitly via :py:meth:`close`, and are released automatically when the - owning communicator is destroyed or aborted. + explicitly via :py:meth:`close`. Resources created through a communicator + are also released when that communicator is destroyed or aborted. """ - def __init__(self, comm_ptr: int): - """Initializes the resource with a communicator pointer. + def __init__(self, comm: _nccl_bindings.Comm): + """Initializes a resource for an NCCL communicator. Args: - comm_ptr: Raw NCCL communicator pointer. + comm: Internal binding object for the owning NCCL communicator. Raises: - NcclInvalid: If comm_ptr is 0 (invalid communicator). + NcclInvalid: If the communicator is null. """ - if comm_ptr == 0: + if not comm: raise NcclInvalid( "Invalid communicator: cannot create resource with communicator ptr=0" ) - self._comm_ptr = comm_ptr + self._comm = comm self._closed = False @abstractmethod @@ -148,7 +157,7 @@ def close(self) -> None: self._deallocate() def _check_valid(self) -> None: - """Raises if the resource has been closed. + """Validates that the resource has not been closed. Raises: RuntimeError: If the resource has been closed. @@ -172,31 +181,31 @@ class RegisteredBufferHandle(CommResource): owning communicator is destroyed or aborted. """ - def __init__(self, comm_ptr: int, buffer_ptr: int, size: int): + def __init__(self, comm: _nccl_bindings.Comm, buffer_ptr: int, size: int): """Creates and registers a buffer with NCCL. Args: - comm_ptr: NCCL communicator raw pointer. + comm: Internal binding object for the owning NCCL communicator. buffer_ptr: Device pointer to the buffer. size: Size of the buffer in bytes. Raises: - NcclInvalid: If comm_ptr is 0 (invalid communicator). + NcclInvalid: If the communicator is null. """ self._buffer_ptr = buffer_ptr self._size = size self._handle: int | None = None - super().__init__(comm_ptr) + super().__init__(comm) self._allocate() def _allocate(self) -> None: """Registers the buffer with NCCL for zero-copy communication.""" - self._handle = _nccl_bindings.comm_register(self._comm_ptr, self._buffer_ptr, self._size) + self._handle = _nccl_bindings.comm_register(self._comm, self._buffer_ptr, self._size) def _deallocate(self) -> None: """Deregisters the buffer from NCCL.""" if self._handle is not None: - _nccl_bindings.comm_deregister(self._comm_ptr, self._handle) + _nccl_bindings.comm_deregister(self._comm, self._handle) self._handle = None @property @@ -228,39 +237,46 @@ class RegisteredWindowHandle(CommResource): Registers a memory window with the communicator for one-sided communication patterns. Created by :py:meth:`Communicator.register_window`. Registration - is collective: all ranks must call - :py:meth:`Communicator.register_window` with equal buffer sizes by - default. Deregistration is local. The window handle can be released - explicitly via :py:meth:`close`, or automatically when the owning - communicator is destroyed or aborted. + is collective: all ranks must call :py:meth:`Communicator.register_window`. + Deregistration is local. The window handle can be released explicitly via + :py:meth:`close`, or automatically when the owning communicator is + destroyed or aborted. """ - def __init__(self, comm_ptr: int, buffer_ptr: int, size: int, flags: WindowFlag | None = None): + _window: _nccl_bindings.Window + _buffer_ptr: int + _size: int + _flags: WindowFlag + + def __init__( + self, + comm: _nccl_bindings.Comm, + buffer_ptr: int, + size: int, + flags: WindowFlag | None = None, + ): """Creates and registers a memory window with NCCL. Args: - comm_ptr: NCCL communicator raw pointer. + comm: Internal binding object for the owning NCCL communicator. buffer_ptr: Device pointer to the buffer. size: Size of the window in bytes. flags: Window registration flags. Defaults to ``None`` (:py:attr:`~nccl.core.WindowFlag.DEFAULT`). Raises: - NcclInvalid: If comm_ptr is 0 (invalid communicator). + NcclInvalid: If the communicator is null. """ self._buffer_ptr = buffer_ptr self._size = size self._flags = flags if flags is not None else WindowFlag.DEFAULT - self._handle = _PointerBox() - self._closed = True - self._comm_ptr = comm_ptr + super().__init__(comm) self._allocate() - super().__init__(comm_ptr) def _allocate(self) -> None: """Collectively registers the window with NCCL.""" - _nccl_bindings.comm_window_register( - self._comm_ptr, self._buffer_ptr, self._size, self._handle.address, self._flags.value + self._window = _nccl_bindings.comm_window_register( + self._comm, self._buffer_ptr, self._size, self._flags.value ) def _deallocate(self) -> None: @@ -269,20 +285,20 @@ def _deallocate(self) -> None: Deregistration is local to the rank. The caller must ensure the buffer is not being accessed by any NCCL operation. """ - if self.handle: - _nccl_bindings.comm_window_deregister(self._comm_ptr, self.handle) - self._handle.ptr = 0 - self._closed = True + if self._window: + _nccl_bindings.comm_window_deregister(self._comm, self._window) + # The wrapper's handle is read-only; swap in a null one to mark it deregistered. + self._window = _nccl_bindings.Window() @property def is_valid(self) -> bool: """Whether the window is still registered (not closed, handle non-null).""" - return not self._closed and bool(self._handle.ptr) + return not self._closed and bool(self._window) @property def handle(self) -> int: - """Window handle for NCCL operations, or ``0`` once deregistered.""" - return int(self._handle.ptr) + """Window handle value, or ``0`` while unavailable or after deregistration.""" + return self._window.handle @property def size(self) -> int: @@ -316,7 +332,7 @@ def get_lsa_multimem_device_pointer(self, offset: int = 0) -> int | None: RuntimeError: If the window has been closed. """ self._check_valid() - ptr = _nccl_bindings.get_lsa_multimem_device_pointer(self.handle, offset) + ptr = _nccl_bindings.get_lsa_multimem_device_pointer(self._window, offset) return ptr if ptr != 0 else None def get_multimem_device_pointer(self, multimem: MultimemHandle, offset: int = 0) -> int | None: @@ -338,7 +354,7 @@ def get_multimem_device_pointer(self, multimem: MultimemHandle, offset: int = 0) RuntimeError: If the window has been closed. """ self._check_valid() - ptr = _nccl_bindings.get_multimem_device_pointer(self.handle, offset, multimem._lowpp.ptr) + ptr = _nccl_bindings.get_multimem_device_pointer(self._window, offset, multimem._lowpp) return ptr if ptr != 0 else None def get_lsa_device_pointer(self, lsa_rank: int, offset: int = 0) -> int: @@ -358,7 +374,7 @@ def get_lsa_device_pointer(self, lsa_rank: int, offset: int = 0) -> int: RuntimeError: If the window has been closed. """ self._check_valid() - return _nccl_bindings.get_lsa_device_pointer(self.handle, offset, lsa_rank) + return _nccl_bindings.get_lsa_device_pointer(self._window, offset, lsa_rank) def get_peer_device_pointer(self, peer: int, offset: int = 0) -> int | None: """Returns a device pointer to a peer's window buffer by world rank. @@ -377,9 +393,84 @@ def get_peer_device_pointer(self, peer: int, offset: int = 0) -> int | None: RuntimeError: If the window has been closed. """ self._check_valid() - ptr = _nccl_bindings.get_peer_device_pointer(self.handle, offset, peer) + ptr = _nccl_bindings.get_peer_device_pointer(self._window, offset, peer) return ptr if ptr != 0 else None + # CFT operations address memory by an ``(le_id, le_offset)`` pair rather + # than by the pointer the ``get_*_device_pointer`` methods above return. + + def get_multimem_le_info(self, offset: int = 0) -> CftLeInfo: + """Returns the multicast logical endpoint address for this window and offset. + + Args: + offset: Byte offset within the window buffer. Defaults to 0. + + Returns: + The logical endpoint address, as a :py:class:`CftLeInfo`. + + Raises: + RuntimeError: If the window has been closed. + + See Also: + :c:func:`ncclGetMultimemDeviceLeInfo` + """ + self._check_valid() + le_id, le_offset = _nccl_bindings.get_multimem_device_le_info( + self._window, offset + ) + return CftLeInfo(le_id, le_offset) + + def get_cft_le_info( + self, peer_cft: int, cft_team: NCCLTeam, offset: int = 0 + ) -> CftLeInfo: + """Returns the logical endpoint address for a peer within ``cft_team``. + + Args: + peer_cft: Rank within ``cft_team``. + cft_team: The CFT team, as returned by + :py:meth:`Communicator.team_cft`. + offset: Byte offset within the window buffer. Defaults to 0. + + Returns: + The logical endpoint address, as a :py:class:`CftLeInfo`. + + Raises: + RuntimeError: If the window has been closed. + + See Also: + :c:func:`ncclGetCftDeviceLeInfo` + """ + self._check_valid() + team_lowpp = cft_team._to_lowpp() + le_id, le_offset = _nccl_bindings.get_cft_device_le_info( + self._window, offset, peer_cft, team_lowpp + ) + return CftLeInfo(le_id, le_offset) + + def get_peer_le_info(self, peer: int, offset: int = 0) -> CftLeInfo: + """Returns the logical endpoint address for a peer by world rank. + + Args: + peer: World rank of the peer. Must fall within this rank's flat + CFT team, as returned by :py:meth:`Communicator.team_cft`; + NCCL rejects a peer outside it. + offset: Byte offset within the window buffer. Defaults to 0. + + Returns: + The logical endpoint address, as a :py:class:`CftLeInfo`. + + Raises: + RuntimeError: If the window has been closed. + + See Also: + :c:func:`ncclGetPeerDeviceLeInfo` + """ + self._check_valid() + le_id, le_offset = _nccl_bindings.get_peer_device_le_info( + self._window, offset, peer + ) + return CftLeInfo(le_id, le_offset) + def __repr__(self) -> str: return f"" @@ -396,7 +487,7 @@ class CustomRedOp(CommResource): def __init__( self, - comm_ptr: int, + comm: _nccl_bindings.Comm, scalar_ptr: int, datatype: NcclDataType, residence: _nccl_bindings.ScalarResidence, @@ -404,33 +495,33 @@ def __init__( """Creates a custom reduction operator. Args: - comm_ptr: NCCL communicator raw pointer. + comm: Internal binding object for the owning NCCL communicator. scalar_ptr: Pointer to the scalar value (host or device memory). datatype: NCCL data type of the scalar and reduction. residence: Indicates scalar memory location (HostImmediate or Device). Raises: - NcclInvalid: If comm_ptr is 0 (invalid communicator). + NcclInvalid: If the communicator is null. """ self._scalar_ptr = scalar_ptr self._datatype = datatype self._residence = residence self._op: int | None = None - super().__init__(comm_ptr) + super().__init__(comm) self._allocate() def _allocate(self) -> None: """Creates the custom reduction operator in NCCL.""" self._op = _nccl_bindings.red_op_create_pre_mul_sum( - self._scalar_ptr, int(self._datatype), int(self._residence), self._comm_ptr + self._scalar_ptr, int(self._datatype), int(self._residence), self._comm ) def _deallocate(self) -> None: """Destroys the custom reduction operator in NCCL.""" if self._op is not None: - _nccl_bindings.red_op_destroy(self._op, self._comm_ptr) + _nccl_bindings.red_op_destroy(self._op, self._comm) self._op = None @property @@ -462,15 +553,25 @@ def __repr__(self) -> str: class DevCommResource(CommResource): """NCCL device communicator resource for device-side operations. - Wraps ``ncclDevComm_t`` and manages its lifecycle. Created by + Owns and manages an :c:type:`ncclDevComm_t ` created by :py:meth:`Communicator.create_dev_comm`. The device communicator is automatically destroyed when the parent communicator is destroyed or aborted. """ + _reqs_lowpp: _nccl_bindings.DevCommRequirements + _team_multimem_lowpp: dict[NCCLTeam, _nccl_bindings.MultimemHandle] + _resource_handle_lowpps: tuple[ + _nccl_bindings.LsaBarrierHandle + | _nccl_bindings.GinBarrierHandle + | _nccl_bindings.LLA2AHandle, + ..., + ] + _dev_comm: _nccl_bindings.DevComm | None + def __init__( self, - comm_ptr: int, + comm: _nccl_bindings.Comm, reqs_lowpp: _nccl_bindings.DevCommRequirements, team_multimem_lowpp: dict[NCCLTeam, _nccl_bindings.MultimemHandle] | None = None, resource_handle_lowpps: tuple[ @@ -484,27 +585,27 @@ def __init__( """Creates a device communicator from an existing host communicator. Args: - comm_ptr: NCCL communicator raw pointer. - reqs_lowpp: Per-create low-level requirements root. Consumed during - construction and released once ncclDevCommCreate has copied it; - the caller keeps its linked-list nodes alive until this - constructor returns. - team_multimem_lowpp: Per-team multimem output storage. The resource - retains these objects because NCCL writes through their - pointers and returned handle facades reference them. - resource_handle_lowpps: Per-resource output handle lowpps in - requirement order. The resource retains the handle storage - because NCCL writes offsets into it during creation and device - kernels read it afterwards. + comm: Internal binding object for the owning NCCL communicator. + reqs_lowpp: Internal device communicator requirements. The caller + keeps the linked-list nodes alive until this constructor + returns; the root object is released after + :c:func:`ncclDevCommCreate` copies it. + team_multimem_lowpp: Internal per-team storage for multimem outputs. + This resource retains the storage because NCCL writes through + it and returned multimem handles reference it. + resource_handle_lowpps: Internal output storage for resource + handles, in requirement order. This resource retains the + storage because NCCL populates it during creation and returned + resource handles reference it. Raises: - NcclInvalid: If comm_ptr is 0 (invalid communicator). + NcclInvalid: If the communicator is null. """ self._reqs_lowpp = reqs_lowpp self._team_multimem_lowpp = team_multimem_lowpp or {} - self._resource_handle_lowpps = resource_handle_lowpps or [] - self._dev_comm: _nccl_bindings.DevComm | None = None - super().__init__(comm_ptr) + self._resource_handle_lowpps = resource_handle_lowpps or () + self._dev_comm = None + super().__init__(comm) # reqs_lowpp is only needed for the create call; drop it afterwards so we # don't retain a struct whose linked-list pointers dangle once the # caller's requirement nodes are freed. @@ -515,17 +616,17 @@ def __init__( def _allocate(self) -> None: """Creates the device communicator via ncclDevCommCreate.""" - self._dev_comm = _nccl_bindings.dev_comm_create(self._comm_ptr, self._reqs_lowpp.ptr) + self._dev_comm = _nccl_bindings.dev_comm_create(self._comm, self._reqs_lowpp.ptr) def _deallocate(self) -> None: """Destroys the device communicator via ncclDevCommDestroy.""" if self._dev_comm is not None: - _nccl_bindings.dev_comm_destroy(self._comm_ptr, self._dev_comm.ptr) + _nccl_bindings.dev_comm_destroy(self._comm, self._dev_comm.ptr) self._dev_comm = None @property def dev_comm(self) -> _nccl_bindings.DevComm: - """DevComm object wrapping :c:type:`ncclDevComm_t `. + """Internal binding object for :c:type:`ncclDevComm_t `. Raises: RuntimeError: If the device communicator has been destroyed. @@ -537,7 +638,8 @@ def dev_comm(self) -> _nccl_bindings.DevComm: @property def ptr(self) -> int: - """Raw pointer to the underlying :c:type:`ncclDevComm_t ` structure. + """Address of the underlying :c:type:`ncclDevComm_t ` + object, represented as a Python integer. Raises: RuntimeError: If the device communicator has been destroyed. @@ -547,9 +649,8 @@ def ptr(self) -> int: def multimem_handle(self, team: NCCLTeam) -> MultimemHandle: """Returns the multimem handle requested for ``team``. - The returned facade wraps this dev comm's per-create output storage; - each lookup creates a new facade over the same storage. It remains - backed by the resource until the device communicator is closed. + The returned handle references storage owned by this resource and + remains valid until the device communicator is closed. Args: team: The team the handle was requested for, as an entry of the diff --git a/projects/rccl/bindings/nccl4py/nccl/core/typing.py b/projects/rccl/bindings/nccl4py/nccl/core/typing.py index 17732a81ee5c..958fc1a48345 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/typing.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/typing.py @@ -13,11 +13,11 @@ from __future__ import annotations -from enum import IntEnum +from enum import IntEnum, IntFlag from typing import Any, Protocol, TypeAlias, Union import numpy as _np -from cuda.core import Buffer, Device, Stream +from cuda.core import Buffer, Device, Event, Stream from cuda.core.typing import IsStreamType __all__ = [ @@ -25,11 +25,16 @@ "NcclRedOp", "NcclGinType", "NcclGinConnectionType", + "NcclHostCftMode", + "NcclNvlsHostMode", + "NcclCftTeamMode", + "NcclCftCap", "NcclCommMemStat", "NcclBufferSpec", "NcclScalarSpec", "NcclDeviceSpec", "NcclStreamSpec", + "NcclEventSpec", "NcclInvalid", # Data type constants "INT8", @@ -139,11 +144,14 @@ class NcclGinType(IntEnum): Reported by :py:attr:`Communicator.gin_type` and :py:attr:`Communicator.railed_gin_type` to indicate which device-side - network transport, if any, is available on the communicator. + network transport, if any, is available on the communicator, and accepted + by :py:attr:`NCCLDevCommRequirements.gin_type` to request one. """ NONE = 0 - """GIN not available on this communicator.""" + """No GIN transport. When reported, none is available; when set on + :py:attr:`NCCLDevCommRequirements.gin_type` it instead means any + available transport is acceptable.""" PROXY = 2 """Proxy-based GIN. Network operations issued from a device kernel are relayed through a CPU proxy thread.""" @@ -154,6 +162,9 @@ class NcclGinType(IntEnum): """GPU-Push Interface. GPU threads push network descriptors directly to a NIC-visible MMIO queue, with no CPU involvement and no memory barriers.""" + EFA_GDA = 5 + """EFA GPUDirect Async. Kernel-initiated network operations on AWS + Elastic Fabric Adapter NICs.""" class NcclGinConnectionType(IntEnum): @@ -173,6 +184,80 @@ class NcclGinConnectionType(IntEnum): RAIL = 2 """Rail-restricted. Ranks must be reachable via GIN only within the same rail (network plane).""" + CUSTOM_STRIDE = 3 + """Strided. Ranks must be reachable via GIN at the stride given by + :py:attr:`NCCLDevCommRequirements.gin_custom_stride`.""" + + +class NcclHostCftMode(IntEnum): + """Host-side Compute Fabric Transport (CFT) mode, mirroring + :c:type:`ncclHostCftMode_t`. + + Set on :py:attr:`NCCLConfig.host_cft_mode` to control whether the + communicator creates the CUDA fabric logical endpoints that back the + host-side CFT queries. + """ + + DEFAULT = -2147483648 + """Use the version-specific default.""" + ENABLE = 1 + """Enable host-side CFT support, creating the communicator's unicast and + multicast logical endpoints during the first window registration.""" + DISABLE = 2 + """Disable host-side CFT support.""" + FALLBACK = 3 + """Try to create the logical endpoints; on error, disable host-side CFT + instead of failing.""" + + +class NcclNvlsHostMode(IntFlag): + """Host-side NVLS mode, mirroring :c:type:`ncclNvlsHostMode_t`. + + Set on :py:attr:`NCCLConfig.nvls_host_mode` to select which host NVLS + components the communicator uses. The two ``DISABLE_`` members combine; + leave the field unset for the library default. + """ + + ENABLE = 0 + """Enable all host NVLS components.""" + DISABLE_TRANSPORT = 1 + """Disable the NVLS transport and the registered-buffer optimization.""" + DISABLE_SYMMETRIC_MULTIMEM = 2 + """Disable multimem in NCCL's symmetric kernels and copy-engine paths.""" + DISABLE = 2147483647 + """Disable all present and future host NVLS components.""" + + +class NcclCftTeamMode(IntEnum): + """CFT team layout, mirroring :c:type:`ncclCftTeamMode_t`. + + Selects which ranks in the CFT unicast group the CFT team + :py:meth:`Communicator.team_cft` includes. + """ + + FLAT = 0 + """Every rank in the CFT unicast group.""" + HIER_MULTIMEM = 1 + """The ranks of the CFT unicast group sharing the same index across + multicast CFT groups.""" + HIER_LSA = 2 + """The ranks of the CFT unicast group sharing the same index across + LSA groups.""" + + +class NcclCftCap(IntFlag): + """Compute Fabric Transport capabilities, mirroring + :c:type:`ncclCftCap_t`. + + Combined as a bitmask on :py:attr:`NCCLDevCommRequirements.cft_caps`. + """ + + NONE = 0x0 + """No CFT capability requested.""" + CFT = 0x1 + """Request unicast CFT logical endpoints.""" + MULTIMEM = 0x2 + """Request multicast CFT operations and multimem CFT barriers.""" class NcclDataType(IntEnum): @@ -407,4 +492,9 @@ def __cuda_array_interface__(self) -> dict[str, Any]: ... NcclStreamSpec: TypeAlias = Union[Stream, IsStreamType, int] """A CUDA stream: a :py:class:`cuda.core.Stream`, an object implementing -``__cuda_stream__``, or an integer stream handle.""" +``__cuda_stream__``, or an integer ``cudaStream_t`` handle. Integer 0 denotes +the default stream.""" + +NcclEventSpec: TypeAlias = Union[Event, int] +"""A CUDA event: a :py:class:`cuda.core.Event` or an integer ``cudaEvent_t`` +handle. Integer 0 is invalid.""" diff --git a/projects/rccl/bindings/nccl4py/nccl/core/utils.py b/projects/rccl/bindings/nccl4py/nccl/core/utils.py index 95b07c8e6f5f..8563ffa446e7 100644 --- a/projects/rccl/bindings/nccl4py/nccl/core/utils.py +++ b/projects/rccl/bindings/nccl4py/nccl/core/utils.py @@ -160,7 +160,7 @@ class UniqueId: * **Bytes**: ``bytes(uid)`` (or :py:attr:`as_bytes`) on the producer, :py:meth:`from_bytes` on receivers. The bytes of unique ID can be - transmitted through any byte-oriented channel — a TCP socket, a + transmitted through any byte-oriented channel -- a TCP socket, a shared filesystem, etc. * **NumPy**: :py:attr:`as_ndarray` returns an in-place view of the underlying buffer, suitable for NumPy-aware buffer transports such diff --git a/projects/rccl/bindings/nccl4py/pyproject.toml b/projects/rccl/bindings/nccl4py/pyproject.toml index 461a9e4a3c00..c0630d909166 100644 --- a/projects/rccl/bindings/nccl4py/pyproject.toml +++ b/projects/rccl/bindings/nccl4py/pyproject.toml @@ -54,15 +54,31 @@ issues = "https://github.com/ROCm/rocm-systems/issues" [build-system] requires = [ - "setuptools>=77.0.3", + "setuptools>=80", + "setuptools-scm>=10.2,<11", "Cython>=3.1" ] build-backend = "setuptools.build_meta" +[tool.setuptools_scm] +# [RCCL] rocm-systems git root, which has no nccl4py-v* tags. +root = "../../../.." +version_scheme = "no-guess-dev" +local_scheme = "node-and-date" +fallback_version = "0.0.0" + +# [RCCL] upstream sets [tool.setuptools_scm.scm.git] pre_parse = "fail_on_shallow"; dropped so +# shallow CI clones of rocm-systems can still build nccl4py. + +[tool.setuptools_scm.tag] +prefix = "nccl4py-v" +strict = true + [tool.setuptools] include-package-data = false [tool.setuptools.packages.find] +where = ["."] # Only the `nccl` package is shipped on disk. The HIP-backed # `cuda.core` surface is registered in `sys.modules` lazily on # `import nccl` (see `nccl/__init__.py` -> `_register_as_cuda_core`) @@ -80,6 +96,7 @@ include = ["nccl*"] [tool.setuptools.package-data] "nccl.bindings" = ["cynccl.pxd", "cynccl_ep.pxd"] +"nccl.core" = ["py.typed"] "nccl.ep" = [ "lib/libnccl_ep.so", "include/*.h", @@ -87,9 +104,6 @@ include = ["nccl*"] "include/device/*.cuh", ] -[tool.setuptools.dynamic] -version = { attr = "nccl._version.__version__" } - [tool.ruff] line-length = 100 @@ -116,7 +130,7 @@ skip = "*-musllinux_*" build-verbosity = 1 [tool.cibuildwheel.linux] -archs = ["native"] # Fast local builds; override in CI for multi-arch +archs = ["native"] # Each local or CI runner builds its native architecture manylinux-x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" repair-wheel-command = "auditwheel repair --exclude libnccl.so.2 --exclude libcuda.so.1 -w {dest_dir} {wheel}" diff --git a/projects/rccl/bindings/nccl4py/setup.py b/projects/rccl/bindings/nccl4py/setup.py index 2a35810b503f..23d03aaa55dc 100644 --- a/projects/rccl/bindings/nccl4py/setup.py +++ b/projects/rccl/bindings/nccl4py/setup.py @@ -108,5 +108,4 @@ def calculate_modules(module: str): compiler_directives=compiler_directives, ), zip_safe=False, - options={"build_ext": {"inplace": False}}, ) diff --git a/projects/rccl/bindings/nccl4py/tests/test_comm_properties_abi.py b/projects/rccl/bindings/nccl4py/tests/test_comm_properties_abi.py index f50cc8af7ab2..1f459377b304 100644 --- a/projects/rccl/bindings/nccl4py/tests/test_comm_properties_abi.py +++ b/projects/rccl/bindings/nccl4py/tests/test_comm_properties_abi.py @@ -88,6 +88,7 @@ class _NcclCommPropertiesRef(ctypes.Structure): ncclGinConnectionType_t ginConnectionType; bool ginSupport[64]; size_t devCommRuntimeVersionSize; + bool cftSupport, cftMulticastSupport, cftCountedSupport; // NCCL 2.32 """ _fields_ = [ @@ -109,6 +110,9 @@ class _NcclCommPropertiesRef(ctypes.Structure): ("ginConnectionType", ctypes.c_int), # ncclGinConnectionType_t ("ginSupport", ctypes.c_uint8 * 64), ("devCommRuntimeVersionSize", ctypes.c_size_t), + ("cftSupport", ctypes.c_uint8), + ("cftMulticastSupport", ctypes.c_uint8), + ("cftCountedSupport", ctypes.c_uint8), ] @@ -160,6 +164,9 @@ class _NcclCommPropertiesBuggyRef(ctypes.Structure): ("ginConnectionType", "gin_connection_type"), ("ginSupport", "gin_support"), ("devCommRuntimeVersionSize", "dev_comm_runtime_version_size"), + ("cftSupport", "cft_support"), + ("cftMulticastSupport", "cft_multicast_support"), + ("cftCountedSupport", "cft_counted_support"), ] @@ -215,6 +222,9 @@ def test_comm_properties_from_buffer_round_trip(gin_type, railed_gin_type): ginConnectionType=1, ginSupport=(ctypes.c_uint8 * 64)(*([0] * 63 + [1])), devCommRuntimeVersionSize=512, + cftSupport=1, + cftMulticastSupport=0, + cftCountedSupport=1, ) props = CommProperties.from_buffer(bytes(ref)) @@ -236,6 +246,9 @@ def test_comm_properties_from_buffer_round_trip(gin_type, railed_gin_type): assert props.gin_connection_type == 1 assert props.gin_support[-1] == 1 assert props.dev_comm_runtime_version_size == 512 + assert bool(props.cft_support) is True + assert bool(props.cft_multicast_support) is False + assert bool(props.cft_counted_support) is True def test_comm_properties_from_buffer_rejects_undersized_buffer(): diff --git a/projects/rccl/bindings/nccl4py/tests/test_loader_stubs.py b/projects/rccl/bindings/nccl4py/tests/test_loader_stubs.py index 90c7086235a5..a150862fb30d 100644 --- a/projects/rccl/bindings/nccl4py/tests/test_loader_stubs.py +++ b/projects/rccl/bindings/nccl4py/tests/test_loader_stubs.py @@ -1,60 +1,75 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Loader / stub tests for the RCCL fork of nccl4py. +"""Loader tests for the RCCL fork of nccl4py. -Verifies that: - -* ``import nccl.bindings`` does not raise even though several - NCCL >= 2.29 host-side symbols are absent from ``librccl.so``. -* Calling each of the following at runtime raises - :class:`NotImplementedError` and the C symbol name appears in the - message:: +Verifies that ``import nccl.bindings`` does not raise, and that each of +the following resolves in ``librccl.so`` and rejects a null +communicator with ``ncclInvalidArgument`` (no crash, no +:class:`NotImplementedError`):: ncclCommGrow -> nccl.bindings.comm_grow ncclCommRevoke -> nccl.bindings.comm_revoke ncclPutSignal -> nccl.bindings.put_signal ncclWaitSignal -> nccl.bindings.wait_signal -The Python wrappers check the ``dlsym`` slot before dereferencing any -pointer argument, so dummy zero values are sufficient for all opaque -handles. +Each call's argument checks reject the null ``Comm`` before any other +pointer is used, so zero values are safe for the remaining arguments. """ import pytest +def _assert_invalid_argument(call): + import nccl.bindings as b + + with pytest.raises(b.NCCLError) as exc: + call() + assert exc.value.status == b.Result.InvalidArgument + + def test_import_bindings_does_not_raise(): import nccl.bindings # noqa: F401 -def test_comm_grow_raises_not_implemented(): +def test_import_top_level_without_ep_bindings(): + import nccl + + version = nccl.get_version() + if version.nccl_ep is not None: + pytest.skip("libnccl_ep loaded on this host") + assert version.nccl.version is not None + + +def test_import_top_level_with_ep_bindings(): + import nccl + + version = nccl.get_version() + if version.nccl_ep is None: + pytest.skip("libnccl_ep not available on this host") + assert version.nccl_ep.version is not None + + +def test_comm_grow_rejects_null_comm(): import nccl.bindings as b - with pytest.raises(NotImplementedError, match="ncclCommGrow"): - b.comm_grow(0, 0, 0, 0, 0) + # n_ranks=0 fails the rank-count check before the comm is read. + _assert_invalid_argument(lambda: b.comm_grow(b.Comm(0), 0, 0, 0, 0)) -def test_comm_revoke_raises_not_implemented(): +def test_comm_revoke_rejects_null_comm(): import nccl.bindings as b - with pytest.raises(NotImplementedError, match="ncclCommRevoke"): - b.comm_revoke(0, 0) + _assert_invalid_argument(lambda: b.comm_revoke(b.Comm(0), 0)) -def test_put_signal_raises_not_implemented(): +def test_put_signal_rejects_null_comm(): import nccl.bindings as b - with pytest.raises(NotImplementedError, match="ncclPutSignal"): - b.put_signal(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + _assert_invalid_argument(lambda: b.put_signal(0, 0, 0, 0, b.Window(0), 0, 0, 0, 0, b.Comm(0), 0)) -def test_wait_signal_raises_not_implemented(): +def test_wait_signal_rejects_null_comm(): import nccl.bindings as b - # Signature: wait_signal(int n_desc, intptr_t signal_descs, - # intptr_t comm, intptr_t stream). - # n_desc=0 + signal_descs=0 is safe: the dlsym slot is checked before - # the pointer is dereferenced, so the call never reaches librccl. - with pytest.raises(NotImplementedError, match="ncclWaitSignal"): - b.wait_signal(0, 0, 0, 0) + _assert_invalid_argument(lambda: b.wait_signal(0, 0, b.Comm(0), 0)) diff --git a/projects/rccl/bindings/nccl4py/tests/test_shim_surface.py b/projects/rccl/bindings/nccl4py/tests/test_shim_surface.py index 04e100e9eac7..75fcce1f30e2 100644 --- a/projects/rccl/bindings/nccl4py/tests/test_shim_surface.py +++ b/projects/rccl/bindings/nccl4py/tests/test_shim_surface.py @@ -5,7 +5,7 @@ Verifies that the HIP-backed ``cuda.core`` shim implements exactly the subset that ``nccl/core/*.py`` imports: :class:`Device`, :class:`Stream`, -:class:`Buffer`, :class:`MemoryResource`, :func:`system.get_num_devices`, +:class:`Event`, :class:`Buffer`, :class:`MemoryResource`, :func:`system.get_num_devices`, :class:`StridedMemoryView`, :func:`args_viewable_as_strided_memory`, and DLPack capsule export with ``kROCM=10``. @@ -22,7 +22,7 @@ pytest.importorskip("hip", reason="hip-python is required for the HIP shim tests") -from cuda.core import Buffer, Device, MemoryResource, Stream, system # noqa: E402 +from cuda.core import Buffer, Device, Event, MemoryResource, Stream, system # noqa: E402 from cuda.core.utils import StridedMemoryView, args_viewable_as_strided_memory # noqa: E402 if system.get_num_devices() == 0: # pragma: no cover - host without GPUs @@ -73,6 +73,37 @@ def test_round_trip_from_handle(self): assert int(second.handle) == 0xCAFE +# --------------------------------------------------------------------------- +# Event (3 tests) +# --------------------------------------------------------------------------- + + +class TestEvent: + def test_create_event_has_a_handle_until_closed(self): + Device(0).set_current() + e = Device(0).create_event() + assert isinstance(e, Event) + assert int(e.handle) != 0 + e.close() + assert int(e.handle) == 0 + + def test_from_handle_round_trip_is_not_owned(self): + e = Event.from_handle(0xFEED) + assert int(e.handle) == 0xFEED + e.close() # borrowed: not destroyed + assert int(e.handle) == 0xFEED + + def test_get_event_ptr_accepts_a_shim_event(self): + from nccl.core.cuda import get_event_ptr + + Device(0).set_current() + e = Device(0).create_event() + try: + assert get_event_ptr(e) == int(e.handle) + finally: + e.close() + + # --------------------------------------------------------------------------- # Buffer (3 tests) # --------------------------------------------------------------------------- diff --git a/projects/rccl/ci/scripts/known_failures.json b/projects/rccl/ci/scripts/known_failures.json index 7e40d5ce3418..ac43bf74709f 100644 --- a/projects/rccl/ci/scripts/known_failures.json +++ b/projects/rccl/ci/scripts/known_failures.json @@ -1,5 +1,5 @@ { - "_comment": "Known PyTorch c10d NCCL test failures on RCCL — not regressions. Tests listed here are excluded from the pass/fail verdict. Update this file when RCCL gains support for the missing feature.", + "_comment": "Known PyTorch c10d NCCL test failures in this ROCm/RCCL CI suite — not regressions. Tests listed here are excluded from the pass/fail verdict. Not every cause is an RCCL gap; some are ROCm runtime behaviour or test assumptions that do not hold on ROCm. Remove each entry once its underlying cause is resolved.", "tests": { "test_extend_nccl_pg_timeout_backend0": "pg_timeout_semantics", "test_extend_nccl_pg_timeout_backend_nccl": "pg_timeout_semantics", @@ -30,7 +30,8 @@ "test_batched_send_recv_op_sizes_per_coalesce1_timing_enabled_True": "missing_duration_ms", "test_coalescing_manager_collective_timing_enabled_True": "missing_duration_ms", "test_individual_send_recv_op_sizes0_timing_enabled_True": "missing_duration_ms", - "test_individual_send_recv_op_sizes1_timing_enabled_True": "missing_duration_ms" + "test_individual_send_recv_op_sizes1_timing_enabled_True": "missing_duration_ms", + "test_extra_cuda_context": "extra_context_heuristic" }, "categories": { "missing_duration_ms": "RCCL flight recorder trace omits duration_ms field", @@ -39,6 +40,7 @@ "trace_timeout": "Flight recorder trace tests exceed pytest-timeout (600s)", "trace_entry_mismatch": "Trace entry structure differs from upstream NCCL", "missing_dump_pipe": "RCCL does not create flight recorder dump pipe", - "pg_timeout_semantics": "extend_pg_timeout adds to existing vs sets absolute value" + "pg_timeout_semantics": "extend_pg_timeout adds to existing vs sets absolute value", + "extra_context_heuristic": "Test infers an extra HIP context from >70% device memory growth across a collective; on ROCm the growth is a ROCr runtime allocation, not a second context and not RCCL memory. Magnitude is build- and arch-dependent, and the ratio also varies with node co-tenancy (AICOMRCCL-2469)" } } diff --git a/projects/rccl/cmake/AsanGpuTargets.cmake b/projects/rccl/cmake/AsanGpuTargets.cmake new file mode 100644 index 000000000000..2f3d7c5eb969 --- /dev/null +++ b/projects/rccl/cmake/AsanGpuTargets.cmake @@ -0,0 +1,63 @@ +# GPU target selection for device AddressSanitizer builds. Device ASAN needs +# xnack, which gfx9 must request explicitly, gfx1250 has always on (and rejects +# the suffix), and no other family has. Cases pinned in +# tests/test_asan_gpu_targets.py; projects/rocshmem repeats the policy inline. + +# Parts where xnack is always on, so naming it is an error. +set(_RCCL_ASAN_XNACK_ALWAYS_ON gfx1250 gfx1250-strict) + +# rccl_asan_adjust_gpu_targets(OUT_VAR gfx942 gfx1100 gfx1250) +# -> OUT_VAR = gfx942:xnack+;gfx1250 +# Uninstrumentable targets are dropped rather than left to produce an +# uninstrumented image inside an ASAN build. +function(rccl_asan_adjust_gpu_targets OUT_VAR) + set(_result "") + set(_dropped "") + + foreach(_target IN LISTS ARGN) + if(_target STREQUAL "") + continue() + endif() + + # A requested suffix says what the caller wants, not what the part + # supports, so classify on the bare name. + string(REGEX REPLACE ":.*" "" _base "${_target}") + # list(FIND) not IN_LIST: the test loads this via cmake -P, where CMP0057 + # is unset. + list(FIND _RCCL_ASAN_XNACK_ALWAYS_ON "${_base}" _always_on) + + # Drop any xnack term the caller wrote before deciding what to do with it: + # it may be ":xnack-", which contradicts the build. Other feature terms + # (e.g. ":sramecc+") are part of the requested target and are kept; xnack + # sorts last, so re-appending it keeps the target ID canonical. + string(REGEX REPLACE ":xnack[+-]" "" _without_xnack "${_target}") + + if(_base MATCHES "^gfx9") + list(APPEND _result "${_without_xnack}:xnack+") + elseif(NOT _always_on EQUAL -1) + # Always on here, so naming xnack at all is an invalid target ID. + list(APPEND _result "${_without_xnack}") + else() + list(APPEND _dropped "${_target}") + endif() + endforeach() + + # Both xnack spellings of one arch normalise together, and a duplicate + # becomes a second add_library() in DeviceLinker.cmake. + if(_result) + list(REMOVE_DUPLICATES _result) + endif() + + if(_dropped) + message(STATUS + "ASAN: dropping GPU target(s) with no xnack support: ${_dropped}") + endif() + if(NOT _result) + message(FATAL_ERROR + "ASAN requires a GPU target that supports xnack (gfx9* with :xnack+, " + "or gfx1250 where it is always on). None of the requested GPU targets " + "do: ${ARGN}") + endif() + + set(${OUT_VAR} "${_result}" PARENT_SCOPE) +endfunction() diff --git a/projects/rccl/cmake/DeviceCoverage.cmake b/projects/rccl/cmake/DeviceCoverage.cmake index 6173f99896a4..b5874728d297 100644 --- a/projects/rccl/cmake/DeviceCoverage.cmake +++ b/projects/rccl/cmake/DeviceCoverage.cmake @@ -10,8 +10,13 @@ set(RCCL_DEVICE_PROFILE_RUNTIME_RELPATHS "lib/amdgcn-amd-amdhsa/libclang_rt.profile.a" "lib/linux/libclang_rt.profile-amdgcn.a") -set(RCCL_HOST_ROCM_PROFILE_RUNTIME_NAME - "libclang_rt.profile_rocm.a") +# Host ROCm profile runtime archive names, in priority order: the per-target +# runtime layout (lib//) first, then the legacy arch-suffixed layout +# (lib/linux/) that compilers built without LLVM_ENABLE_PER_TARGET_RUNTIME_DIR use. +cmake_host_system_information(RESULT _rccl_host_arch QUERY OS_PLATFORM) +set(RCCL_HOST_ROCM_PROFILE_RUNTIME_NAMES + "libclang_rt.profile_rocm.a" + "libclang_rt.profile_rocm-${_rccl_host_arch}.a") # Report whether accepts the device coverage instrumentation flags. @@ -91,18 +96,20 @@ endfunction() # compiler-rt packages. Older packages include the ROCm collection objects in # the generic profile runtime, so an absent archive is not an error. function(rccl_find_host_rocm_profile_runtime compiler output_runtime) - execute_process( - COMMAND "${compiler}" "-print-file-name=${RCCL_HOST_ROCM_PROFILE_RUNTIME_NAME}" - OUTPUT_VARIABLE runtime - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE runtime_result) - - if(runtime_result EQUAL 0 AND IS_ABSOLUTE "${runtime}" AND EXISTS "${runtime}") - set(${output_runtime} "${runtime}" PARENT_SCOPE) - else() - set(${output_runtime} "" PARENT_SCOPE) - endif() + foreach(_name IN LISTS RCCL_HOST_ROCM_PROFILE_RUNTIME_NAMES) + execute_process( + COMMAND "${compiler}" "-print-file-name=${_name}" + OUTPUT_VARIABLE runtime + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE runtime_result) + + if(runtime_result EQUAL 0 AND IS_ABSOLUTE "${runtime}" AND EXISTS "${runtime}") + set(${output_runtime} "${runtime}" PARENT_SCOPE) + return() + endif() + endforeach() + set(${output_runtime} "" PARENT_SCOPE) endfunction() diff --git a/projects/rccl/cmake/DeviceLinker.cmake b/projects/rccl/cmake/DeviceLinker.cmake index 8670af49d78e..bfb53916a40b 100644 --- a/projects/rccl/cmake/DeviceLinker.cmake +++ b/projects/rccl/cmake/DeviceLinker.cmake @@ -19,13 +19,32 @@ message(STATUS "Device Linker: assembly-extract pipeline enabled (RCCLDEV langua list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") enable_language(RCCLDEV) -# Tell the driver where to find the real compiler. +# Device coverage requires the same ROCm 7.15+ compiler selected for the +# main build, including its packaged device profile runtime. get_filename_component(_dl_compiler_dir "${CMAKE_CXX_COMPILER}" DIRECTORY) -find_program(DL_CLANG NAMES amdclang++ clang++ - HINTS "${_dl_compiler_dir}" "${ROCM_PATH}/bin" REQUIRED) +if(ENABLE_FULL_COVERAGE) + set(DL_CLANG "${CMAKE_CXX_COMPILER}") + message(STATUS "Device Linker: DL_CLANG pinned to CMAKE_CXX_COMPILER for device coverage = ${DL_CLANG}") +else() + find_program(DL_CLANG NAMES amdclang++ clang++ + HINTS "${_dl_compiler_dir}" "${ROCM_PATH}/bin" REQUIRED) +endif() +# Pick clang-offload-bundler from the SAME toolchain as DL_CLANG: a bundler from a +# different LLVM build can load a foreign libclang-cpp.so and crash at startup. +get_filename_component(_dl_clang_dir "${DL_CLANG}" REALPATH) +get_filename_component(_dl_clang_dir "${_dl_clang_dir}" DIRECTORY) +unset(DL_BUNDLER CACHE) +unset(DL_BUNDLER) find_program(DL_BUNDLER NAMES clang-offload-bundler - HINTS "${_dl_compiler_dir}" "${_dl_compiler_dir}/../lib/llvm/bin" - "${ROCM_PATH}/llvm/bin" REQUIRED) + HINTS "${_dl_clang_dir}" "${_dl_compiler_dir}/../lib/llvm/bin" + "${ROCM_PATH}/llvm/bin") +if(NOT DL_BUNDLER) + message(FATAL_ERROR + "Device Linker: clang-offload-bundler was not found next to the selected " + "compiler '${DL_CLANG}' (searched '${_dl_clang_dir}', " + "'${_dl_compiler_dir}/../lib/llvm/bin', '${ROCM_PATH}/llvm/bin').") +endif() +message(STATUS "Device Linker: clang-offload-bundler = ${DL_BUNDLER}") # Extract --hip-path and --hip-device-lib-path from CMAKE_CXX_FLAGS. # TheRock's amd-hip toolchain injects these so amdclang++ can locate HIP @@ -85,6 +104,17 @@ if(_rccl_copts) endif() message(STATUS "Device Linker: inherited compile options: ${DL_INHERITED_FLAGS}") +# The --link step is the one consumer that cannot take a -fvisibility= option: +# the ncclDevFunc_* device functions have to stay visible, or the device link +# fails with "recompile with -fPIC". The host objects below still need it. +set(DL_LINK_INHERITED_FLAGS "") +foreach(_opt IN LISTS DL_INHERITED_FLAGS) + if(_opt MATCHES "^-fvisibility=") + continue() + endif() + list(APPEND DL_LINK_INHERITED_FLAGS "${_opt}") +endforeach() + # --------------------------------------------------------------------------- # Parse GPU_TARGETS: strip target features, build offload-arch flag list # --------------------------------------------------------------------------- @@ -100,12 +130,141 @@ message(STATUS "Device Linker: GPU targets = ${DL_GPU_TARGETS}") # --------------------------------------------------------------------------- # Optimization flags (passed to both compile and link modes of the driver) # --------------------------------------------------------------------------- -if(CMAKE_BUILD_TYPE MATCHES "Debug") +if(ENABLE_FULL_COVERAGE) + # Drop DWARF (-g0) even under --debug. LLVM source-based coverage lives in + # __llvm_covfun/__llvm_covmap + __llvm_prf_*, not DWARF, so llvm-cov needs + # only device.elf + .profdata. With thousands of specialized kernels, keeping + # debug info pushes the per-arch device.elf past the ~2 GB (INT32_MAX) HIP + # fat-binary unbundling limit, after which every kernel launch fails. + set(DL_OPT_FLAGS -O1 -g0) +elseif(CMAKE_BUILD_TYPE MATCHES "Debug") set(DL_OPT_FLAGS -O1 -g) else() set(DL_OPT_FLAGS -O3) endif() +# --------------------------------------------------------------------------- +# Device-side coverage instrumentation (ENABLE_FULL_COVERAGE). +# +# We thread -fprofile-instr-generate / -fcoverage-mapping through every +# device compile in this pipeline (per-kernel RCCLDEV compiles, dispatcher +# compile inside --link, and the standalone fat-object compiles below). +# The assembly-extract pass passes them through to the underlying clang +# invocation; the LLVM profile metadata sections (__llvm_prf_*, __llvm_cov*) +# survive extraction because the extractor only strips the amdhsa_kernel / +# amdgpu_metadata ranges. +# +# For the per-arch device.elf to link cleanly we must also resolve the +# `__llvm_profile_runtime` reference emitted by each instrumented TU. We +# locate the per-arch device profile runtime archive (libclang_rt.profile.a +# under the amdgcn-amd-amdhsa resource sub-directory) and pass it to the +# rccl-device-compile driver, which forwards it to ld.lld. +# --------------------------------------------------------------------------- +set(DL_COVERAGE_FLAGS "") +set(DL_DEVICE_PROFILE_RT "") +if(ENABLE_FULL_COVERAGE) + set(DL_COVERAGE_FLAGS -fprofile-instr-generate -fcoverage-mapping) + set(DL_DEVICE_PROFILE_RT "${RCCL_DEVICE_PROFILE_RT}") + if(NOT EXISTS "${DL_DEVICE_PROFILE_RT}") + message(FATAL_ERROR + "Device Linker: the preflighted device profile runtime is missing: " + "'${DL_DEVICE_PROFILE_RT}'. Reconfigure the build.") + endif() + message(STATUS + "Device Linker: coverage enabled, device profile RT = ${DL_DEVICE_PROFILE_RT}") +endif() + +# Profile section anchor (ENABLE_FULL_COVERAGE). +# +# A coverage-instrumented device TU that emits no counters (e.g. a fat object +# with no device kernels, like sym_reduce_scatter.o) still emits an +# __llvm_profile_sections descriptor referencing __start/__stop___llvm_prf_{cnts,data}. +# With no __llvm_prf_cnts/__llvm_prf_data input section, the linker cannot synthesize +# those boundary symbols and the device link fails with "undefined hidden symbol: +# __start___llvm_prf_cnts". We link a tiny anchor that defines empty, retained +# __llvm_prf_{cnts,data} sections so the boundary symbols are always defined; when a +# TU does emit counters the empty sections merge in as a zero-byte no-op. The anchor +# is generic amdgcn bitcode (no mcpu) so every per-arch device link LTO-compiles it. +set(DL_DEVICE_PROFILE_ANCHOR "") +if(DL_DEVICE_PROFILE_RT) + file(MAKE_DIRECTORY "${DEVICE_BUILD_DIR}") + set(_dl_anchor_src "${DEVICE_BUILD_DIR}/prf_section_anchor.c") + set(_dl_anchor_bc "${DEVICE_BUILD_DIR}/prf_section_anchor.bc") + file(WRITE "${_dl_anchor_src}" +"/* Auto-generated by cmake/DeviceLinker.cmake for ENABLE_FULL_COVERAGE.\n" +" Empty, retained profile counter/data sections so the device linker always\n" +" defines __start/__stop___llvm_prf_{cnts,data}, even for instrumented TUs that\n" +" reference those boundaries but emit no counters (e.g. no device kernels). */\n" +"__attribute__((used, retain, section(\"__llvm_prf_cnts\")))\n" +"static char __rccl_prf_cnts_anchor[0];\n" +"__attribute__((used, retain, section(\"__llvm_prf_data\")))\n" +"static char __rccl_prf_data_anchor[0];\n") + execute_process( + COMMAND ${DL_CLANG} --target=amdgcn-amd-amdhsa -c -emit-llvm -O1 + -o "${_dl_anchor_bc}" "${_dl_anchor_src}" + RESULT_VARIABLE _dl_anchor_rc + ERROR_VARIABLE _dl_anchor_err) + if(NOT _dl_anchor_rc EQUAL 0 OR NOT EXISTS "${_dl_anchor_bc}") + message(FATAL_ERROR + "Device Linker: failed to build profile section anchor bitcode.\n${_dl_anchor_err}") + endif() + set(DL_DEVICE_PROFILE_ANCHOR "${_dl_anchor_bc}") + message(STATUS "Device Linker: profile section anchor = ${DL_DEVICE_PROFILE_ANCHOR}") +endif() + +# Standalone fat-object compiles (onerank.o, collectives.o, dda_all_reduce_ipc.o, +# sym_*.o) are built as full `-x hip` objects, so each -c step performs its own +# embedded device link (clang-linker-wrapper -> ld.lld). That link must resolve +# __llvm_profile_instrument_gpu from the device profile RT. The rccl-device-compile +# --link path passes the archive to ld.lld directly; here we forward it to the +# clang driver's offload linker via -Xoffload-linker. +# +# The profile RT is a separate bitcode LTO module, so clang's auto-injected +# -amdgpu-internalize-symbols dead-strips the hidden __llvm_profile_instrument_gpu; disable it. +# The anchor supplies the __llvm_prf_{cnts,data} boundary sections (see above). +set(DL_DEVICE_PROFILE_RT_LINK_FLAGS "") +if(DL_DEVICE_PROFILE_RT) + set(DL_DEVICE_PROFILE_RT_LINK_FLAGS + -Xoffload-linker -plugin-opt=-amdgpu-internalize-symbols=false + -Xoffload-linker "${DL_DEVICE_PROFILE_ANCHOR}" + -Xoffload-linker "${DL_DEVICE_PROFILE_RT}") +endif() + +# Instrumentation plus the profile-runtime link flags for the standalone `-x hip` +# fat-object compiles below. Both expand to nothing unless ENABLE_FULL_COVERAGE. +set(DL_DEVICE_COVERAGE_FLAGS ${DL_COVERAGE_FLAGS} ${DL_DEVICE_PROFILE_RT_LINK_FLAGS}) + +# --------------------------------------------------------------------------- +# Main-module coverage drain: deterministic compilation-unit ID (-cuid). +# +# The device profile runtime drains a per-arch device.elf by looking up a host +# "shadow" variable __llvm_profile_sections_ (registered from the host +# TU via __hipRegisterVar) and resolving it to the identically-named device +# global with hipGetSymbolAddress. The device global points at the module-wide +# __start/__stop___llvm_prf_{cnts,data,names} boundaries, so draining through +# ANY ONE descriptor drains the entire linked device.elf. +# +# The main RDC module is compiled in two separate steps: the device side +# (dispatcher common.cu.cpp inside --link, plus the per-kernel TUs) and the host +# side (common.cu.cpp --offload-host-only). With the default -fuse-cuid=hash each +# step derives a different CUID from its own file path + command line, so the +# host shadow name (e.g. a2f3...) never matches any of the device descriptors and +# hipGetSymbolAddress fails -> device counters read back as zero. +# +# Pin BOTH the dispatcher device compile and the host common.cu.cpp compile to +# the same explicit -cuid so their __llvm_profile_sections_ names match +# (CUIDHash = MD5(cuid), identical across separate invocations). This is the +# compiler's intended lever for split host/device compilation (see the codegen +# test clang/test/CodeGenHIP/offload-pgo-sections.hip, which passes -cuid=abc to +# both cc1 jobs). It must NOT be applied to the per-kernel device TUs: they are +# distinct compilation units and a shared CUID would collide on the (non-weak) +# __llvm_profile_sections_ device global at link time. +set(DL_MAIN_MODULE_CUID_FLAGS "") +if(ENABLE_FULL_COVERAGE) + set(DL_MAIN_MODULE_CUID_FLAGS -cuid=rccl_main_module) + message(STATUS "Device Linker: main-module coverage CUID pinned = rccl_main_module") +endif() + # --------------------------------------------------------------------------- # INTERFACE library: shared definitions and includes for device compilation. # Reads from the rccl target (already fully configured) and directory scope. @@ -184,14 +343,15 @@ function(dl_evaluate_guard GUARD GPU_TARGET RESULT_VAR) set(${RESULT_VAR} FALSE PARENT_SCOPE) return() endif() - string(REGEX MATCHALL "__gfx[0-9a-z]+__" _guard_archs "${GUARD}") + string(REGEX MATCHALL "__gfx[0-9a-z_]+__" _guard_archs "${GUARD}") + string(REPLACE "-" "_" _gpu_macro "${GPU_TARGET}") if(NOT _guard_archs) set(${RESULT_VAR} TRUE PARENT_SCOPE) return() endif() foreach(_ga ${_guard_archs}) string(REGEX REPLACE "^__(.+)__$" "\\1" _arch "${_ga}") - if("${_arch}" STREQUAL "${GPU_TARGET}") + if("${_arch}" STREQUAL "${_gpu_macro}") set(${RESULT_VAR} TRUE PARENT_SCOPE) return() endif() @@ -267,6 +427,7 @@ foreach(DL_GPU_TARGET ${DL_GPU_TARGETS}) --arch=${DL_GPU_TARGET} --clang=${DL_CLANG} ${DL_OPT_FLAGS} + ${DL_COVERAGE_FLAGS} -std=c++17 ${DL_HIP_COMPILER_FLAGS} # -fPIC is required so amdclang++ emits GOT-relative relocations for @@ -357,6 +518,11 @@ foreach(DL_GPU_TARGET ${DL_GPU_TARGETS}) endif() endif() + set(_dl_profile_rt_arg "") + if(DL_DEVICE_PROFILE_RT) + set(_dl_profile_rt_arg "--profile-rt=${DL_DEVICE_PROFILE_RT}") + endif() + add_custom_command( OUTPUT ${ARCH_DEVICE_ELF} COMMAND ${CMAKE_RCCLDEV_COMPILER} @@ -366,9 +532,13 @@ foreach(DL_GPU_TARGET ${DL_GPU_TARGETS}) ${DL_HIP_COMPILER_FLAGS} --dispatcher=${HIPIFY_DIR}/src/device/common.cu.cpp ${_rocshmem_bitcode_arg} + ${_dl_profile_rt_arg} ${_link_def_flags} ${_link_inc_flags} ${DL_OPT_FLAGS} + ${DL_LINK_INHERITED_FLAGS} + ${DL_COVERAGE_FLAGS} + ${DL_MAIN_MODULE_CUID_FLAGS} -std=c++17 -o ${ARCH_DEVICE_ELF} @${_link_rsp} @@ -382,6 +552,21 @@ foreach(DL_GPU_TARGET ${DL_GPU_TARGETS}) list(APPEND DL_BUNDLER_TARGETS "hip-amdgcn-amd-amdhsa--${DL_GPU_TARGET}") list(APPEND DL_BUNDLER_INPUTS "--input=${ARCH_DEVICE_ELF}") + # Friendly symlink next to librccl.so: device-${arch}.elf -> device_build//device.elf + # Lets coverage workflows (and inspection in general) reference the per-arch + # device ELF without the leading-hyphen directory wart we use for CDNA build + # scheduling. Same byte content that ends up packed into .hip_fatbin; llvm-cov + # only needs this file + the merged .profdata for device-side reports. + set(_dev_elf_link "${PROJECT_BINARY_DIR}/device-${DL_GPU_TARGET}.elf") + add_custom_command( + OUTPUT ${_dev_elf_link} + COMMAND ${CMAKE_COMMAND} -E create_symlink ${ARCH_DEVICE_ELF} ${_dev_elf_link} + DEPENDS ${ARCH_DEVICE_ELF} + COMMENT "DL [${DL_GPU_TARGET}] symlink: device-${DL_GPU_TARGET}.elf -> device_build/.../device.elf" + VERBATIM + ) + list(APPEND DEVICE_ELF_SYMLINKS "${_dev_elf_link}") + # ========================================================================= # Optional: emit LLVM IR for specialized kernels (ninja device_ir) # ========================================================================= @@ -487,6 +672,8 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_COVERAGE_FLAGS} + ${DL_MAIN_MODULE_CUID_FLAGS} -std=c++17 -fPIC ${DL_HOST_COMPRESS} @@ -512,6 +699,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${ONERANK_FAT_OBJ} @@ -546,6 +734,7 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.20") ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -MD -MF ${COLLECTIVES_DEPFILE} @@ -567,6 +756,7 @@ else() ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${COLLECTIVES_FAT_OBJ} @@ -596,6 +786,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${DIAG_P2P_FAT_OBJ} @@ -620,6 +811,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${DDA_ALL_REDUCE_IPC_FAT_OBJ} @@ -639,6 +831,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${DDA_REDUCE_SCATTER_IPC_FAT_OBJ} @@ -658,6 +851,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${DDA_ALL_GATHER_IPC_FAT_OBJ} @@ -677,6 +871,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${DDA_ALLTOALL_IPC_FAT_OBJ} @@ -709,6 +904,7 @@ if(ENABLE_ROCSHMEM_GIN) ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${GIN_ALLTOALL_SDMA_FAT_OBJ} @@ -742,6 +938,7 @@ if(ENABLE_ROCSHMEM_GIN) ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -c -o ${GIN_ALLREDUCE_SDMA_FAT_OBJ} @@ -751,6 +948,36 @@ if(ENABLE_ROCSHMEM_GIN) VERBATIM ) endif() + +# =========================================================================== +# gin_anvil_conn_check_device.cc: GIN-SDMA LSA connectivity-check kernels. +# This source must bypass the rccl target's --offload-host-only compilation. +# =========================================================================== +set(GIN_ANVIL_CONN_CHECK_FAT_OBJ "") +if(ENABLE_ROCSHMEM_GIN) + set(GIN_ANVIL_CONN_CHECK_FAT_OBJ "${DEVICE_BUILD_DIR}/gin_anvil_conn_check_device.o") + add_custom_command( + OUTPUT ${GIN_ANVIL_CONN_CHECK_FAT_OBJ} + COMMAND ${DL_CLANG} + -x hip ${DL_OFFLOAD_ARCH_FLAGS} + ${DL_HIP_COMPILER_FLAGS} + -DRCCL_DEVICE_LINKER + -DENABLE_ROCSHMEM_GIN + ${_link_def_flags} + ${_host_inc_flags} + ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} + -std=c++17 + -fPIC + -c -o ${GIN_ANVIL_CONN_CHECK_FAT_OBJ} + ${HIPIFY_DIR}/src/gin/gin_anvil_conn_check_device.cc + DEPENDS ${HIPIFY_DIR}/src/gin/gin_anvil_conn_check_device.cc + COMMENT "DL compile: gin_anvil_conn_check_device.cc (GIN-SDMA conn-check kernels)" + VERBATIM + ) +endif() + # =========================================================================== # dda_all_reduce_fabric.cu.cpp: fabric/VMM counterpart of the IPC file above. # =========================================================================== @@ -766,6 +993,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -w @@ -788,9 +1016,10 @@ add_custom_command( ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${DDA_ALL_REDUCE_FABRIC_LL_FAT_OBJ} ${HIPIFY_DIR}/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll.cu.cpp @@ -808,9 +1037,10 @@ add_custom_command( ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${DDA_ALL_REDUCE_FABRIC_LL128_FAT_OBJ} ${HIPIFY_DIR}/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu.cpp @@ -839,6 +1069,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -w @@ -859,6 +1090,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -w @@ -878,9 +1110,10 @@ add_custom_command( ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${DDA_ALL_GATHER_FABRIC_LL_FAT_OBJ} ${HIPIFY_DIR}/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu.cpp @@ -898,9 +1131,10 @@ add_custom_command( ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${DDA_ALL_GATHER_FABRIC_LL128_FAT_OBJ} ${HIPIFY_DIR}/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu.cpp @@ -919,6 +1153,7 @@ add_custom_command( ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC -w @@ -938,9 +1173,10 @@ add_custom_command( ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${DDA_ALLTOALL_FABRIC_LL_FAT_OBJ} ${HIPIFY_DIR}/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu.cpp @@ -958,9 +1194,10 @@ add_custom_command( ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${DDA_ALLTOALL_FABRIC_LL128_FAT_OBJ} ${HIPIFY_DIR}/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu.cpp @@ -978,9 +1215,10 @@ add_custom_command( ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${DDA_REDUCE_SCATTER_FABRIC_LL_FAT_OBJ} ${HIPIFY_DIR}/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu.cpp @@ -998,9 +1236,10 @@ add_custom_command( ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${DDA_REDUCE_SCATTER_FABRIC_LL128_FAT_OBJ} ${HIPIFY_DIR}/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll128.cu.cpp @@ -1037,9 +1276,10 @@ foreach(_ce_reduce_src IN LISTS _ce_reduce_srcs) ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} + ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC - ${DL_INHERITED_FLAGS} -w -c -o ${_ce_reduce_obj} ${_ce_reduce_src} @@ -1141,6 +1381,7 @@ if(GENERATE_SYM_KERNELS) ${_host_inc_flags} ${DL_OPT_FLAGS} ${DL_INHERITED_FLAGS} + ${DL_DEVICE_COVERAGE_FLAGS} -std=c++17 -fPIC ${_this_bc_flag} @@ -1158,7 +1399,7 @@ endif() # Top-level target # =========================================================================== add_custom_target(device_linker_build ALL - DEPENDS ${COMMON_FAT_OBJ} ${ONERANK_FAT_OBJ} ${COLLECTIVES_FAT_OBJ} ${DIAG_P2P_FAT_OBJ} ${DDA_ALL_REDUCE_IPC_FAT_OBJ} ${DDA_REDUCE_SCATTER_IPC_FAT_OBJ} ${DDA_ALL_GATHER_IPC_FAT_OBJ} ${DDA_ALLTOALL_IPC_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_LL_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_LL128_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_LL_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_LL128_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_LL_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_LL128_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_LL_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_LL128_FAT_OBJ} ${CE_REDUCE_FAT_OBJS} ${SYM_FAT_OBJS} ${GIN_ALLTOALL_SDMA_FAT_OBJ} ${GIN_ALLREDUCE_SDMA_FAT_OBJ} ${RCCL_EP_FAT_OBJ} ${DEVICE_ELF_SYMLINKS} + DEPENDS ${COMMON_FAT_OBJ} ${ONERANK_FAT_OBJ} ${COLLECTIVES_FAT_OBJ} ${DIAG_P2P_FAT_OBJ} ${DDA_ALL_REDUCE_IPC_FAT_OBJ} ${DDA_REDUCE_SCATTER_IPC_FAT_OBJ} ${DDA_ALL_GATHER_IPC_FAT_OBJ} ${DDA_ALLTOALL_IPC_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_LL_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_LL128_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_LL_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_LL128_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_LL_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_LL128_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_LL_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_LL128_FAT_OBJ} ${CE_REDUCE_FAT_OBJS} ${SYM_FAT_OBJS} ${GIN_ALLTOALL_SDMA_FAT_OBJ} ${GIN_ALLREDUCE_SDMA_FAT_OBJ} ${GIN_ANVIL_CONN_CHECK_FAT_OBJ} ${RCCL_EP_FAT_OBJ} ${DEVICE_ELF_SYMLINKS} ) add_dependencies(device_linker_build hipify_all copy_nccl_device_headers) if(ENABLE_ROCSHMEM_GIN AND TARGET copy_rocshmem_headers) @@ -1195,6 +1436,7 @@ set(DEVICE_LINKER_OBJECTS ${SYM_FAT_OBJS} ${GIN_ALLTOALL_SDMA_FAT_OBJ} ${GIN_ALLREDUCE_SDMA_FAT_OBJ} + ${GIN_ANVIL_CONN_CHECK_FAT_OBJ} ) # =========================================================================== diff --git a/projects/rccl/cmake/Findrocshmem_static.cmake b/projects/rccl/cmake/Findrocshmem_static.cmake index 27f4c3a0edef..b7ae805e5469 100644 --- a/projects/rccl/cmake/Findrocshmem_static.cmake +++ b/projects/rccl/cmake/Findrocshmem_static.cmake @@ -27,9 +27,5 @@ find_path(ROCSHMEM_INCLUDE_DIR find_library(ROCSHMEM_LIBRARY NAMES rocshmem HINTS ${ROCSHMEM_INSTALL_DIR}/lib) - -## -- todo --- what to do with verbs? add to handle args call below? -- ## -find_library(IBVERBS ibverbs) find_package_handle_standard_args(rocshmem_static DEFAULT_MSG ROCSHMEM_INCLUDE_DIR ROCSHMEM_LIBRARY) -## mark_as_advanced(MSCCLPP_INCLUDE_DIRS MSCCLPP_NCCL_STATIC_LIB) add this for Rocshmem? diff --git a/projects/rccl/cmake/ROCSHMEM.cmake b/projects/rccl/cmake/ROCSHMEM.cmake index 558aafcce106..15fcde5d4908 100644 --- a/projects/rccl/cmake/ROCSHMEM.cmake +++ b/projects/rccl/cmake/ROCSHMEM.cmake @@ -24,18 +24,11 @@ include(ExternalProject) # Pinned mono-repo commit for rocshmem source checkout. # Used by both install.sh (setup_rocshmem_worktree) and cmake (auto-detect below). -set(ROCSHMEM_MONO_HASH "33d980d7ca1f0bf90cfe4ff9106310abcf47b550" CACHE STRING +set(ROCSHMEM_MONO_HASH "f636cc5be66716349dd9ed42eebb7b4d651fbb20" CACHE STRING "Pinned rocm-systems commit hash for rocshmem source checkout") function(add_rocshmem_targets) - # Common dependency: libibverbs is required for all rocSHMEM paths - find_library(_IBVERBS ibverbs) - if(NOT _IBVERBS) - message(FATAL_ERROR "libibverbs not found (install rdma-core/libibverbs-dev)") - endif() - set(IBVERBS ${_IBVERBS} PARENT_SCOPE) - # ----------------------------------------------------------------- # Auto-detect ROCSHMEM_SOURCE_DIR if not provided. # Runs first so source headers are available regardless of whether @@ -104,9 +97,19 @@ function(add_rocshmem_targets) endif() find_package(rocshmem_static) if(rocshmem_static_FOUND) + # This path returns before the -DASAN=ON propagation below, so an + # ASAN rccl links whatever the pre-built tree happens to contain. + if(BUILD_ADDRESS_SANITIZER) + message(WARNING + "ASAN build using pre-built rocSHMEM at ${ROCSHMEM_INSTALL_DIR}. " + "It is linked into librccl.so as-is; if it was not itself built " + "with -DASAN=ON, rccl gets an uninstrumented rocSHMEM inside an " + "instrumented library. Unset ROCSHMEM_INSTALL_DIR to build it " + "from source with ASAN propagated.") + endif() set(ROCSHMEM_INCLUDE_DIR "${ROCSHMEM_INCLUDE_DIR}" PARENT_SCOPE) set(ROCSHMEM_LIBRARY "${ROCSHMEM_LIBRARY}" PARENT_SCOPE) - set(ROCSHMEM_SOURCE_DIR "${ROCSHMEM_SOURCE_DIR}" PARENT_SCOPE) + set(ROCSHMEM_SOURCE_DIR "${ROCSHMEM_SOURCE_DIR}" CACHE INTERNAL "rocSHMEM source directory") return() endif() endif() @@ -131,6 +134,12 @@ function(add_rocshmem_targets) if(ENABLE_ROCSHMEM_GIN) set(_rocshmem_sdma_opt "-DUSE_SDMA=ON") endif() + # librocshmem.a is linked into librccl.so, so without this an ASAN + # build gets an uninstrumented rocSHMEM inside an instrumented library. + set(_rocshmem_asan_opt "") + if(BUILD_ADDRESS_SANITIZER) + set(_rocshmem_asan_opt "-DASAN=ON") + endif() message(STATUS "rocSHMEM: building from ${ROCSHMEM_SOURCE_DIR}") ExternalProject_Add(rocshmem_ext @@ -148,10 +157,18 @@ function(add_rocshmem_targets) CONFIGURE_COMMAND "" BUILD_COMMAND ${CMAKE_COMMAND} -E make_directory build - && ${CMAKE_COMMAND} -E chdir build bash -lc "INSTALL_PREFIX=${ROCSHMEM_INSTALL_DIR} ../scripts/build_configs/gda -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DUSE_EXTERNAL_MPI=OFF -DGDA_MLX5=ON -DGDA_BNXT=ON -DGDA_IONIC=ON -DBUILD_EXAMPLES=OFF -DBUILD_FUNCTIONAL_TESTS=OFF -DBUILD_UNIT_TESTS=OFF -DBUILD_CTESTS=OFF -DBUILD_TOOLS=OFF -DGPU_TARGETS=${_rocshmem_gpu_targets} ${_rocshmem_sdma_opt} ${_rocshmem_cmake_opts} " + && ${CMAKE_COMMAND} -E chdir build bash -lc "INSTALL_PREFIX=${ROCSHMEM_INSTALL_DIR} ../scripts/build_configs/gda -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DUSE_EXTERNAL_MPI=OFF -DGDA_MLX5=ON -DGDA_BNXT=ON -DGDA_IONIC=ON -DBUILD_EXAMPLES=OFF -DBUILD_FUNCTIONAL_TESTS=OFF -DBUILD_UNIT_TESTS=OFF -DBUILD_CTESTS=OFF -DBUILD_TOOLS=OFF -DGPU_TARGETS=${_rocshmem_gpu_targets} ${_rocshmem_sdma_opt} ${_rocshmem_asan_opt} ${_rocshmem_cmake_opts} " INSTALL_COMMAND "" ) + # Rebuild when the sub-build's options change (e.g. ASAN): its build + # stamp ignores BUILD_COMMAND. The stamp sits beside the shared install, + # so the other config tree overwriting librocshmem.a invalidates it too. + set(_rocshmem_opts_stamp "${ROCSHMEM_INSTALL_DIR}/rccl_build_options.txt") + file(GENERATE OUTPUT "${_rocshmem_opts_stamp}" + CONTENT "${CMAKE_BUILD_TYPE} ${_rocshmem_gpu_targets} ${_rocshmem_sdma_opt} ${_rocshmem_asan_opt} ${_rocshmem_cmake_opts}\n") + ExternalProject_Add_StepDependencies(rocshmem_ext build "${_rocshmem_opts_stamp}") + set(ROCSHMEM_INSTALL_DIR "${ROCSHMEM_INSTALL_DIR}" PARENT_SCOPE) set(ROCSHMEM_INCLUDE_DIR "${ROCSHMEM_INSTALL_DIR}/include" PARENT_SCOPE) set(ROCSHMEM_LIBRARY "${ROCSHMEM_INSTALL_DIR}/lib/librocshmem.a" PARENT_SCOPE) @@ -159,7 +176,9 @@ function(add_rocshmem_targets) add_custom_target(rocshmem_static ALL DEPENDS rocshmem_ext) endif() - set(ROCSHMEM_SOURCE_DIR "${ROCSHMEM_SOURCE_DIR}" PARENT_SCOPE) + # CACHE INTERNAL (not PARENT_SCOPE): SOURCE_DIR is consumed by the + # top-level CMakeLists.txt install rules, outside add_subdirectory(src). + set(ROCSHMEM_SOURCE_DIR "${ROCSHMEM_SOURCE_DIR}" CACHE INTERNAL "rocSHMEM source directory") endfunction() @@ -174,6 +193,8 @@ function(copy_files FILE_LIST_VAR SRC_DIR DST_DIR OUTPUT_LIST_VAR) OUTPUT "${dst_file}" COMMAND ${CMAKE_COMMAND} -E make_directory "${dst_file_dir}" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${src_file}" "${dst_file}" + # Qualify log.hpp include to prevent shadowing by consumer headers. + COMMAND sed -i "s|#include \"log\\.hpp\"|#include \"nccl_device/gin/rocshmem_gda/log.hpp\"|g" "${dst_file}" DEPENDS "${src_file}" COMMENT "Copying ${src_file} -> ${dst_file}" VERBATIM diff --git a/projects/rccl/contrib/nccl_checkpoint/Makefile b/projects/rccl/contrib/nccl_checkpoint/Makefile index fdeedbbb6329..ee12e4d524bb 100644 --- a/projects/rccl/contrib/nccl_checkpoint/Makefile +++ b/projects/rccl/contrib/nccl_checkpoint/Makefile @@ -35,11 +35,11 @@ CUDALIBS := -L$(CUDA_LIB) -lcudart SHIM_SO := $(LIBDIR)/libnccl-checkpoint-shim.so PUBLIC_HEADER := nccl_checkpoint.h NCCL_HEADER := $(GENDIR)/nccl.h -SHIM_HEADERS := $(NCCL_HEADER) $(NCCL_SRC)/include/nccl_device/core.h +SHIM_HEADERS := $(NCCL_HEADER) $(NCCL_SRC)/include/nccl_device/host.h SHIM_AUTO := $(GENDIR)/shim_auto.cc SHIM_EXPORTS := $(GENDIR)/shim_exports.map -INCFLAGS = -I. -I$(GENDIR) -I$(NCCL_SRC)/include -I$(NCCL_SRC)/include/plugin $(HIREDIS_CFLAGS) +INCFLAGS = -I. -I$(GENDIR) -I$(NCCL_SRC)/include $(HIREDIS_CFLAGS) CXXFLAGS += $(INCFLAGS) -fPIC -fvisibility=default # Dependency generation flags for automatic header tracking DEPFLAGS = -MMD -MP diff --git a/projects/rccl/contrib/nccl_checkpoint/README.md b/projects/rccl/contrib/nccl_checkpoint/README.md index 3ef4922d714e..425ee244345b 100644 --- a/projects/rccl/contrib/nccl_checkpoint/README.md +++ b/projects/rccl/contrib/nccl_checkpoint/README.md @@ -152,15 +152,14 @@ make install PREFIX=/path/to/install ### Dependencies and Build Configuration -The NCCL Checkpoint library has build-dependencies on CUDA, hiredis, and NCCL -itself. The checkpointing library depends on both public and internal NCCL -headers from `NCCL_SRC`. It is recommended the version of NCCL found in the -`NCCL_SRC` path precisely matches the version of the NCCL library used at -runtime. +The NCCL Checkpoint library has build-dependencies on CUDA, hiredis, and public +NCCL headers from `NCCL_SRC`. It is recommended the version of NCCL found in +the `NCCL_SRC` path precisely matches the version of the NCCL library used at +runtime. NCCL Checkpoint requires NCCL 2.31.0 or newer. | Variable | Purpose | Default Value | | --- | --- | --- | -| `NCCL_SRC` | NCCL source tree. Used for both internal and public NCCL headers. | `../../src` | +| `NCCL_SRC` | NCCL source tree. Used for public NCCL headers and NCCL makefile metadata. | `../../src` | | `BUILDDIR` | NCCL Checkpoint build directory for generated shim code, objects, libraries, and test binaries. | `./build` | | `PREFIX` | Install prefix used by `make install`; the shim installs under `$PREFIX/lib`. | `/usr/local` | | `CUDA_HOME` | CUDA Toolkit root inherited from the NCCL common Makefile. | `/usr/local/cuda` | @@ -224,7 +223,14 @@ services). `NCCL_CHECKPOINT_KVS_TIMEOUT` controls how long the restore path waits for the redis server to accept connections and for restore keys to appear. The value is -in seconds and defaults to 300. +in seconds and defaults to 300. This timeout only covers the checkpoint shim's +redis rendezvous. Once communicator replay has entered NCCL transport setup, +lower-level NCCL transport calls use their normal NCCL behavior and may require +their own transport-specific diagnostics. + +For restore diagnostics, set `NCCL_DEBUG=INFO` to print checkpoint progress +messages. Set `NCCL_DEBUG=TRACE NCCL_DEBUG_SUBSYS=CHECKPOINT` for more detailed +checkpoint shim stage logs. ## Additional Documentation diff --git a/projects/rccl/contrib/nccl_checkpoint/gen_shim.py b/projects/rccl/contrib/nccl_checkpoint/gen_shim.py index e3ae8873b913..e90ae896858e 100644 --- a/projects/rccl/contrib/nccl_checkpoint/gen_shim.py +++ b/projects/rccl/contrib/nccl_checkpoint/gen_shim.py @@ -380,7 +380,7 @@ def emit_file(fns: List[NcclFunction], header_path: str) -> str: "// AUTO-GENERATED by gen_shim.py — DO NOT EDIT", "", '#include ', - '#include "nccl_device/core.h"', + '#include "nccl_device/host.h"', '#include "shim_core.h"', "#include ", "", diff --git a/projects/rccl/contrib/nccl_checkpoint/python/tests/Makefile b/projects/rccl/contrib/nccl_checkpoint/python/tests/Makefile index 01d8e6ad6429..b008a4de02c0 100644 --- a/projects/rccl/contrib/nccl_checkpoint/python/tests/Makefile +++ b/projects/rccl/contrib/nccl_checkpoint/python/tests/Makefile @@ -12,8 +12,7 @@ CPPFLAGS := \ -I$(CUDA_HOME)/include \ -I$(CUDA_HOME)/include/cccl \ -I$(NCCL_SRC)/../build/include \ - -I$(NCCL_SRC)/include \ - -I$(NCCL_SRC)/include/plugin + -I$(NCCL_SRC)/include LDFLAGS := \ -Wl,--gc-sections \ diff --git a/projects/rccl/contrib/nccl_checkpoint/shim.cc b/projects/rccl/contrib/nccl_checkpoint/shim.cc index 1ff4c466d845..3391e6e71456 100644 --- a/projects/rccl/contrib/nccl_checkpoint/shim.cc +++ b/projects/rccl/contrib/nccl_checkpoint/shim.cc @@ -25,16 +25,21 @@ static CommInitParams* allocInitParams(int nranks, int rank, const ncclConfig_t* params->configProvided = configPtr != nullptr; params->net_name[0] = '\0'; params->comm_name[0] = '\0'; + params->config = NCCL_CONFIG_INITIALIZER; if (configPtr) { - params->config = *configPtr; - if (configPtr->netName) { - strncpy(params->net_name, configPtr->netName, sizeof(params->net_name) - 1); + size_t configSize = configPtr->size; + if (configSize > sizeof(ncclConfig_t)) configSize = sizeof(ncclConfig_t); + memcpy(¶ms->config, configPtr, configSize); + if (params->config.netName) { + strncpy(params->net_name, params->config.netName, sizeof(params->net_name) - 1); + params->net_name[sizeof(params->net_name) - 1] = '\0'; + params->config.netName = params->net_name; } - if (configPtr->commName) { - strncpy(params->comm_name, configPtr->commName, sizeof(params->comm_name) - 1); + if (params->config.commName) { + strncpy(params->comm_name, params->config.commName, sizeof(params->comm_name) - 1); + params->comm_name[sizeof(params->comm_name) - 1] = '\0'; + params->config.commName = params->comm_name; } - } else { - params->config = NCCL_CONFIG_INITIALIZER; } params->config.size = sizeof(ncclConfig_t); params->config.magic = NCCL_API_MAGIC; @@ -113,16 +118,9 @@ static ncclResult_t captureCommRuntimeState(ncclComm_t synthComm, ncclComm_t rea const CommHandleEntry* entry = nullptr; NCCLCHECK(g_commHandles.find(synthComm, &entry)); if (entry->config == nullptr) return ncclSuccess; - if (realComm->endMagic != NCCL_MAGIC) { - WARN("Compile and runtime version mismatch (Magic Mismatch). libnccl-checkpoint-shim.so detected" - " that NCCL runtime is a different version. Please recompile with the same versions."); - return ncclInternalError; - } CommInitParams* params = g_commHandles[synthComm].config; - params->commHash = realComm->commHash; - params->cudaDev = realComm->cudaDev; - return ncclSuccess; + return captureCommRuntimeProperties(realComm, params); } // CREATE:comm — records init params for restore diff --git a/projects/rccl/contrib/nccl_checkpoint/shim_checkpoint.cc b/projects/rccl/contrib/nccl_checkpoint/shim_checkpoint.cc index a735c070a37b..ab7b1b50455c 100644 --- a/projects/rccl/contrib/nccl_checkpoint/shim_checkpoint.cc +++ b/projects/rccl/contrib/nccl_checkpoint/shim_checkpoint.cc @@ -37,7 +37,7 @@ static ncclResult_t replayWindow(ncclComm_t synthComm, ncclWindow_t synthWin, Wi } static ncclResult_t isCommBlocking(CommInitParams* params, bool* isBlocking) { - if (params->commParent != nullptr) { + if (!params->configProvided && params->commParent != nullptr) { const CommHandleEntry* parentEntry = nullptr; NCCLCHECK(g_commHandles.find(params->commParent, &parentEntry)); if (parentEntry->config == nullptr) return ncclInternalError; @@ -83,21 +83,6 @@ static ncclResult_t replayCommResources(ncclComm_t synthComm) { return ncclSuccess; } -static ncclResult_t commGetHash(ncclComm_t realComm, uint64_t* commHash) { - if (realComm->endMagic != NCCL_MAGIC) { - int runtime_version; - NCCLCHECK(ncclGetVersion(&runtime_version)); - WARN("Detected memory sentinel mismatch! Likely the NCCL runtime version " - "(%d) is not compatible with the libnccl-checkpoint-shim.so version (%d), " - "or the communicator has experienced memory corruption." - " Please recompile with the same versions.", - runtime_version, NCCL_VERSION_CODE); - return ncclInternalError; - } - *commHash = realComm->commHash; - return ncclSuccess; -} - static bool commIsActiveForUser(const CommHandleEntry* entry) { assert(entry != nullptr); return entry->userState == comm_user_active; @@ -424,20 +409,34 @@ static ncclResult_t destroyRestoredUserDestroyedComms() { extern "C" ncclResult_t ncclCheckpointPrepare(void) { using finalize_t = ncclResult_t (*)(ncclComm_t); using destroy_t = ncclResult_t (*)(ncclComm_t); + using group_t = ncclResult_t (*)(); static finalize_t real_finalize = nullptr; static destroy_t real_destroy = nullptr; - ncclResult_t ret; - + static group_t ncclGroupStart = nullptr; + static group_t ncclGroupEnd = nullptr; + bool currentGroupBlocking = true; + bool haveComms = false; + ncclResult_t ret, retGroup; g_CommCheckpointCount = 0; NCCLCHECK(g_commHandles.forEachHandle([&](ncclComm_t synthComm, const CommHandleEntry* entry) { - if (!entry->restoreUnsafe) return ncclSuccess; - WARN("ncclCheckpointPrepare: communicator %p cannot be checkpointed: %s", synthComm, - entry->restoreUnsafeReason ? entry->restoreUnsafeReason : "restore-unsafe API was used"); - return ncclInvalidUsage; + if (entry->restoreUnsafe) { + WARN("ncclCheckpointPrepare: communicator %p cannot be checkpointed: %s", synthComm, + entry->restoreUnsafeReason ? entry->restoreUnsafeReason : "restore-unsafe API was used"); + return ncclInvalidUsage; + } + haveComms = true; + return ncclSuccess; })); - NCCLCHECK(g_commHandles.forEachHandle([&](ncclComm_t synthComm, const CommHandleEntry* entry) { + if (!haveComms) { + goto end; + } + NCCLCHECK(resolveRealFunction("ncclGroupStart", &ncclGroupStart)); + NCCLCHECK(resolveRealFunction("ncclGroupEnd", &ncclGroupEnd)); + NCCLCHECK(ncclGroupStart()); + + ret = g_commHandles.forEachHandle([&](ncclComm_t synthComm, const CommHandleEntry* entry) { CommInitParams* params = entry->config; if (!commNeedsRestore(entry)) return ncclSuccess; g_CommCheckpointCount++; @@ -451,11 +450,19 @@ extern "C" ncclResult_t ncclCheckpointPrepare(void) { return ncclSuccess; } ncclResult_t ret; + bool thisCommBlocking; + NCCLCHECK(isCommBlocking(params, &thisCommBlocking)); + if (thisCommBlocking != currentGroupBlocking) { + /* not valid to group blocking and non-blocking comms together: start new group */ + NCCLCHECK(ncclGroupEnd()); + NCCLCHECK(ncclGroupStart()); + currentGroupBlocking = thisCommBlocking; + } + NCCLCHECK(ncclCommGetAsyncError(realComm, &ret)); NCCLCHECK_WAIT(ret, realComm); - // extract hash at prepare time rather than blocking during init time. - NCCLCHECK(commGetHash(realComm, ¶ms->commHash)); - params->cudaDev = realComm->cudaDev; + // Extract runtime properties at prepare time rather than blocking during init time. + NCCLCHECK(captureCommRuntimeProperties(realComm, params)); TRACE(NCCL_CHECKPOINT, "prepare finalize comm %p real=%p hash=0x%016lx cudaDev=%d", synthComm, realComm, params->commHash, params->cudaDev); if (isActive) { @@ -464,7 +471,11 @@ extern "C" ncclResult_t ncclCheckpointPrepare(void) { } TRACE(NCCL_CHECKPOINT, "prepare waited for comm %p status=%d", synthComm, ret); return ret; - })); + }); + retGroup = ncclGroupEnd(); /* end the group no matter what forEachHandle returned*/ + NCCLCHECK(ret); + NCCLCHECK(retGroup); + NCCLCHECK(g_commHandles.forEachHandle([&](ncclComm_t synthComm, const CommHandleEntry* entry) { if (!commNeedsRestore(entry)) return ncclSuccess; @@ -495,6 +506,8 @@ extern "C" ncclResult_t ncclCheckpointPrepare(void) { NCCLCHECK(ret); return ncclSuccess; })); + + end: markCheckpointPrepared(); return ncclSuccess; } diff --git a/projects/rccl/contrib/nccl_checkpoint/shim_core.cc b/projects/rccl/contrib/nccl_checkpoint/shim_core.cc index f9d2cccaca18..5ba910044a01 100644 --- a/projects/rccl/contrib/nccl_checkpoint/shim_core.cc +++ b/projects/rccl/contrib/nccl_checkpoint/shim_core.cc @@ -8,8 +8,6 @@ #include "shim_core.h" -thread_local int ncclDebugNoWarn = 0; - #include namespace nccl_checkpoint { diff --git a/projects/rccl/contrib/nccl_checkpoint/shim_core.h b/projects/rccl/contrib/nccl_checkpoint/shim_core.h index 109fd4879ee8..4fa8487d121f 100644 --- a/projects/rccl/contrib/nccl_checkpoint/shim_core.h +++ b/projects/rccl/contrib/nccl_checkpoint/shim_core.h @@ -15,17 +15,11 @@ #include #include #include +#include "nccl_device.h" #include #include #include -/* Internal NCCL headers used: - * comm.h — ncclComm::commHash (uint64_t) */ -#include "comm.h" -#undef WARN -#undef INFO -#undef TRACE - #define NCCL_CHECKPOINT_MAJOR 0 #define NCCL_CHECKPOINT_MINOR 1 #define NCCL_CHECKPOINT_PATCH 0 @@ -128,6 +122,30 @@ static inline void ncclCheckpointWarn(const char* file, const char* func, int li #define TRACE(_dummy, ...) \ ::nccl_checkpoint::ncclCheckpointLog(::nccl_checkpoint::ncclCheckpointLogTrace, __func__, __LINE__, __VA_ARGS__) +#define NCCLCHECK(cmd) \ + do { \ + ncclResult_t res = cmd; \ + if (res != ncclSuccess && res != ncclInProgress) { \ + WARN("-> %d\n",res); \ + return res; \ + } \ + } while (0) + +#define NCCLCHECKGOTO(cmd, RES, label) \ + do { \ + RES = cmd; \ + if (RES != ncclSuccess && RES != ncclInProgress) goto label; \ + } while (0) + +#define CUDACHECK(cmd) \ + do { \ + cudaError_t err = cmd; \ + if (err != cudaSuccess) { \ + WARN("Cuda failure '%s'", cudaGetErrorString(err)); \ + return ncclUnhandledCudaError; \ + } \ + } while (0) + template static inline ncclResult_t resolveRealFunction(const char* name, FnT* fn) { if (*fn != nullptr) return ncclSuccess; @@ -190,6 +208,17 @@ struct CommInitParams { } }; +static inline ncclResult_t captureCommRuntimeProperties(ncclComm_t realComm, CommInitParams* params) { + using query_t = ncclResult_t (*)(ncclComm_t, ncclCommProperties_t*); + static query_t real_ncclCommQueryProperties = nullptr; + ncclCommProperties_t props = NCCL_COMM_PROPERTIES_INITIALIZER; + NCCLCHECK(resolveRealFunction("ncclCommQueryProperties", &real_ncclCommQueryProperties)); + NCCLCHECK(real_ncclCommQueryProperties(realComm, &props)); + params->commHash = props.commHash; + params->cudaDev = props.cudaDev; + return ncclSuccess; +} + struct NoConfig {}; struct RegConfig { diff --git a/projects/rccl/contrib/nccl_ep/README.md b/projects/rccl/contrib/nccl_ep/README.md index 84680688bd4b..180457c845cd 100644 --- a/projects/rccl/contrib/nccl_ep/README.md +++ b/projects/rccl/contrib/nccl_ep/README.md @@ -391,7 +391,7 @@ Maintains state for a sequence of related MoE operations, i.e. dispatch and comb The group's RDMA buffer is sized by `config.rdma_buffer_size`: -- **`NCCL_EP_AUTO`** (recommended for most users): the buffer is **not** allocated at `ncclEpCreateGroup` time. +- **`NCCL_EP_AUTO`** (recommended for most users): the buffer is **not** allocated at `ncclEpCreateGroup` time. The first `ncclEpInitHandle` allocates it sized to that handle's actual `(layout, num_topk)`. A later `ncclEpInitHandle` whose layout needs a larger buffer (for example, the first `EXPERT_MAJOR` handle on a group that previously only hosted `RANK_MAJOR` handles, or a handle with a larger `num_topk`) @@ -837,13 +837,13 @@ cudaMalloc(&out_topk_idx_data, num_recv_tokens * top_k * sizeof(int64_t)); // Fast in-place initialization of static tensors // go away after Dispatch invocation -ncclEpTensor_t in_tokens = { +ncclEpTensor_t in_tokens = { NCCL_EP_TENSOR_INIT_INLINE, .ndim = 2, .datatype = ncclBfloat16, .data = in_tokens_data, .sizes = in_tokens_dims }; -ncclEpTensor_t out_tokens = { +ncclEpTensor_t out_tokens = { NCCL_EP_TENSOR_INIT_INLINE, .ndim = 2, .datatype = ncclBfloat16, @@ -894,13 +894,13 @@ void* combine_in_data = nullptr; void* combine_out_data = nullptr; cudaMalloc(&combine_in_data, num_recv_tokens * hidden * sizeof(uint16_t)); cudaMalloc(&combine_out_data, num_tokens * hidden * sizeof(uint16_t)); -ncclEpTensor_t combine_in_tokens = { +ncclEpTensor_t combine_in_tokens = { NCCL_EP_TENSOR_INIT_INLINE, - .ndim = 2, + .ndim = 2, .datatype = ncclBfloat16, .data = combine_in_data, .sizes = combine_in_dims }; -ncclEpTensor_t combine_out_tokens = { +ncclEpTensor_t combine_out_tokens = { NCCL_EP_TENSOR_INIT_INLINE, .ndim = 2, .datatype = ncclBfloat16, diff --git a/projects/rccl/contrib/nccl_ubx/bench/README.md b/projects/rccl/contrib/nccl_ubx/bench/README.md index b67209f6d6e6..a3124fe5bae0 100644 --- a/projects/rccl/contrib/nccl_ubx/bench/README.md +++ b/projects/rccl/contrib/nccl_ubx/bench/README.md @@ -4,7 +4,7 @@ Benchmark harness for UB-X collectives. Reports latency and bandwidth in the same column layout as nccl-tests, so you can paste UBX numbers into the same comparison sheets you already use. -CUDA graph mode is the default (`-G 10000`). Pass `--no-cudagraph` for +CUDA graph mode is the default (`-G 1000`). Pass `--no-cudagraph` for eager mode when you want per-op launch + sync overhead measured as well. ## Install @@ -60,7 +60,7 @@ srun -n 4 python -m ubx_bench a2av_dispatch \ | `-i SIZE` | none | Additive step. | | `-n N` | `20` | Timed iterations per measurement. | | `-w N` | `5` | Warmup iterations. | -| `-G N` | `10000` | Number of CUDA-graph replays per measurement. | +| `-G N` | `1000` | Number of CUDA-graph replays per measurement. | | `--no-cudagraph` | (off) | Disable CUDA graph capture; run in eager mode. | | `--backend NAME[,NAME…]` | `all` | `ubx`, `nccl`, `all`, or comma-separated list. | | `--kernel NAME` | `auto` | UB-X kernel: `mc`, `uc`, `lamport`, `auto` (collective-dependent). | diff --git a/projects/rccl/contrib/nccl_ubx/bench/ubx_bench/__main__.py b/projects/rccl/contrib/nccl_ubx/bench/ubx_bench/__main__.py index c7ec67949b3e..cb5ab35096ca 100644 --- a/projects/rccl/contrib/nccl_ubx/bench/ubx_bench/__main__.py +++ b/projects/rccl/contrib/nccl_ubx/bench/ubx_bench/__main__.py @@ -106,8 +106,8 @@ def main(): "approximates real MoE expert load skew)") # Graph/Correctness - parser.add_argument("-G", "--cudagraph", type=int, default=10000, - help="CUDA graph replays per measurement (default: 10000). " + parser.add_argument("-G", "--cudagraph", type=int, default=1000, + help="CUDA graph replays per measurement (default: 1000). " "Pass 0 or use --no-cudagraph for eager mode.") parser.add_argument("--no-cudagraph", dest="cudagraph", action="store_const", const=0, diff --git a/projects/rccl/contrib/nccl_ubx/bench/ubx_bench/configs.py b/projects/rccl/contrib/nccl_ubx/bench/ubx_bench/configs.py index 255e00b9c792..72abdfeb4026 100644 --- a/projects/rccl/contrib/nccl_ubx/bench/ubx_bench/configs.py +++ b/projects/rccl/contrib/nccl_ubx/bench/ubx_bench/configs.py @@ -65,7 +65,7 @@ class BenchConfig: hidden_size: int = 0 # Graph/Correctness - cudagraph: int = 10000 + cudagraph: int = 1000 check: int = 1 # Output diff --git a/projects/rccl/contrib/nccl_ubx/tests/conftest.py b/projects/rccl/contrib/nccl_ubx/tests/conftest.py index a4466424c538..2c7c4cde5b00 100644 --- a/projects/rccl/contrib/nccl_ubx/tests/conftest.py +++ b/projects/rccl/contrib/nccl_ubx/tests/conftest.py @@ -28,6 +28,26 @@ def pytest_collection_modifyitems(config, items): requires_sm9 = pytest.mark.requires_sm9 +@pytest.fixture +def nccl_backend(): + """UB-X's nccl4py backend module, with one skip-vs-fail policy. + + Skip only when nccl4py is genuinely not installed. When it IS installed but + UB-X can no longer import what it needs, fail: that is the shape of the + nccl4py 0.3.0 change, which shipped precisely because a broken build still + looked healthy. Every test touching the backend goes through here, so the + two cases can never drift apart between test classes. + """ + from ubx import _nccl_backend as backend + + if backend.nccl4py_available(): + return backend + err = backend.nccl4py_import_error() + if backend.nccl4py_missing(): + pytest.skip(f"nccl4py is not installed here: {err!r}") + pytest.fail(f"nccl4py is installed but UB-X cannot import from it: {err!r}") + + @pytest.fixture def cuda_device(): """Provide a CUDA device for testing.""" diff --git a/projects/rccl/contrib/nccl_ubx/tests/test_backend_detection.py b/projects/rccl/contrib/nccl_ubx/tests/test_backend_detection.py index 1da9aeadf51a..eeeb9f98fc6d 100644 --- a/projects/rccl/contrib/nccl_ubx/tests/test_backend_detection.py +++ b/projects/rccl/contrib/nccl_ubx/tests/test_backend_detection.py @@ -35,3 +35,231 @@ def test_sm_version_positive(self): from ubx._common import _get_cuda_sm_version major, minor = _get_cuda_sm_version() assert major > 0 + + + +class TestNccl4pyBackend: + """Backend health -- the checks this module is named for. + + It previously covered only version strings and hardware probes, so it + stayed green on a build where every UB-X collective died at its first + symmetric-pool allocation. + """ + + def test_allocator_symbol_is_bound(self, nccl_backend): + """UB-X's allocation entry point must actually resolve. + + No GPU needed. This is the assertion that separates the broken build + from the fixed one. + """ + assert callable(nccl_backend._mem_alloc) + + def test_consumes_versioned_api_only(self): + """UB-X must reach nccl4py only through its versioned surface. + + ``nccl.core.__init__`` states that semantic-versioning guarantees cover + exactly the names in its ``__all__``, and that all other modules "are + internal implementation details and are subject to change without + notice". That rules out both ``nccl.bindings`` -- the private Cython + layer whose 0.3.0 reshuffle caused this -- and ``nccl.core.``, + which carries no more promise than ``nccl.bindings`` did. Only bare + ``nccl.core`` is a contract. + """ + import ast + import inspect + from ubx import _nccl_backend + + tree = ast.parse(inspect.getsource(_nccl_backend)) + modules = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + modules.add(node.module) + + offenders = sorted( + m for m in modules if m.split(".")[0] == "nccl" and m != "nccl.core" + ) + assert not offenders, ( + f"_nccl_backend reaches outside nccl4py's versioned surface: " + f"{offenders} -- import these from 'nccl.core' instead" + ) + + def test_missing_means_absent_not_relocated(self): + """``nccl4py_missing`` must not read a moved submodule as "not installed". + + This is the hole that would let the next 0.3.0 through. A deleted or + renamed ``nccl.core`` submodule raises ``ModuleNotFoundError`` exactly + like an uninstalled package does; classifying the two alike would skip + the regression instead of failing on it. + """ + from ubx import _nccl_backend as backend + + def classify(err): + saved = backend._NCCL4PY_IMPORT_ERROR + backend._NCCL4PY_IMPORT_ERROR = err + try: + return backend.nccl4py_missing() + finally: + backend._NCCL4PY_IMPORT_ERROR = saved + + # nccl4py genuinely absent -> an environment gap, safe to skip. + assert classify(ModuleNotFoundError("No module named 'nccl'", name="nccl")) + # nccl4py present but reorganised -> a real breakage, must not skip. + assert not classify( + ModuleNotFoundError( + "No module named 'nccl.core.buffer'", name="nccl.core.buffer" + ) + ) + # nccl4py present, symbol gone -> likewise. + assert not classify( + ImportError("cannot import name 'mem_alloc' from 'nccl.core'") + ) + + +class TestDeviceIndexResolution: + """``_resolve_device_index``: torch.device -> the ordinal mem_alloc wants. + + ``mem_alloc`` takes an ``NcclDeviceSpec``, so a ``torch.device`` is not a + valid argument. An index-less device silently means "current", which would + otherwise put the pool on a different GPU than the tensor view over it. + """ + + def test_explicit_index_is_preserved(self): + import torch + from ubx._nccl_backend import _resolve_device_index + assert _resolve_device_index(torch.device("cuda:1")) == 1 + + def test_accepts_a_device_string(self): + """Annotated ``torch.device``, but nothing stops a caller passing a str.""" + from ubx._nccl_backend import _resolve_device_index + assert _resolve_device_index("cuda:2") == 2 + + def test_returns_a_plain_int(self): + from ubx._nccl_backend import _resolve_device_index + assert type(_resolve_device_index("cuda:0")) is int + + @pytest.mark.requires_cuda + def test_indexless_device_uses_current(self): + """Bare "cuda" must resolve to the *current* device, not to 0. + + Asserting against device 0 would also pass a hardcoded ``return 0``, so + move the current device first. On a single-GPU box the two are + indistinguishable and this degenerates to the weaker check. + """ + import torch + from ubx._nccl_backend import _resolve_device_index + + target = 1 if torch.cuda.device_count() > 1 else 0 + previous = torch.cuda.current_device() + torch.cuda.set_device(target) + try: + assert _resolve_device_index(torch.device("cuda")) == target + finally: + torch.cuda.set_device(previous) + + +class TestBufferOwnership: + """``mem_alloc`` returns a Buffer that OWNS its allocation. + + Unlike the bare integer pointer it replaced, it frees on ``close()`` *and* + on garbage collection -- a lifetime constraint the pool now has to respect. + """ + + @pytest.mark.requires_cuda + def test_handle_converts_to_a_nonzero_address(self, nccl_backend): + """``int(buf.handle)`` is what ``NcclSymPool`` stores as ``_raw_ptr``. + + ``allocator.py`` does pointer arithmetic on that value and passes it to + the pybind11 extension, so the conversion has to work and must not + yield NULL. + """ + buf = nccl_backend._mem_alloc(1024, device=0) + try: + ptr = int(buf.handle) + finally: + buf.close() + assert ptr != 0 + + @pytest.mark.requires_cuda + def test_close_is_idempotent(self, nccl_backend): + """``NcclSymPool.close()`` drops its Buffer reference even when + ``close()`` raised, which leaves the finalizer to call it a second + time. That is only safe if the second call is a no-op. + """ + buf = nccl_backend._mem_alloc(1024, device=0) + buf.close() + buf.close() + + @pytest.mark.requires_cuda + def test_dropped_buffers_are_reclaimed(self, nccl_backend): + """Buffers dropped without ``close()`` must still be freed. + + Device 0 is named explicitly: ``mem_get_info()`` defaults to the + *current* device, which need not be the one allocated on below. + + The first half is a positive control. ``mem_get_info`` reports + driver-level free memory, and nothing guarantees a priori that it + observes ``ncclMemAlloc`` -- if it did not, a finalizer that freed + nothing would leave free memory unchanged and pass, which is also + exactly what success looks like. Proving the allocations are visible + first is what makes the reclaim assertion mean anything. + """ + import gc + import torch + + count, chunk = 50, 8 * 1024 * 1024 # 400 MiB in flight + torch.cuda.init() + gc.collect() + free_start, _ = torch.cuda.mem_get_info(0) + + held = [nccl_backend._mem_alloc(chunk, device=0) for _ in range(count)] + free_held, _ = torch.cuda.mem_get_info(0) + observed_mib = (free_start - free_held) / 1024 ** 2 + assert observed_mib > 256, ( + f"mem_get_info saw only {observed_mib:.1f} MiB while holding " + f"{count} x {chunk >> 20} MiB of ncclMemAlloc -- it does not " + f"observe this allocator, so the reclaim check below is vacuous" + ) + + del held # deliberately no close(): this is the finalizer's job + gc.collect() + free_end, _ = torch.cuda.mem_get_info(0) + + leaked_mib = (free_start - free_end) / 1024 ** 2 + assert leaked_mib < 64, ( + f"{count} unclosed buffers leaked {leaked_mib:.1f} MiB" + ) + + @pytest.mark.requires_cuda + def test_finalizer_at_interpreter_exit_is_silent(self, nccl_backend): + """A Buffer left to shutdown must not spew "Exception ignored in". + + Nothing calls ``close()`` here: the finalizer runs during interpreter + teardown, by which point the nccl4py module state it reaches through + may already be gone. If it raises there, Python swallows the traceback + to stderr and the allocation leaks with no other signal. UB-X cannot + catch this one -- ``NcclSymPool.close()``'s ``except`` only covers the + explicit path. + """ + import subprocess + import sys + import textwrap + + code = textwrap.dedent( + """ + import torch + from ubx._nccl_backend import _mem_alloc + torch.cuda.init() + buf = _mem_alloc(1024, device=0) # deliberately never closed + print("ALLOCATED") + """ + ) + proc = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, timeout=120 + ) + assert "ALLOCATED" in proc.stdout, f"probe never ran:\n{proc.stderr}" + assert "Exception ignored" not in proc.stderr, ( + f"finalizer raised during interpreter shutdown:\n{proc.stderr}" + ) + assert proc.returncode == 0, f"rc={proc.returncode}\n{proc.stderr}" diff --git a/projects/rccl/contrib/nccl_ubx/ubx/_nccl_backend.py b/projects/rccl/contrib/nccl_ubx/ubx/_nccl_backend.py index 4b19015aa78b..46b948eda4a3 100644 --- a/projects/rccl/contrib/nccl_ubx/ubx/_nccl_backend.py +++ b/projects/rccl/contrib/nccl_ubx/ubx/_nccl_backend.py @@ -21,11 +21,21 @@ # nccl4py is container-provided. Import is best-effort so unit tests can # inspect the module shape on machines without nccl4py installed. +# +# Import from ``nccl.core`` itself, never from its submodules. nccl4py's own +# __init__ states that semantic-versioning guarantees cover exactly the names +# in ``nccl.core.__all__``, and that every other module is an internal detail +# free to move without notice -- which is what ``nccl.core.buffer`` and its +# siblings are. All six names below are in that ``__all__``. try: - import nccl.bindings as _nccl_bindings - from nccl.core.communicator import Communicator, NCCLDevCommRequirements - from nccl.core.constants import WindowFlag - from nccl.core.utils import UniqueId, get_unique_id + from nccl.core import ( + Communicator, + NCCLDevCommRequirements, + UniqueId, + WindowFlag, + get_unique_id, + mem_alloc as _mem_alloc, + ) _NCCL4PY_AVAILABLE = True _NCCL4PY_IMPORT_ERROR: Optional[BaseException] = None @@ -39,6 +49,31 @@ def nccl4py_available() -> bool: return _NCCL4PY_AVAILABLE +def nccl4py_import_error() -> Optional[BaseException]: + """The exception that made nccl4py unusable, or None if it is usable.""" + return _NCCL4PY_IMPORT_ERROR + + +def nccl4py_missing() -> bool: + """Whether nccl4py itself is absent, as opposed to present but unusable. + + ``nccl4py_available()`` collapses two very different situations into one + False, and callers need to tell them apart. Exactly one is benign: the + ``nccl`` package is not installed at all, an environment gap a test may + reasonably skip. + + Everything else must fail loudly, including a ``ModuleNotFoundError`` that + names a *sub*module. ``No module named 'nccl.core.buffer'`` means nccl4py + IS installed but no longer laid out the way UB-X imports it -- the exact + shape of the 0.3.0 change that shipped UB-X unable to allocate. Reading + that as "not installed" would skip the regression this guard exists to + catch, so the check is on the missing module's name, not on the exception + type alone. + """ + err = _NCCL4PY_IMPORT_ERROR + return isinstance(err, ModuleNotFoundError) and getattr(err, "name", None) == "nccl" + + def _debug_enabled() -> bool: return bool(os.environ.get("UBX_DEBUG")) @@ -148,6 +183,33 @@ def get_or_create_nccl_comm( # Pool: ncclMemAlloc + window register + devcomm create # --------------------------------------------------------------------------- +def _resolve_device_index(device) -> int: + """Resolve a torch device to the integer ordinal ``mem_alloc`` expects. + + ``mem_alloc`` takes an ``NcclDeviceSpec`` -- an int or a + ``cuda.core.Device`` -- so a ``torch.device`` is not a valid argument. + Routing through ``torch.device()`` also tolerates a plain ``"cuda:0"`` + string, which the annotations forbid but callers can still pass. An + index-less device means "current", the same GPU ``mem_alloc(device=None)`` + would have picked; naming it explicitly keeps the pool and the tensor view + built over it on one device. + + A non-CUDA device is rejected rather than resolved. ``torch.device("cpu")`` + also has ``index is None``, so falling through would quietly allocate a + symmetric pool on the current GPU for a caller that asked for host memory. + """ + dev = torch.device(device) + if dev.type != "cuda": + raise ValueError( + f"symmetric pools are CUDA-only; got device {dev!r} " + f"of type {dev.type!r}" + ) + index = dev.index + if index is None: + index = torch.cuda.current_device() + return int(index) + + def _wrap_device_ptr_as_tensor( ptr: int, size: int, device: torch.device, ) -> torch.Tensor: @@ -180,14 +242,18 @@ class NcclSymPool: Lifecycle: - 1. ``ncclMemAlloc(size_bytes)`` -> raw device pointer + 1. ``nccl.core.mem_alloc(size_bytes)`` -> owning ``Buffer``; its device + pointer is what the rest of the pool is built on 2. Wrap as ``torch.Tensor`` via __cuda_array_interface__ 3. ``comm.register_window(...)`` -> RegisteredWindowHandle 4. ``comm.create_dev_comm(...)`` -> DevCommResource (with LSA multimem) 5. Resolve LSA multimem ptr (offset 0) and per-rank LSA peer ptrs - Tear-down (close()): destroy devcomm -> deregister window -> - ``ncclMemFree``. + Tear-down (close()): destroy devcomm -> deregister window -> drop the + tensor view -> release the ``Buffer`` (``ncclMemFree``). The Buffer is + held for the pool's whole lifetime: it also frees on garbage collection, + so letting it go early would free the pool while the window still + references it. """ def __init__( @@ -209,6 +275,7 @@ def __init__( self._world_size = int(world_size) # Pre-init for safe close() on partial construction failure self._raw_ptr: int = 0 + self._buf = None self._internal_pool: Optional[torch.Tensor] = None self._window = None self._dev_comm = None @@ -218,10 +285,14 @@ def __init__( _r = int(os.environ.get("RANK", "-1")) _diag = os.environ.get("UBX_INIT_DIAG", "0") == "1" - # 1. ncclMemAlloc + # 1. ncclMemAlloc, through nccl4py's public allocator. if _diag: print(f"[r{_r} NcclSymPool] step1 PRE mem_alloc({self._size})", flush=True) - self._raw_ptr = int(_nccl_bindings.mem_alloc(self._size)) + dev_index = _resolve_device_index(device) + # Buffer owns the allocation and also frees on GC -- see the class + # docstring; self._buf must outlive the registered window. + self._buf = _mem_alloc(self._size, device=dev_index) + self._raw_ptr = int(self._buf.handle) if self._raw_ptr == 0: raise RuntimeError(f"ncclMemAlloc({self._size}) returned NULL") if _diag: @@ -333,15 +404,25 @@ def close(self) -> None: except Exception: pass self._window = None - # Drop the tensor view BEFORE freeing the underlying allocation, + # Drop the tensor view BEFORE releasing the underlying allocation, # so torch can't hand the pointer to a kernel after free. self._internal_pool = None - if self._raw_ptr != 0: + # Release the Buffer last, once nothing else can reach the memory. + # buf.close() is exactly what nccl.core.mem_free does; calling it + # directly saves importing a second symbol here. It still reaches into + # nccl4py module state, so this is not free of that state at + # interpreter shutdown -- hence the except below, which is + # load-bearing rather than defensive. Dropping the reference after a + # failed close hands the retry to the Buffer's own finalizer; + # test_close_is_idempotent covers only close-after-success, so treat + # that retry as best-effort rather than a guarantee. + if self._buf is not None: try: - _nccl_bindings.mem_free(self._raw_ptr) + self._buf.close() except Exception: pass - self._raw_ptr = 0 + self._buf = None + self._raw_ptr = 0 def __del__(self): # Best-effort; explicit close() is preferred and verifiable in tests. diff --git a/projects/rccl/contrib/nccl_ubx/ubx/allocator.py b/projects/rccl/contrib/nccl_ubx/ubx/allocator.py index 73f206ffc8b4..06d4a0b17d16 100644 --- a/projects/rccl/contrib/nccl_ubx/ubx/allocator.py +++ b/projects/rccl/contrib/nccl_ubx/ubx/allocator.py @@ -319,6 +319,13 @@ def close(self) -> None: garbage collection but should be invoked explicitly in tests / lifecycle-sensitive code. """ + # Drop the external aliases FIRST. NcclSymPool.close() frees the + # pool, and these views are built over that memory -- nulling them + # afterwards leaves a live tensor over a freed allocation for the + # rest of this function. + self.internal_pool = None + self.pool_ptr = 0 + self.mc0_ptr = 0 if getattr(self, "_nccl_pool", None) is not None: self._nccl_pool.close() self._nccl_pool = None @@ -333,10 +340,6 @@ def close(self) -> None: pass self._nccl_comm = None self._owns_comm = False - # Drop external aliases to fail loudly if anyone keeps using them. - self.internal_pool = None - self.pool_ptr = 0 - self.mc0_ptr = 0 def __del__(self): try: diff --git a/projects/rccl/contrib/niin/.gitignore b/projects/rccl/contrib/niin/.gitignore index 1df9b84386b2..9e9e97ab5835 100644 --- a/projects/rccl/contrib/niin/.gitignore +++ b/projects/rccl/contrib/niin/.gitignore @@ -2,3 +2,6 @@ /include/nccl.h /include/nccl_device.h /include/nccl_device +/include/gpunetio +/gpunetio/build*/ +/perftest/build*/ diff --git a/projects/rccl/contrib/niin/Makefile b/projects/rccl/contrib/niin/Makefile index 358399bea470..aab71b7ebd7b 100644 --- a/projects/rccl/contrib/niin/Makefile +++ b/projects/rccl/contrib/niin/Makefile @@ -14,11 +14,42 @@ NCCL_LIB := $(abspath $(NCCL_HOME)/build/lib/libnccl.so) NCCL_RUNTIME_LIB := $(LIBDIR)/libnccl.so.2 NVSHMEM_HOST_LIB := $(LIBDIR)/libnvshmem_host.so NVSHMEM_LIB := $(LIBDIR)/libnvshmem.so +GPUNETIO_ATOMICS_LIB := $(LIBDIR)/libniin_gpunetio_atomics.a NCCL_HEADER_LINKS := $(INCDIR)/nccl.h $(INCDIR)/nccl_device.h $(INCDIR)/nccl_device +GPUNETIO_HEADER_LINK := $(INCDIR)/gpunetio +# Required when GPUNETIO_HOME names an NCCL-vendored provider but NCCL_HOME is +# an install/build prefix rather than the matching source checkout. +NCCL_SOURCE_ROOT ?= $(NCCL_HOME) +# NCCL vendors the GPUNetIO sources GDAKI is built against, so the atomic +# provider builds out of the box. Point this elsewhere to use another +# unmodified GPUNetIO tree. +GPUNETIO_HOME ?= $(NCCL_SOURCE_ROOT)/src/transport/net_ib/gdaki/doca-gpunetio +GPUNETIO_BUILD_DIR ?= $(CURDIR)/gpunetio/build +# Empty retains CMake's default. Set a target-specific value such as `89-real` +# when the resulting CUDA artifacts will run only on L40S GPUs. +GPUNETIO_CUDA_ARCHITECTURES ?= +NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS ?= OFF +GDRCOPY_INCLUDE_DIR ?= +IBVERBS_INCLUDE_DIR ?= +IBVERBS_LIBRARY ?= -.PHONY: all clean +ifneq ($(strip $(GDRCOPY_INCLUDE_DIR)),) +GDRCOPY_CMAKE_ARG := -DGDRCOPY_INCLUDE_DIR="$(abspath $(GDRCOPY_INCLUDE_DIR))" +endif +ifneq ($(strip $(IBVERBS_INCLUDE_DIR)),) +IBVERBS_INCLUDE_CMAKE_ARG := -DIBVERBS_INCLUDE_DIR="$(abspath $(IBVERBS_INCLUDE_DIR))" +endif +ifneq ($(strip $(IBVERBS_LIBRARY)),) +IBVERBS_LIBRARY_CMAKE_ARG := -DIBVERBS_LIBRARY="$(abspath $(IBVERBS_LIBRARY))" +endif +ifneq ($(strip $(GPUNETIO_CUDA_ARCHITECTURES)),) +GPUNETIO_CUDA_ARCHITECTURES_CMAKE_ARG := -DCMAKE_CUDA_ARCHITECTURES="$(GPUNETIO_CUDA_ARCHITECTURES)" +endif -all: $(NCCL_HEADER_LINKS) $(NVSHMEM_HOST_LIB) $(NVSHMEM_LIB) $(NCCL_RUNTIME_LIB) +.PHONY: all clean gpunetio-atomics + +all: $(NCCL_HEADER_LINKS) $(GPUNETIO_HEADER_LINK) $(NVSHMEM_HOST_LIB) $(NVSHMEM_LIB) \ + $(NCCL_RUNTIME_LIB) gpunetio-atomics $(INCDIR)/nccl.h: @if [ ! -e "$(NCCL_INCLUDE)/nccl.h" ]; then \ @@ -56,5 +87,41 @@ $(NVSHMEM_HOST_LIB) $(NCCL_RUNTIME_LIB): $(NVSHMEM_LIB): $(NVSHMEM_HOST_LIB) ln -sfn "$(notdir $(NVSHMEM_HOST_LIB))" "$@" +# The device side of the atomic provider is header-only. Linking the GPUNetIO +# include tree into NIIN's own include/ means `-I$(NIIN_HOME)/include` is enough +# for a CUDA consumer to get it, with no extra include path and no define. +# +# NCCL ships those sources, so a missing tree means NCCL_HOME/NCCL_SOURCE_ROOT +# points somewhere that is not an NCCL source checkout. That is a build +# configuration error, not a supported configuration: failing here beats +# shipping a NIIN whose network atomics are quietly unimplemented. +$(GPUNETIO_HEADER_LINK): + @if [ ! -d "$(GPUNETIO_HOME)/include" ]; then \ + echo "GPUNetIO sources not found: $(GPUNETIO_HOME)/include"; \ + echo "NIIN builds from an NCCL source checkout. Pass NCCL_SOURCE_ROOT=/path/to/nccl"; \ + echo "(or GPUNETIO_HOME=/path/to/gpunetio-open) when NCCL_HOME is a build or install prefix."; \ + exit 1; \ + fi + @mkdir -p "$(INCDIR)" + ln -sfn "$(abspath $(GPUNETIO_HOME)/include)" "$@" + @echo "NIIN GPUNetIO device headers: $@" + +gpunetio-atomics: + @if [ ! -d "$(GPUNETIO_HOME)" ]; then \ + echo "GPUNetIO sources not found: $(GPUNETIO_HOME)"; \ + echo "Pass GPUNETIO_HOME=/path/to/unmodified/gpunetio-open"; \ + exit 1; \ + fi + cmake -S "$(CURDIR)/gpunetio" -B "$(GPUNETIO_BUILD_DIR)" \ + -DNCCL_HOME="$(abspath $(NCCL_HOME))" \ + -DNCCL_SOURCE_ROOT="$(abspath $(NCCL_SOURCE_ROOT))" \ + -DGPUNETIO_HOME="$(abspath $(GPUNETIO_HOME))" \ + -DNIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS="$(NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS)" $(GPUNETIO_CUDA_ARCHITECTURES_CMAKE_ARG) $(GDRCOPY_CMAKE_ARG) $(IBVERBS_INCLUDE_CMAKE_ARG) $(IBVERBS_LIBRARY_CMAKE_ARG) + cmake --build "$(GPUNETIO_BUILD_DIR)" --target niin_gpunetio_compile_check + @mkdir -p "$(LIBDIR)" + ln -sfn "$(abspath $(GPUNETIO_BUILD_DIR))/libniin_gpunetio_atomics.a" "$(GPUNETIO_ATOMICS_LIB)" + @echo "NIIN GPUNetIO atomics archive: $(GPUNETIO_ATOMICS_LIB)" + clean: - rm -f "$(NVSHMEM_HOST_LIB)" "$(NVSHMEM_LIB)" "$(NCCL_RUNTIME_LIB)" $(NCCL_HEADER_LINKS) + rm -f "$(NVSHMEM_HOST_LIB)" "$(NVSHMEM_LIB)" "$(GPUNETIO_ATOMICS_LIB)" \ + "$(NCCL_RUNTIME_LIB)" "$(GPUNETIO_HEADER_LINK)" $(NCCL_HEADER_LINKS) diff --git a/projects/rccl/contrib/niin/README.md b/projects/rccl/contrib/niin/README.md index 0afefe3f7cfc..57b2c544fbc0 100644 --- a/projects/rccl/contrib/niin/README.md +++ b/projects/rccl/contrib/niin/README.md @@ -59,6 +59,10 @@ int main() { - NCCL 2.29+ with device API support for windows, GIN, and barriers - CUDA 12.2+ with `--expt-relaxed-constexpr` and C++17 +- A shared filesystem for multi-PE `nvshmem_init()` (the default is + `$HOME/.niin`; set `NIIN_BOOTSTRAP_DIR` when home is not shared). The + launcher must provide rank and size metadata plus a job ID, or set + `NIIN_BOOTSTRAP_ID` explicitly. - MPI for multi-PE applications that call `nvshmemx_init_attr()` with `NVSHMEMX_INIT_WITH_MPI_COMM`; compile those applications with `-DNIIN_HAS_MPI`. UID-based initialization can use the application's own bootstrap mechanism. @@ -88,17 +92,30 @@ nvcc my_app.cu -o my_app \ --expt-relaxed-constexpr -std=c++17 -arch=sm_89 ``` +For device code spanning multiple CUDA translation units, compile those units +with `-rdc=true` and perform one device link. This lets NIIN share its device +context across the consumer's translation units. + Set the runtime library path or link with an rpath: ```bash export LD_LIBRARY_PATH=${NIIN_HOME}/lib:${LD_LIBRARY_PATH} ``` -NIIN does not build or require a separate `libnvshmem_device.a`. Device-side -NVSHMEM functions are compiled from NIIN's headers into the application's CUDA -translation units. During `nvshmem_init()`/`nvshmemx_init_attr()`, NIIN creates -the NCCL device communicator and symmetric heap state, then publishes the device -context pointer to NIIN's `__device__` global with `cudaMemcpyToSymbol`. +NIIN does not build or require a separate device archive. Device-side NVSHMEM +functions are compiled from NIIN's headers into the application's CUDA +translation units. The GIN transport routines -- `niin_gin_put`/`get`/`put_value`, +the remote signal delivery, the GIN flush, and both put-with-signal remote forms +-- are marked `__noinline__`, so they compile to one device function per program +instead of expanding a network descriptor build into every caller: kernels that +only ever reach NVLink peers do not pay for that code or its registers. The +NVLink and self paths stay force-inlined, being a peer-pointer copy plus a store +where a call would cost more than it saves. During +`nvshmem_init()`/`nvshmemx_init_attr()`, NIIN creates the NCCL device +communicator and symmetric heap state, then publishes the device context pointer +to NIIN's `__device__` global with `cudaMemcpyToSymbol`. Multi-translation-unit +CUDA applications must use relocatable device code (`-rdc=true` or CMake CUDA +separable compilation); a single-TU non-RDC program is supported as well. ## Source Layout @@ -111,6 +128,7 @@ contrib/niin/include/ types.h # NVSHMEM constants, X-macro type generation context.h # niinContext struct, __device__ global, helpers query.h # nvshmem_my_pe, nvshmem_n_pes, nvshmem_ptr (__host__ __device__) + tma.h # TMA-backed RMA (give_smem/release_smem, cp.async.bulk) rma.h # All put/get variants (typed, sized, mem, nbi, strided) signaling.h # put_signal, signal_fetch, signal_wait_until sync.h # fence, quiet, wait_until, test (+ all/any/some + vector) @@ -129,12 +147,16 @@ contrib/niin/include/ `nvshmem_init()` performs the following: -1. Detects rank and world size from an MPI communicator or supported launcher-provided rank metadata. Falls back to single-PE mode if none found. +1. Detects rank and world size from supported launcher-provided metadata. Falls back to single-PE mode if none is found. 2. Selects the local GPU from launcher-provided local rank metadata or `rank % nDevices`. -3. Creates an NCCL communicator (`ncclCommInitRank` for multi-PE with MPI, `ncclCommInitAll` for single-PE). -4. Allocates the symmetric heap via `ncclMemAlloc` (default 256 MB, configurable via `NVSHMEM_SYMMETRIC_SIZE`). -5. Registers the heap as an NCCL window (`ncclCommWindowRegister`) and creates a device communicator (`ncclDevCommCreate`) with GIN resources. -6. Sets NIIN's `__device__` global context pointer via `cudaMemcpyToSymbol` so all subsequent kernel launches have access to the NIIN context automatically. +3. For multi-PE jobs, has PE 0 publish an NCCL unique ID in a small record in + `NIIN_BOOTSTRAP_DIR` (default `$HOME/.niin`) and waits for it on the other + PEs. The record name is derived from the Slurm, PMI, or PMIx job ID; set + `NIIN_BOOTSTRAP_ID` to override it. This needs no MPI initialization. +4. Creates an NCCL communicator with `ncclCommInitRankConfig`. +5. Allocates the symmetric heap via `ncclMemAlloc` (default 256 MB, configurable via `NVSHMEM_SYMMETRIC_SIZE`). +6. Registers the heap as an NCCL window (`ncclCommWindowRegister`) and creates a device communicator (`ncclDevCommCreate`) with GIN resources. +7. Sets NIIN's `__device__` global context pointer via `cudaMemcpyToSymbol` so all subsequent kernel launches have access to the NIIN context automatically. For multi-PE with MPI, use: @@ -153,6 +175,31 @@ the complete pattern: make -C ${NIIN_HOME}/perftest NCCL_HOME=${NCCL_HOME} MPI_HOME=${MPI_HOME} ``` +`shmem_atomic_bw` links NIIN's GPUNetIO atomic provider, which the NIIN build +produces by default from NCCL's vendored GPUNetIO sources, so the commands above +build it too. Keep the provider's GPU architecture and the perftest `ARCH` in +step -- the `89-real`/`sm_89` pair below is for L40S: + +```bash +make -C ${NIIN_HOME} NCCL_HOME=${NCCL_HOME} GPUNETIO_CUDA_ARCHITECTURES=89-real +make -C ${NIIN_HOME}/perftest NCCL_HOME=${NCCL_HOME} MPI_HOME=${MPI_HOME} ARCH=sm_89 +``` + +Pass `GPUNETIO_HOME=` to build against a different unmodified GPUNetIO tree, and +`IBVERBS_LIBRARY=` when the verbs library is not on the default search path. + +Run `build/shmem_atomic_bw` with two PEs on different LSA domains (normally +one GPU rank on each of two nodes), for example with +`-a fetch_add`. It accepts the upstream operation names `inc`, `fetch_inc`, +`set`, `add`, `fetch_add`, `and`, `fetch_and`, `or`, `fetch_or`, `xor`, +`fetch_xor`, `swap`, and `compare_swap`. The current provider serializes each +destination PE's response slot, so this reports serialized direct-AMO payload +throughput rather than NIC wire bandwidth; its table includes payload GB/s and +millions of atomic operations per second. Its defaults are a 8 B--64 KiB sweep +with 10 warmup and 10 timed iterations; `--blocks` and `--threads` must remain +one. Make the selected NCCL, CUDA, MPI, and ibverbs libraries available at +runtime (or preserve the matching `LD_LIBRARY_PATH` used for the build). + For UID-based bootstrap, generate a NIIN/NCCL unique ID on one PE, distribute it with the application's bootstrap mechanism, and pass the rank metadata through `nvshmemx_set_attr_uniqueid_args()`: @@ -179,6 +226,60 @@ Scalar `nvshmem_*_g()` over non-LSA peers is out of scope for now because it returns a value directly rather than writing into a symmetric destination buffer. Use block get APIs for network GET paths. +### Multi-QP (GIN contexts) + +A NCCL GIN context owns one QP per peer, so a single context serializes all of +a PE's network traffic onto one QP. NIIN requests several contexts at init +(`NIIN_NUM_QPS`, default 4) and round-robins operations across them: every +network put, get, or signal takes the next context from a rotating cursor, the +way NVSHMEM's IBGDA and IBRC transports pick a QP. Because NCCL derives a +context's `connectionId` from its index, consecutive contexts also land on +different NICs on multi-rail nodes. + +NCCL rounds the requested count up to a multiple of the connection count, so the +effective number of QPs can exceed `NIIN_NUM_QPS`. NIIN reads the real value back +from `ncclDevComm::ginContextCount` rather than assuming its request was honored +verbatim. + +Selection is per operation rather than per CTA, so any grid shape -- a single +CTA included -- spreads across every QP. An operation that has to stay ordered +internally reserves one context and reuses it: a `put_signal` issues its +payload, the flush that orders it, and its signal on the same QP, and the fused +GIN path drives the whole descriptor from one context. + +Two QPs are not ordered against each other at the receiving NIC, so ordering an +earlier put ahead of a later one means completing it: + +- **`nvshmem_quiet()`** drains every context: it has to complete operations, + and after rotation a thread's operations can be on any of them. +- **`nvshmem_fence()`** only orders, so on a single context it is a local + memory fence with no completion wait -- one RC QP per peer already delivers + that PE's operations in order. It falls back to quiet's drain when there is + more than one context, or when the GPUNetIO atomic sidecar is bound, since + its QPs are a separate network domain from GIN. +- **Barriers** run on context 0 on every PE: GIN barrier signals are + per-context state, so a barrier only converges if all PEs drive it from the + same index. The barrier drains the rotating contexts before synchronizing. +- **`nvshmemx_quiet_on_stream()` and the host `nvshmem_quiet()`** drain every + context as well. Their kernel never issued the puts it is being asked to + complete, and a put can still be in flight on a context after the kernel that + issued it has retired, so waiting on the kernel alone does not complete it. + +#### Tuning `NIIN_NUM_QPS` + +| Situation | Guidance | +|---|---| +| Large single transfers (>= 1 MB) already at line rate | Extra QPs do not help; the link, not the QP, is the limit | +| Many mid-sized transfers (4 KB - 512 KB) | The sweet spot. This is where per-QP issue rate dominates | +| Fence- or quiet-heavy code | Every fence and quiet drains all contexts, so fewer QPs can win | +| Debugging a suspected ordering bug | Set `NIIN_NUM_QPS=1` to collapse onto one QP and compare | + +Raising the count is not free: each context is a real QP per peer, so memory and +connection-setup cost grow with it, and the drains above are O(contexts x peers). + +Note that `NIIN_NUM_QPS=1` does not necessarily produce one context. NCCL rounds +the request up to a multiple of the GIN connection count, so on a 2-connection +node the smallest achievable count is 2. ### Memory Allocation `nvshmem_malloc()` returns pointers into a single pre-allocated symmetric heap registered as an NCCL window. A free-list allocator manages sub-allocations (aligned to 256 bytes) with first-fit search, block splitting, and coalescing on `nvshmem_free()`. Freed memory is immediately reusable — freeing all allocations coalesces the heap back into a single block. @@ -205,13 +306,103 @@ signal` by setting `NIIN_PUT_SIGNAL_MODE=separate` (aliases: `split`, `fence_signal`). The default `auto` mode keeps fused signaling where safe and uses the split path where peer pointer atomics are not reliable. +### TMA (Tensor Memory Accelerator) + +NIIN implements NVSHMEM's TMA feature: an application lends a slice of its CTA +shared memory to the runtime, and NIIN's LSA (NVLink) put/get paths then move +data with `cp.async.bulk` instead of vectorized load/store. TMA is off by +default and requires sm_90 or newer. + +Enable it with `NVSHMEM_TMA_POLICY=ENABLE` (or `FORCE`), then register shared +memory inside the kernel: + +```cpp +// int4 element type pins the dynamic shared memory base to 16-byte alignment. +extern __shared__ int4 smem[]; + +__global__ void my_kernel(float *dst, const float *src, int nelem, int peer) { + nvshmemx_give_smem(smem, nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM)); + __syncthreads(); // required: publishes the registration + + nvshmemx_float_put_block(dst, src, nelem, peer); // goes out over TMA + nvshmem_quiet(); + + nvshmemx_release_smem(); // required before the kernel returns + __syncthreads(); +} + +// Host: size the launch from the same query. +int smemBytes = nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM); +my_kernel<<>>(dst, src, nelem, peer); +``` + +`nvshmemx_ask_smem(NVSHMEMX_SMEM_RECOMMENDED)` returns 64 KB, which exceeds the +48 KB default cap on dynamic shared memory. Launching with that much requires +opting in first: + +```cpp +cudaFuncSetAttribute(my_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + nvshmemx_ask_smem(NVSHMEMX_SMEM_RECOMMENDED)); +``` + +#### Routing + +The route is chosen from where the local buffer lives. NIIN reserves +`NIIN_SMEM_DATA_REGION_OFFSET` (512) bytes at the base of the registered buffer +for mbarriers; staging tiles occupy the remainder. + +| Local buffer | Route | Notes | +|---|---|---| +| Shared memory (put source) | `smem -> peer gmem` | Direct `cp.async.bulk`, no staging | +| Shared memory (get destination) | `peer gmem -> smem` | Direct `cp.async.bulk` with an mbarrier | +| Global memory, thread/warp scope | Staged through the tile | Single issuer, one chunk at a time | +| Global memory, block scope | Staged, double-buffered | Warp-specialized load/store warps; needs at least two full warps | + +Routing is opportunistic. A transfer that TMA cannot take falls back to the +regular vectorized load/store path, so the put/get contract is unchanged either +way. TMA is skipped when any of these hold: + +- Source or destination is not 16-byte aligned, or the size is not a multiple of 16 +- Either buffer is in local (stack) memory, which the TMA engine cannot address +- The CTA never called `nvshmemx_give_smem()`, or its buffer was smaller than + `nvshmemx_ask_smem(NVSHMEMX_SMEM_BARRIERS_ONLY)`, or the buffer was not 16-byte aligned +- The CTA's linear block id is at or beyond `NIIN_TMA_MAX_BLOCKS` (4096) +- Block-scoped staging in a CTA with fewer than two full warps +- The target PE is reached over GIN rather than LSA + +`nvshmem_fence()` and `nvshmem_quiet()` drain the calling thread's outstanding +bulk group before their usual ordering work, so TMA-initiated transfers are +ordered alongside the load/store and GIN paths. + +#### Caveats inherited from NVSHMEM + +- **A put sourced from shared memory needs a proxy fence.** After writing the + staging buffer with ordinary stores, issue `fence.proxy.async.shared::cta` + (NIIN exposes `niin_tma_fence_proxy_async_shared_cta()`, a no-op below sm_90) + before the put, or the TMA engine may read stale data. One fence covers a run + of puts as long as the source is not rewritten in between. +- **The staging tile and its mbarriers are per-CTA state.** At most one + thread-scoped staged transfer may be in flight per CTA. Two threads of the + same CTA issuing `nvshmem_putmem()` concurrently would corrupt each other's + staging; use the warp or block variants for multi-threaded transfers. +- **`give_smem` must be followed by a synchronization** before any RMA, and + every CTA that calls it must call `nvshmemx_release_smem()` before returning. + A stale registration would send a later kernel down the TMA path through a + dangling shared-memory pointer. +- **The warp and block APIs require a converged threadgroup.** Electing the + issuing thread uses `elect.sync`/`__shfl_sync` over the full warp, so + `nvshmemx_*_warp` must be called with all 32 lanes active and + `nvshmemx_*_block` with the whole CTA. That is already NVSHMEM's contract for + those APIs; without TMA, NIIN's cooperative copy happened to tolerate partial + groups. + ## API Coverage ### Host-Side API | NVSHMEM API | NIIN Status | Notes | |---|---|---| -| `nvshmem_init()` | Full | Auto-detects rank metadata; single-PE fallback | +| `nvshmem_init()` | Full | Auto-detects launcher metadata and self-bootstraps multi-PE jobs through a shared directory; single-PE fallback | | `nvshmemx_init_attr(MPI_COMM)` | Full | Requires `-DNIIN_HAS_MPI` | | `nvshmemx_init_attr(UNIQUEID)` | Full | Uses `nvshmemx_get_uniqueid()` and caller-distributed UID | | `nvshmem_finalize()` | Full | | @@ -226,11 +417,12 @@ uses the split path where peer pointer atomics are not reliable. | `nvshmem_ptr(ptr, pe)` | Full | Host and device; nullptr for non-LSA peers | | `nvshmem_barrier_all()` | Full | Host: ncclAllReduce barrier; Device: ncclBarrierSession | | `nvshmem_sync_all()` | Full | Alias for barrier_all | -| `nvshmem_fence()` | Full | Host: cudaDeviceSynchronize; Device: __threadfence_system | -| `nvshmem_quiet()` | Full | Host: same as fence (`cudaDeviceSynchronize`); Device: GIN flush + threadfence | +| `nvshmem_fence()` | Full | Host: cudaDeviceSynchronize; Device: `__threadfence_system()` on a single GIN context, drains every context once they rotate | +| `nvshmem_quiet()` | Full | Host: `cudaDeviceSynchronize` then an all-contexts GIN drain (a put can outlive the kernel that issued it); Device: drains every GIN context + threadfence | | `nvshmem_info_get_name()` | Full | Returns "NIIN (NVSHMEM Implemented In NCCL)" | | `nvshmem_info_get_version()` | Full | Reports 3.0 | -| `nvshmemx_barrier_all_on_stream()` | Full | | +| `nvshmemx_barrier_all_on_stream()` | WORLD only | Stream-ordered NCCL world barrier | +| `nvshmemx_sync_all_on_stream()` | WORLD only | Stream-ordered NCCL world sync | | `nvshmemx_quiet_on_stream()` | Full | Stream-ordered device quiet kernel: GIN flush + `__threadfence_system()` | | `nvshmemx_collective_launch()` | Stub | Redirects to cudaLaunchKernel | | `nvshmemx_cumodule_init/finalize()` | Stub | No-op (compatibility) | @@ -238,14 +430,21 @@ uses the split path where peer pointer atomics are not reliable. ### Host Stream-Ordered APIs (`nvshmemx.h`) -All `nvshmemx_*_on_stream` variants are implemented. These APIs match -NVSHMEM's host on-stream contract: callers provide a `cudaStream_t`, but do not -choose grid or CTA geometry. NIIN owns the internal launch geometry; block -put/get and put-signal fallbacks use a size-based number of eight-warp CTAs for -self and LSA transfers, capped by `NVSHMEM_MAX_CTAS`, so the LSA path can use -the cooperative block RMA implementation. Non-LSA GIN transfers remain -single-CTA until NIIN can assign independent GIN contexts per CTA. Scalar, -strided, signal, and wait wrappers use compact control kernels. +The supported `nvshmemx_*_on_stream` variants below match NVSHMEM's host +on-stream contract: callers provide a `cudaStream_t`, but do not choose grid or +CTA geometry. NIIN owns the internal launch geometry; block put/get and +put-signal fallbacks use a size-based number of eight-warp CTAs for self and +LSA transfers, capped by `NVSHMEM_MAX_CTAS`, so the LSA path can use the +cooperative block RMA implementation. Non-LSA GIN transfers remain single-CTA, +so a single on-stream transfer issues from one CTA; its operations still rotate +across QPs as described in +[Multi-QP (GIN contexts)](#multi-qp-gin-contexts). Scalar, strided, signal, and +wait wrappers use compact control kernels. + +NIIN currently owns one host NCCL communicator, for `NVSHMEM_TEAM_WORLD`. +The host collectives below reject non-WORLD teams rather than issue a partial +collective on that communicator; per-team NCCL communicators are required to +extend this support safely. | NVSHMEM API | NIIN Status | Notes | |---|---|---| @@ -262,6 +461,8 @@ strided, signal, and wait wrappers use compact control kernels. | `nvshmemx__wait_until_on_stream()` | Full | 13 wait types | | `nvshmemx__wait_until_all_on_stream()` | Full | 13 wait types | | `nvshmemx__wait_until_all_vector_on_stream()` | Full | 13 wait types | +| `nvshmemx_int32/int64__reduce_on_stream()` | WORLD only | NCCL AllReduce | +| `nvshmemx_alltoallmem_on_stream()` | WORLD only | NCCL AlltoAll, byte count | ### Device-Side Query @@ -293,6 +494,22 @@ strided, signal, and wait wrappers use compact control kernels. All typed variants are generated via X-macros for 24 standard RMA types: `float`, `double`, `char`, `schar`, `short`, `int`, `long`, `longlong`, `uchar`, `ushort`, `uint`, `ulong`, `ulonglong`, `int8`, `int16`, `int32`, `int64`, `uint8`, `uint16`, `uint32`, `uint64`, `size`, `ptrdiff`. +### TMA Shared-Memory Registration + +Off by default; set `NVSHMEM_TMA_POLICY=ENABLE` and use sm_90 or newer. See +[TMA (Tensor Memory Accelerator)](#tma-tensor-memory-accelerator) for routing +rules and caveats. + +| NVSHMEM API | NIIN Status | Notes | +|---|---|---| +| `nvshmemx_ask_smem(flag)` | Full | Host and device; 64 KB / 32 KB / barriers-only | +| `nvshmemx_give_smem(ptr, size)` | Full | Per-CTA registration, up to 4096 CTAs | +| `nvshmemx_release_smem()` | Full | | +| `NVSHMEM_TMA_POLICY` | Full | `DISABLE` (default), `ENABLE`, `FORCE` | +| TMA-backed put/get over LSA | Full | smem->gmem, gmem->smem, and staged gmem->gmem | +| TMA-backed put/get over GIN | N/A | Network transfers go through GIN, not `cp.async.bulk` | +| `nvshmemx_flush()` | Not impl | NIIN's nbi RMA is already blocking, so source buffers are reusable on return | + ### Device-Side Signaling | NVSHMEM API | LSA | Network | Notes | @@ -308,9 +525,9 @@ All typed variants are generated via X-macros for 24 standard RMA types: `float` | NVSHMEM API | NIIN Status | Notes | |---|---|---| -| `nvshmem_fence()` | Full | Local system-scope ordering via `__threadfence_system()`; not a collective barrier | -| `nvshmem_quiet()` | Full | `gin.flush()` + `__threadfence_system()` | -| `nvshmem_barrier_all()` | Full | `ncclBarrierSession` (world team) with acquire/release barrier ordering | +| `nvshmem_fence()` | Full | `__threadfence_system()` alone on a single GIN context, where the RC QP already orders that PE's operations; an all-contexts drain once operations rotate across QPs or the atomic sidecar is bound. Not a collective barrier | +| `nvshmem_quiet()` | Full | All-contexts GIN drain + `__threadfence_system()` | +| `nvshmem_barrier_all()` | Full | All-contexts `ncclGinBarrierSession` (world team) with acquire/release ordering and a `Put\|Get` fence, so barrier_all implies quiet as NVSHMEM requires | | `nvshmem_sync_all()` | Full | Collective synchronization; currently uses the same NCCL barrier path as barrier_all | | `nvshmem_barrier(TEAM_WORLD)` | Full | Redirects to barrier_all | | `nvshmem_barrier(other team)` | Not impl | **Team mapping gap** | @@ -327,19 +544,45 @@ Wait/test operations are generated for 13 types: `short`, `int`, `long`, `longlo | NVSHMEM API | LSA | Network | Notes | |---|---|---|---| -| `nvshmem__atomic_fetch_add()` | Full | **Not impl** | **No network atomics in NCCL GIN** | -| `nvshmem__atomic_add()` | Full | **Not impl** | | -| `nvshmem__atomic_compare_swap()` | Full | **Not impl** | | -| `nvshmem__atomic_swap()` | Full | **Not impl** | | -| `nvshmem__atomic_fetch()` | Full | **Not impl** | `atomicAdd(ptr, 0)` | -| `nvshmem__atomic_set()` | Full | **Not impl** | `atomicExch` | -| `nvshmem__atomic_inc/fetch_inc()` | Full | **Not impl** | | -| `nvshmem__atomic_fetch_and/or/xor()` | Full | **Not impl** | Bitwise types only | -| `nvshmem__atomic_and/or/xor()` | Full | **Not impl** | | - -Standard AMO types (11): `int`, `long`, `longlong`, `uint`, `ulong`, `ulonglong`, `int32`, `int64`, `uint32`, `uint64`, `size`. +| `nvshmem__atomic_fetch_add()` | Full | Optional native | Enabled by default when the provider is linked; not a GIN op | +| `nvshmem__atomic_add()` | Full | Optional native | Same provider | +| `nvshmem__atomic_compare_swap()` | Full | Optional native | Same provider | +| `nvshmem__atomic_swap()` | Full | Optional native | Same provider | +| `nvshmem__atomic_fetch()` | Full | Optional native | Same provider | +| `nvshmem__atomic_set()` | Full | Optional native | Same provider | +| `nvshmem__atomic_inc/fetch_inc()` | Full | Optional native | Same provider | +| `nvshmem__atomic_fetch_and/or/xor()` | Full | Optional native | Native mode uses masked 4/8-byte WQEs; bitwise types only | +| `nvshmem__atomic_and/or/xor()` | Full | Optional native | Same provider | +| `nvshmem_float/double_atomic_{swap,fetch,set}()` | Full | Experimental only | Remote support is an unsupported SRQ-proxy prototype | +| `nvshmemx_{half,float,double}_atomic_{fetch_add,add}()` | Full | Experimental only | Remote support is an unsupported SRQ-proxy prototype | + +Standard AMO types (12): `int`, `long`, `longlong`, `uint`, `ulong`, `ulonglong`, `int32`, `int64`, `uint32`, `uint64`, `size`, `ptrdiff`. Bitwise AMO types (7): `uint`, `ulong`, `ulonglong`, `int32`, `int64`, `uint32`, `uint64`. +GIN does not expose atomic operations, so the provider in +[`gpunetio/`](gpunetio/README.md) creates NIIN-owned atomic-only QPs and a +separate remote-atomic registration of the symmetric heap. The device half is header-only and compiled in by default: `make -C +contrib/niin` links the GPUNetIO include tree into `$NIIN_HOME/include`, so +`-I$NIIN_HOME/include` is all a CUDA consumer needs -- no define, no extra +include path. Compile with `NIIN_GPUNETIO_ENABLE=0` to opt out and leave +network AMOs unimplemented. + +NCCL ships the GPUNetIO sources, so NIIN builds from an NCCL source checkout. +Building against a build or install prefix fails with a message naming +`NCCL_SOURCE_ROOT` rather than quietly producing a NIIN whose network atomics +are unimplemented. + +The host half stays a separate object, `lib/libniin_gpunetio_atomics.a`, which +the same `make` builds. Link it and `nvshmem_init()` brings the provider up and +binds it automatically whenever the job has peers outside the LSA domain, so +network AMOs work with no application change; `NIIN_GPUNETIO_ATOMICS=0` turns it +off and `=1` reports why it did not come up. Its entry points are weak symbols, +so an application that does not link the archive is unaffected and network AMOs +stay fail-closed -- and because NIIN references them weakly, link the archive +with `-Wl,--whole-archive`. On systems without +NIC atomic support, and for operations the NIC cannot express such as +floating-point atomics, a GDRCopy-based implementation is available. + ### Device-Side Collectives | NVSHMEM API | NIIN Status | Notes | @@ -369,6 +612,8 @@ All `nvshmemx_*_warp` and `nvshmemx_*_block` variants are implemented with coope | `nvshmemx__iget_warp/block()` | Full | 24 typed strided | | `nvshmemx__put_signal_warp/block()` | Full | 24 typed + putmem | | `nvshmemx__put_signal_nbi_warp/block()` | Full | 24 typed + putmem | +| `nvshmemx_sync_all_block()` | Full | One matching CTA collective per PE at a time | +| `nvshmemx_barrier_all_block()` | Full | CTA release/acquire ordering; one matching CTA collective per PE at a time | ### Team Management @@ -437,18 +682,18 @@ handler. The behavior is controlled at compile time by defining | Limitation | Impact | Notes | |---|---|---| | Unsupported NVSHMEM-specific environment variables are not interpreted | NVSHMEM transport, bootstrap, affinity, debug, and tuning variables do not configure NIIN | NIIN honors only the NVSHMEM-named variables listed in [Environment and Runtime Configuration](#environment-and-runtime-configuration). Use NCCL environment variables such as `NCCL_DEBUG`, `NCCL_IB_HCA`, `NCCL_IB_DISABLE`, `NCCL_P2P_DISABLE`, and `NCCL_SHM_DISABLE` for NCCL runtime behavior. | -| No standalone NVSHMEM bootstrap runtime | Multi-PE setup is driven by MPI, UID-based initialization, or launcher-provided rank metadata | Use `nvshmemx_init_attr(...MPI_COMM...)` with `-DNIIN_HAS_MPI`, or use `nvshmemx_get_uniqueid()` with `NVSHMEMX_INIT_WITH_UNIQUEID` and an application-provided bootstrap. | +| `nvshmem_init()` bootstrap needs shared storage | Normal multi-PE initialization cannot proceed if `$HOME/.niin` is not shared | Set `NIIN_BOOTSTRAP_DIR` to a job-writable shared directory. The launcher must set rank/size and a job ID, or set `NIIN_BOOTSTRAP_ID` explicitly. MPI and caller-managed UNIQUEID initialization remain available alternatives. | | One NCCL PE per GPU | Multi-process-per-GPU NVSHMEM use cases are not represented | `NVSHMEMI_TEAM_SAME_GPU` is always a single-PE team. | | No arbitrary NVSHMEM team creation | Applications cannot create teams with arbitrary PE membership, layouts, or team-specific contexts | NIIN represents predefined teams plus constrained host-side strided/2D split teams only. User-created teams are not available to device APIs. | | NCCL memory/window model | Symmetric allocations come from one NCCL-registered heap | Set `NVSHMEM_SYMMETRIC_SIZE` before initialization when a larger heap is required. | -| NCCL device transport constraints | API support depends on whether the target PE is LSA-accessible or reachable through GIN | Block gets can use GIN; scalar network gets and network atomics are not implemented. | -| Performance is not yet tuned | Supported APIs are currently optimized for correctness first | Expect performance to change as NIIN's NCCL-backed paths and launch heuristics are tuned. | +| NCCL device transport constraints | API support depends on whether the target PE is LSA-accessible or reachable through GIN | Block gets use GIN; scalar network gets remain unimplemented. Network integral AMOs require NIIN's optional native GPUNetIO provider; network floating AMOs remain experimental because GIN has no AMO API. TMA applies to the LSA path only. | +| Performance is not yet tuned | Supported APIs are currently optimized for correctness first | Expect performance to change as NIIN's NCCL-backed paths and launch heuristics are tuned. The main knobs available today are `NIIN_NUM_QPS` (see [Multi-QP (GIN contexts)](#multi-qp-gin-contexts)) and `NVSHMEM_TMA_POLICY` (see [TMA (Tensor Memory Accelerator)](#tma-tensor-memory-accelerator)). | ### Cannot be implemented with current NCCL APIs | Feature | Reason | |---|---| -| Network atomics | NCCL GIN does not expose remote atomic operations | +| GIN-native network atomics | NCCL GIN does not expose remote atomic operations. NIIN's optional direct GPUNetIO sidecar covers integral AMOs without adding a GIN operation. A separate SRQ/GDRCopy floating-AMO prototype is not part of NIIN's supported coverage. | | Device-side collectives | NCCL does not expose device-kernel-callable NVSHMEM collectives such as reductions, broadcast, alltoall, or fcollect | ### Out of scope for now @@ -456,13 +701,15 @@ handler. The behavior is controlled at compile time by defining | Feature | Notes | |---|---| | Scalar network get (`nvshmem__g`) | Block gets use GIN; scalar get returns a value directly and has no symmetric destination buffer | -| Tile put/get APIs | Not part of NIIN's current supported RMA subset | +| Tile put/get APIs (`nvshmemx::tile_put`/`tile_get`) | Not part of NIIN's current supported RMA subset. Unrelated to the TMA APIs, which are supported | +| `nvshmemx_flush()` | NIIN's non-blocking RMA is already blocking, so source buffers are reusable when a put returns | ### Not yet implemented (could be added) | Feature | Notes | |---|---| | Device-side custom (user-created) team query | Device supports all 6 predefined teams but not `team_split_strided` results | +| TMA for GIN transfers | TMA covers LSA peers; network transfers still go through GIN | ### NCCL infrastructure limitations (non-API-surface) @@ -479,7 +726,7 @@ These are fundamental differences between NCCL and NVSHMEM's underlying infrastr | Feature | Reason | |---|---| -| Queue Pair (QP) APIs | NVSHMEM-specific API; out of scope at this time | +| Queue Pair (QP) APIs | NVSHMEM-specific API; out of scope at this time. NIIN does use multiple QPs internally -- see [Multi-QP (GIN contexts)](#multi-qp-gin-contexts) -- but does not expose NVSHMEM's QP-management calls. | | SHMEM interop (`nvshmemx_init_attr(SHMEM)`) | No OpenSHMEM runtime in NCCL | | EGM / fabric memory handles | NCCL handles its own memory registration | @@ -495,10 +742,16 @@ by NIIN. |---|---| | `NVSHMEM_SYMMETRIC_SIZE` | Symmetric heap size; default is `256M`; supports K/M/G suffixes | | `NVSHMEM_MAX_CTAS` | Maximum CTAs for NIIN's internal host on-stream block RMA launch heuristic; default is `16` | +| `NVSHMEM_TMA_POLICY` | TMA usage for LSA put/get: `DISABLE` (default), `ENABLE`, `FORCE`. `ENABLE` below sm_90 warns and disables; `FORCE` below sm_90 fails initialization | NIIN also provides the NIIN-specific `NIIN_PUT_SIGNAL_MODE` variable to select put-signal routing: `auto`, `separate`, `split`, or `fence_signal`. +`NIIN_NUM_QPS` sets the number of GIN contexts (QPs per peer) NIIN round-robins +operations across; the default is `4`, NCCL rounds it up to a multiple of the GIN +connection count, and `1` restores single-QP behavior. See +[Multi-QP (GIN contexts)](#multi-qp-gin-contexts). + For NCCL runtime behavior, use NCCL environment variables. Common examples: | NCCL variable | Description | @@ -518,9 +771,35 @@ mpirun -np 2 ./my_app ## Testing +### Multi-QP coverage + +The network (GIN) paths only engage when the peer is not LSA-accessible, so +multi-QP has to be exercised across nodes. On a multi-node NVLink system such as +GB200 NVL72 that also means disabling multi-node NVLink, or peers on separate +trays remain LSA and never reach GIN at all: + +```bash +# 2 nodes x 1 PE; NCCL_MNNVL_ENABLE=0 forces the IB/GIN path +NIIN_NUM_QPS=8 NCCL_MNNVL_ENABLE=0 \ + srun -N2 --ntasks-per-node=1 --mpi=pmix \ + ./build/shmem_put_bw -n 16 -t 256 -b 4096 -e 8388608 +``` + +Confirm the contexts were actually created -- NCCL logs the real count, which +may be rounded up from `NIIN_NUM_QPS`: + +```bash +NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=INIT ... 2>&1 | grep "creating .* contexts" +# devCommCreate: creating 8 contexts: 2 GIN connections with 4 contexts each +``` + +Sweep `NIIN_NUM_QPS` over 1, 2, 4, 8 and compare; run the counts interleaved +rather than in blocks so drift cannot masquerade as a trend. Operations rotate +across QPs independently of the grid, so the CTA count does not have to track +the QP count. ### Functional tests (`contrib/niin/test/niin_test.cu`) -Multi-GPU test using 2 GPUs via the low-level `niinInit`/`niinCommit` API. 24 test categories, 109 checks: +Multi-GPU test using 2 GPUs via the low-level `niinInit`/`niinCommit` API. 30 test categories, 121 checks: | Test | What it validates | |---|---| @@ -548,6 +827,12 @@ Multi-GPU test using 2 GPUs via the low-level `niinInit`/`niinCommit` API. 24 te | Odd-size block put | 1000 bytes (not multiple of 16), exercises tail handling | | Block get | `nvshmemx_int_get_block()` 256 ints cooperative get cross-GPU | | Misaligned block get | 999 bytes at +5 byte offset cross-GPU | +| TMA block put (64KB) | `nvshmemx_int_put_block()` with registered smem — double-buffered staging | +| TMA thread put (8KB) | `nvshmem_int_put()` with registered smem — single-issuer staging | +| TMA put from shared memory | Block put whose source is the registered smem tile | +| TMA get into shared memory | `nvshmemx_int_get_block()` landing directly in smem | +| TMA unaligned fallback | 1000 bytes at +3 with smem registered — must fall back and still be correct | +| nvshmemx_ask_smem | Host-side sizing query returns ordered, self-consistent values | | Team operations | `team_split_strided`, `team_split_2d`, `team_translate_pe`, `team_destroy` | | Stream-based RMA | `nvshmemx_int_put_on_stream()`, `nvshmemx_int_p_on_stream()` | @@ -566,6 +851,10 @@ nvcc contrib/niin/test/niin_test.cu -o niin_test \ ./niin_test # requires >= 2 GPUs ``` +The TMA tests enable TMA automatically when the GPUs are sm_90 or newer. Below +that they still run and cover the load/store fallback, since `give_smem` is +inert there. + ### NVSHMEM-compatible API test Single-PE test using the pure NVSHMEM API surface (no `niin_*` calls): diff --git a/projects/rccl/contrib/niin/gpunetio/CMakeLists.txt b/projects/rccl/contrib/niin/gpunetio/CMakeLists.txt new file mode 100644 index 000000000000..7e71b212a08f --- /dev/null +++ b/projects/rccl/contrib/niin/gpunetio/CMakeLists.txt @@ -0,0 +1,271 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.24) + +project(niin_gpunetio_atomics LANGUAGES CXX CUDA) + +option(NIIN_GPUNETIO_BUILD_DEVICE_COMPILE_TEST + "Compile the NIIN public atomic API against GPUNetIO device headers" ON) +option(NIIN_GPUNETIO_BUILD_SMOKE_TEST + "Build the MPI/Slurm multi-node direct-atomic smoke test when MPI is available" ON) +option(NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS + "Build NIIN's experimental raw-verbs/GDRCopy SRQ AMO prototype" OFF) + +set(NCCL_HOME "" CACHE PATH "NCCL build or install prefix") +set(NCCL_SOURCE_ROOT "" CACHE PATH "NCCL source checkout (needed only for its vendored GPUNetIO tree)") +set(GPUNETIO_HOME "" CACHE PATH "Unmodified GPUNetIO Open source tree") +set(GDRCOPY_INCLUDE_DIR "" CACHE PATH "Directory containing gdrapi.h (needed only for SRQ proxy atomics)") + +if(NOT GPUNETIO_HOME OR NOT EXISTS "${GPUNETIO_HOME}/include") + message(FATAL_ERROR + "Set GPUNETIO_HOME to an unmodified GPUNetIO Open source tree. " + "It may be a standalone tree with CMakeLists.txt or NCCL's vendored tree; " + "NIIN builds it as a customer dependency and never patches it.") +endif() + +find_package(CUDAToolkit REQUIRED) +find_package(Threads REQUIRED) + +# CMake's imported CUDA targets normally propagate this directory. Some +# relocatable toolkit installations report the runtime library correctly but +# leave CUDAToolkit_INCLUDE_DIRS empty for C++ object targets, while GPUNetIO's +# host sources include cuda_runtime.h directly. Derive the canonical include +# directory from nvcc only in that case. +if(NOT CUDAToolkit_INCLUDE_DIRS) + get_filename_component(NIIN_CUDA_BIN_DIR "${CMAKE_CUDA_COMPILER}" DIRECTORY) + get_filename_component(NIIN_CUDA_TOOLKIT_ROOT "${NIIN_CUDA_BIN_DIR}" DIRECTORY) + set(CUDAToolkit_INCLUDE_DIRS "${NIIN_CUDA_TOOLKIT_ROOT}/include") +endif() +if(NOT EXISTS "${CUDAToolkit_INCLUDE_DIRS}/cuda_runtime.h") + message(FATAL_ERROR "Could not locate cuda_runtime.h below the selected CUDA toolkit") +endif() + +find_path(NCCL_INCLUDE_DIR nccl.h + HINTS "${NCCL_HOME}/build/include" "${NCCL_HOME}/include" + REQUIRED) +find_path(NCCL_DEVICE_INCLUDE_DIR nccl_device.h + HINTS "${NCCL_HOME}/build/include" "${NCCL_HOME}/include" + REQUIRED) +find_library(NCCL_LIBRARY NAMES nccl + HINTS "${NCCL_HOME}/build/lib" "${NCCL_HOME}/lib" "${NCCL_HOME}/lib64" + REQUIRED) +find_path(IBVERBS_INCLUDE_DIR infiniband/verbs.h REQUIRED) +find_library(IBVERBS_LIBRARY NAMES ibverbs REQUIRED) + +# NIIN directly emits GPUNetIO WQEs, while the surrounding NIIN headers also +# include NCCL's vendored GPUNetIO device ABI through nccl_device.h. The two +# must be exactly the same provider release: mixing their identically named +# device structs/functions is an ABI bug even when both headers compile. +set(NCCL_GPUNETIO_VERSION_HEADER + "${NCCL_DEVICE_INCLUDE_DIR}/nccl_device/gin/gdaki/doca_gpunetio/common/doca_gpunetio_verbs_def.h") +if(EXISTS "${GPUNETIO_HOME}/include/gpunetio/common/doca_gpunetio_verbs_def.h") + set(NIIN_GPUNETIO_VERSION_HEADER + "${GPUNETIO_HOME}/include/gpunetio/common/doca_gpunetio_verbs_def.h") +elseif(EXISTS "${GPUNETIO_HOME}/include/common/doca_gpunetio_verbs_def.h") + set(NIIN_GPUNETIO_VERSION_HEADER + "${GPUNETIO_HOME}/include/common/doca_gpunetio_verbs_def.h") +else() + message(FATAL_ERROR "Could not locate GPUNetIO's public device-version header") +endif() +if(NOT EXISTS "${NCCL_GPUNETIO_VERSION_HEADER}") + message(FATAL_ERROR + "NCCL's generated GPUNetIO device headers are missing from ${NCCL_DEVICE_INCLUDE_DIR}; " + "build NCCL before configuring this optional NIIN provider") +endif() + +function(niin_read_gpunetio_version component header output) + file(STRINGS "${header}" version_line + REGEX "^#define[ \t]+DOCA_GPUNETIO_VERSION_${component}[ \t]+[0-9]+") + string(REGEX REPLACE ".*[^0-9]([0-9]+)$" "\\1" version_value "${version_line}") + if(NOT version_value MATCHES "^[0-9]+$") + message(FATAL_ERROR "Could not read GPUNetIO ${component} version from ${header}") + endif() + set(${output} "${version_value}" PARENT_SCOPE) +endfunction() + +foreach(component MAJOR MINOR PATCH) + niin_read_gpunetio_version("${component}" "${NCCL_GPUNETIO_VERSION_HEADER}" nccl_version) + niin_read_gpunetio_version("${component}" "${NIIN_GPUNETIO_VERSION_HEADER}" niin_version) + if(NOT nccl_version STREQUAL niin_version) + message(FATAL_ERROR + "GPUNetIO ${component} version mismatch: NCCL device ABI is ${nccl_version}, " + "but GPUNETIO_HOME supplies ${niin_version}. Use the exact provider release NCCL was built with.") + endif() +endforeach() + +get_filename_component(NIIN_ROOT "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE) +set(NIIN_INCLUDE_DIR "${NIIN_ROOT}/include") + +# A public GPUNetIO Open checkout is normally a CMake subproject. Current +# NCCL dev instead vendors the same provider sources directly and does not +# ship their CMakeLists.txt. Both are unmodified customer dependencies; for +# the latter NIIN owns only a tiny object-library wrapper around NCCL's own +# canonical source list. This does not expose an NCCL/GDAKI API or modify the +# vendored provider. +set(NIIN_GPUNETIO_PROVIDER_OBJECTS) +if(EXISTS "${GPUNETIO_HOME}/CMakeLists.txt") + # GPUNetIO Open uses DOCA_HOME as its source-root input. Keep it local to + # configuring the external subproject rather than relying on it elsewhere. + set(DOCA_HOME "${GPUNETIO_HOME}") + add_subdirectory("${GPUNETIO_HOME}" "${CMAKE_CURRENT_BINARY_DIR}/gpunetio-open") + unset(DOCA_HOME) + # Some relocatable CUDA installations leave the imported cudart target + # without its include directory even though nvcc and the runtime library + # were found. GPUNetIO's object target includes cuda_runtime.h directly, so + # make the canonical toolkit include path explicit at this consumer + # boundary. This augments the target; it does not modify GPUNetIO. + target_include_directories(doca_gpunetio PRIVATE "${CUDAToolkit_INCLUDE_DIRS}") + list(APPEND NIIN_GPUNETIO_PROVIDER_OBJECTS $) +else() + if(NOT NCCL_SOURCE_ROOT AND EXISTS "${NCCL_HOME}/src/include/compiler.h") + set(NCCL_SOURCE_ROOT "${NCCL_HOME}") + endif() + if(NOT NCCL_SOURCE_ROOT OR NOT EXISTS "${NCCL_SOURCE_ROOT}/src/include/compiler.h") + message(FATAL_ERROR + "GPUNETIO_HOME has no CMakeLists.txt, so it is treated as NCCL's vendored provider. " + "Set NCCL_SOURCE_ROOT to the matching NCCL checkout (it supplies src/include/compiler.h).") + endif() + if(EXISTS "${IBVERBS_INCLUDE_DIR}/infiniband/mlx5dv.h") + set(MLX5DV_INCLUDE_DIR "${IBVERBS_INCLUDE_DIR}") + else() + find_path(MLX5DV_INCLUDE_DIR infiniband/mlx5dv.h REQUIRED) + endif() + + set(NIIN_GPUNETIO_OPEN_SOURCES + doca_verbs_qp.cpp + doca_verbs_cq.cpp + doca_verbs_device_attr.cpp + doca_verbs_umem.cpp + doca_verbs_srq.cpp + doca_verbs_uar.cpp + doca_verbs_dev.cpp + doca_gpunetio.cpp + doca_gpunetio_log.cpp + doca_gpunetio_high_level.cpp + doca_gpunetio_cuda_wrapper.cpp + doca_verbs_mlx5dv_wrapper.cpp + doca_verbs_ibv_wrapper.cpp + doca_gpunetio_gdrcopy.cpp + doca_gpunetio_sdk_wrapper.cpp + doca_verbs_dev_sdk_wrapper.cpp + doca_verbs_umem_sdk_wrapper.cpp + doca_verbs_uar_sdk_wrapper.cpp + doca_verbs_cq_sdk_wrapper.cpp + doca_verbs_qp_sdk_wrapper.cpp) + list(TRANSFORM NIIN_GPUNETIO_OPEN_SOURCES PREPEND "${GPUNETIO_HOME}/src/") + + add_library(niin_gpunetio_open_obj OBJECT ${NIIN_GPUNETIO_OPEN_SOURCES}) + set_target_properties(niin_gpunetio_open_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_features(niin_gpunetio_open_obj PRIVATE cxx_std_17) + target_compile_definitions(niin_gpunetio_open_obj PRIVATE + DOCA_GPUNETIO_USE_CUDA_WRAPPER + DOCA_VERBS_USE_NET_WRAPPER) + target_include_directories(niin_gpunetio_open_obj PRIVATE + "${GPUNETIO_HOME}/include" + "${GPUNETIO_HOME}/src" + "${NCCL_SOURCE_ROOT}/src/include" + "${CUDAToolkit_INCLUDE_DIRS}" + "${IBVERBS_INCLUDE_DIR}" + "${MLX5DV_INCLUDE_DIR}") + target_link_libraries(niin_gpunetio_open_obj PRIVATE + CUDA::cudart + Threads::Threads + ${CMAKE_DL_LIBS}) + list(APPEND NIIN_GPUNETIO_PROVIDER_OBJECTS $) +endif() + +set(NIIN_GPUNETIO_ATOMIC_SOURCES src/atomics_host.cc) +if(NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS) + find_path(GDRCOPY_INCLUDE_DIR gdrapi.h + HINTS "${GDRCOPY_INCLUDE_DIR}" + "$ENV{GDRCOPY_HOME}/include" + "$ENV{GDRCOPY_INCLUDE_DIR}" + REQUIRED) + # The proxy is deliberately a NIIN source directory rather than a GPUNetIO + # extension. It dynamically loads libgdrapi at runtime, so no GDRCopy + # library is linked into ordinary direct-QP applications. + list(APPEND NIIN_GPUNETIO_ATOMIC_SOURCES proxy_atomics/src/proxy_atomics_host.cc) +endif() + +add_library(niin_gpunetio_atomics STATIC + ${NIIN_GPUNETIO_ATOMIC_SOURCES} + ${NIIN_GPUNETIO_PROVIDER_OBJECTS}) + +target_compile_features(niin_gpunetio_atomics PUBLIC cxx_std_17) +target_compile_definitions(niin_gpunetio_atomics PUBLIC NIIN_GPUNETIO_ENABLE=1) +if(NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS) + target_compile_definitions(niin_gpunetio_atomics PUBLIC NIIN_GPUNETIO_SRQ_PROXY_ATOMICS_ENABLE=1) +endif() +target_include_directories(niin_gpunetio_atomics + PUBLIC + "${NIIN_INCLUDE_DIR}" + # `nvshmem.h`/NIIN device headers include NCCL's public device ABI, so a + # CMake consumer must see the same generated NCCL headers while compiling + # CUDA call sites, not only while compiling this archive's host source. + "${NCCL_INCLUDE_DIR}" + "${NCCL_DEVICE_INCLUDE_DIR}" + # Consumers compile the header-only device AMO dispatch, so GPUNetIO's + # public device headers are part of this target's public build interface. + "${GPUNETIO_HOME}/include" + PRIVATE + "${IBVERBS_INCLUDE_DIR}") +if(NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS) + target_include_directories(niin_gpunetio_atomics PRIVATE "${GDRCOPY_INCLUDE_DIR}") +endif() +target_link_libraries(niin_gpunetio_atomics + PUBLIC + "${NCCL_LIBRARY}" + PRIVATE + CUDA::cudart + CUDA::cuda_driver + "${IBVERBS_LIBRARY}" + Threads::Threads + dl) + +# This is compile-only by design. It instantiates every current public NIIN +# atomic API at both 4 and 8 bytes, including the masked/extended WQEs, but +# does not require a NIC or launch a kernel during a normal build. +if(NIIN_GPUNETIO_BUILD_DEVICE_COMPILE_TEST) + add_library(niin_gpunetio_atomic_device_compile OBJECT tests/device_compile.cu) + target_compile_features(niin_gpunetio_atomic_device_compile PRIVATE cxx_std_17) + target_compile_definitions(niin_gpunetio_atomic_device_compile PRIVATE NIIN_GPUNETIO_ENABLE=1) + target_include_directories(niin_gpunetio_atomic_device_compile PRIVATE + "${NIIN_INCLUDE_DIR}" + "${NCCL_INCLUDE_DIR}" + "${NCCL_DEVICE_INCLUDE_DIR}" + "${GPUNETIO_HOME}/include") + set_target_properties(niin_gpunetio_atomic_device_compile PROPERTIES + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON) + add_custom_target(niin_gpunetio_compile_check + DEPENDS niin_gpunetio_atomics niin_gpunetio_atomic_device_compile) +endif() + +if(NIIN_GPUNETIO_BUILD_SMOKE_TEST) + find_package(MPI QUIET COMPONENTS CXX) + if(MPI_CXX_FOUND) + add_executable(niin_gpunetio_atomic_smoke tests/atomic_smoke.cu) + target_compile_features(niin_gpunetio_atomic_smoke PRIVATE cxx_std_17) + target_link_libraries(niin_gpunetio_atomic_smoke PRIVATE + niin_gpunetio_atomics + MPI::MPI_CXX + CUDA::cudart) + set_target_properties(niin_gpunetio_atomic_smoke PROPERTIES + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON) + if(NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS) + add_executable(niin_gpunetio_proxy_atomic_smoke proxy_atomics/tests/proxy_atomic_smoke.cu) + target_compile_features(niin_gpunetio_proxy_atomic_smoke PRIVATE cxx_std_17) + target_link_libraries(niin_gpunetio_proxy_atomic_smoke PRIVATE + niin_gpunetio_atomics + MPI::MPI_CXX + CUDA::cudart) + set_target_properties(niin_gpunetio_proxy_atomic_smoke PROPERTIES + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON) + endif() + else() + message(STATUS "MPI C++ was not found; skipping niin_gpunetio_atomic_smoke") + endif() +endif() diff --git a/projects/rccl/contrib/niin/gpunetio/README.md b/projects/rccl/contrib/niin/gpunetio/README.md new file mode 100644 index 000000000000..48f19185fd54 --- /dev/null +++ b/projects/rccl/contrib/niin/gpunetio/README.md @@ -0,0 +1,217 @@ +# NIIN GPUNetIO atomics and SRQ proxy + +This is a deliberately narrow, NIIN-owned escape hatch for atomic operations +that GIN does not currently expose. It is not a new GIN API and it does not +replace any existing NIIN transport path: + +- NIIN puts, gets, signals, barriers, and all existing RMA continue to use + GIN exactly as before. +- In native-direct mode, only remote integral AMOs use the GPUNetIO WQE + sidecar; self and LSA AMOs remain CUDA atomics. +- An optional raw-verbs/SRQ proxy source tree is retained for development of + floating AMOs; it is not part of the supported native-direct provider. +- GPUNetIO is consumed as an external dependency. No GPUNetIO source is + modified, and there are no NCCL core or GIN changes. + +Keeping this code here means it can be removed cleanly when GIN exposes native +atomic operations. + +> **SRQ proxy status:** `NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS` is +> experimental. The native direct provider below is the supported result of +> this work. The proxy has no peer-visible recovery when a target response +> fails, and a CPU GDRCopy BAR update cannot be ordered with a target GPU +> kernel that is already running. Do not enable it for normal applications. + +## What it owns + +The provider creates a small private atomic fabric: + +- one outbound RC QP per remote PE, configured for + `DOCA_VERBS_QP_ATOMIC_MODE_UP_TO_8BYTES`; +- one independent registration of the existing NIIN symmetric heap, with + remote-atomic access only; +- one registered 16-byte response slot and device lock per destination PE; +- a public-NCCL all-gather of only its own QP and heap-MR metadata; +- a collective setup gate that requires GPUNetIO GPU-SM doorbells. + +The response slot serialization is intentional for the first functional +implementation: a PE's dedicated QP and response buffer are never reused +until its completion is observed. It can later grow into a ticketed ring +without changing the public NIIN AMO API. + +The provider verifies the selected HCA supports standard and masked +fetch-add/compare-swap, plus 4- and 8-byte RC-QP atomics, before exposing any +endpoint. It fails closed rather than advertise extended WQEs on unsupported +hardware. + +## Native-direct coverage + +The direct GPUNetIO provider covers 4- or 8-byte integral AMOs when it is +bound: + +- fetch-add/add, compare-swap, swap/set, fetch, increment/fetch-increment; +- fetch and non-fetch AND, OR, and XOR; +- masked 32-bit operations and the two-WQEBB masked 64-bit compare-swap + encodings needed by set/swap/AND/OR. + +Self and LSA peers still use CUDA atomics. When no provider is bound, remote +AMOs keep NIIN's previous fail-closed behavior. + +`ncclMemAlloc` normally returns a CUDA VMM heap, for which CUDA rejects the +legacy `CU_POINTER_ATTRIBUTE_SYNC_MEMOPS` pointer attribute. The sidecar does +not require that attribute: its direct and proxy paths use explicit completion +and system-fence ordering. This still does not make unsafely concurrent target +kernel access valid. As with other GPUDirect RDMA updates, applications must +use their normal CUDA/NVSHMEM phase synchronization before a target kernel +consumes a remote atomic update. + +## Experimental SRQ proxy + +Native NIC atomics cannot express floating-point addition. When built with +`NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS=ON`, the same library also includes a +separate NIIN-owned raw-verbs/GDRCopy SRQ prototype. Its current code explores +proxy-all routing for the integral API, raw-bit float/double fetch/swap/set, +and the six +`nvshmemx_{half,float,double}_atomic_{fetch_add,add}` operations. + +Initialize and bind either `niinGpunetioAtomic*` or +`niinGpunetioProxyAtomic*` for a given device context, never both. The +GPUNetIO `CPU_PROXY` handler remains unrelated: it only forwards GPUNetIO +doorbells and is not the semantic SRQ AMO fallback. The native direct provider +therefore rejects CPU-doorbell QPs collectively; request GPU-SM doorbells when +bring-up requires a deterministic choice. + +The proxy maps each target's existing symmetric heap through GDRCopy and +serializes AMOs on that target's NIIN progress thread. It requires a working +GDRCopy kernel driver and runtime library on every PE. It is intentionally +kept outside the supported coverage matrix until it has a safe target-GPU +ordering handoff and source-visible transport-failure recovery. In particular, +do not use proxy-all self or LSA AMOs: the host BAR update cannot be ordered +with the waiting GPU kernel. + +## Build + +Point the optional target at an unmodified GPUNetIO Open source tree and an +NCCL build/install prefix. Its version must exactly match the GPUNetIO device +headers used to build NCCL; the CMake configuration rejects an ABI mismatch: + +```bash +make -C contrib/niin gpunetio-atomics \ + NCCL_HOME=/path/to/nccl \ + GPUNETIO_HOME=/path/to/gpunetio-open \ + GPUNETIO_CUDA_ARCHITECTURES=89-real +``` + +Set `GPUNETIO_CUDA_ARCHITECTURES` to the deployment GPU architecture (the +example is an L40S-only `sm_89` artifact). It is forwarded as CMake's +`CMAKE_CUDA_ARCHITECTURES`; choosing a `-real` architecture embeds native SASS +and avoids relying on a driver to JIT PTX emitted by a newer CUDA toolkit. + +For current NCCL development trees, GPUNetIO may be vendored without its own +`CMakeLists.txt`. That is also supported without changing its sources; point +the build at the matching NCCL checkout as well: + +```bash +cmake -S contrib/niin/gpunetio -B build/niin-gpunetio \ + -DNCCL_HOME=/path/to/nccl-build-or-prefix \ + -DNCCL_SOURCE_ROOT=/path/to/nccl-source \ + -DGPUNETIO_HOME=/path/to/nccl-source/src/transport/net_ib/gdaki/doca-gpunetio +cmake --build build/niin-gpunetio --target niin_gpunetio_compile_check +``` + +For development-only compilation of the SRQ proxy, enable it when GDRCopy +headers are available: + +```bash +cmake -S contrib/niin/gpunetio -B build/niin-gpunetio \ + -DNCCL_HOME=/path/to/nccl-build-or-prefix \ + -DNCCL_SOURCE_ROOT=/path/to/nccl-source \ + -DGPUNETIO_HOME=/path/to/nccl-source/src/transport/net_ib/gdaki/doca-gpunetio \ + -DNIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS=ON \ + -DGDRCOPY_INCLUDE_DIR=/path/to/gdrcopy/include +cmake --build build/niin-gpunetio --target niin_gpunetio_proxy_atomic_smoke +``` + +The Make helper forwards the same options: + +```bash +make -C contrib/niin gpunetio-atomics \ + NCCL_HOME=/path/to/nccl \ + GPUNETIO_HOME=/path/to/gpunetio-open \ + NIIN_GPUNETIO_ENABLE_SRQ_PROXY_ATOMICS=ON \ + GDRCOPY_INCLUDE_DIR=/path/to/gdrcopy/include +``` + +If rdma-core is not installed in a standard compiler location, the Make helper +also accepts `IBVERBS_INCLUDE_DIR=/path/to/include` and +`IBVERBS_LIBRARY=/path/to/libibverbs.so`. + +The target builds `libniin_gpunetio_atomics.a` and a CUDA compile-only check +that instantiates every current public NIIN AMO, including the masked WQEs. +The Make target also exposes the archive as +`contrib/niin/lib/libniin_gpunetio_atomics.a`. Prefer consuming the CMake +target so its CUDA, ibverbs, thread, dynamic-loader, and NCCL link +dependencies propagate correctly. It is intentionally separate from +`make -C contrib/niin all`, so ordinary NIIN users do not acquire a GPUNetIO +dependency. + +Applications that use the provider must compile their CUDA code with +`NIIN_GPUNETIO_ENABLE=1` and the GPUNetIO public include directory. The CMake +target exports both requirements to CMake consumers. + +## Lifecycle + +Every PE calls the atomic initialization collectively, after the normal NIIN +two-phase setup has completed and outside `ncclGroupStart/End`: + +```cpp +niinContext_host hostCtx; +niinContext* deviceCtx; +// niinInit(...), ncclGroupEnd(), and niinCommit(&hostCtx, deviceCtx) first. + +niinGpunetioAtomicOptions opts = NIIN_GPUNETIO_ATOMIC_OPTIONS_INITIALIZER; +niinGpunetioAtomicHostContext* atomics = nullptr; +NCCLCHECK(niinGpunetioAtomicInit(comm, heapBase, heapBytes, &opts, &atomics)); +NCCLCHECK(niinGpunetioAtomicBind(atomics, deviceCtx)); + +// Launch kernels using deviceCtx. + +cudaDeviceSynchronize(); +NCCLCHECK(niinGpunetioAtomicFinalize(atomics)); +NCCLCHECK(ncclGroupStart()); +NCCLCHECK(niinFinalize(comm, deviceCtx)); +NCCLCHECK(ncclGroupEnd()); +``` + +The experimental proxy has a parallel lifecycle API, but is deliberately not +shown as an application recipe until its target-GPU ordering and failure +recovery semantics are complete. + +`niinGpunetioAtomicInit` uses only public NCCL/CUDA/ibverbs/GPUNetIO APIs. It +does not inspect a GIN context, borrow GIN QPs, reuse a GIN MR, or issue any +direct put/get/RMA operation. + +Select the HCA/port/GID when automatic selection is not appropriate: + +- `NIIN_GPUNETIO_IB_DEV=mlx5_0` +- `NIIN_GPUNETIO_IB_PORT=1` +- `NIIN_GPUNETIO_GID_INDEX=` + +The experimental SRQ proxy has separate selection variables: + +- `NIIN_GPUNETIO_PROXY_IB_DEV=mlx5_0` +- `NIIN_GPUNETIO_PROXY_IB_PORT=1` +- `NIIN_GPUNETIO_PROXY_GID_INDEX=` + +The selected GPU allocation must be registerable by ibverbs (for example via +the site GPU peer-memory driver or DMA-BUF support). A direct atomic provider +cannot reuse GIN's private MR registration without new NCCL-core plumbing. + +## Validation + +The compile-only target instantiates the public AMO API. The direct smoke test +calls the native adapter directly and covers standard/masked 4/8-byte +operations and concurrent callers to one destination PE. The optional proxy +smoke is a compile-time development harness only. A forced GPUNetIO +CPU-doorbell handler is an expected collective rejection, rather than a kernel +launch. diff --git a/projects/rccl/contrib/niin/gpunetio/proxy_atomics/API_COVERAGE.md b/projects/rccl/contrib/niin/gpunetio/proxy_atomics/API_COVERAGE.md new file mode 100644 index 000000000000..da09b7140914 --- /dev/null +++ b/projects/rccl/contrib/niin/gpunetio/proxy_atomics/API_COVERAGE.md @@ -0,0 +1,113 @@ +# NIIN proxy atomic API coverage contract + +This document records the intended API contract for NIIN's experimental SRQ +proxy atomic plane. It deliberately lives beside the NIIN-owned proxy, rather +than in GIN or GPUNetIO: existing NIIN RMA, signaling, and synchronization +remain GIN operations. + +The native GPUNetIO provider is the supported atomic path. This proxy remains +development-only until it has a safe CUDA ordering handoff for target heap +access and peer-visible failure recovery. In particular, a CPU GDRCopy BAR RMW +cannot safely service a self or LSA AMO issued by a still-running target GPU +kernel. + +`include/niin/gpunetio/proxy_atomics/api.h` provides the corresponding +value-class and operation checks for the device dispatcher. + +## Public operations and value classes + +The prototype adapter aims to implement the integral rows below and the listed +floating base/extended forms. The proxy protocol carries raw 4- or 8-byte +integral values, so aliases such as `long`, `size_t`, and `ptrdiff_t` do not +need separate wire encodings. + +| Public value class | Public API spelling(s) | Current NIIN surface | Required proxy operations | +| --- | --- | --- | --- | +| 32-bit integral | public aliases whose actual width is 32, such as `int`, `uint`, `int32`, `uint32` | yes | fetch-add/add, compare-swap, swap/set, fetch, inc/fetch-inc, fetch/non-fetch AND/OR/XOR | +| 64-bit integral | public aliases whose actual width is 64, such as `long`, `longlong`, `ulong`, `ulonglong`, `int64`, `uint64`, `size`, `ptrdiff` | yes | same integral operation set | +| `ptrdiff_t` | `nvshmem_ptrdiff_atomic_*` | yes | Encode as its actual 4- or 8-byte two's-complement representation; do not introduce a platform-width `ptrdiff` wire tag | +| `float` | base `nvshmem_float_atomic_{swap,fetch,set}` and extended `nvshmemx_float_atomic_{fetch_add,add}` | yes | raw-bit swap/fetch/set; target-side floating add/fetch-add | +| `double` | base `nvshmem_double_atomic_{swap,fetch,set}` and extended `nvshmemx_double_atomic_{fetch_add,add}` | yes | raw-bit swap/fetch/set; target-side floating add/fetch-add | +| `__half` | extended `nvshmemx_half_atomic_{fetch_add,add}` | yes | target-side binary16 add/fetch-add only | + +The upstream NVSHMEM extended floating API is exactly the six +`nvshmemx_{half,float,double}_atomic_{fetch_add,add}` functions. `__half` +is not a base `nvshmem_half_atomic_*` type. In particular, this contract +does **not** invent floating compare-swap or bitwise AMOs that NVSHMEM does +not expose. + +The floating adapter must not instantiate NIIN's existing generic +`niin_atomicAdd` helper: that helper intentionally implements integral +modulo addition by passing raw bits through an unsigned CUDA atomic. The +floating rows require their actual CUDA `atomicAdd(float*)`, +`atomicAdd(double*)`, and supported `atomicAdd(__half*)` overloads for +self/LSA access. Likewise, a floating atomic fetch must use a bit-preserving +integer fetch (or the proxy) rather than `atomicAdd(value, 0)`, which can +canonicalize a NaN or change signed-zero representation. + +The native GPUNetIO path continues to cover only the integral classes. The +prototype currently routes every AMO in the table through the proxy plane, +including self and LSA targets and otherwise-native integral ones. That is not +a safe public contract: a target CPU BAR RMW cannot be ordered with a target +GPU kernel that is already running. Mixing a target CPU read-modify-write +proxy with NIC AMOs against the same heap location is also not linearizable, +so a context binds one provider mode at a time. + +## Wire-value rules + +`niinGpunetioProxyAtomicType` identifies the *representation* rather than a +C typedef spelling: + +- integral values are 32 or 64 bits; +- `float` is IEEE-754 binary32 and `double` is IEEE-754 binary64; +- `__half` is IEEE-754 binary16. + +Operands, compare operands, and returned values are raw bits in 64-bit +protocol fields. Only the low `sizeof(T)` bytes are meaningful. Producers +and consumers must use `memcpy`/bit conversion rather than pointer casts so +that request-ring alignment does not become a language-aliasing requirement. +The request fields themselves need an explicitly specified byte order if the +protocol is ever used across heterogeneous hosts; the initial supported +deployment is homogeneous little-endian CUDA/IB hosts. + +`ptrdiff_t` is a normal integral value on the wire. The setup protocol must +collectively reject a PE set whose `sizeof(ptrdiff_t)` is not uniform instead +of assuming LP64 from a public symbol name. + +## Required semantics + +- Fetching operations return the value observed immediately before the + target-side update. Non-fetching operations still wait for the proxy + completion before returning so program order is preserved. +- Compare-swap compares raw integral bits. For the proxy's raw-bit + float/double swap/fetch/set operations, NaN payloads and signed zero are + copied unchanged. +- Integral add/inc use modulo-2^N arithmetic, matching the raw native AMO + encoding. Signedness does not change the wire opcode. +- Floating add is serialized at the target using the host's ordinary IEEE + arithmetic and binary16 conversion for `__half`; it is not a source-GPU + update or a NIC integer AMO. Normal finite operands are smoke-tested. Until + dedicated edge tests exist, the proxy does not promise CUDA-bit-identical + NaN, denormal, signed-zero, or half-rounding behavior for floating adds. +- A completion body (`resultBits`, `status`) must become visible before its + ticket/valid word. A two-write completion protocol with the ticket written + last is the simplest safe design for a GPU polling the completion ring. + +## ABI and routing boundaries + +The proxy request needs an operation code, representation tag, target PE, +remote heap offset, operand bits, compare bits, slot, and monotonically +checked ticket. Its completion needs result bits, status, and ticket. Keep +both structures versioned and fixed-width; never place host pointers, +`size_t`, or CUDA/ibverbs structures on this wire. + +The prototype device API currently selects an attached proxy-all context +before the self/LSA/native-direct branches. That preserves one CPU-RMW domain, +but is precisely why the current self/LSA behavior is development-only. A +supported design needs an explicit CUDA ordering handoff and a separate +failure-reporting path. The native-only mode retains the existing routing +exactly: self/LSA CUDA atomics and remote integral GPUNetIO WQEs. + +Neither mode may borrow GIN state, QPs, MRs, or request formats. The proxy +uses NIIN-owned queues/QPs/SRQ and is allowed to depend on public CUDA, +ibverbs, and GDRCopy APIs only. diff --git a/projects/rccl/contrib/niin/gpunetio/proxy_atomics/src/proxy_atomics_host.cc b/projects/rccl/contrib/niin/gpunetio/proxy_atomics/src/proxy_atomics_host.cc new file mode 100644 index 000000000000..260f8186c49e --- /dev/null +++ b/projects/rccl/contrib/niin/gpunetio/proxy_atomics/src/proxy_atomics_host.cc @@ -0,0 +1,1195 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +// NIIN's software-atomic proxy. This source deliberately contains the entire +// layering escape hatch: raw public ibverbs for its RC/SRQ plane, dynamic +// GDRCopy for target CPU access, and public NCCL only for bootstrap. It neither +// includes nor inspects GPUNetIO/GDAKI/GIN/NCCL transport-private state. + +#include "niin/gpunetio/proxy_atomics/host.h" + +#include "niin/context_abi.h" +#include "niin/gpunetio/proxy_atomics/device.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint32_t kBootstrapMagic = 0x4e495053u; // "NIPS" +constexpr uint16_t kBootstrapVersion = 1; +constexpr uint32_t kMinQueueDepth = 16; +constexpr uint32_t kMaxQueueDepth = 32768; +constexpr uint8_t kQpPsn = 0; +constexpr uint8_t kRetryCount = 7; +constexpr uint8_t kRnrRetryCount = 7; +constexpr uint8_t kAckTimeout = 20; +constexpr uint8_t kMinRnrTimer = 12; +constexpr size_t kCompletionInlineBytes = sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t); +constexpr size_t kTicketInlineBytes = sizeof(uint64_t); +constexpr int kCqPollBatch = 16; + +struct alignas(64) ProxyReceiveBuffer { + struct niinGpunetioProxyAtomicRequest request; +}; + +struct alignas(8) ProxyWire { + uint32_t magic; + uint16_t version; + uint16_t reserved0; + uint32_t sourcePe; + uint32_t destinationPe; + uint32_t qpn; + uint16_t lid; + uint8_t gid[16]; + uint8_t linkLayer; + uint8_t activeMtu; + uint16_t reserved1; + uint64_t completionBase; + uint64_t completionBytes; + uint32_t completionRkey; + uint32_t completionStride; + uint32_t completionOffset; + uint32_t completionSlots; + uint64_t heapBytes; + uint32_t ptrdiffBytes; + uint32_t reserved2; +}; + +static_assert(sizeof(ProxyWire) % alignof(uint64_t) == 0, + "NIIN proxy bootstrap wire must remain 64-bit aligned"); + +bool isPowerOfTwo(uint32_t value) { + return value != 0 && (value & (value - 1)) == 0; +} + +int readEnvInt(const char* name, int fallback) { + const char* text = std::getenv(name); + if (text == nullptr || *text == '\0') return fallback; + char* end = nullptr; + const long value = std::strtol(text, &end, 10); + if (end == text || *end != '\0' || value < std::numeric_limits::min() || + value > std::numeric_limits::max()) + return fallback; + return static_cast(value); +} + +ncclResult_t cudaToNccl(cudaError_t status) { + return status == cudaSuccess ? ncclSuccess : ncclSystemError; +} + +using GdrOpen = gdr_t (*)(); +using GdrClose = int (*)(gdr_t); +using GdrPinBuffer = int (*)(gdr_t, unsigned long, size_t, uint64_t, uint32_t, gdr_mh_t*); +using GdrUnpinBuffer = int (*)(gdr_t, gdr_mh_t); +using GdrGetInfo = int (*)(gdr_t, gdr_mh_t, gdr_info_t*); +using GdrMap = int (*)(gdr_t, gdr_mh_t, void**, size_t); +using GdrUnmap = int (*)(gdr_t, gdr_mh_t, void*, size_t); +using GdrRuntimeGetVersion = void (*)(int*, int*); + +struct GdrCopyMapping { + void* library = nullptr; + gdr_t descriptor = nullptr; + GdrClose close = nullptr; + GdrPinBuffer pinBuffer = nullptr; + GdrUnpinBuffer unpinBuffer = nullptr; + GdrGetInfo getInfo = nullptr; + GdrMap map = nullptr; + GdrUnmap unmap = nullptr; + gdr_mh_t memoryHandle = {}; + bool pinned = false; + bool mapped = false; + void* mappedBase = nullptr; + size_t mappedBytes = 0; + void* heapCpuBase = nullptr; + size_t heapBytes = 0; + + ~GdrCopyMapping() { reset(); } + + void reset() { + if (mapped && unmap != nullptr) (void)unmap(descriptor, memoryHandle, mappedBase, mappedBytes); + if (pinned && unpinBuffer != nullptr) (void)unpinBuffer(descriptor, memoryHandle); + if (descriptor != nullptr && close != nullptr) (void)close(descriptor); + if (library != nullptr) dlclose(library); + library = nullptr; + descriptor = nullptr; + close = nullptr; + pinBuffer = nullptr; + unpinBuffer = nullptr; + getInfo = nullptr; + map = nullptr; + unmap = nullptr; + memoryHandle = {}; + pinned = false; + mapped = false; + mappedBase = nullptr; + mappedBytes = 0; + heapCpuBase = nullptr; + heapBytes = 0; + } + + template + bool loadSymbol(const char* name, Fn* target) { + *target = reinterpret_cast(dlsym(library, name)); + return *target != nullptr; + } + + bool initialize(const char* requestedLibrary, void* heapBase, size_t bytes) { + if (heapBase == nullptr || bytes == 0) return false; + const char* candidates[] = {requestedLibrary, "libgdrapi.so.2", "libgdrapi.so"}; + for (const char* candidate : candidates) { + if (candidate == nullptr || *candidate == '\0') continue; + library = dlopen(candidate, RTLD_NOW | RTLD_LOCAL); + if (library != nullptr) break; + } + if (library == nullptr) return false; + + GdrOpen open = nullptr; + GdrRuntimeGetVersion runtimeGetVersion = nullptr; + if (!loadSymbol("gdr_open", &open) || !loadSymbol("gdr_close", &close) || + !loadSymbol("gdr_pin_buffer", &pinBuffer) || !loadSymbol("gdr_unpin_buffer", &unpinBuffer) || + !loadSymbol("gdr_get_info", &getInfo) || !loadSymbol("gdr_map", &map) || + !loadSymbol("gdr_unmap", &unmap) || !loadSymbol("gdr_runtime_get_version", &runtimeGetVersion)) { + reset(); + return false; + } + int runtimeMajor = 0; + int runtimeMinor = 0; + runtimeGetVersion(&runtimeMajor, &runtimeMinor); + if (runtimeMajor != GDR_API_MAJOR_VERSION) { + std::fprintf(stderr, "NIIN proxy atomics: unsupported GDRCopy runtime %d.%d (need major %d)\n", + runtimeMajor, runtimeMinor, GDR_API_MAJOR_VERSION); + reset(); + return false; + } + descriptor = open(); + if (descriptor == nullptr) { + reset(); + return false; + } + if (pinBuffer(descriptor, reinterpret_cast(heapBase), bytes, 0, 0, &memoryHandle) != 0) { + reset(); + return false; + } + pinned = true; + if (map(descriptor, memoryHandle, &mappedBase, bytes) != 0 || mappedBase == nullptr) { + reset(); + return false; + } + mapped = true; + mappedBytes = bytes; + gdr_info_t info = {}; + if (getInfo(descriptor, memoryHandle, &info) != 0 || reinterpret_cast(heapBase) < info.va) { + reset(); + return false; + } + const uint64_t offset = reinterpret_cast(heapBase) - info.va; + if (offset > info.mapped_size || bytes > info.mapped_size - offset || + reinterpret_cast(mappedBase) > UINTPTR_MAX - offset) { + reset(); + return false; + } + heapCpuBase = reinterpret_cast(reinterpret_cast(mappedBase) + offset); + heapBytes = bytes; + return true; + } +}; + +template +T bitsToValue(uint64_t bits) { + T value; + std::memcpy(&value, &bits, sizeof(value)); + return value; +} + +template +uint64_t valueToBits(T value) { + uint64_t bits = 0; + std::memcpy(&bits, &value, sizeof(value)); + return bits; +} + +float halfBitsToFloat(uint16_t bits) { + const uint32_t sign = static_cast(bits & UINT16_C(0x8000)) << 16; + uint32_t exponent = (bits >> 10) & UINT16_C(0x1f); + uint32_t mantissa = bits & UINT16_C(0x03ff); + uint32_t result = 0; + if (exponent == 0) { + if (mantissa == 0) { + result = sign; + } else { + int exp = -14; + while ((mantissa & UINT32_C(0x0400)) == 0) { + mantissa <<= 1; + --exp; + } + mantissa &= UINT32_C(0x03ff); + result = sign | static_cast(exp + 127) << 23 | mantissa << 13; + } + } else if (exponent == 31) { + result = sign | UINT32_C(0x7f800000) | mantissa << 13; + } else { + result = sign | (exponent + 112) << 23 | mantissa << 13; + } + return bitsToValue(result); +} + +uint16_t floatToHalfBits(float value) { + const uint32_t bits = static_cast(valueToBits(value)); + const uint16_t sign = static_cast((bits >> 16) & UINT32_C(0x8000)); + const uint32_t exponent = (bits >> 23) & UINT32_C(0xff); + uint32_t mantissa = bits & UINT32_C(0x007fffff); + if (exponent == 255) { + if (mantissa == 0) return static_cast(sign | UINT16_C(0x7c00)); + // Preserve a nonzero payload so a NaN remains a NaN after narrowing. + return static_cast(sign | UINT16_C(0x7c00) | + std::max(UINT32_C(1), mantissa >> 13)); + } + int halfExponent = static_cast(exponent) - 127 + 15; + if (halfExponent >= 31) return static_cast(sign | UINT16_C(0x7c00)); + if (halfExponent <= 0) { + if (halfExponent < -10) return sign; + mantissa |= UINT32_C(0x00800000); + const uint32_t shift = static_cast(14 - halfExponent); + const uint32_t rounded = (mantissa + (UINT32_C(1) << (shift - 1)) + + ((mantissa >> shift) & UINT32_C(1))) >> shift; + return static_cast(sign | rounded); + } + const uint32_t rounded = (mantissa + UINT32_C(0x00001000) + + ((mantissa >> 13) & UINT32_C(1))) >> 13; + return static_cast(sign | (static_cast(halfExponent) << 10) | rounded); +} + +} // namespace + +struct niinGpunetioProxyAtomicHostContext { + ncclComm_t comm = nullptr; + int rank = -1; + int nPes = 0; + int port = 0; + int gidIndex = 0; + uint16_t pkeyIndex = 0; + uint8_t serviceLevel = 0; + struct ibv_context* verbsContext = nullptr; + struct ibv_pd* pd = nullptr; + struct ibv_port_attr portAttr = {}; + union ibv_gid localGid = {}; + struct ibv_cq* sendCq = nullptr; + struct ibv_cq* receiveCq = nullptr; + struct ibv_srq* srq = nullptr; + struct ibv_mr* receiveMr = nullptr; + struct ibv_mr* slotsMr = nullptr; + std::vector qps; + std::vector peers; + std::vector receiveBuffers; + struct niinGpunetioProxyAtomicSlot* slotsHost = nullptr; + struct niinGpunetioProxyAtomicSlot* slotsDevice = nullptr; + uint32_t* progressFailureHost = nullptr; + uint32_t* progressFailureDevice = nullptr; + size_t slotCount = 0; + uint32_t slotsPerPe = 1; + std::vector lastTickets; + int* endpointLocksDevice = nullptr; + struct niinGpunetioProxyAtomicDeviceContext proxyContextHost = {}; + struct niinGpunetioProxyAtomicDeviceContext* proxyContextDevice = nullptr; + struct niinGpunetioAtomicContext atomicContextHost = {}; + struct niinGpunetioAtomicContext* atomicContextDevice = nullptr; + struct niinContext* boundDeviceContext = nullptr; + struct GdrCopyMapping gdr; + std::atomic stopProgress{false}; + std::atomic progressFailed{false}; + std::thread progressThread; + std::mutex progressMutex; + + ~niinGpunetioProxyAtomicHostContext() { cleanup(); } + + void markProgressFailed() { + progressFailed.store(true, std::memory_order_release); + if (progressFailureHost != nullptr) + __atomic_store_n(progressFailureHost, UINT32_C(1), __ATOMIC_RELEASE); + } + + void cleanup() { + // In normal use Finalize is called after kernel completion. Publishing an + // error first also makes an accidental late device waiter fail closed + // before the mapped proxy state is released. + markProgressFailed(); + stopProgress.store(true, std::memory_order_release); + if (progressThread.joinable()) progressThread.join(); + if (boundDeviceContext != nullptr) { + const struct niinGpunetioAtomicContext* currentContext = nullptr; + const struct niinGpunetioAtomicContext* nullContext = nullptr; + // Do not clear a provider that was explicitly installed after this + // proxy. The normal bind API rejects that misuse, but match the direct + // provider's teardown rule so cleanup cannot erase unrelated state. + const cudaError_t readStatus = cudaMemcpy( + ¤tContext, + reinterpret_cast(boundDeviceContext) + + offsetof(struct niinContext, gpunetioAtomicContext), + sizeof(currentContext), cudaMemcpyDeviceToHost); + if (readStatus == cudaSuccess && currentContext == atomicContextDevice) { + (void)cudaMemcpy(reinterpret_cast(boundDeviceContext) + + offsetof(struct niinContext, gpunetioAtomicContext), + &nullContext, sizeof(nullContext), cudaMemcpyHostToDevice); + } + boundDeviceContext = nullptr; + } + if (atomicContextDevice != nullptr) cudaFree(atomicContextDevice); + if (proxyContextDevice != nullptr) cudaFree(proxyContextDevice); + if (endpointLocksDevice != nullptr) cudaFree(endpointLocksDevice); + atomicContextDevice = nullptr; + proxyContextDevice = nullptr; + endpointLocksDevice = nullptr; + for (struct ibv_qp* qp : qps) { + if (qp != nullptr) ibv_destroy_qp(qp); + } + qps.clear(); + if (srq != nullptr) ibv_destroy_srq(srq); + if (slotsMr != nullptr) ibv_dereg_mr(slotsMr); + if (receiveMr != nullptr) ibv_dereg_mr(receiveMr); + if (sendCq != nullptr) ibv_destroy_cq(sendCq); + if (receiveCq != nullptr) ibv_destroy_cq(receiveCq); + srq = nullptr; + slotsMr = nullptr; + receiveMr = nullptr; + sendCq = nullptr; + receiveCq = nullptr; + if (slotsHost != nullptr) cudaFreeHost(slotsHost); + if (progressFailureHost != nullptr) cudaFreeHost(progressFailureHost); + slotsHost = nullptr; + slotsDevice = nullptr; + progressFailureHost = nullptr; + progressFailureDevice = nullptr; + receiveBuffers.clear(); + lastTickets.clear(); + peers.clear(); + gdr.reset(); + if (pd != nullptr) ibv_dealloc_pd(pd); + if (verbsContext != nullptr) ibv_close_device(verbsContext); + pd = nullptr; + verbsContext = nullptr; + } +}; + +namespace { + +ncclResult_t openVerbsDevice(niinGpunetioProxyAtomicHostContext* provider, + const struct niinGpunetioProxyAtomicOptions& options) { + const char* requestedDevice = options.ibDevice; + if (requestedDevice == nullptr || *requestedDevice == '\0') + requestedDevice = std::getenv("NIIN_GPUNETIO_PROXY_IB_DEV"); + int deviceCount = 0; + struct ibv_device** devices = ibv_get_device_list(&deviceCount); + if (devices == nullptr || deviceCount <= 0) return ncclSystemError; + + for (int index = 0; index < deviceCount && provider->verbsContext == nullptr; ++index) { + if (requestedDevice != nullptr && *requestedDevice != '\0' && + std::strcmp(ibv_get_device_name(devices[index]), requestedDevice) != 0) + continue; + struct ibv_context* context = ibv_open_device(devices[index]); + if (context == nullptr) continue; + const int firstPort = options.ibPort > 0 ? options.ibPort : 1; + const int lastPort = options.ibPort > 0 ? options.ibPort : 255; + struct ibv_port_attr portAttr = {}; + int selectedPort = 0; + for (int port = firstPort; port <= lastPort; ++port) { + if (ibv_query_port(context, static_cast(port), &portAttr) != 0) { + if (options.ibPort > 0) break; + continue; + } + if (portAttr.state == IBV_PORT_ACTIVE) { + selectedPort = port; + break; + } + if (options.ibPort > 0) break; + } + if (selectedPort == 0) { + ibv_close_device(context); + continue; + } + provider->verbsContext = context; + provider->port = selectedPort; + provider->portAttr = portAttr; + } + ibv_free_device_list(devices); + if (provider->verbsContext == nullptr) return ncclInvalidUsage; + + provider->gidIndex = options.gidIndex >= 0 + ? options.gidIndex + : readEnvInt("NIIN_GPUNETIO_PROXY_GID_INDEX", 0); + if (provider->gidIndex < 0 || + ibv_query_gid(provider->verbsContext, static_cast(provider->port), provider->gidIndex, + &provider->localGid) != 0) + return ncclInvalidUsage; + provider->pd = ibv_alloc_pd(provider->verbsContext); + return provider->pd == nullptr ? ncclSystemError : ncclSuccess; +} + +ncclResult_t createMappedSlots(niinGpunetioProxyAtomicHostContext* provider) { + if (provider->nPes <= 0 || provider->slotsPerPe == 0 || + static_cast(provider->nPes) > + std::numeric_limits::max() / provider->slotsPerPe) + return ncclInvalidArgument; + provider->slotCount = static_cast(provider->nPes) * provider->slotsPerPe; + if (provider->slotCount > std::numeric_limits::max() / sizeof(*provider->slotsHost)) + return ncclInvalidArgument; + const size_t slotBytes = provider->slotCount * sizeof(*provider->slotsHost); + cudaError_t cudaStatus = cudaHostAlloc(reinterpret_cast(&provider->slotsHost), slotBytes, + cudaHostAllocMapped | cudaHostAllocPortable); + if (cudaStatus != cudaSuccess) return ncclSystemError; + std::memset(provider->slotsHost, 0, slotBytes); + cudaStatus = cudaHostGetDevicePointer(reinterpret_cast(&provider->slotsDevice), provider->slotsHost, 0); + if (cudaStatus != cudaSuccess) return ncclSystemError; + cudaStatus = cudaHostAlloc(reinterpret_cast(&provider->progressFailureHost), + sizeof(*provider->progressFailureHost), + cudaHostAllocMapped | cudaHostAllocPortable); + if (cudaStatus != cudaSuccess) return ncclSystemError; + *provider->progressFailureHost = 0; + cudaStatus = cudaHostGetDevicePointer(reinterpret_cast(&provider->progressFailureDevice), + provider->progressFailureHost, 0); + if (cudaStatus != cudaSuccess) return ncclSystemError; + provider->slotsMr = ibv_reg_mr(provider->pd, provider->slotsHost, slotBytes, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + if (provider->slotsMr == nullptr) return ncclSystemError; + cudaStatus = cudaMalloc(&provider->endpointLocksDevice, provider->nPes * sizeof(int)); + if (cudaStatus != cudaSuccess) return ncclSystemError; + cudaStatus = cudaMemset(provider->endpointLocksDevice, 0, provider->nPes * sizeof(int)); + if (cudaStatus != cudaSuccess) return ncclSystemError; + provider->lastTickets.assign(provider->slotCount, 0); + return ncclSuccess; +} + +int postReceive(niinGpunetioProxyAtomicHostContext* provider, struct ProxyReceiveBuffer* buffer) { + struct ibv_sge sge = {}; + sge.addr = reinterpret_cast(&buffer->request); + sge.length = sizeof(buffer->request); + sge.lkey = provider->receiveMr->lkey; + struct ibv_recv_wr wr = {}; + wr.wr_id = reinterpret_cast(buffer); + wr.sg_list = &sge; + wr.num_sge = 1; + struct ibv_recv_wr* bad = nullptr; + return ibv_post_srq_recv(provider->srq, &wr, &bad); +} + +ncclResult_t createSrqResources(niinGpunetioProxyAtomicHostContext* provider, + const struct niinGpunetioProxyAtomicOptions& options) { + const uint64_t peerCount = provider->nPes > 0 ? static_cast(provider->nPes - 1) : 0; + // The send CQ is shared by every RC QP. With one leased source slot per + // destination, an all-peer burst can have one signaled SEND and one + // signaled completion-ticket write outstanding per peer at once. + const uint64_t sendCqEntries = std::max(UINT64_C(32), 2 * peerCount + 8); + if (options.srqDepth > static_cast(INT_MAX - 8) || + sendCqEntries > static_cast(INT_MAX)) + return ncclInvalidArgument; + provider->receiveCq = ibv_create_cq(provider->verbsContext, static_cast(options.srqDepth + 8), + nullptr, nullptr, 0); + provider->sendCq = ibv_create_cq(provider->verbsContext, static_cast(sendCqEntries), nullptr, + nullptr, 0); + if (provider->receiveCq == nullptr || provider->sendCq == nullptr) return ncclSystemError; + struct ibv_srq_init_attr srqAttr = {}; + srqAttr.attr.max_wr = options.srqDepth; + srqAttr.attr.max_sge = 1; + provider->srq = ibv_create_srq(provider->pd, &srqAttr); + if (provider->srq == nullptr) return ncclSystemError; + provider->receiveBuffers.resize(options.srqDepth); + provider->receiveMr = ibv_reg_mr(provider->pd, provider->receiveBuffers.data(), + provider->receiveBuffers.size() * sizeof(provider->receiveBuffers[0]), + IBV_ACCESS_LOCAL_WRITE); + if (provider->receiveMr == nullptr) return ncclSystemError; + for (struct ProxyReceiveBuffer& buffer : provider->receiveBuffers) { + if (postReceive(provider, &buffer) != 0) return ncclSystemError; + } + return ncclSuccess; +} + +ncclResult_t transitionQpInit(niinGpunetioProxyAtomicHostContext* provider, struct ibv_qp* qp) { + struct ibv_qp_attr attr = {}; + attr.qp_state = IBV_QPS_INIT; + attr.pkey_index = provider->pkeyIndex; + attr.port_num = static_cast(provider->port); + attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE; + const int mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS; + return ibv_modify_qp(qp, &attr, mask) == 0 ? ncclSuccess : ncclSystemError; +} + +ncclResult_t createPeerQps(niinGpunetioProxyAtomicHostContext* provider, + const struct niinGpunetioProxyAtomicOptions& options, + std::vector* local) { + provider->qps.assign(provider->nPes, nullptr); + local->assign(provider->nPes, {}); + for (int peer = 0; peer < provider->nPes; ++peer) { + struct ProxyWire& wire = (*local)[peer]; + wire.magic = kBootstrapMagic; + wire.version = kBootstrapVersion; + wire.sourcePe = provider->rank; + wire.destinationPe = peer; + wire.lid = provider->portAttr.lid; + std::memcpy(wire.gid, provider->localGid.raw, sizeof(wire.gid)); + wire.linkLayer = static_cast(provider->portAttr.link_layer); + wire.activeMtu = static_cast(provider->portAttr.active_mtu); + wire.completionBase = reinterpret_cast(provider->slotsHost); + wire.completionBytes = provider->slotCount * sizeof(*provider->slotsHost); + wire.completionRkey = provider->slotsMr->rkey; + wire.completionStride = sizeof(*provider->slotsHost); + wire.completionOffset = offsetof(struct niinGpunetioProxyAtomicSlot, completion); + wire.completionSlots = provider->slotCount; + wire.heapBytes = provider->gdr.heapBytes; + wire.ptrdiffBytes = sizeof(ptrdiff_t); + if (peer == provider->rank) continue; + + struct ibv_qp_init_attr init = {}; + init.send_cq = provider->sendCq; + init.recv_cq = provider->receiveCq; + init.srq = provider->srq; + init.qp_type = IBV_QPT_RC; + init.cap.max_send_wr = options.sendQueueDepth; + init.cap.max_recv_wr = 0; + init.cap.max_send_sge = 1; + init.cap.max_recv_sge = 1; + init.cap.max_inline_data = sizeof(struct niinGpunetioProxyAtomicCompletion); + struct ibv_qp* qp = ibv_create_qp(provider->pd, &init); + if (qp == nullptr) return ncclSystemError; + if (init.cap.max_inline_data < kCompletionInlineBytes || + init.cap.max_inline_data < kTicketInlineBytes) { + ibv_destroy_qp(qp); + return ncclSystemError; + } + provider->qps[peer] = qp; + ncclResult_t status = transitionQpInit(provider, qp); + if (status != ncclSuccess) return status; + wire.qpn = qp->qp_num; + } + return ncclSuccess; +} + +ncclResult_t allGatherSetupStatus(ncclComm_t comm, int nPes, ncclResult_t localStatus) { + if (nPes <= 0) return ncclInvalidArgument; + uint32_t local = static_cast(localStatus); + std::vector remote(nPes, static_cast(ncclSystemError)); + void* send = nullptr; + void* recv = nullptr; + cudaStream_t stream = nullptr; + ncclResult_t result = cudaToNccl(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMalloc(&send, sizeof(local))); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMalloc(&recv, remote.size() * sizeof(remote[0]))); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMemcpyAsync(send, &local, sizeof(local), cudaMemcpyHostToDevice, stream)); + if (result != ncclSuccess) goto cleanup; + result = ncclAllGather(send, recv, sizeof(local), ncclUint8, comm, stream); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMemcpyAsync(remote.data(), recv, remote.size() * sizeof(remote[0]), + cudaMemcpyDeviceToHost, stream)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaStreamSynchronize(stream)); + if (result != ncclSuccess) goto cleanup; + for (uint32_t status : remote) { + if (status != static_cast(ncclSuccess)) { + result = localStatus == ncclSuccess ? ncclSystemError : localStatus; + goto cleanup; + } + } + result = ncclSuccess; +cleanup: + if (recv != nullptr) cudaFree(recv); + if (send != nullptr) cudaFree(send); + if (stream != nullptr) cudaStreamDestroy(stream); + return result; +} + +ncclResult_t allGatherWire(ncclComm_t comm, const std::vector& local, int nPes, + std::vector* remote) { + if (local.size() != static_cast(nPes)) return ncclInvalidArgument; + const size_t localBytes = local.size() * sizeof(local[0]); + if (nPes <= 0 || localBytes > std::numeric_limits::max() / nPes) return ncclInvalidArgument; + void* send = nullptr; + void* recv = nullptr; + cudaStream_t stream = nullptr; + ncclResult_t result = cudaToNccl(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMalloc(&send, localBytes)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMalloc(&recv, localBytes * static_cast(nPes))); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMemcpyAsync(send, local.data(), localBytes, cudaMemcpyHostToDevice, stream)); + if (result != ncclSuccess) goto cleanup; + result = ncclAllGather(send, recv, localBytes, ncclUint8, comm, stream); + if (result != ncclSuccess) goto cleanup; + remote->resize(static_cast(nPes) * nPes); + result = cudaToNccl(cudaMemcpyAsync(remote->data(), recv, remote->size() * sizeof((*remote)[0]), + cudaMemcpyDeviceToHost, stream)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaStreamSynchronize(stream)); +cleanup: + if (recv != nullptr) cudaFree(recv); + if (send != nullptr) cudaFree(send); + if (stream != nullptr) cudaStreamDestroy(stream); + return result; +} + +bool remoteCompletionLayoutIsValid(const niinGpunetioProxyAtomicHostContext* provider, + const struct ProxyWire& remote) { + constexpr uint64_t slotBytes = sizeof(struct niinGpunetioProxyAtomicSlot); + constexpr uint64_t completionOffset = offsetof(struct niinGpunetioProxyAtomicSlot, completion); + if (provider->slotCount > UINT64_MAX / slotBytes) return false; + const uint64_t expectedSlots = static_cast(provider->slotCount); + const uint64_t expectedBytes = expectedSlots * slotBytes; + return remote.completionBase != 0 && remote.completionRkey != 0 && + (remote.completionBase % alignof(struct niinGpunetioProxyAtomicSlot)) == 0 && + remote.completionStride == slotBytes && remote.completionOffset == completionOffset && + remote.completionSlots == expectedSlots && remote.completionBytes == expectedBytes && + remote.completionBase <= UINT64_MAX - remote.completionBytes; +} + +ncclResult_t connectPeerQp(niinGpunetioProxyAtomicHostContext* provider, int peer, + const struct ProxyWire& remote) { + if (peer == provider->rank || peer < 0 || peer >= provider->nPes || provider->qps[peer] == nullptr || + remote.magic != kBootstrapMagic || remote.version != kBootstrapVersion || + remote.sourcePe != static_cast(peer) || remote.destinationPe != static_cast(provider->rank) || + remote.qpn == 0 || remote.linkLayer != static_cast(provider->portAttr.link_layer) || + !remoteCompletionLayoutIsValid(provider, remote) || remote.heapBytes != provider->gdr.heapBytes || + remote.ptrdiffBytes != sizeof(ptrdiff_t)) + return ncclInvalidUsage; + + struct ibv_qp_attr attr = {}; + attr.qp_state = IBV_QPS_RTR; + attr.path_mtu = static_cast(std::min(provider->portAttr.active_mtu, remote.activeMtu)); + attr.dest_qp_num = remote.qpn; + attr.rq_psn = kQpPsn; + attr.max_dest_rd_atomic = 1; + attr.min_rnr_timer = kMinRnrTimer; + attr.ah_attr.dlid = remote.lid; + attr.ah_attr.sl = provider->serviceLevel; + attr.ah_attr.port_num = static_cast(provider->port); + const bool isIb = provider->portAttr.link_layer == IBV_LINK_LAYER_INFINIBAND; + const bool sameIbSubnet = + isIb && std::memcmp(&provider->localGid.global.subnet_prefix, remote.gid, + sizeof(provider->localGid.global.subnet_prefix)) == 0; + attr.ah_attr.is_global = !isIb || !sameIbSubnet || (provider->portAttr.flags & IBV_QPF_GRH_REQUIRED) != 0; + if (attr.ah_attr.is_global) { + std::memcpy(&attr.ah_attr.grh.dgid, remote.gid, sizeof(remote.gid)); + attr.ah_attr.grh.sgid_index = provider->gidIndex; + attr.ah_attr.grh.hop_limit = 255; + } + int mask = IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | + IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; + if (ibv_modify_qp(provider->qps[peer], &attr, mask) != 0) return ncclSystemError; + std::memset(&attr, 0, sizeof(attr)); + attr.qp_state = IBV_QPS_RTS; + attr.sq_psn = kQpPsn; + attr.timeout = kAckTimeout; + attr.retry_cnt = kRetryCount; + attr.rnr_retry = kRnrRetryCount; + attr.max_rd_atomic = 1; + mask = IBV_QP_STATE | IBV_QP_SQ_PSN | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | + IBV_QP_MAX_QP_RD_ATOMIC; + return ibv_modify_qp(provider->qps[peer], &attr, mask) == 0 ? ncclSuccess : ncclSystemError; +} + +ncclResult_t exportDeviceContext(niinGpunetioProxyAtomicHostContext* provider) { + provider->proxyContextHost.version = NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_VERSION; + provider->proxyContextHost.flags = NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_READY | + NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_PROXY_ALL; + provider->proxyContextHost.nPes = provider->nPes; + provider->proxyContextHost.localPe = provider->rank; + provider->proxyContextHost.heapBytes = provider->gdr.heapBytes; + provider->proxyContextHost.slotsPerPe = provider->slotsPerPe; + provider->proxyContextHost.slots = provider->slotsDevice; + provider->proxyContextHost.endpointLocks = provider->endpointLocksDevice; + provider->proxyContextHost.progressFailure = provider->progressFailureDevice; + ncclResult_t result = cudaToNccl(cudaMalloc(&provider->proxyContextDevice, sizeof(provider->proxyContextHost))); + if (result != ncclSuccess) return result; + result = cudaToNccl(cudaMemcpy(provider->proxyContextDevice, &provider->proxyContextHost, + sizeof(provider->proxyContextHost), cudaMemcpyHostToDevice)); + if (result != ncclSuccess) return result; + + provider->atomicContextHost.version = NIIN_GPUNETIO_ATOMIC_CONTEXT_VERSION; + provider->atomicContextHost.flags = NIIN_GPUNETIO_ATOMIC_CONTEXT_READY | NIIN_GPUNETIO_ATOMIC_CONTEXT_PROXY; + provider->atomicContextHost.nPes = provider->nPes; + provider->atomicContextHost.localPe = provider->rank; + provider->atomicContextHost.proxyContext = provider->proxyContextDevice; + result = cudaToNccl(cudaMalloc(&provider->atomicContextDevice, sizeof(provider->atomicContextHost))); + if (result != ncclSuccess) return result; + return cudaToNccl(cudaMemcpy(provider->atomicContextDevice, &provider->atomicContextHost, + sizeof(provider->atomicContextHost), cudaMemcpyHostToDevice)); +} + +void publishLocalCompletion(struct niinGpunetioProxyAtomicSlot* slot, uint64_t result, uint32_t status, + uint64_t ticket) { + slot->completion.resultBits = result; + slot->completion.status = status; + slot->completion.reserved = 0; + __atomic_thread_fence(__ATOMIC_RELEASE); + __atomic_store_n(&slot->completion.ticket, ticket, __ATOMIC_RELEASE); +} + +bool executeIntegral(void* address, uint8_t type, const struct niinGpunetioProxyAtomicRequest& request, + uint64_t* result, uint32_t* status) { + if (type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL32) { + volatile uint32_t* target = static_cast(address); + const uint32_t old = *target; + const uint32_t operand = static_cast(request.operandBits); + const uint32_t compare = static_cast(request.compareBits); + uint32_t next = old; + bool write = true; + switch (request.op) { + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD: next = old + operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_COMPARE_SWAP: next = old == compare ? operand : old; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SET: next = operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH: write = false; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_INC: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_INC: next = old + 1; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_AND: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_AND: next = old & operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_OR: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_OR: next = old | operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_XOR: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_XOR: next = old ^ operand; break; + default: *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_UNSUPPORTED; return false; + } + if (write) *target = next; + *result = old; + } else if (type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL64) { + volatile uint64_t* target = static_cast(address); + const uint64_t old = *target; + const uint64_t operand = request.operandBits; + const uint64_t compare = request.compareBits; + uint64_t next = old; + bool write = true; + switch (request.op) { + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD: next = old + operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_COMPARE_SWAP: next = old == compare ? operand : old; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SET: next = operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH: write = false; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_INC: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_INC: next = old + 1; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_AND: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_AND: next = old & operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_OR: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_OR: next = old | operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_XOR: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_XOR: next = old ^ operand; break; + default: *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_UNSUPPORTED; return false; + } + if (write) *target = next; + *result = old; + } else { + *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_UNSUPPORTED; + return false; + } + __sync_synchronize(); + *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_SUCCESS; + return true; +} + +bool executeFloating(void* address, uint8_t type, const struct niinGpunetioProxyAtomicRequest& request, + uint64_t* result, uint32_t* status) { + if (type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT32) { + volatile float* target = static_cast(address); + const float old = *target; + const float operand = bitsToValue(request.operandBits); + float next = old; + bool write = true; + switch (request.op) { + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD: next = old + operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SET: next = operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH: write = false; break; + default: *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_UNSUPPORTED; return false; + } + if (write) *target = next; + *result = valueToBits(old); + } else if (type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT64) { + volatile double* target = static_cast(address); + const double old = *target; + const double operand = bitsToValue(request.operandBits); + double next = old; + bool write = true; + switch (request.op) { + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD: next = old + operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SET: next = operand; break; + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH: write = false; break; + default: *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_UNSUPPORTED; return false; + } + if (write) *target = next; + *result = valueToBits(old); + } else if (type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT16) { + volatile uint16_t* target = static_cast(address); + const uint16_t old = *target; + if (request.op != NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD && + request.op != NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD) { + *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_UNSUPPORTED; + return false; + } + const float sum = halfBitsToFloat(old) + halfBitsToFloat(static_cast(request.operandBits)); + *target = floatToHalfBits(sum); + *result = old; + } else { + *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_UNSUPPORTED; + return false; + } + __sync_synchronize(); + *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_SUCCESS; + return true; +} + +bool executeRequest(niinGpunetioProxyAtomicHostContext* provider, + const struct niinGpunetioProxyAtomicRequest& request, uint64_t* result, + uint32_t* status) { + if (!niinGpunetioProxyAtomicRequestIsWellFormed(request) || + request.targetPe != static_cast(provider->rank)) { + *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_BAD_REQUEST; + return false; + } + const size_t bytes = niinGpunetioProxyAtomicElementBytes(request.type); + if (request.remoteOffset > provider->gdr.heapBytes || bytes > provider->gdr.heapBytes - request.remoteOffset) { + *status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_BAD_ADDRESS; + return false; + } + void* address = static_cast(provider->gdr.heapCpuBase) + request.remoteOffset; + if (niinGpunetioProxyAtomicTypeIsIntegral(request.type)) + return executeIntegral(address, request.type, request, result, status); + return executeFloating(address, request.type, request, result, status); +} + +bool postRequest(niinGpunetioProxyAtomicHostContext* provider, + struct niinGpunetioProxyAtomicSlot* slot, int targetPe) { + if (targetPe < 0 || targetPe >= provider->nPes || targetPe == provider->rank || + provider->qps[targetPe] == nullptr) + return false; + struct ibv_sge sge = {}; + sge.addr = reinterpret_cast(&slot->request); + sge.length = sizeof(slot->request); + sge.lkey = provider->slotsMr->lkey; + struct ibv_send_wr wr = {}; + wr.wr_id = reinterpret_cast(slot); + wr.sg_list = &sge; + wr.num_sge = 1; + wr.opcode = IBV_WR_SEND; + wr.send_flags = IBV_SEND_SIGNALED; + struct ibv_send_wr* bad = nullptr; + return ibv_post_send(provider->qps[targetPe], &wr, &bad) == 0; +} + +bool postRemoteCompletion(niinGpunetioProxyAtomicHostContext* provider, int sourcePe, + uint32_t sourceSlot, uint64_t result, uint32_t status, uint64_t ticket) { + if (sourcePe < 0 || sourcePe >= provider->nPes || sourcePe == provider->rank || + provider->qps[sourcePe] == nullptr || sourcePe >= static_cast(provider->peers.size())) + return false; + const struct ProxyWire& peer = provider->peers[sourcePe]; + if (!remoteCompletionLayoutIsValid(provider, peer) || sourceSlot >= peer.completionSlots) + return false; + const uint64_t slotOffset = static_cast(sourceSlot) * peer.completionStride; + if (slotOffset > peer.completionBytes || peer.completionOffset > peer.completionBytes - slotOffset || + sizeof(struct niinGpunetioProxyAtomicCompletion) > + peer.completionBytes - slotOffset - peer.completionOffset || + peer.completionBase > UINT64_MAX - slotOffset - peer.completionOffset) + return false; + const uint64_t remoteBase = peer.completionBase + slotOffset + peer.completionOffset; + if (remoteBase > UINT64_MAX - sizeof(uint64_t)) return false; + struct { + uint64_t resultBits; + uint32_t completionStatus; + uint32_t reserved; + } body = {result, status, 0}; + struct ibv_sge bodySge = {}; + bodySge.addr = reinterpret_cast(&body); + bodySge.length = sizeof(body); + struct ibv_sge ticketSge = {}; + ticketSge.addr = reinterpret_cast(&ticket); + ticketSge.length = sizeof(ticket); + struct ibv_send_wr bodyWr = {}; + bodyWr.wr_id = static_cast(NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH); + bodyWr.sg_list = &bodySge; + bodyWr.num_sge = 1; + bodyWr.opcode = IBV_WR_RDMA_WRITE; + bodyWr.send_flags = IBV_SEND_INLINE; + bodyWr.wr.rdma.remote_addr = remoteBase; + bodyWr.wr.rdma.rkey = peer.completionRkey; + struct ibv_send_wr ticketWr = {}; + ticketWr.wr_id = static_cast(NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH); + ticketWr.sg_list = &ticketSge; + ticketWr.num_sge = 1; + ticketWr.opcode = IBV_WR_RDMA_WRITE; + ticketWr.send_flags = IBV_SEND_INLINE | IBV_SEND_SIGNALED; + ticketWr.wr.rdma.remote_addr = remoteBase + niinGpunetioProxyAtomicCompletionBodyBytes(); + ticketWr.wr.rdma.rkey = peer.completionRkey; + bodyWr.next = &ticketWr; + struct ibv_send_wr* bad = nullptr; + return ibv_post_send(provider->qps[sourcePe], &bodyWr, &bad) == 0; +} + +bool processLocalSlots(niinGpunetioProxyAtomicHostContext* provider, bool* didWork) { + for (size_t index = 0; index < provider->slotCount; ++index) { + struct niinGpunetioProxyAtomicSlot* slot = provider->slotsHost + index; + const uint64_t ticket = __atomic_load_n(&slot->request.ticket, __ATOMIC_ACQUIRE); + if (ticket == NIIN_GPUNETIO_PROXY_ATOMIC_UNPUBLISHED_TICKET || ticket == provider->lastTickets[index]) continue; + const struct niinGpunetioProxyAtomicRequest request = slot->request; + provider->lastTickets[index] = ticket; + *didWork = true; + uint64_t result = 0; + uint32_t status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_BAD_REQUEST; + if (request.ticket != ticket || request.sourcePe != static_cast(provider->rank) || + request.sourceSlot != index || request.targetPe >= static_cast(provider->nPes) || + !niinGpunetioProxyAtomicRequestIsWellFormed(request)) { + publishLocalCompletion(slot, result, status, ticket); + continue; + } + if (request.targetPe == static_cast(provider->rank)) { + (void)executeRequest(provider, request, &result, &status); + publishLocalCompletion(slot, result, status, ticket); + } else if (!postRequest(provider, slot, static_cast(request.targetPe))) { + publishLocalCompletion(slot, result, NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_INTERNAL_ERROR, ticket); + provider->markProgressFailed(); + } + } + return true; +} + +int peerFromReceiveQpn(const niinGpunetioProxyAtomicHostContext* provider, uint32_t qpNumber) { + for (int peer = 0; peer < provider->nPes; ++peer) { + if (peer != provider->rank && provider->qps[peer] != nullptr && + provider->qps[peer]->qp_num == qpNumber) + return peer; + } + return -1; +} + +bool processReceiveCq(niinGpunetioProxyAtomicHostContext* provider, bool* didWork) { + struct ibv_wc completions[kCqPollBatch]; + // Do not drain this CQ indefinitely: each received request posts a + // signaled completion-ticket write onto the shared send CQ. Returning after + // one bounded batch lets progressOnce() drain that CQ before a sustained + // stream can overrun it. + const int count = ibv_poll_cq(provider->receiveCq, kCqPollBatch, completions); + if (count < 0) return false; + for (int index = 0; index < count; ++index) { + struct ibv_wc& completion = completions[index]; + auto* buffer = reinterpret_cast(completion.wr_id); + if (buffer == nullptr) return false; + *didWork = true; + const int sourcePe = peerFromReceiveQpn(provider, completion.qp_num); + if (completion.status != IBV_WC_SUCCESS || completion.opcode != IBV_WC_RECV || + completion.byte_len != sizeof(struct niinGpunetioProxyAtomicRequest) || sourcePe < 0) { + return false; + } + const struct niinGpunetioProxyAtomicRequest request = buffer->request; + const uint64_t expectedSlot64 = + static_cast(provider->rank) * static_cast(provider->slotsPerPe); + if (expectedSlot64 > UINT32_MAX) return false; + const uint32_t expectedSlot = static_cast(expectedSlot64); + uint64_t result = 0; + uint32_t status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_BAD_REQUEST; + if (request.sourcePe == static_cast(sourcePe) && + request.targetPe == static_cast(provider->rank) && request.sourceSlot == expectedSlot && + request.ticket != NIIN_GPUNETIO_PROXY_ATOMIC_UNPUBLISHED_TICKET) { + (void)executeRequest(provider, request, &result, &status); + } + // Complete to the peer identified by the trusted local QP number, not a + // request-supplied source identity. A malformed but ticketed request + // receives BAD_REQUEST on the slot where its sender is waiting. + if (request.ticket == NIIN_GPUNETIO_PROXY_ATOMIC_UNPUBLISHED_TICKET || + !postRemoteCompletion(provider, sourcePe, expectedSlot, result, status, request.ticket)) + return false; + if (postReceive(provider, buffer) != 0) return false; + } + return true; +} + +bool processSendCq(niinGpunetioProxyAtomicHostContext* provider, bool* didWork) { + struct ibv_wc completions[kCqPollBatch]; + for (;;) { + const int count = ibv_poll_cq(provider->sendCq, kCqPollBatch, completions); + if (count < 0) return false; + if (count == 0) return true; + *didWork = true; + for (int index = 0; index < count; ++index) { + if (completions[index].status != IBV_WC_SUCCESS) return false; + } + } +} + +ncclResult_t progressOnce(niinGpunetioProxyAtomicHostContext* provider, bool* didWork = nullptr) { + std::lock_guard guard(provider->progressMutex); + bool progressed = false; + if (!processSendCq(provider, &progressed) || !processReceiveCq(provider, &progressed) || + !processLocalSlots(provider, &progressed)) { + provider->markProgressFailed(); + return ncclSystemError; + } + if (didWork != nullptr) *didWork = progressed; + return ncclSuccess; +} + +void startProgressThread(niinGpunetioProxyAtomicHostContext* provider) { + provider->stopProgress.store(false, std::memory_order_release); + provider->progressThread = std::thread([provider] { + while (!provider->stopProgress.load(std::memory_order_acquire)) { + bool progressed = false; + if (progressOnce(provider, &progressed) != ncclSuccess) break; + if (!progressed) std::this_thread::yield(); + } + }); +} + +} // namespace + +ncclResult_t niinGpunetioProxyAtomicInit( + ncclComm_t comm, void* heapBase, size_t heapBytes, + const struct niinGpunetioProxyAtomicOptions* suppliedOptions, + struct niinGpunetioProxyAtomicHostContext** out) { + if (comm == nullptr || heapBase == nullptr || heapBytes == 0 || out == nullptr) return ncclInvalidArgument; + *out = nullptr; + struct niinGpunetioProxyAtomicOptions options = NIIN_GPUNETIO_PROXY_ATOMIC_OPTIONS_INITIALIZER; + if (suppliedOptions != nullptr) options = *suppliedOptions; + if (options.mode != NIIN_GPUNETIO_PROXY_ATOMIC_EXECUTION_PROXY_ALL || options.serviceLevel > 15 || + options.requestSlotsPerPe != 1 || !isPowerOfTwo(options.srqDepth) || + !isPowerOfTwo(options.sendQueueDepth) || options.srqDepth < kMinQueueDepth || + options.sendQueueDepth < kMinQueueDepth || options.srqDepth > kMaxQueueDepth || + options.sendQueueDepth > kMaxQueueDepth) + return ncclInvalidArgument; + + auto* provider = new (std::nothrow) niinGpunetioProxyAtomicHostContext; + if (provider == nullptr) return ncclSystemError; + provider->comm = comm; + provider->slotsPerPe = options.requestSlotsPerPe; + ncclResult_t result = ncclCommUserRank(comm, &provider->rank); + if (result != ncclSuccess) goto fail; + result = ncclCommCount(comm, &provider->nPes); + if (result != ncclSuccess || provider->nPes <= 0 || provider->nPes > INT32_MAX) { + if (result == ncclSuccess) result = ncclInvalidArgument; + goto fail; + } + provider->pkeyIndex = options.pkeyIndex; + provider->serviceLevel = options.serviceLevel; + + { + ncclResult_t local = ncclSuccess; + const char* setupStage = "heap alignment/SRQ sizing"; + if ((reinterpret_cast(heapBase) & (alignof(uint64_t) - 1)) != 0 || + (provider->nPes > 1 && static_cast(provider->nPes - 1) > options.srqDepth)) { + local = ncclInvalidArgument; + } + if (local == ncclSuccess) { + setupStage = "verbs device setup"; + local = openVerbsDevice(provider, options); + } + if (local == ncclSuccess) { + setupStage = "GDRCopy CPU mapping setup"; + if (!provider->gdr.initialize(options.gdrCopyLibrary, heapBase, heapBytes)) { + std::fprintf(stderr, + "NIIN proxy atomics: GDRCopy CPU mapping is unavailable; proxy-all AMOs require it\n"); + local = ncclInvalidUsage; + } + } + if (local == ncclSuccess) { + setupStage = "mapped request-slot setup"; + local = createMappedSlots(provider); + } + if (local == ncclSuccess) { + setupStage = "SRQ setup"; + local = createSrqResources(provider, options); + } + if (local != ncclSuccess) + std::fprintf(stderr, "NIIN proxy atomics [rank %d]: %s failed: %s\n", provider->rank, + setupStage, ncclGetErrorString(local)); + result = allGatherSetupStatus(comm, provider->nPes, local); + if (result != ncclSuccess) goto fail; + } + { + std::vector localWire; + std::vector remoteWire; + ncclResult_t local = createPeerQps(provider, options, &localWire); + result = allGatherSetupStatus(comm, provider->nPes, local); + if (result != ncclSuccess) goto fail; + result = allGatherWire(comm, localWire, provider->nPes, &remoteWire); + if (result != ncclSuccess) goto fail; + provider->peers.assign(provider->nPes, {}); + local = ncclSuccess; + for (int peer = 0; peer < provider->nPes; ++peer) { + const struct ProxyWire& remote = remoteWire[static_cast(peer) * provider->nPes + provider->rank]; + provider->peers[peer] = remote; + if (peer != provider->rank && (local = connectPeerQp(provider, peer, remote)) != ncclSuccess) break; + } + result = allGatherSetupStatus(comm, provider->nPes, local); + if (result != ncclSuccess) goto fail; + local = exportDeviceContext(provider); + result = allGatherSetupStatus(comm, provider->nPes, local); + if (result != ncclSuccess) goto fail; + } + startProgressThread(provider); + *out = provider; + return ncclSuccess; + +fail: + delete provider; + return result; +} + +ncclResult_t niinGpunetioProxyAtomicBind(struct niinGpunetioProxyAtomicHostContext* provider, + struct niinContext* deviceContext) { + if (provider == nullptr || provider->atomicContextDevice == nullptr || deviceContext == nullptr) + return ncclInvalidArgument; + if (provider->boundDeviceContext != nullptr && provider->boundDeviceContext != deviceContext) + return ncclInvalidArgument; + const struct niinGpunetioAtomicContext* existing = nullptr; + if (cudaMemcpy(&existing, reinterpret_cast(deviceContext) + + offsetof(struct niinContext, gpunetioAtomicContext), + sizeof(existing), cudaMemcpyDeviceToHost) != cudaSuccess) + return ncclSystemError; + if (existing != nullptr && existing != provider->atomicContextDevice) return ncclInvalidUsage; + const struct niinGpunetioAtomicContext* context = provider->atomicContextDevice; + if (cudaMemcpy(reinterpret_cast(deviceContext) + offsetof(struct niinContext, gpunetioAtomicContext), + &context, sizeof(context), cudaMemcpyHostToDevice) != cudaSuccess) + return ncclSystemError; + provider->boundDeviceContext = deviceContext; + return ncclSuccess; +} + +ncclResult_t niinGpunetioProxyAtomicProgress(struct niinGpunetioProxyAtomicHostContext* provider) { + if (provider == nullptr) return ncclInvalidArgument; + if (provider->progressFailed.load(std::memory_order_acquire)) return ncclSystemError; + return progressOnce(provider); +} + +ncclResult_t niinGpunetioProxyAtomicFinalize(struct niinGpunetioProxyAtomicHostContext* provider) { + if (provider == nullptr) return ncclInvalidArgument; + // Every rank must arrive here only after all of its AMO-producing kernels + // complete. The private RC/SRQ fabric is peer-owned, so this collective + // quiesce prevents one PE from destroying a target QP/SRQ while another PE + // is still in the normal teardown path. + const ncclResult_t quiesce = allGatherSetupStatus(provider->comm, provider->nPes, ncclSuccess); + if (quiesce != ncclSuccess) return quiesce; + delete provider; + return ncclSuccess; +} diff --git a/projects/rccl/contrib/niin/gpunetio/proxy_atomics/tests/proxy_atomic_smoke.cu b/projects/rccl/contrib/niin/gpunetio/proxy_atomics/tests/proxy_atomic_smoke.cu new file mode 100644 index 000000000000..3182d3798692 --- /dev/null +++ b/projects/rccl/contrib/niin/gpunetio/proxy_atomics/tests/proxy_atomic_smoke.cu @@ -0,0 +1,387 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +// Two-rank smoke for NIIN's proxy-all AMO sidecar. Unlike the direct-WQE +// smoke, this deliberately sends a self AMO as well as remote integral and +// float/double/half AMOs: ProxyAll must put all of those operations through +// the same CPU RMW service rather than mixing local CUDA or NIC atomics. + +#include + +#include +#include + +#include + +#include "niin/host.h" +#include "niin/gpunetio/proxy_atomics/host.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr size_t kHeapBytes = 4096; +constexpr size_t kIntOffset = 0; +constexpr size_t kInt64Offset = 16; +constexpr size_t kFloatOffset = 64; +constexpr size_t kDoubleOffset = 72; +constexpr size_t kHalfOffset = 96; +constexpr size_t kSelfIntOffset = 128; +constexpr size_t kConcurrentOffset = 160; +constexpr int kConcurrentThreads = 32; + +struct Results { + uint32_t failures; + uint32_t remoteFetchAdd; + uint32_t remoteCompareSwap; + uint32_t remoteSwap; + uint32_t remoteFetch; + uint32_t remoteFetchInc; + uint32_t remoteFetchAnd; + uint32_t remoteFetchOr; + uint32_t remoteFetchXor; + uint64_t remoteFetchAdd64; + uint64_t remoteCompareSwap64; + uint64_t remoteSwap64; + uint64_t remoteFetch64; + uint64_t remoteFetchInc64; + uint64_t remoteFetchAnd64; + uint64_t remoteFetchOr64; + uint64_t remoteFetchXor64; + uint32_t selfFetchAdd; + float floatFetchAdd; + float floatSwap; + float floatFetch; + double doubleFetchAdd; + double doubleSwap; + double doubleFetch; + uint16_t halfFetchAdd; +}; + +[[noreturn]] void fail(int rank, const char* where, const char* detail) { + std::fprintf(stderr, "NIIN proxy atomic smoke [rank %d]: %s: %s\n", rank, where, detail); + MPI_Abort(MPI_COMM_WORLD, 1); + std::abort(); +} + +void checkCuda(int rank, cudaError_t status, const char* where) { + if (status != cudaSuccess) fail(rank, where, cudaGetErrorString(status)); +} + +void checkNccl(int rank, ncclResult_t status, const char* where) { + if (status != ncclSuccess) fail(rank, where, ncclGetErrorString(status)); +} + +int selectCudaDevice(int rank) { + int deviceCount = 0; + checkCuda(rank, cudaGetDeviceCount(&deviceCount), "cudaGetDeviceCount"); + if (deviceCount <= 0) fail(rank, "cudaGetDeviceCount", "no visible CUDA device"); + + MPI_Comm localComm = MPI_COMM_NULL; + if (MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &localComm) != MPI_SUCCESS) + fail(rank, "MPI_Comm_split_type", "could not discover the node-local rank"); + int localRank = 0; + if (MPI_Comm_rank(localComm, &localRank) != MPI_SUCCESS) + fail(rank, "MPI_Comm_rank", "could not read the node-local rank"); + MPI_Comm_free(&localComm); + + // A launcher may expose one GPU per process, while a local development + // launch can expose every GPU on the node. + return deviceCount == 1 ? 0 : localRank % deviceCount; +} + +void finalizeNiinContext(int rank, ncclComm_t comm, niinContext* deviceContext) { + checkNccl(rank, ncclGroupStart(), "ncclGroupStart(niinFinalize)"); + checkNccl(rank, niinFinalize(comm, deviceContext), "niinFinalize"); + checkNccl(rank, ncclGroupEnd(), "ncclGroupEnd(niinFinalize)"); + checkCuda(rank, cudaFree(deviceContext), "cudaFree(device context)"); +} + +__global__ void issueProxyAtomics(niinContext* context, Results* results) { + if (blockIdx.x != 0 || threadIdx.x != 0) return; + // Exercise the public NIIN/NVSHMEM spelling, not just the transport + // adapter. ProxyAll must intercept this before the normal self/LSA/direct + // branches so all AMOs share one target CPU RMW ordering domain. + niin_g_ctx = context; + const int selfPe = nvshmem_my_pe(); + const int targetPe = (selfPe + 1) % nvshmem_n_pes(); + char* heap = static_cast(niin_heap_base()); + uint32_t* intTarget = reinterpret_cast(heap + kIntOffset); + uint64_t* int64Target = reinterpret_cast(heap + kInt64Offset); + float* floatTarget = reinterpret_cast(heap + kFloatOffset); + double* doubleTarget = reinterpret_cast(heap + kDoubleOffset); + __half* halfTarget = reinterpret_cast<__half*>(heap + kHalfOffset); + uint32_t* selfIntTarget = reinterpret_cast(heap + kSelfIntOffset); + + uint32_t previous32 = nvshmem_uint32_atomic_fetch_add(intTarget, UINT32_C(5), targetPe); + results->remoteFetchAdd = previous32; + nvshmem_uint32_atomic_add(intTarget, UINT32_C(5), targetPe); + previous32 = nvshmem_uint32_atomic_compare_swap(intTarget, UINT32_C(20), UINT32_C(30), targetPe); + results->remoteCompareSwap = previous32; + previous32 = nvshmem_uint32_atomic_swap(intTarget, UINT32_C(40), targetPe); + results->remoteSwap = previous32; + previous32 = nvshmem_uint32_atomic_fetch(intTarget, targetPe); + results->remoteFetch = previous32; + nvshmem_uint32_atomic_set(intTarget, UINT32_C(50), targetPe); + nvshmem_uint32_atomic_inc(intTarget, targetPe); + previous32 = nvshmem_uint32_atomic_fetch_inc(intTarget, targetPe); + results->remoteFetchInc = previous32; + previous32 = nvshmem_uint32_atomic_fetch_and(intTarget, UINT32_C(0x3f), targetPe); + results->remoteFetchAnd = previous32; + nvshmem_uint32_atomic_and(intTarget, UINT32_C(0x0f), targetPe); + previous32 = nvshmem_uint32_atomic_fetch_or(intTarget, UINT32_C(0x10), targetPe); + results->remoteFetchOr = previous32; + nvshmem_uint32_atomic_or(intTarget, UINT32_C(0x20), targetPe); + previous32 = nvshmem_uint32_atomic_fetch_xor(intTarget, UINT32_C(0x3), targetPe); + results->remoteFetchXor = previous32; + nvshmem_uint32_atomic_xor(intTarget, UINT32_C(0x1), targetPe); + + uint64_t previous64 = nvshmem_uint64_atomic_fetch_add(int64Target, UINT64_C(5), targetPe); + results->remoteFetchAdd64 = previous64; + nvshmem_uint64_atomic_add(int64Target, UINT64_C(5), targetPe); + previous64 = + nvshmem_uint64_atomic_compare_swap(int64Target, UINT64_C(20), UINT64_C(30), targetPe); + results->remoteCompareSwap64 = previous64; + previous64 = nvshmem_uint64_atomic_swap(int64Target, UINT64_C(40), targetPe); + results->remoteSwap64 = previous64; + previous64 = nvshmem_uint64_atomic_fetch(int64Target, targetPe); + results->remoteFetch64 = previous64; + nvshmem_uint64_atomic_set(int64Target, UINT64_C(50), targetPe); + nvshmem_uint64_atomic_inc(int64Target, targetPe); + previous64 = nvshmem_uint64_atomic_fetch_inc(int64Target, targetPe); + results->remoteFetchInc64 = previous64; + previous64 = nvshmem_uint64_atomic_fetch_and(int64Target, UINT64_C(0x3f), targetPe); + results->remoteFetchAnd64 = previous64; + nvshmem_uint64_atomic_and(int64Target, UINT64_C(0x0f), targetPe); + previous64 = nvshmem_uint64_atomic_fetch_or(int64Target, UINT64_C(0x10), targetPe); + results->remoteFetchOr64 = previous64; + nvshmem_uint64_atomic_or(int64Target, UINT64_C(0x20), targetPe); + previous64 = nvshmem_uint64_atomic_fetch_xor(int64Target, UINT64_C(0x3), targetPe); + results->remoteFetchXor64 = previous64; + nvshmem_uint64_atomic_xor(int64Target, UINT64_C(0x1), targetPe); + + previous32 = nvshmem_uint32_atomic_fetch_add(selfIntTarget, UINT32_C(1), selfPe); + results->selfFetchAdd = previous32; + + float previousFloat = nvshmemx_float_atomic_fetch_add(floatTarget, 1.5f, targetPe); + results->floatFetchAdd = previousFloat; + nvshmemx_float_atomic_add(floatTarget, 0.5f, targetPe); + previousFloat = nvshmem_float_atomic_swap(floatTarget, 5.0f, targetPe); + results->floatSwap = previousFloat; + previousFloat = nvshmem_float_atomic_fetch(floatTarget, targetPe); + results->floatFetch = previousFloat; + nvshmem_float_atomic_set(floatTarget, 6.0f, targetPe); + + double previousDouble = nvshmemx_double_atomic_fetch_add(doubleTarget, 1.5, targetPe); + results->doubleFetchAdd = previousDouble; + nvshmemx_double_atomic_add(doubleTarget, 0.5, targetPe); + previousDouble = nvshmem_double_atomic_swap(doubleTarget, 5.0, targetPe); + results->doubleSwap = previousDouble; + previousDouble = nvshmem_double_atomic_fetch(doubleTarget, targetPe); + results->doubleFetch = previousDouble; + nvshmem_double_atomic_set(doubleTarget, 6.0, targetPe); + + const __half halfOne = __float2half_rn(1.0f); + __half previousHalf = nvshmemx_half_atomic_fetch_add(halfTarget, halfOne, targetPe); + results->halfFetchAdd = __half_as_ushort(previousHalf); + nvshmemx_half_atomic_add(halfTarget, __float2half_rn(0.5f), targetPe); + + results->failures = 0; +} + +// The first proxy implementation intentionally has one source slot and one +// GPU lock per destination. Exercise that serialization explicitly: all 32 +// callers must complete, while the target observes one linearized sum. +__global__ void issueConcurrentProxyAdds(niinContext* context) { + if (threadIdx.x == 0) niin_g_ctx = context; + __syncthreads(); + if (threadIdx.x >= kConcurrentThreads) return; + const int selfPe = nvshmem_my_pe(); + const int targetPe = (selfPe + 1) % nvshmem_n_pes(); + auto* target = reinterpret_cast(static_cast(niin_heap_base()) + kConcurrentOffset); + nvshmem_uint32_atomic_add(target, UINT32_C(1), targetPe); +} + +__global__ void readTarget(const uint32_t* remoteInt, const uint64_t* remoteInt64, + const uint32_t* remoteConcurrent, + const float* remoteFloat, + const double* remoteDouble, const uint16_t* remoteHalf, + uint32_t* intValue, uint64_t* int64Value, uint32_t* concurrentValue, + float* floatValue, double* doubleValue, uint16_t* halfValue) { + if (blockIdx.x != 0 || threadIdx.x != 0) return; + __threadfence_system(); + *intValue = *remoteInt; + *int64Value = *remoteInt64; + *concurrentValue = *remoteConcurrent; + *floatValue = *remoteFloat; + *doubleValue = *remoteDouble; + *halfValue = *remoteHalf; +} + +bool equalFloat(float a, float b) { return std::fabs(a - b) < 1.0e-6f; } +bool equalDouble(double a, double b) { return std::fabs(a - b) < 1.0e-12; } + +} // namespace + +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + int rank = -1; + int nPes = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nPes); + if (nPes != 2) fail(rank, "startup", "run with exactly two ranks"); + checkCuda(rank, cudaSetDevice(selectCudaDevice(rank)), "cudaSetDevice"); + + ncclUniqueId id; + if (rank == 0) checkNccl(rank, ncclGetUniqueId(&id), "ncclGetUniqueId"); + if (MPI_Bcast(&id, sizeof(id), MPI_BYTE, 0, MPI_COMM_WORLD) != MPI_SUCCESS) + fail(rank, "MPI_Bcast", "unique-id exchange failed"); + ncclComm_t comm = nullptr; + checkNccl(rank, ncclCommInitRank(&comm, nPes, id, rank), "ncclCommInitRank"); + + void* heap = nullptr; + checkNccl(rank, ncclMemAlloc(&heap, kHeapBytes), "ncclMemAlloc"); + std::array initial = {}; + if (rank == 1) { + const uint32_t intInitial = 10; + const uint64_t int64Initial = 10; + const uint32_t concurrentInitial = 0; + const float floatInitial = 2.0f; + const double doubleInitial = 2.0; + const uint16_t halfInitial = UINT16_C(0x3c00); + std::memcpy(initial.data() + kIntOffset, &intInitial, sizeof(intInitial)); + std::memcpy(initial.data() + kInt64Offset, &int64Initial, sizeof(int64Initial)); + std::memcpy(initial.data() + kConcurrentOffset, &concurrentInitial, sizeof(concurrentInitial)); + std::memcpy(initial.data() + kFloatOffset, &floatInitial, sizeof(floatInitial)); + std::memcpy(initial.data() + kDoubleOffset, &doubleInitial, sizeof(doubleInitial)); + std::memcpy(initial.data() + kHalfOffset, &halfInitial, sizeof(halfInitial)); + } else { + const uint32_t selfInitial = 7; + std::memcpy(initial.data() + kSelfIntOffset, &selfInitial, sizeof(selfInitial)); + } + checkCuda(rank, cudaMemcpy(heap, initial.data(), initial.size(), cudaMemcpyHostToDevice), "heap initialization"); + + // Exercise proxy routing through a normal committed NIIN context. Disable + // GIN here because the proxy is intentionally independent of it. + niinContext_host hostContext = {}; + checkNccl(rank, ncclGroupStart(), "ncclGroupStart(niinInit)"); + checkNccl(rank, niinInit(comm, heap, kHeapBytes, &hostContext, false), "niinInit"); + checkNccl(rank, ncclGroupEnd(), "ncclGroupEnd(niinInit)"); + niinContext* deviceContext = nullptr; + checkCuda(rank, cudaMalloc(&deviceContext, sizeof(*deviceContext)), "cudaMalloc(device context)"); + checkNccl(rank, niinCommit(&hostContext, deviceContext), "niinCommit"); + + niinGpunetioProxyAtomicOptions options = NIIN_GPUNETIO_PROXY_ATOMIC_OPTIONS_INITIALIZER; + niinGpunetioProxyAtomicHostContext* proxy = nullptr; + checkNccl(rank, niinGpunetioProxyAtomicInit(comm, heap, kHeapBytes, &options, &proxy), + "niinGpunetioProxyAtomicInit"); + checkNccl(rank, niinGpunetioProxyAtomicBind(proxy, deviceContext), "niinGpunetioProxyAtomicBind"); + if (MPI_Barrier(MPI_COMM_WORLD) != MPI_SUCCESS) fail(rank, "MPI_Barrier", "before AMOs"); + + Results* deviceResults = nullptr; + if (rank == 0) { + checkCuda(rank, cudaMalloc(&deviceResults, sizeof(*deviceResults)), "cudaMalloc(results)"); + issueProxyAtomics<<<1, 1>>>(deviceContext, deviceResults); + checkCuda(rank, cudaGetLastError(), "issueProxyAtomics launch"); + checkCuda(rank, cudaDeviceSynchronize(), "issueProxyAtomics completion"); + issueConcurrentProxyAdds<<<1, kConcurrentThreads>>>(deviceContext); + checkCuda(rank, cudaGetLastError(), "issueConcurrentProxyAdds launch"); + checkCuda(rank, cudaDeviceSynchronize(), "issueConcurrentProxyAdds completion"); + } + if (MPI_Barrier(MPI_COMM_WORLD) != MPI_SUCCESS) fail(rank, "MPI_Barrier", "after AMOs"); + + uint32_t finalInt = 0; + uint64_t finalInt64 = 0; + uint32_t finalConcurrent = 0; + float finalFloat = 0.0f; + double finalDouble = 0.0; + uint16_t finalHalf = 0; + if (rank == 1) { + uint32_t* dInt = nullptr; + uint64_t* dInt64 = nullptr; + uint32_t* dConcurrent = nullptr; + float* dFloat = nullptr; + double* dDouble = nullptr; + uint16_t* dHalf = nullptr; + checkCuda(rank, cudaMalloc(&dInt, sizeof(*dInt)), "cudaMalloc(final int)"); + checkCuda(rank, cudaMalloc(&dInt64, sizeof(*dInt64)), "cudaMalloc(final int64)"); + checkCuda(rank, cudaMalloc(&dConcurrent, sizeof(*dConcurrent)), "cudaMalloc(final concurrent)"); + checkCuda(rank, cudaMalloc(&dFloat, sizeof(*dFloat)), "cudaMalloc(final float)"); + checkCuda(rank, cudaMalloc(&dDouble, sizeof(*dDouble)), "cudaMalloc(final double)"); + checkCuda(rank, cudaMalloc(&dHalf, sizeof(*dHalf)), "cudaMalloc(final half)"); + readTarget<<<1, 1>>>(static_cast(heap) + kIntOffset / sizeof(uint32_t), + reinterpret_cast(static_cast(heap) + kInt64Offset), + reinterpret_cast(static_cast(heap) + kConcurrentOffset), + reinterpret_cast(static_cast(heap) + kFloatOffset), + reinterpret_cast(static_cast(heap) + kDoubleOffset), + reinterpret_cast(static_cast(heap) + kHalfOffset), + dInt, dInt64, dConcurrent, dFloat, dDouble, dHalf); + checkCuda(rank, cudaGetLastError(), "readTarget launch"); + checkCuda(rank, cudaDeviceSynchronize(), "readTarget completion"); + checkCuda(rank, cudaMemcpy(&finalInt, dInt, sizeof(finalInt), cudaMemcpyDeviceToHost), "copy final int"); + checkCuda(rank, cudaMemcpy(&finalInt64, dInt64, sizeof(finalInt64), cudaMemcpyDeviceToHost), + "copy final int64"); + checkCuda(rank, cudaMemcpy(&finalConcurrent, dConcurrent, sizeof(finalConcurrent), cudaMemcpyDeviceToHost), + "copy final concurrent"); + checkCuda(rank, cudaMemcpy(&finalFloat, dFloat, sizeof(finalFloat), cudaMemcpyDeviceToHost), "copy final float"); + checkCuda(rank, cudaMemcpy(&finalDouble, dDouble, sizeof(finalDouble), cudaMemcpyDeviceToHost), "copy final double"); + checkCuda(rank, cudaMemcpy(&finalHalf, dHalf, sizeof(finalHalf), cudaMemcpyDeviceToHost), "copy final half"); + cudaFree(dHalf); + cudaFree(dDouble); + cudaFree(dFloat); + cudaFree(dConcurrent); + cudaFree(dInt64); + cudaFree(dInt); + } + + int localPass = 1; + if (rank == 0) { + Results actual = {}; + checkCuda(rank, cudaMemcpy(&actual, deviceResults, sizeof(actual), cudaMemcpyDeviceToHost), "copy results"); + localPass = actual.failures == 0 && actual.remoteFetchAdd == 10 && actual.remoteCompareSwap == 20 && + actual.remoteSwap == 30 && actual.remoteFetch == 40 && actual.remoteFetchInc == 51 && + actual.remoteFetchAnd == 52 && actual.remoteFetchOr == 4 && actual.remoteFetchXor == 52 && + actual.remoteFetchAdd64 == 10 && actual.remoteCompareSwap64 == 20 && + actual.remoteSwap64 == 30 && actual.remoteFetch64 == 40 && actual.remoteFetchInc64 == 51 && + actual.remoteFetchAnd64 == 52 && actual.remoteFetchOr64 == 4 && + actual.remoteFetchXor64 == 52 && + actual.selfFetchAdd == 7 && equalFloat(actual.floatFetchAdd, 2.0f) && + equalFloat(actual.floatSwap, 4.0f) && equalFloat(actual.floatFetch, 5.0f) && + equalDouble(actual.doubleFetchAdd, 2.0) && equalDouble(actual.doubleSwap, 4.0) && + equalDouble(actual.doubleFetch, 5.0) && actual.halfFetchAdd == UINT16_C(0x3c00); + if (!localPass) std::fprintf(stderr, "NIIN proxy atomic smoke: source results mismatch\n"); + } + if (rank == 1 && (finalInt != 54 || finalInt64 != 54 || finalConcurrent != kConcurrentThreads || + !equalFloat(finalFloat, 6.0f) || + !equalDouble(finalDouble, 6.0) || finalHalf != UINT16_C(0x4100))) { + std::fprintf(stderr, + "NIIN proxy atomic smoke: targets int=%u int64=%llu concurrent=%u float=%f double=%f half=0x%04x\n", + finalInt, static_cast(finalInt64), finalConcurrent, finalFloat, + finalDouble, finalHalf); + localPass = 0; + } + int globalPass = 0; + if (MPI_Allreduce(&localPass, &globalPass, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD) != MPI_SUCCESS) + fail(rank, "MPI_Allreduce", "result reduction failed"); + + if (deviceResults != nullptr) cudaFree(deviceResults); + if (MPI_Barrier(MPI_COMM_WORLD) != MPI_SUCCESS) fail(rank, "MPI_Barrier", "before finalization"); + checkNccl(rank, niinGpunetioProxyAtomicFinalize(proxy), "niinGpunetioProxyAtomicFinalize"); + finalizeNiinContext(rank, comm, deviceContext); + checkNccl(rank, ncclMemFree(heap), "ncclMemFree"); + checkNccl(rank, ncclCommDestroy(comm), "ncclCommDestroy"); + MPI_Finalize(); + if (!globalPass) return 1; + std::fprintf(stderr, "NIIN proxy atomic smoke: PASS\n"); + return 0; +} diff --git a/projects/rccl/contrib/niin/gpunetio/src/atomics_host.cc b/projects/rccl/contrib/niin/gpunetio/src/atomics_host.cc new file mode 100644 index 000000000000..70dfd892ace1 --- /dev/null +++ b/projects/rccl/contrib/niin/gpunetio/src/atomics_host.cc @@ -0,0 +1,904 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +// NIIN's direct GPUNetIO atomic provider. +// +// This file intentionally uses only public NCCL, CUDA, ibverbs, and +// GPUNetIO APIs. In particular, it must not include NCCL transport headers +// or inspect a GIN context: GIN remains the implementation for NIIN RMA and +// signaling, while this provider owns only its atomic QPs and registrations. + +#include "niin/gpunetio/host.h" + +#include "niin/context_abi.h" +#include "niin/gpunetio/context.h" + +#include +#include +#if __has_include() +#include +#elif __has_include() +#include +#else +#error "NIIN GPUNetIO atomics requires GPUNetIO's public host header" +#endif +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint32_t kWireMagic = 0x4e49414du; // "NIAM" +constexpr uint16_t kWireVersion = 1; +constexpr uint32_t kMinSqDepth = 64; +constexpr uint32_t kMaxSqDepth = 32768; +// Eight bytes hold the response value and byte eight is the independent +// scratch byte used by a fenced DUMP. Keep the stride naturally aligned so +// both 32- and 64-bit response loads remain aligned. +constexpr uint32_t kResponseStride = 16; +constexpr uint32_t kCstScratchOffset = sizeof(uint64_t); +constexpr uint8_t kQpPsn = 0; +constexpr uint16_t kAckTimeout = 20; +constexpr uint16_t kRetryCount = 7; +constexpr uint16_t kRnrRetryCount = 7; +constexpr uint16_t kMinRnrTimer = 12; +constexpr uint8_t kMaxOutstandingAtomics = 1; + +// The public GPUNetIO headers expose the MLX5 capability layout but purposefully +// do not expose a capability-query convenience API. Query the standard +// mlx5dv DevX entry point dynamically so this NIIN client remains compatible +// with GPUNetIO's own dynamic libmlx5 loading model and does not need to use +// GPUNetIO private source headers. +using Mlx5dvDevxGeneralCmd = int (*)(struct ibv_context*, const void*, size_t, void*, size_t); + +constexpr uint32_t kMlx5AtomicOpCas = 0x1; +constexpr uint32_t kMlx5AtomicOpFetchAdd = 0x2; +constexpr uint32_t kMlx5AtomicOpMaskedCas = 0x4; +constexpr uint32_t kMlx5AtomicOpMaskedFetchAdd = 0x8; +constexpr uint32_t kMlx5AtomicOpsRequired = + kMlx5AtomicOpCas | kMlx5AtomicOpFetchAdd | kMlx5AtomicOpMaskedCas | + kMlx5AtomicOpMaskedFetchAdd; +constexpr uint32_t kMlx5AtomicSize4B = 0x4; +constexpr uint32_t kMlx5AtomicSize8B = 0x8; +constexpr uint32_t kMlx5AtomicSizesRequired = kMlx5AtomicSize4B | kMlx5AtomicSize8B; + +struct Mlx5AtomicCapabilities { + // 4-byte atomic responses are always network byte order. Some HCAs can + // report 8-byte responses in host/device byte order; retain that negotiated + // mode in NIIN's device ABI rather than assuming one family of NICs. + bool atomic8BResultHostEndian = false; +}; + +// `mlx5_ifc.h` models bit fields as byte arrays whose offsetof/sizeof values +// are bit offsets/sizes. Keep the tiny encoder/decoder private to this file +// instead of reaching into GPUNetIO's private DevX wrapper. +bool mlx5SetField(uint8_t* command, size_t bitOffset, size_t bitSize, uint32_t value) { + if (bitSize == 0 || bitSize > 32 || (bitOffset & 31u) + bitSize > 32) return false; + const uint32_t fieldMask = bitSize == 32 ? UINT32_MAX : (uint32_t{1} << bitSize) - 1; + uint32_t wordBe = 0; + std::memcpy(&wordBe, command + (bitOffset / 32) * sizeof(wordBe), sizeof(wordBe)); + const uint32_t shift = 32 - static_cast(bitSize) - static_cast(bitOffset & 31u); + const uint32_t word = be32toh(wordBe); + wordBe = htobe32((word & ~(fieldMask << shift)) | ((value & fieldMask) << shift)); + std::memcpy(command + (bitOffset / 32) * sizeof(wordBe), &wordBe, sizeof(wordBe)); + return true; +} + +bool mlx5GetField(const uint8_t* command, size_t bitOffset, size_t bitSize, uint32_t* value) { + if (value == nullptr || bitSize == 0 || bitSize > 32 || (bitOffset & 31u) + bitSize > 32) return false; + const uint32_t fieldMask = bitSize == 32 ? UINT32_MAX : (uint32_t{1} << bitSize) - 1; + uint32_t wordBe = 0; + std::memcpy(&wordBe, command + (bitOffset / 32) * sizeof(wordBe), sizeof(wordBe)); + const uint32_t shift = 32 - static_cast(bitSize) - static_cast(bitOffset & 31u); + *value = (be32toh(wordBe) >> shift) & fieldMask; + return true; +} + +bool queryRequiredMlx5AtomicCaps(struct ibv_context* verbsContext, Mlx5AtomicCapabilities* capabilities) { + if (verbsContext == nullptr || capabilities == nullptr) return false; + + alignas(uint32_t) uint8_t commandIn[MLX5_ST_SZ_BYTES(query_hca_cap_in)] = {}; + alignas(uint32_t) uint8_t commandOut[MLX5_ST_SZ_BYTES(query_hca_cap_out)] = {}; + if (!mlx5SetField(commandIn, offsetof(struct mlx5_ifc_query_hca_cap_in_bits, opcode), + MLX5_FLD_SZ_BITS(query_hca_cap_in, opcode), MLX5_CMD_OP_QUERY_HCA_CAP) || + !mlx5SetField(commandIn, offsetof(struct mlx5_ifc_query_hca_cap_in_bits, op_mod), + MLX5_FLD_SZ_BITS(query_hca_cap_in, op_mod), + MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE | (MLX5_CAP_ATOMIC << 1) | + HCA_CAP_OPMOD_GET_CUR)) + return false; + + void* mlx5 = dlopen("libmlx5.so.1", RTLD_NOW | RTLD_LOCAL); + if (mlx5 == nullptr) mlx5 = dlopen("libmlx5.so", RTLD_NOW | RTLD_LOCAL); + if (mlx5 == nullptr) return false; + const auto generalCmd = reinterpret_cast(dlsym(mlx5, "mlx5dv_devx_general_cmd")); + const int status = generalCmd == nullptr + ? -1 + : generalCmd(verbsContext, commandIn, sizeof(commandIn), commandOut, + sizeof(commandOut)); + dlclose(mlx5); + if (status != 0) return false; + + const uint8_t* atomicCaps = commandOut + MLX5_BYTE_OFF(query_hca_cap_out, capability); + uint32_t operations = 0; + uint32_t qpSizes = 0; + uint32_t atomic8BEndiannessMode = 0; + uint32_t atomic8BHostEndianSupported = 0; + if (!mlx5GetField(atomicCaps, offsetof(struct mlx5_ifc_atomic_caps_bits, atomic_operations), + MLX5_FLD_SZ_BITS(atomic_caps, atomic_operations), &operations) || + !mlx5GetField(atomicCaps, offsetof(struct mlx5_ifc_atomic_caps_bits, atomic_size_qp), + MLX5_FLD_SZ_BITS(atomic_caps, atomic_size_qp), &qpSizes) || + !mlx5GetField(atomicCaps, offsetof(struct mlx5_ifc_atomic_caps_bits, + atomic_req_8B_endianness_mode), + MLX5_FLD_SZ_BITS(atomic_caps, atomic_req_8B_endianness_mode), + &atomic8BEndiannessMode) || + !mlx5GetField(atomicCaps, offsetof(struct mlx5_ifc_atomic_caps_bits, + supported_atomic_req_8B_endianness_mode_1), + MLX5_FLD_SZ_BITS(atomic_caps, supported_atomic_req_8B_endianness_mode_1), + &atomic8BHostEndianSupported)) + return false; + + // NIIN creates RC QPs, so only QP (not DC) atomic-size capability is a + // requirement. All four operation bits are required because the public + // NIIN surface includes masked set/bitwise AMOs as well as standard CAS/FA. + if ((operations & kMlx5AtomicOpsRequired) != kMlx5AtomicOpsRequired || + (qpSizes & kMlx5AtomicSizesRequired) != kMlx5AtomicSizesRequired) + return false; + + // This is the same capability interpretation used by NVSHMEM's transport: + // only a supported, nonzero mode permits host-endian 8-byte responses. + capabilities->atomic8BResultHostEndian = + atomic8BHostEndianSupported != 0 && atomic8BEndiannessMode != 0; + return true; +} + +struct niinGpunetioAtomicWire { + uint32_t magic; + uint16_t version; + uint16_t sourcePe; + uint16_t destinationPe; + uint16_t lid; + uint32_t qpn; + uint8_t gid[DOCA_VERBS_GID_BYTE_LENGTH]; + uint8_t linkLayer; + uint8_t activeMtu; + uint16_t reserved0; + uint64_t heapBase; + uint64_t heapBytes; + // This is in the mkey byte order expected by GPUNetIO's device WQE helpers. + uint32_t heapRkey; + uint32_t reserved1; +}; + +static_assert(sizeof(niinGpunetioAtomicWire) % alignof(uint64_t) == 0, + "NIIN GPUNetIO wire metadata must retain 64-bit alignment"); + +bool isPowerOfTwo(uint32_t value) { + return value != 0 && (value & (value - 1)) == 0; +} + +int readEnvInt(const char* name, int fallback) { + const char* text = std::getenv(name); + if (text == nullptr || *text == '\0') return fallback; + char* end = nullptr; + const long value = std::strtol(text, &end, 10); + if (end == text || *end != '\0' || value < std::numeric_limits::min() || + value > std::numeric_limits::max()) + return fallback; + return static_cast(value); +} + +uint32_t toGpunetioMkey(uint32_t key) { +#if DOCA_GPUNETIO_VERBS_MKEY_SWAPPED == 1 + return __builtin_bswap32(key); +#else + return key; +#endif +} + +enum doca_verbs_mtu_size toDocaMtu(int ibvMtu) { + switch (ibvMtu) { + case IBV_MTU_256: return DOCA_VERBS_MTU_SIZE_256_BYTES; + case IBV_MTU_512: return DOCA_VERBS_MTU_SIZE_512_BYTES; + case IBV_MTU_1024: return DOCA_VERBS_MTU_SIZE_1K_BYTES; + case IBV_MTU_2048: return DOCA_VERBS_MTU_SIZE_2K_BYTES; + case IBV_MTU_4096: return DOCA_VERBS_MTU_SIZE_4K_BYTES; + default: return DOCA_VERBS_MTU_SIZE_256_BYTES; + } +} + +ncclResult_t cudaToNccl(cudaError_t status) { + return status == cudaSuccess ? ncclSuccess : ncclSystemError; +} + +} // namespace + +struct niinGpunetioAtomicHostContext { + ncclComm_t comm = nullptr; + int rank = -1; + int nPes = 0; + + struct ibv_context* verbsContext = nullptr; + struct ibv_pd* pd = nullptr; + struct ibv_port_attr portAttr = {}; + union ibv_gid localGid = {}; + int port = 0; + int gidIndex = 0; + uint16_t pkeyIndex = 0; + uint8_t serviceLevel = 0; + bool atomic8BResultHostEndian = false; + + doca_gpu_t* gpu = nullptr; + doca_dev_t* netDev = nullptr; + + struct ibv_mr* heapMr = nullptr; + struct ibv_mr* responseMr = nullptr; + void* responseBase = nullptr; + size_t responseBytes = 0; + + // qps[pe] is the private outbound atomic QP to pe. The local-PE entry is + // intentionally null: NIIN continues to use CUDA atomics for self/LSA. + std::vector qps; + std::vector exportedQps; + struct doca_gpu_dev_verbs_qp* deviceQps = nullptr; + + std::vector endpointHost; + struct niinGpunetioAtomicEndpoint* endpointDevice = nullptr; + int* endpointLocksDevice = nullptr; + struct niinGpunetioAtomicContext atomicContextHost = {}; + struct niinGpunetioAtomicContext* atomicContextDevice = nullptr; + struct niinContext* boundDeviceContext = nullptr; + + std::atomic stopProgress{false}; + std::atomic progressFailed{false}; + std::thread progressThread; + + ~niinGpunetioAtomicHostContext() { cleanup(); } + + bool hasCpuProxyQp() const { + for (const auto* qp : qps) { + if (qp != nullptr && qp->qp_gverbs != nullptr && qp->qp_gverbs->qp_cpu != nullptr && + (qp->qp_gverbs->qp_cpu->nic_handler & + DOCA_GPUNETIO_VERBS_NIC_HANDLER_FLAG_CPU_PROXY) != 0) + return true; + } + return false; + } + + ncclResult_t progressOnce(bool* didProgress = nullptr) { + bool progressed = false; + for (const auto* qp : qps) { + if (qp == nullptr || qp->qp_gverbs == nullptr || qp->qp_gverbs->qp_cpu == nullptr || + (qp->qp_gverbs->qp_cpu->nic_handler & + DOCA_GPUNETIO_VERBS_NIC_HANDLER_FLAG_CPU_PROXY) == 0) + continue; + bool thisQpProgressed = false; + if (doca_gpu_verbs_cpu_proxy_progress(qp->qp_gverbs, &thisQpProgressed) != DOCA_SUCCESS) { + progressFailed.store(true, std::memory_order_release); + return ncclSystemError; + } + progressed = progressed || thisQpProgressed; + } + if (didProgress != nullptr) *didProgress = progressed; + return ncclSuccess; + } + + void startProgressThread() { + if (!hasCpuProxyQp()) return; + stopProgress.store(false, std::memory_order_release); + progressThread = std::thread([this] { + while (!stopProgress.load(std::memory_order_acquire)) { + bool progressed = false; + if (progressOnce(&progressed) != ncclSuccess) return; + if (!progressed) std::this_thread::yield(); + } + }); + } + + void clearBinding() { + if (boundDeviceContext == nullptr) return; + const struct niinGpunetioAtomicContext* currentContext = nullptr; + const cudaError_t readStatus = cudaMemcpy( + ¤tContext, + reinterpret_cast(boundDeviceContext) + + offsetof(struct niinContext, gpunetioAtomicContext), + sizeof(currentContext), cudaMemcpyDeviceToHost); + const struct niinGpunetioAtomicContext* nullContext = nullptr; + // Do not clear a newer provider which was bound after this one. Bind() + // rejects that normal misuse, but this comparison also makes teardown + // robust when a caller has updated the context explicitly. + if (readStatus == cudaSuccess && currentContext == atomicContextDevice) { + cudaMemcpy(reinterpret_cast(boundDeviceContext) + + offsetof(struct niinContext, gpunetioAtomicContext), + &nullContext, sizeof(nullContext), cudaMemcpyHostToDevice); + } + boundDeviceContext = nullptr; + } + + void cleanup() { + stopProgress.store(true, std::memory_order_release); + if (progressThread.joinable()) progressThread.join(); + + clearBinding(); + + if (atomicContextDevice != nullptr) cudaFree(atomicContextDevice); + atomicContextDevice = nullptr; + if (endpointLocksDevice != nullptr) cudaFree(endpointLocksDevice); + endpointLocksDevice = nullptr; + if (endpointDevice != nullptr) cudaFree(endpointDevice); + endpointDevice = nullptr; + + if (deviceQps != nullptr && gpu != nullptr && !exportedQps.empty()) { + doca_gpu_verbs_unexport_multi_qps_dev(gpu, exportedQps.data(), exportedQps.size(), deviceQps); + } + deviceQps = nullptr; + exportedQps.clear(); + + for (auto*& qp : qps) { + if (qp != nullptr) doca_gpu_verbs_destroy_qp_hl(qp); + qp = nullptr; + } + qps.clear(); + + if (responseMr != nullptr) ibv_dereg_mr(responseMr); + responseMr = nullptr; + if (heapMr != nullptr) ibv_dereg_mr(heapMr); + heapMr = nullptr; + if (responseBase != nullptr) cudaFree(responseBase); + responseBase = nullptr; + + if (netDev != nullptr) doca_verbs_dev_close(netDev); + netDev = nullptr; + if (pd != nullptr) ibv_dealloc_pd(pd); + pd = nullptr; + if (verbsContext != nullptr) ibv_close_device(verbsContext); + verbsContext = nullptr; + if (gpu != nullptr) doca_gpu_destroy(gpu); + gpu = nullptr; + } +}; + +namespace { + +ncclResult_t openVerbsDevice(niinGpunetioAtomicHostContext* provider, + const struct niinGpunetioAtomicOptions& options) { + const char* requestedDevice = options.ibDevice; + if (requestedDevice == nullptr || *requestedDevice == '\0') { + requestedDevice = std::getenv("NIIN_GPUNETIO_IB_DEV"); + } + const int requestedPort = options.ibPort != 0 ? options.ibPort : readEnvInt("NIIN_GPUNETIO_IB_PORT", 0); + const int requestedGidIndex = + options.gidIndex >= 0 ? options.gidIndex : readEnvInt("NIIN_GPUNETIO_GID_INDEX", 0); + if (requestedPort < 0 || requestedGidIndex < 0 || requestedGidIndex > UINT8_MAX) return ncclInvalidArgument; + + int deviceCount = 0; + struct ibv_device** devices = ibv_get_device_list(&deviceCount); + if (devices == nullptr) return ncclSystemError; + + ncclResult_t result = ncclSystemError; + for (int deviceIndex = 0; deviceIndex < deviceCount && provider->verbsContext == nullptr; ++deviceIndex) { + if (requestedDevice != nullptr && std::strcmp(ibv_get_device_name(devices[deviceIndex]), requestedDevice) != 0) + continue; + + struct ibv_context* context = ibv_open_device(devices[deviceIndex]); + if (context == nullptr) continue; + + struct ibv_device_attr deviceAttr = {}; + if (ibv_query_device(context, &deviceAttr) != 0) { + ibv_close_device(context); + continue; + } + + for (int port = 1; port <= deviceAttr.phys_port_cnt; ++port) { + if (requestedPort != 0 && port != requestedPort) continue; + struct ibv_port_attr portAttr = {}; + if (ibv_query_port(context, port, &portAttr) != 0 || portAttr.state != IBV_PORT_ACTIVE) continue; + + union ibv_gid gid = {}; + if (ibv_query_gid(context, port, requestedGidIndex, &gid) != 0) continue; + + struct ibv_pd* pd = ibv_alloc_pd(context); + if (pd == nullptr) continue; + + // GPUNetIO Open 4.1's public high-level QP path resolves the RoCE + // address using port 1. An IB port can use any active port, but reject + // an Ethernet port other than 1 rather than construct a QP that may + // resolve the wrong MAC address. + if (portAttr.link_layer == IBV_LINK_LAYER_ETHERNET && port != 1) { + ibv_dealloc_pd(pd); + continue; + } + + provider->verbsContext = context; + provider->pd = pd; + provider->portAttr = portAttr; + provider->localGid = gid; + provider->port = port; + provider->gidIndex = requestedGidIndex; + result = ncclSuccess; + break; + } + + if (provider->verbsContext == nullptr) ibv_close_device(context); + } + + ibv_free_device_list(devices); + return result; +} + +ncclResult_t openGpunetio(niinGpunetioAtomicHostContext* provider) { + int cudaDevice = -1; + if (cudaGetDevice(&cudaDevice) != cudaSuccess) return ncclSystemError; + + char pciBusId[32] = {}; + if (cudaDeviceGetPCIBusId(pciBusId, sizeof(pciBusId), cudaDevice) != cudaSuccess) return ncclSystemError; + if (doca_gpu_create(pciBusId, &provider->gpu) != DOCA_SUCCESS) return ncclSystemError; + if (doca_verbs_dev_open(provider->pd, &provider->netDev) != DOCA_SUCCESS) return ncclSystemError; + return ncclSuccess; +} + +ncclResult_t registerAtomicMemory(niinGpunetioAtomicHostContext* provider, void* heapBase, + size_t heapBytes) { + // This provider exposes atomic WQEs only. Do not grant ordinary remote + // read/write access to either its symmetric heap registration or its local + // response ring. + const int heapAccess = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_ATOMIC; + provider->heapMr = ibv_reg_mr(provider->pd, heapBase, heapBytes, heapAccess); + if (provider->heapMr == nullptr) return ncclSystemError; + + provider->responseBytes = static_cast(provider->nPes) * kResponseStride; + if (cudaMalloc(&provider->responseBase, provider->responseBytes) != cudaSuccess) return ncclSystemError; + if (cudaMemset(provider->responseBase, 0, provider->responseBytes) != cudaSuccess) return ncclSystemError; + + // The response ring receives Atomic response DMA and is also the source of + // the terminal DUMP used to establish response visibility. Giving it the + // same access flags keeps registration semantics simple; no remote address + // from this MR is ever exchanged or used for NIIN RMA. + provider->responseMr = ibv_reg_mr(provider->pd, provider->responseBase, provider->responseBytes, + heapAccess); + return provider->responseMr != nullptr ? ncclSuccess : ncclSystemError; +} + +enum doca_gpu_dev_verbs_nic_handler selectNicHandler(enum niinGpunetioAtomicNicHandler requested) { + switch (requested) { + case NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_GPU_SM_DB: + return DOCA_GPUNETIO_VERBS_NIC_HANDLER_GPU_SM_DB; + case NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_CPU_PROXY: + return DOCA_GPUNETIO_VERBS_NIC_HANDLER_CPU_PROXY; + case NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_AUTO: + default: + return DOCA_GPUNETIO_VERBS_NIC_HANDLER_AUTO; + } +} + +ncclResult_t createPrivateQps(niinGpunetioAtomicHostContext* provider, + const struct niinGpunetioAtomicOptions& options, + std::vector* localWire) { + struct doca_gpu_verbs_qp_init_attr_hl initAttr = {}; + initAttr.gpu_dev = provider->gpu; + initAttr.net_dev = provider->netDev; + initAttr.ibpd = provider->pd; + initAttr.sq_nwqe = static_cast(options.sqDepth); + initAttr.nic_handler = selectNicHandler(options.nicHandler); + initAttr.mreg_type = DOCA_GPUNETIO_VERBS_MEM_REG_TYPE_DEFAULT; + // Keep the exported device-QP CQ type in sync with the hardware CQ setup. + // A zero-initialized cq_type means normal 64B CQEs even when cq_collapsed is + // true, which makes the generic device poller decode collapsed CQEs wrong. + initAttr.cq_type = DOCA_GPUNETIO_VERBS_CQ_64B_COLLAPSED; + initAttr.cq_collapsed = true; + + provider->qps.resize(provider->nPes, nullptr); + localWire->assign(provider->nPes, {}); + for (int pe = 0; pe < provider->nPes; ++pe) { + struct niinGpunetioAtomicWire& wire = (*localWire)[pe]; + wire.magic = kWireMagic; + wire.version = kWireVersion; + wire.sourcePe = static_cast(provider->rank); + wire.destinationPe = static_cast(pe); + wire.lid = provider->portAttr.lid; + std::memcpy(wire.gid, &provider->localGid, sizeof(wire.gid)); + wire.linkLayer = static_cast(provider->portAttr.link_layer); + wire.activeMtu = static_cast(provider->portAttr.active_mtu); + wire.heapBase = reinterpret_cast(provider->heapMr->addr); + wire.heapBytes = provider->heapMr->length; + wire.heapRkey = toGpunetioMkey(provider->heapMr->rkey); + + if (pe == provider->rank) continue; + if (doca_gpu_verbs_create_qp_hl(&initAttr, &provider->qps[pe]) != DOCA_SUCCESS) + return ncclSystemError; + if (doca_verbs_qp_get_qpn(provider->qps[pe]->qp, &wire.qpn) != DOCA_SUCCESS) + return ncclSystemError; + } + + // The direct NIIN provider has been validated with GPU SM doorbells. The + // GPUNetIO CPU doorbell proxy is a different host-visible publication path; + // on the provider version paired with this NCCL build it can leave a device + // AMO waiting indefinitely. Reject it collectively during setup instead + // of publishing a context that can hang a user kernel. This is unrelated + // to NIIN's future SRQ software-atomic proxy. + if (provider->hasCpuProxyQp()) { + std::fprintf(stderr, + "NIIN GPUNetIO atomics: CPU doorbell proxy is not supported by the direct " + "atomic provider; require GPU SM doorbells\n"); + return ncclInvalidUsage; + } + return ncclSuccess; +} + +ncclResult_t allGatherWire(ncclComm_t comm, const std::vector& local, + int nPes, std::vector* remote) { + if (local.size() != static_cast(nPes)) return ncclInvalidArgument; + const size_t localBytes = local.size() * sizeof(niinGpunetioAtomicWire); + const size_t remoteBytes = localBytes * static_cast(nPes); + void* deviceSend = nullptr; + void* deviceRecv = nullptr; + cudaStream_t stream = nullptr; + + ncclResult_t result = cudaToNccl(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMalloc(&deviceSend, localBytes)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMalloc(&deviceRecv, remoteBytes)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMemcpyAsync(deviceSend, local.data(), localBytes, cudaMemcpyHostToDevice, stream)); + if (result != ncclSuccess) goto cleanup; + result = ncclAllGather(deviceSend, deviceRecv, localBytes, ncclUint8, comm, stream); + if (result != ncclSuccess) goto cleanup; + + remote->resize(static_cast(nPes) * static_cast(nPes)); + result = cudaToNccl( + cudaMemcpyAsync(remote->data(), deviceRecv, remoteBytes, cudaMemcpyDeviceToHost, stream)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaStreamSynchronize(stream)); + +cleanup: + if (deviceRecv != nullptr) cudaFree(deviceRecv); + if (deviceSend != nullptr) cudaFree(deviceSend); + if (stream != nullptr) cudaStreamDestroy(stream); + return result; +} + +// `niinGpunetioAtomicInit` is collective even though most of its setup is +// local. Do not let one PE return early after a local verbs/GPUNetIO failure +// while another continues into the descriptor all-gather: that would turn an +// ordinary unsupported-HCA error into a peer hang. A byte all-gather keeps +// this bootstrap independent of NCCL reduction datatype/operator support. +// +// CUDA/NCCL failures while executing this helper are necessarily fatal to the +// collective itself, but all ordinary local setup failures reach this point on +// every PE before the next stage begins. +ncclResult_t allGatherSetupStatus(ncclComm_t comm, int nPes, ncclResult_t localStatus) { + if (nPes <= 0) return ncclInvalidArgument; + + uint32_t localCode = static_cast(localStatus); + std::vector remoteCodes(static_cast(nPes), static_cast(ncclSystemError)); + void* deviceSend = nullptr; + void* deviceRecv = nullptr; + cudaStream_t stream = nullptr; + + ncclResult_t result = cudaToNccl(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMalloc(&deviceSend, sizeof(localCode))); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMalloc(&deviceRecv, remoteCodes.size() * sizeof(remoteCodes[0]))); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMemcpyAsync(deviceSend, &localCode, sizeof(localCode), cudaMemcpyHostToDevice, stream)); + if (result != ncclSuccess) goto cleanup; + result = ncclAllGather(deviceSend, deviceRecv, sizeof(localCode), ncclUint8, comm, stream); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaMemcpyAsync(remoteCodes.data(), deviceRecv, + remoteCodes.size() * sizeof(remoteCodes[0]), cudaMemcpyDeviceToHost, + stream)); + if (result != ncclSuccess) goto cleanup; + result = cudaToNccl(cudaStreamSynchronize(stream)); + if (result != ncclSuccess) goto cleanup; + + for (uint32_t code : remoteCodes) { + if (code != static_cast(ncclSuccess)) { + // Preserve the most useful local error on the failing PE. A peer only + // learns that setup failed collectively; it cannot safely infer which + // external resource (NIC, GPU mapping, or registration) failed there. + result = localStatus == ncclSuccess ? ncclSystemError : localStatus; + goto cleanup; + } + } + result = ncclSuccess; + +cleanup: + if (deviceRecv != nullptr) cudaFree(deviceRecv); + if (deviceSend != nullptr) cudaFree(deviceSend); + if (stream != nullptr) cudaStreamDestroy(stream); + return result; +} + +ncclResult_t connectPeerQp(niinGpunetioAtomicHostContext* provider, int peer, + const struct niinGpunetioAtomicWire& remote) { + if (peer == provider->rank || provider->qps[peer] == nullptr || remote.magic != kWireMagic || + remote.version != kWireVersion || remote.sourcePe != peer || remote.destinationPe != provider->rank || + remote.qpn == 0 || remote.heapBase == 0 || remote.heapBytes == 0 || + remote.linkLayer != static_cast(provider->portAttr.link_layer)) + return ncclInvalidArgument; + + struct doca_verbs_ah_attr_t* ah = nullptr; + doca_verbs_qp_attr_t* qpAttr = nullptr; + auto fail = [&] { + if (qpAttr != nullptr) doca_verbs_qp_attr_destroy(qpAttr); + if (ah != nullptr) doca_verbs_ah_attr_destroy(ah); + return ncclSystemError; + }; + + if (doca_verbs_ah_attr_create(provider->netDev, &ah) != DOCA_SUCCESS) return fail(); + if (doca_verbs_ah_attr_set_sl(ah, provider->serviceLevel) != DOCA_SUCCESS) return fail(); + + struct doca_verbs_gid remoteGid = {}; + std::memcpy(remoteGid.raw, remote.gid, sizeof(remoteGid.raw)); + const bool isIb = provider->portAttr.link_layer == IBV_LINK_LAYER_INFINIBAND; + // Match normal verbs path selection instead of forcing GRH for every IB + // connection. A same-subnet IB link ordinarily uses a LID-only AH. Forcing + // GRH there can leave a QP apparently connected yet unable to complete its + // first WQE. RoCE always needs the GID-backed path. + const bool sameIbSubnet = + isIb && std::memcmp(&provider->localGid.global.subnet_prefix, + &remoteGid.raw[0], sizeof(provider->localGid.global.subnet_prefix)) == 0; + const bool useIbGrh = isIb && (!sameIbSubnet || (provider->portAttr.flags & IBV_QPF_GRH_REQUIRED) != 0); + if (doca_verbs_ah_attr_set_addr_type( + ah, isIb ? (useIbGrh ? DOCA_VERBS_ADDR_TYPE_IB_GRH : DOCA_VERBS_ADDR_TYPE_IB_NO_GRH) + : DOCA_VERBS_ADDR_TYPE_IPv4) != DOCA_SUCCESS || + doca_verbs_ah_attr_set_dlid(ah, remote.lid) != DOCA_SUCCESS) + return fail(); + if ((!isIb || useIbGrh) && + (doca_verbs_ah_attr_set_gid(ah, remoteGid) != DOCA_SUCCESS || + doca_verbs_ah_attr_set_sgid_index(ah, static_cast(provider->gidIndex)) != DOCA_SUCCESS || + doca_verbs_ah_attr_set_hop_limit(ah, 255) != DOCA_SUCCESS)) + return fail(); + + if (doca_verbs_qp_attr_create(&qpAttr) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_rq_psn(qpAttr, kQpPsn) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_sq_psn(qpAttr, kQpPsn) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_pkey_index(qpAttr, provider->pkeyIndex) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_path_mtu( + qpAttr, toDocaMtu(std::min(provider->portAttr.active_mtu, remote.activeMtu))) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_port_num(qpAttr, provider->port) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_ack_timeout(qpAttr, kAckTimeout) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_retry_cnt(qpAttr, kRetryCount) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_rnr_retry(qpAttr, kRnrRetryCount) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_min_rnr_timer(qpAttr, kMinRnrTimer) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_next_state(qpAttr, DOCA_VERBS_QP_STATE_INIT) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_allow_remote_write(qpAttr, 0) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_allow_remote_read(qpAttr, 0) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_atomic_mode(qpAttr, DOCA_VERBS_QP_ATOMIC_MODE_UP_TO_8BYTES) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_ah_attr(qpAttr, ah) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_dest_qp_num(qpAttr, remote.qpn) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_max_rd_atomic(qpAttr, kMaxOutstandingAtomics) != DOCA_SUCCESS || + doca_verbs_qp_attr_set_max_dest_rd_atomic(qpAttr, kMaxOutstandingAtomics) != DOCA_SUCCESS) + return fail(); + + struct doca_verbs_qp_t* qp = provider->qps[peer]->qp; + if (doca_verbs_qp_modify(qp, qpAttr, + DOCA_VERBS_QP_ATTR_NEXT_STATE | DOCA_VERBS_QP_ATTR_ALLOW_REMOTE_WRITE | + DOCA_VERBS_QP_ATTR_ALLOW_REMOTE_READ | DOCA_VERBS_QP_ATTR_PKEY_INDEX | + DOCA_VERBS_QP_ATTR_PORT_NUM | + DOCA_VERBS_QP_ATTR_ATOMIC_MODE) != DOCA_SUCCESS) + return fail(); + if (doca_verbs_qp_attr_set_next_state(qpAttr, DOCA_VERBS_QP_STATE_RTR) != DOCA_SUCCESS || + doca_verbs_qp_modify(qp, qpAttr, + DOCA_VERBS_QP_ATTR_NEXT_STATE | DOCA_VERBS_QP_ATTR_RQ_PSN | + DOCA_VERBS_QP_ATTR_DEST_QP_NUM | DOCA_VERBS_QP_ATTR_PATH_MTU | + DOCA_VERBS_QP_ATTR_AH_ATTR | DOCA_VERBS_QP_ATTR_ATOMIC_MODE | + DOCA_VERBS_QP_ATTR_MIN_RNR_TIMER | + DOCA_VERBS_QP_ATTR_MAX_DEST_RD_ATOMIC) != DOCA_SUCCESS) + return fail(); + if (doca_verbs_qp_attr_set_next_state(qpAttr, DOCA_VERBS_QP_STATE_RTS) != DOCA_SUCCESS || + doca_verbs_qp_modify(qp, qpAttr, + DOCA_VERBS_QP_ATTR_NEXT_STATE | DOCA_VERBS_QP_ATTR_SQ_PSN | + DOCA_VERBS_QP_ATTR_ACK_TIMEOUT | DOCA_VERBS_QP_ATTR_RETRY_CNT | + DOCA_VERBS_QP_ATTR_RNR_RETRY | + DOCA_VERBS_QP_ATTR_MAX_QP_RD_ATOMIC) != DOCA_SUCCESS) + return fail(); + + doca_verbs_qp_attr_destroy(qpAttr); + doca_verbs_ah_attr_destroy(ah); + return ncclSuccess; +} + +ncclResult_t exportDeviceContext(niinGpunetioAtomicHostContext* provider, + const std::vector& remote) { + provider->exportedQps.clear(); + for (int pe = 0; pe < provider->nPes; ++pe) { + if (pe != provider->rank) provider->exportedQps.push_back(provider->qps[pe]->qp_gverbs); + } + if (!provider->exportedQps.empty() && + doca_gpu_verbs_export_multi_qps_dev(provider->gpu, provider->exportedQps.data(), + provider->exportedQps.size(), &provider->deviceQps) != DOCA_SUCCESS) + return ncclSystemError; + + provider->endpointHost.assign(provider->nPes, {}); + int deviceQpIndex = 0; + for (int pe = 0; pe < provider->nPes; ++pe) { + if (pe == provider->rank) continue; + const struct niinGpunetioAtomicWire& peerWire = + remote[static_cast(pe) * provider->nPes + provider->rank]; + struct niinGpunetioAtomicEndpoint& endpoint = provider->endpointHost[pe]; + endpoint.deviceQp = provider->deviceQps + deviceQpIndex++; + endpoint.remoteHeapBase = peerWire.heapBase; + endpoint.remoteHeapBytes = peerWire.heapBytes; + endpoint.remoteHeapRkey = peerWire.heapRkey; + endpoint.flags = NIIN_GPUNETIO_ATOMIC_ENDPOINT_CONNECTED | + NIIN_GPUNETIO_ATOMIC_ENDPOINT_ATOMIC_4B | + NIIN_GPUNETIO_ATOMIC_ENDPOINT_ATOMIC_8B | + NIIN_GPUNETIO_ATOMIC_ENDPOINT_EXTENDED; + } + + ncclResult_t result = cudaToNccl( + cudaMalloc(&provider->endpointDevice, provider->endpointHost.size() * sizeof(provider->endpointHost[0]))); + if (result != ncclSuccess) return result; + result = cudaToNccl(cudaMemcpy(provider->endpointDevice, provider->endpointHost.data(), + provider->endpointHost.size() * sizeof(provider->endpointHost[0]), + cudaMemcpyHostToDevice)); + if (result != ncclSuccess) return result; + result = cudaToNccl(cudaMalloc(&provider->endpointLocksDevice, provider->nPes * sizeof(int))); + if (result != ncclSuccess) return result; + result = cudaToNccl(cudaMemset(provider->endpointLocksDevice, 0, provider->nPes * sizeof(int))); + if (result != ncclSuccess) return result; + + provider->atomicContextHost.version = NIIN_GPUNETIO_ATOMIC_CONTEXT_VERSION; + provider->atomicContextHost.flags = NIIN_GPUNETIO_ATOMIC_CONTEXT_READY | + NIIN_GPUNETIO_ATOMIC_CONTEXT_DIRECT | + NIIN_GPUNETIO_ATOMIC_CONTEXT_EXTENDED; + if (provider->atomic8BResultHostEndian) + provider->atomicContextHost.flags |= NIIN_GPUNETIO_ATOMIC_CONTEXT_8B_RESULT_HOST_ENDIAN; + if (provider->hasCpuProxyQp()) + provider->atomicContextHost.flags |= NIIN_GPUNETIO_ATOMIC_CONTEXT_CPU_PROXY_DOORBELL; + provider->atomicContextHost.nPes = provider->nPes; + provider->atomicContextHost.localPe = provider->rank; + provider->atomicContextHost.endpoints = provider->endpointDevice; + provider->atomicContextHost.responseBase = provider->responseBase; + provider->atomicContextHost.responseBytes = provider->responseBytes; + provider->atomicContextHost.responseStride = kResponseStride; + provider->atomicContextHost.cstScratchOffset = kCstScratchOffset; + provider->atomicContextHost.responseLkey = toGpunetioMkey(provider->responseMr->lkey); + provider->atomicContextHost.endpointLocks = provider->endpointLocksDevice; + provider->atomicContextHost.proxyContext = nullptr; + + result = cudaToNccl(cudaMalloc(&provider->atomicContextDevice, sizeof(provider->atomicContextHost))); + if (result != ncclSuccess) return result; + return cudaToNccl(cudaMemcpy(provider->atomicContextDevice, &provider->atomicContextHost, + sizeof(provider->atomicContextHost), cudaMemcpyHostToDevice)); +} + +} // namespace + +ncclResult_t niinGpunetioAtomicInit( + ncclComm_t comm, void* heapBase, size_t heapBytes, + const struct niinGpunetioAtomicOptions* suppliedOptions, + struct niinGpunetioAtomicHostContext** out) { + if (comm == nullptr || heapBase == nullptr || heapBytes == 0 || out == nullptr) return ncclInvalidArgument; + *out = nullptr; + + struct niinGpunetioAtomicOptions options = NIIN_GPUNETIO_ATOMIC_OPTIONS_INITIALIZER; + if (suppliedOptions != nullptr) options = *suppliedOptions; + if (!isPowerOfTwo(options.sqDepth) || options.sqDepth < kMinSqDepth || options.sqDepth > kMaxSqDepth) + return ncclInvalidArgument; + if (options.serviceLevel > 15) return ncclInvalidArgument; + + auto* provider = new (std::nothrow) niinGpunetioAtomicHostContext; + if (provider == nullptr) return ncclSystemError; + provider->comm = comm; + + ncclResult_t result = ncclCommUserRank(comm, &provider->rank); + ncclResult_t localStatus = ncclSuccess; + Mlx5AtomicCapabilities mlx5AtomicCapabilities = {}; + if (result != ncclSuccess) goto fail; + result = ncclCommCount(comm, &provider->nPes); + if (result != ncclSuccess || provider->nPes <= 0 || provider->nPes > UINT16_MAX) { + if (result == ncclSuccess) result = ncclInvalidArgument; + goto fail; + } + provider->pkeyIndex = options.pkeyIndex; + provider->serviceLevel = options.serviceLevel; + + // Each phase first performs its local work, then collectively reports the + // outcome before any PE can enter a later NCCL/verbs operation. This is + // especially important for the capability gate: HCAs without masked + // atomics must reject the provider cleanly rather than leave peers waiting + // in the QP descriptor all-gather. + localStatus = openVerbsDevice(provider, options); + if (localStatus == ncclSuccess && + !queryRequiredMlx5AtomicCaps(provider->verbsContext, &mlx5AtomicCapabilities)) { + std::fprintf(stderr, + "NIIN GPUNetIO atomics: selected HCA lacks required 4/8-byte " + "standard and masked atomic QP capabilities\n"); + localStatus = ncclInvalidUsage; + } + if (localStatus == ncclSuccess) + provider->atomic8BResultHostEndian = mlx5AtomicCapabilities.atomic8BResultHostEndian; + if (localStatus == ncclSuccess) localStatus = openGpunetio(provider); + if (localStatus == ncclSuccess) localStatus = registerAtomicMemory(provider, heapBase, heapBytes); + result = allGatherSetupStatus(comm, provider->nPes, localStatus); + if (result != ncclSuccess) goto fail; + + { + std::vector localWire; + std::vector remoteWire; + localStatus = createPrivateQps(provider, options, &localWire); + result = allGatherSetupStatus(comm, provider->nPes, localStatus); + if (result != ncclSuccess) goto fail; + result = allGatherWire(comm, localWire, provider->nPes, &remoteWire); + if (result != ncclSuccess) goto fail; + + localStatus = ncclSuccess; + for (int peer = 0; peer < provider->nPes; ++peer) { + if (peer == provider->rank) continue; + const struct niinGpunetioAtomicWire& wire = + remoteWire[static_cast(peer) * provider->nPes + provider->rank]; + localStatus = connectPeerQp(provider, peer, wire); + if (localStatus != ncclSuccess) break; + } + result = allGatherSetupStatus(comm, provider->nPes, localStatus); + if (result != ncclSuccess) goto fail; + + localStatus = exportDeviceContext(provider, remoteWire); + result = allGatherSetupStatus(comm, provider->nPes, localStatus); + if (result != ncclSuccess) goto fail; + } + + provider->startProgressThread(); + *out = provider; + return ncclSuccess; + +fail: + delete provider; + return result; +} + +ncclResult_t niinGpunetioAtomicBind(struct niinGpunetioAtomicHostContext* provider, + struct niinContext* deviceContext) { + if (provider == nullptr || provider->atomicContextDevice == nullptr || deviceContext == nullptr) + return ncclInvalidArgument; + if (provider->boundDeviceContext != nullptr && provider->boundDeviceContext != deviceContext) + return ncclInvalidArgument; + + const struct niinGpunetioAtomicContext* existingContext = nullptr; + cudaError_t status = cudaMemcpy( + &existingContext, + reinterpret_cast(deviceContext) + offsetof(struct niinContext, gpunetioAtomicContext), + sizeof(existingContext), cudaMemcpyDeviceToHost); + if (status != cudaSuccess) return ncclSystemError; + if (existingContext != nullptr && existingContext != provider->atomicContextDevice) + return ncclInvalidUsage; + + const struct niinGpunetioAtomicContext* context = provider->atomicContextDevice; + status = cudaMemcpy( + reinterpret_cast(deviceContext) + offsetof(struct niinContext, gpunetioAtomicContext), &context, + sizeof(context), cudaMemcpyHostToDevice); + if (status != cudaSuccess) return ncclSystemError; + provider->boundDeviceContext = deviceContext; + return ncclSuccess; +} + +ncclResult_t niinGpunetioAtomicProgress(struct niinGpunetioAtomicHostContext* provider) { + if (provider == nullptr) return ncclInvalidArgument; + if (provider->progressFailed.load(std::memory_order_acquire)) return ncclSystemError; + return provider->progressOnce(); +} + +ncclResult_t niinGpunetioAtomicFinalize(struct niinGpunetioAtomicHostContext* provider) { + if (provider == nullptr) return ncclInvalidArgument; + delete provider; + return ncclSuccess; +} diff --git a/projects/rccl/contrib/niin/gpunetio/tests/atomic_smoke.cu b/projects/rccl/contrib/niin/gpunetio/tests/atomic_smoke.cu new file mode 100644 index 000000000000..12eca164a82c --- /dev/null +++ b/projects/rccl/contrib/niin/gpunetio/tests/atomic_smoke.cu @@ -0,0 +1,428 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +// Two-rank, one-GPU-per-rank smoke test for NIIN's atomic-only GPUNetIO +// provider. It calls the direct adapter rather than the normal NIIN routing, +// so it exercises the private atomic QPs even when both ranks are local. A +// two-node run is still preferred when validating a routed fabric path. + +#include + +#include +#include +#include + +#include "niin/context_abi.h" +#include "niin/gpunetio/atomics.h" +#include "niin/gpunetio/host.h" +#include "niin/host.h" + +#include +#include +#include +#include +#include + +namespace { + +constexpr int kConcurrentThreads = 32; + +[[noreturn]] void fail(int rank, const char* where, const char* detail) { + std::fprintf(stderr, "NIIN GPUNetIO atomic smoke [rank %d]: %s: %s\n", rank, where, detail); + MPI_Abort(MPI_COMM_WORLD, 1); + std::abort(); +} + +void checkCuda(int rank, cudaError_t status, const char* where) { + if (status != cudaSuccess) fail(rank, where, cudaGetErrorString(status)); +} + +void checkNccl(int rank, ncclResult_t status, const char* where) { + if (status != ncclSuccess) fail(rank, where, ncclGetErrorString(status)); +} + +void phase(int rank, const char* name) { + std::fprintf(stderr, "NIIN GPUNetIO atomic smoke [rank %d]: %s\n", rank, name); + std::fflush(stderr); +} + +// The atomic provider bootstraps its own QPs with public NCCL all-gather. Do +// a same-shaped byte all-gather first so a launcher/NCCL failure is reported +// separately from GPUNetIO setup. +void verifyMetadataCollective(int rank, int nPes, ncclComm_t comm) { + constexpr size_t kBytes = 64; + std::array sendHost = {}; + std::array recvHost = {}; + std::memset(sendHost.data(), rank + 1, sendHost.size()); + + void* sendDevice = nullptr; + void* recvDevice = nullptr; + cudaStream_t stream = nullptr; + checkCuda(rank, cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), "metadata stream"); + checkCuda(rank, cudaMalloc(&sendDevice, kBytes), "metadata send allocation"); + checkCuda(rank, cudaMalloc(&recvDevice, kBytes * static_cast(nPes)), "metadata receive allocation"); + checkCuda(rank, cudaMemcpyAsync(sendDevice, sendHost.data(), kBytes, cudaMemcpyHostToDevice, stream), + "metadata send copy"); + checkNccl(rank, ncclAllGather(sendDevice, recvDevice, kBytes, ncclUint8, comm, stream), + "NCCL metadata all-gather"); + checkCuda(rank, cudaMemcpyAsync(recvHost.data(), recvDevice, kBytes * static_cast(nPes), + cudaMemcpyDeviceToHost, stream), + "metadata receive copy"); + checkCuda(rank, cudaStreamSynchronize(stream), "metadata all-gather completion"); + for (int pe = 0; pe < nPes; ++pe) { + for (size_t byte = 0; byte < kBytes; ++byte) { + if (recvHost[static_cast(pe) * kBytes + byte] != static_cast(pe + 1)) + fail(rank, "NCCL metadata all-gather", "incorrect gathered rank marker"); + } + } + cudaFree(recvDevice); + cudaFree(sendDevice); + cudaStreamDestroy(stream); +} + +struct AtomicResults { + uint32_t failures; + uint32_t fetchAdd32; + uint32_t compareSwap32; + uint32_t fetchAfterCompareSwap32; + uint32_t swap32; + uint32_t fetch32; + uint32_t fetchInc32; + uint32_t fetchAnd32; + uint32_t fetchOr32; + uint32_t fetchXor32; + uint64_t fetchAdd64; + uint64_t compareSwap64; + uint64_t fetchAfterCompareSwap64; + uint64_t swap64; + uint64_t fetch64; + uint64_t fetchInc64; + uint64_t fetchAnd64; + uint64_t fetchOr64; + uint64_t fetchXor64; +}; + +struct ConcurrentResults { + uint32_t failures; + uint32_t previous[kConcurrentThreads]; +}; + +__global__ void issueAllAtomics(niinContext* context, int targetPe, AtomicResults* results) { + if (blockIdx.x != 0 || threadIdx.x != 0) return; + const niinGpunetioAtomicContext* atomics = context->gpunetioAtomicContext; + uint32_t failures = 0; + +#define NIIN_SMOKE_TRY(expression) \ + do { \ + if (!(expression)) ++failures; \ + } while (0) + + // A bad symmetric-pointer cast must be rejected before it reserves a WQE; + // otherwise an invalid hardware AMO could poison this PE's dedicated QP. + NIIN_SMOKE_TRY(!niin_gpunetio_atomic_fetch_add_try(atomics, size_t{1}, UINT32_C(1), targetPe, + &results->fetchAdd32)); + + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_add_try(atomics, size_t{0}, UINT32_C(5), targetPe, + &results->fetchAdd32)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_add_try(atomics, size_t{0}, UINT32_C(5), targetPe)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_compare_swap_try(atomics, size_t{0}, UINT32_C(20), UINT32_C(30), + targetPe, &results->compareSwap32)); + NIIN_SMOKE_TRY( + niin_gpunetio_atomic_fetch_try(atomics, size_t{0}, targetPe, &results->fetchAfterCompareSwap32)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_swap_try(atomics, size_t{0}, UINT32_C(40), targetPe, + &results->swap32)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_try(atomics, size_t{0}, targetPe, &results->fetch32)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_set_try(atomics, size_t{0}, UINT32_C(50), targetPe)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_inc_try(atomics, size_t{0}, targetPe)); + NIIN_SMOKE_TRY( + niin_gpunetio_atomic_fetch_inc_try(atomics, size_t{0}, targetPe, &results->fetchInc32)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_and_try(atomics, size_t{0}, UINT32_C(0x3f), targetPe, + &results->fetchAnd32)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_and_try(atomics, size_t{0}, UINT32_C(0x0f), targetPe)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_or_try(atomics, size_t{0}, UINT32_C(0x10), targetPe, + &results->fetchOr32)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_or_try(atomics, size_t{0}, UINT32_C(0x20), targetPe)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_xor_try(atomics, size_t{0}, UINT32_C(0x3), targetPe, + &results->fetchXor32)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_xor_try(atomics, size_t{0}, UINT32_C(0x1), targetPe)); + + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_add_try(atomics, size_t{8}, UINT64_C(5), targetPe, + &results->fetchAdd64)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_add_try(atomics, size_t{8}, UINT64_C(5), targetPe)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_compare_swap_try(atomics, size_t{8}, UINT64_C(20), UINT64_C(30), + targetPe, &results->compareSwap64)); + NIIN_SMOKE_TRY( + niin_gpunetio_atomic_fetch_try(atomics, size_t{8}, targetPe, &results->fetchAfterCompareSwap64)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_swap_try(atomics, size_t{8}, UINT64_C(40), targetPe, + &results->swap64)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_try(atomics, size_t{8}, targetPe, &results->fetch64)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_set_try(atomics, size_t{8}, UINT64_C(50), targetPe)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_inc_try(atomics, size_t{8}, targetPe)); + NIIN_SMOKE_TRY( + niin_gpunetio_atomic_fetch_inc_try(atomics, size_t{8}, targetPe, &results->fetchInc64)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_and_try(atomics, size_t{8}, UINT64_C(0x3f), targetPe, + &results->fetchAnd64)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_and_try(atomics, size_t{8}, UINT64_C(0x0f), targetPe)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_or_try(atomics, size_t{8}, UINT64_C(0x10), targetPe, + &results->fetchOr64)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_or_try(atomics, size_t{8}, UINT64_C(0x20), targetPe)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_fetch_xor_try(atomics, size_t{8}, UINT64_C(0x3), targetPe, + &results->fetchXor64)); + NIIN_SMOKE_TRY(niin_gpunetio_atomic_xor_try(atomics, size_t{8}, UINT64_C(0x1), targetPe)); + +#undef NIIN_SMOKE_TRY + results->failures = failures; +} + +// Every thread targets the same remote word. The provider has exactly one +// response slot and QP lock per destination, so this checks that contention +// serializes callers without reusing an in-flight result buffer. +__global__ void issueConcurrentFetchAdds(niinContext* context, int targetPe, ConcurrentResults* results) { + const int thread = static_cast(threadIdx.x); + if (blockIdx.x != 0 || thread >= kConcurrentThreads) return; + + uint32_t previous = 0; + const bool succeeded = niin_gpunetio_atomic_fetch_add_try( + context->gpunetioAtomicContext, size_t{16}, UINT32_C(1), targetPe, &previous); + results->previous[thread] = previous; + if (!succeeded) atomicAdd(&results->failures, UINT32_C(1)); +} + +__global__ void readTargetValues(const uint32_t* word32, const uint64_t* word64, uint32_t* result32, + uint64_t* result64) { + if (blockIdx.x != 0 || threadIdx.x != 0) return; + cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); + *result32 = *word32; + *result64 = *word64; +} + +bool validateResults(const AtomicResults& actual) { + return actual.failures == 0 && actual.fetchAdd32 == UINT32_C(10) && actual.compareSwap32 == UINT32_C(20) && + actual.fetchAfterCompareSwap32 == UINT32_C(30) && actual.swap32 == UINT32_C(30) && + actual.fetch32 == UINT32_C(40) && + actual.fetchInc32 == UINT32_C(51) && actual.fetchAnd32 == UINT32_C(52) && + actual.fetchOr32 == UINT32_C(4) && actual.fetchXor32 == UINT32_C(52) && + actual.fetchAdd64 == UINT64_C(10) && actual.compareSwap64 == UINT64_C(20) && + actual.fetchAfterCompareSwap64 == UINT64_C(30) && actual.swap64 == UINT64_C(30) && + actual.fetch64 == UINT64_C(40) && + actual.fetchInc64 == UINT64_C(51) && actual.fetchAnd64 == UINT64_C(52) && + actual.fetchOr64 == UINT64_C(4) && actual.fetchXor64 == UINT64_C(52); +} + +bool validateConcurrentResults(const ConcurrentResults& actual) { + if (actual.failures != 0) return false; + std::array seen = {}; + for (int thread = 0; thread < kConcurrentThreads; ++thread) { + const uint32_t previous = actual.previous[thread]; + if (previous >= static_cast(kConcurrentThreads) || seen[previous]) return false; + seen[previous] = true; + } + return true; +} + +void printResults(const AtomicResults& actual) { + std::fprintf(stderr, + "NIIN GPUNetIO atomic smoke: failures=%u; 32b [fa=%u cs=%u fcs=%u sw=%u f=%u fi=%u " + "and=%u or=%u xor=%u]; 64b [fa=%llu cs=%llu fcs=%llu sw=%llu f=%llu fi=%llu and=%llu or=%llu xor=%llu]\n", + actual.failures, actual.fetchAdd32, actual.compareSwap32, actual.fetchAfterCompareSwap32, + actual.swap32, actual.fetch32, + actual.fetchInc32, actual.fetchAnd32, actual.fetchOr32, actual.fetchXor32, + static_cast(actual.fetchAdd64), + static_cast(actual.compareSwap64), + static_cast(actual.fetchAfterCompareSwap64), + static_cast(actual.swap64), static_cast(actual.fetch64), + static_cast(actual.fetchInc64), + static_cast(actual.fetchAnd64), static_cast(actual.fetchOr64), + static_cast(actual.fetchXor64)); +} + +int selectCudaDevice(int rank) { + int deviceCount = 0; + checkCuda(rank, cudaGetDeviceCount(&deviceCount), "cudaGetDeviceCount"); + if (deviceCount <= 0) fail(rank, "cudaGetDeviceCount", "no visible CUDA device"); + + MPI_Comm localComm = MPI_COMM_NULL; + if (MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &localComm) != MPI_SUCCESS) + fail(rank, "MPI_Comm_split_type", "could not discover the node-local rank"); + int localRank = 0; + if (MPI_Comm_rank(localComm, &localRank) != MPI_SUCCESS) fail(rank, "MPI_Comm_rank", "could not read local rank"); + MPI_Comm_free(&localComm); + + // Launchers that set CUDA_VISIBLE_DEVICES to one GPU per process report one + // visible device, while unrestricted launchers expose every local GPU. + return deviceCount == 1 ? 0 : localRank % deviceCount; +} + +void finalizeNiinContext(int rank, ncclComm_t comm, niinContext* deviceContext) { + checkNccl(rank, ncclGroupStart(), "ncclGroupStart(niinFinalize)"); + checkNccl(rank, niinFinalize(comm, deviceContext), "niinFinalize"); + checkNccl(rank, ncclGroupEnd(), "ncclGroupEnd(niinFinalize)"); + checkCuda(rank, cudaFree(deviceContext), "cudaFree(device context)"); +} + +} // namespace + +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + int rank = -1; + int nPes = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nPes); + if (nPes != 2) fail(rank, "startup", "run with exactly two MPI ranks"); + + checkCuda(rank, cudaSetDevice(selectCudaDevice(rank)), "cudaSetDevice"); + + ncclUniqueId id; + if (rank == 0) checkNccl(rank, ncclGetUniqueId(&id), "ncclGetUniqueId"); + if (MPI_Bcast(&id, sizeof(id), MPI_BYTE, 0, MPI_COMM_WORLD) != MPI_SUCCESS) + fail(rank, "MPI_Bcast", "NCCL unique-id exchange failed"); + + ncclComm_t comm = nullptr; + checkNccl(rank, ncclCommInitRank(&comm, nPes, id, rank), "ncclCommInitRank"); + phase(rank, "NCCL communicator initialized"); + verifyMetadataCollective(rank, nPes, comm); + phase(rank, "NCCL metadata all-gather completed"); + + constexpr size_t kHeapBytes = 4096; + void* heap = nullptr; + checkNccl(rank, ncclMemAlloc(&heap, kHeapBytes), "ncclMemAlloc"); + checkCuda(rank, cudaMemset(heap, 0, kHeapBytes), "cudaMemset(heap)"); + + auto* word32 = static_cast(heap); + auto* word64 = reinterpret_cast(static_cast(heap) + 8); + const uint32_t initial32 = UINT32_C(10); + const uint64_t initial64 = UINT64_C(10); + checkCuda(rank, cudaMemcpy(word32, &initial32, sizeof(initial32), cudaMemcpyHostToDevice), "init word32"); + checkCuda(rank, cudaMemcpy(word64, &initial64, sizeof(initial64), cudaMemcpyHostToDevice), "init word64"); + + niinContext_host hostContext = {}; + checkNccl(rank, ncclGroupStart(), "ncclGroupStart(niinInit)"); + checkNccl(rank, niinInit(comm, heap, kHeapBytes, &hostContext), "niinInit"); + checkNccl(rank, ncclGroupEnd(), "ncclGroupEnd(niinInit)"); + niinContext* deviceContext = nullptr; + checkCuda(rank, cudaMalloc(&deviceContext, sizeof(*deviceContext)), "cudaMalloc(device context)"); + checkNccl(rank, niinCommit(&hostContext, deviceContext), "niinCommit"); + phase(rank, "heap and device context initialized"); + + niinGpunetioAtomicOptions options = NIIN_GPUNETIO_ATOMIC_OPTIONS_INITIALIZER; + // AUTO is the normal coverage. `gpu` requires the validated GPU-SM + // doorbell path. `cpu` is retained to verify that setup fails collectively + // instead of allowing an unsupported CPU doorbell configuration to hang a + // device AMO. + const char* handler = std::getenv("NIIN_GPUNETIO_SMOKE_NIC_HANDLER"); + if (handler != nullptr && std::strcmp(handler, "gpu") == 0) { + options.nicHandler = NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_GPU_SM_DB; + } else if (handler != nullptr && std::strcmp(handler, "cpu") == 0) { + options.nicHandler = NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_CPU_PROXY; + } else if (handler != nullptr && std::strcmp(handler, "auto") != 0) { + fail(rank, "NIIN_GPUNETIO_SMOKE_NIC_HANDLER", "expected auto, gpu, or cpu"); + } + niinGpunetioAtomicHostContext* atomics = nullptr; + phase(rank, "starting direct-provider bootstrap"); + const ncclResult_t atomicInit = niinGpunetioAtomicInit(comm, heap, kHeapBytes, &options, &atomics); + if (handler != nullptr && std::strcmp(handler, "cpu") == 0) { + const int localExpectedRejection = + atomicInit == ncclInvalidUsage && atomics == nullptr ? 1 : 0; + int allExpectedRejection = 0; + if (MPI_Allreduce(&localExpectedRejection, &allExpectedRejection, 1, MPI_INT, MPI_MIN, + MPI_COMM_WORLD) != MPI_SUCCESS) + fail(rank, "MPI_Allreduce", "CPU-doorbell rejection reduction failed"); + finalizeNiinContext(rank, comm, deviceContext); + checkNccl(rank, ncclMemFree(heap), "ncclMemFree"); + checkNccl(rank, ncclCommDestroy(comm), "ncclCommDestroy"); + if (rank == 0) + std::printf("NIIN GPUNetIO atomic smoke: %s (CPU doorbell rejection)\n", + allExpectedRejection ? "PASS" : "FAIL"); + MPI_Finalize(); + return allExpectedRejection ? 0 : 1; + } + checkNccl(rank, atomicInit, "niinGpunetioAtomicInit"); + phase(rank, "direct-provider bootstrap completed"); + checkNccl(rank, niinGpunetioAtomicBind(atomics, deviceContext), "niinGpunetioAtomicBind"); + phase(rank, "direct-provider bound"); + + if (MPI_Barrier(MPI_COMM_WORLD) != MPI_SUCCESS) fail(rank, "MPI_Barrier", "before atomics"); + + AtomicResults* deviceResults = nullptr; + ConcurrentResults* deviceConcurrentResults = nullptr; + if (rank == 0) { + checkCuda(rank, cudaMalloc(&deviceResults, sizeof(*deviceResults)), "cudaMalloc(results)"); + checkCuda(rank, cudaMemset(deviceResults, 0, sizeof(*deviceResults)), "cudaMemset(results)"); + phase(rank, "issuing direct atomics"); + issueAllAtomics<<<1, 1>>>(deviceContext, 1, deviceResults); + checkCuda(rank, cudaGetLastError(), "issueAllAtomics launch"); + checkCuda(rank, cudaDeviceSynchronize(), "issueAllAtomics completion"); + checkCuda(rank, cudaMalloc(&deviceConcurrentResults, sizeof(*deviceConcurrentResults)), + "cudaMalloc(concurrent results)"); + checkCuda(rank, cudaMemset(deviceConcurrentResults, 0, sizeof(*deviceConcurrentResults)), + "cudaMemset(concurrent results)"); + issueConcurrentFetchAdds<<<1, kConcurrentThreads>>>(deviceContext, 1, deviceConcurrentResults); + checkCuda(rank, cudaGetLastError(), "issueConcurrentFetchAdds launch"); + checkCuda(rank, cudaDeviceSynchronize(), "issueConcurrentFetchAdds completion"); + phase(rank, "direct atomics completed"); + } + + if (MPI_Barrier(MPI_COMM_WORLD) != MPI_SUCCESS) fail(rank, "MPI_Barrier", "after atomics"); + + uint32_t final32 = 0; + uint32_t finalConcurrent32 = 0; + uint64_t final64 = 0; + uint32_t* deviceFinal32 = nullptr; + uint64_t* deviceFinal64 = nullptr; + if (rank == 1) { + checkCuda(rank, cudaMalloc(&deviceFinal32, sizeof(*deviceFinal32)), "cudaMalloc(final32)"); + checkCuda(rank, cudaMalloc(&deviceFinal64, sizeof(*deviceFinal64)), "cudaMalloc(final64)"); + readTargetValues<<<1, 1>>>(word32, word64, deviceFinal32, deviceFinal64); + checkCuda(rank, cudaGetLastError(), "readTargetValues launch"); + checkCuda(rank, cudaDeviceSynchronize(), "readTargetValues completion"); + checkCuda(rank, cudaMemcpy(&final32, deviceFinal32, sizeof(final32), cudaMemcpyDeviceToHost), "copy final32"); + checkCuda(rank, cudaMemcpy(&finalConcurrent32, static_cast(heap) + 16, sizeof(finalConcurrent32), + cudaMemcpyDeviceToHost), + "copy final concurrent32"); + checkCuda(rank, cudaMemcpy(&final64, deviceFinal64, sizeof(final64), cudaMemcpyDeviceToHost), "copy final64"); + } + + int localPass = 1; + if (rank == 0) { + AtomicResults results = {}; + ConcurrentResults concurrentResults = {}; + checkCuda(rank, cudaMemcpy(&results, deviceResults, sizeof(results), cudaMemcpyDeviceToHost), "copy results"); + checkCuda(rank, cudaMemcpy(&concurrentResults, deviceConcurrentResults, sizeof(concurrentResults), + cudaMemcpyDeviceToHost), + "copy concurrent results"); + localPass = validateResults(results) && validateConcurrentResults(concurrentResults) ? 1 : 0; + if (!localPass) { + std::fprintf(stderr, "NIIN GPUNetIO atomic smoke: fetch-result or contention mismatch\n"); + printResults(results); + } + } + if (rank == 1 && (final32 != UINT32_C(54) || final64 != UINT64_C(54) || + finalConcurrent32 != static_cast(kConcurrentThreads))) { + std::fprintf(stderr, + "NIIN GPUNetIO atomic smoke: target values were %u, %llu, and %u; expected 54, 54, and %d\n", + final32, static_cast(final64), finalConcurrent32, kConcurrentThreads); + localPass = 0; + } + + int globalPass = 0; + if (MPI_Allreduce(&localPass, &globalPass, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD) != MPI_SUCCESS) + fail(rank, "MPI_Allreduce", "result reduction failed"); + + if (deviceResults != nullptr) cudaFree(deviceResults); + if (deviceConcurrentResults != nullptr) cudaFree(deviceConcurrentResults); + if (deviceFinal32 != nullptr) cudaFree(deviceFinal32); + if (deviceFinal64 != nullptr) cudaFree(deviceFinal64); + cudaDeviceSynchronize(); + checkNccl(rank, niinGpunetioAtomicFinalize(atomics), "niinGpunetioAtomicFinalize"); + finalizeNiinContext(rank, comm, deviceContext); + checkNccl(rank, ncclMemFree(heap), "ncclMemFree"); + checkNccl(rank, ncclCommDestroy(comm), "ncclCommDestroy"); + + if (rank == 0) std::printf("NIIN GPUNetIO atomic smoke: %s\n", globalPass ? "PASS" : "FAIL"); + MPI_Finalize(); + return globalPass ? 0 : 1; +} diff --git a/projects/rccl/contrib/niin/gpunetio/tests/device_compile.cu b/projects/rccl/contrib/niin/gpunetio/tests/device_compile.cu new file mode 100644 index 000000000000..6428b9b9f498 --- /dev/null +++ b/projects/rccl/contrib/niin/gpunetio/tests/device_compile.cu @@ -0,0 +1,96 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +// Compile-only coverage for the public NIIN atomic surface. Do not launch +// this kernel: its purpose is to instantiate the direct GPUNetIO WQE paths +// without requiring a particular fabric in a build environment. + +#include + +#include +#include + +extern "C" __global__ void niin_gpunetio_atomic_compile_surface( + niinContext* context, int* i32, uint32_t* u32, uint64_t* u64, ptrdiff_t* pdiff, + float* f32, double* f64, __half* f16) { + niin_g_ctx = context; + + int oldI32 = 0; + uint32_t oldU32 = 0; + uint64_t oldU64 = 0; + ptrdiff_t oldPdiff = 0; + + oldI32 = nvshmem_int_atomic_fetch_add(i32, 1, 1); + nvshmem_int_atomic_add(i32, 1, 1); + oldI32 = nvshmem_int_atomic_compare_swap(i32, 0, 1, 1); + oldI32 = nvshmem_int_atomic_swap(i32, 1, 1); + oldI32 = nvshmem_int_atomic_fetch(i32, 1); + nvshmem_int_atomic_set(i32, 1, 1); + nvshmem_int_atomic_inc(i32, 1); + oldI32 = nvshmem_int_atomic_fetch_inc(i32, 1); + + oldU32 = nvshmem_uint32_atomic_fetch_and(u32, UINT32_C(0xff), 1); + nvshmem_uint32_atomic_and(u32, UINT32_C(0xff), 1); + oldU32 = nvshmem_uint32_atomic_fetch_or(u32, UINT32_C(1), 1); + nvshmem_uint32_atomic_or(u32, UINT32_C(1), 1); + oldU32 = nvshmem_uint32_atomic_fetch_xor(u32, UINT32_C(1), 1); + nvshmem_uint32_atomic_xor(u32, UINT32_C(1), 1); + + oldU64 = nvshmem_uint64_atomic_fetch_add(u64, UINT64_C(1), 1); + nvshmem_uint64_atomic_add(u64, UINT64_C(1), 1); + oldU64 = nvshmem_uint64_atomic_compare_swap(u64, UINT64_C(0), UINT64_C(1), 1); + oldU64 = nvshmem_uint64_atomic_swap(u64, UINT64_C(1), 1); + oldU64 = nvshmem_uint64_atomic_fetch(u64, 1); + nvshmem_uint64_atomic_set(u64, UINT64_C(1), 1); + nvshmem_uint64_atomic_inc(u64, 1); + oldU64 = nvshmem_uint64_atomic_fetch_inc(u64, 1); + oldU64 = nvshmem_uint64_atomic_fetch_and(u64, UINT64_C(0xff), 1); + nvshmem_uint64_atomic_and(u64, UINT64_C(0xff), 1); + oldU64 = nvshmem_uint64_atomic_fetch_or(u64, UINT64_C(1), 1); + nvshmem_uint64_atomic_or(u64, UINT64_C(1), 1); + oldU64 = nvshmem_uint64_atomic_fetch_xor(u64, UINT64_C(1), 1); + nvshmem_uint64_atomic_xor(u64, UINT64_C(1), 1); + + // ptrdiff is part of NVSHMEM's standard integer AMO spelling. It is a + // 4- or 8-byte integral type on supported CUDA platforms and therefore + // belongs on the native direct-QP path alongside size_t. + oldPdiff = nvshmem_ptrdiff_atomic_fetch_add(pdiff, ptrdiff_t{1}, 1); + nvshmem_ptrdiff_atomic_add(pdiff, ptrdiff_t{1}, 1); + oldPdiff = nvshmem_ptrdiff_atomic_compare_swap(pdiff, ptrdiff_t{0}, ptrdiff_t{1}, 1); + oldPdiff = nvshmem_ptrdiff_atomic_swap(pdiff, ptrdiff_t{1}, 1); + oldPdiff = nvshmem_ptrdiff_atomic_fetch(pdiff, 1); + nvshmem_ptrdiff_atomic_set(pdiff, ptrdiff_t{1}, 1); + nvshmem_ptrdiff_atomic_inc(pdiff, 1); + oldPdiff = nvshmem_ptrdiff_atomic_fetch_inc(pdiff, 1); + + // Base floating fetch/swap/set preserve raw representations. The extended + // NVSHMEMX add/fetch-add calls instantiate the NIIN SRQ proxy adapter; they + // intentionally do not instantiate a GPUNetIO integer AMO WQE. + float oldF32 = nvshmem_float_atomic_swap(f32, 1.0f, 1); + oldF32 = nvshmem_float_atomic_fetch(f32, 1); + nvshmem_float_atomic_set(f32, 2.0f, 1); + double oldF64 = nvshmem_double_atomic_swap(f64, 1.0, 1); + oldF64 = nvshmem_double_atomic_fetch(f64, 1); + nvshmem_double_atomic_set(f64, 2.0, 1); + const __half oneHalf = __float2half_rn(1.0f); + __half oldF16 = nvshmemx_half_atomic_fetch_add(f16, oneHalf, 1); + nvshmemx_half_atomic_add(f16, oneHalf, 1); + oldF32 = nvshmemx_float_atomic_fetch_add(f32, 1.0f, 1); + nvshmemx_float_atomic_add(f32, 1.0f, 1); + oldF64 = nvshmemx_double_atomic_fetch_add(f64, 1.0, 1); + nvshmemx_double_atomic_add(f64, 1.0, 1); + + // Preserve the temporary values so device compilation cannot erase every + // call before the WQE template bodies are instantiated. + if (oldI32 == -1) *i32 = oldI32; + if (oldU32 == UINT32_MAX) *u32 = oldU32; + if (oldU64 == UINT64_MAX) *u64 = oldU64; + if (oldPdiff == -1) *pdiff = oldPdiff; + if (oldF32 == -1.0f) *f32 = oldF32; + if (oldF64 == -1.0) *f64 = oldF64; + if (__half2float(oldF16) == -1.0f) *f16 = oldF16; +} diff --git a/projects/rccl/contrib/niin/include/niin/atomics.h b/projects/rccl/contrib/niin/include/niin/atomics.h index dabbba97e3c3..a180d4a85622 100644 --- a/projects/rccl/contrib/niin/include/niin/atomics.h +++ b/projects/rccl/contrib/niin/include/niin/atomics.h @@ -9,6 +9,7 @@ #define NIIN_ATOMICS_H_ #include "niin/context.h" +#include "niin/gpunetio/atomics.h" #include // --------------------------------------------------------------------------- @@ -32,6 +33,58 @@ __device__ __forceinline__ T niin_atomicAdd(T* addr, T val) { } } +template +__device__ __forceinline__ void niin_atomicAddNoReturn(T* addr, T val) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "niin_atomicAddNoReturn: unsupported type size"); + if constexpr (sizeof(T) == 4) { + (void)atomicAdd((unsigned int*)addr, *(unsigned int*)&val); + } else { + (void)atomicAdd((unsigned long long*)addr, *(unsigned long long*)&val); + } +} + +template +__device__ __forceinline__ T niin_atomicAddSystem(T* addr, T val) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "niin_atomicAddSystem: unsupported type size"); + if constexpr (sizeof(T) == 4) { + unsigned int r = atomicAdd_system((unsigned int*)addr, *(unsigned int*)&val); + T result; memcpy(&result, &r, 4); return result; + } else { + unsigned long long r = atomicAdd_system((unsigned long long*)addr, *(unsigned long long*)&val); + T result; memcpy(&result, &r, 8); return result; + } +} + +template +__device__ __forceinline__ void niin_atomicAddSystemNoReturn(T* addr, T val) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "niin_atomicAddSystemNoReturn: unsupported type size"); + if constexpr (sizeof(T) == 4) { + (void)atomicAdd_system((unsigned int*)addr, *(unsigned int*)&val); + } else { + (void)atomicAdd_system((unsigned long long*)addr, *(unsigned long long*)&val); + } +} + +template +NIIN_NOINLINE_DEVICE void niin_atomic_add_slow(T* dest, T value, int pe, size_t offset) { + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { + if (niin_gpunetio_atomic_add_try(atomicContext, offset, value, pe)) return; + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_atomic_add (proxy)"); + return; + } + if (pe == nvshmem_my_pe()) { + niin_atomicAddNoReturn(dest, value); + return; + } + if (niin_is_lsa_peer(pe)) { + niin_atomicAddSystemNoReturn((T*)niin_get_peer_ptr(offset, pe), value); + return; + } + if (niin_gpunetio_atomic_add_try(atomicContext, offset, value, pe)) return; + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_atomic_add (network)"); +} + template __device__ __forceinline__ T niin_atomicCAS(T* addr, T cmp, T val) { static_assert(sizeof(T) == 4 || sizeof(T) == 8, "niin_atomicCAS: unsupported type size"); @@ -93,16 +146,29 @@ __device__ __forceinline__ T niin_atomicXor(T* addr, T val) { } // --------------------------------------------------------------------------- -// nvshmem_TYPE_atomic_fetch_add: NVLink = atomicAdd via peer ptr, Network = NOT_IMPLEMENTED +// nvshmem_TYPE_atomic_fetch_add: self/LSA use CUDA atomics; a bound NIIN +// GPUNetIO provider handles the network case. An unbound provider retains the +// existing fail-closed network behavior. // --------------------------------------------------------------------------- #define NIIN_DEFINE_ATOMIC_FETCH_ADD(TYPENAME, TYPE) \ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_fetch_add( \ TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_add_try(atomicContext, offset, value, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_add (proxy)", (TYPE)0); \ + } \ if (pe == nvshmem_my_pe()) \ return niin_atomicAdd(dest, value); \ - size_t offset = niin_sym_offset(dest); \ if (niin_is_lsa_peer(pe)) \ - return niin_atomicAdd((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + return niin_atomicAddSystem((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_add_try(atomicContext, \ + offset, value, pe, &previous)) \ + return previous; \ NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_add (network)", (TYPE)0); \ } @@ -115,7 +181,15 @@ NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_FETCH_ADD) #define NIIN_DEFINE_ATOMIC_ADD(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmem_##TYPENAME##_atomic_add( \ TYPE* dest, TYPE value, int pe) { \ - (void)nvshmem_##TYPENAME##_atomic_fetch_add(dest, value, pe); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_world_is_lsa_only()) { \ + if (pe == niin_rank()) \ + niin_atomicAddNoReturn(dest, value); \ + else \ + niin_atomicAddSystemNoReturn((TYPE*)niin_get_peer_ptr_lsa_only(offset, pe), value); \ + return; \ + } \ + niin_atomic_add_slow(dest, value, pe, offset); \ } NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_ADD) @@ -127,11 +201,22 @@ NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_ADD) #define NIIN_DEFINE_ATOMIC_COMPARE_SWAP(TYPENAME, TYPE) \ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_compare_swap( \ TYPE* dest, TYPE cond, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_atomic_compare_swap_try(atomicContext, offset, cond, value, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_compare_swap (proxy)", (TYPE)0); \ + } \ if (pe == nvshmem_my_pe()) \ return niin_atomicCAS(dest, cond, value); \ - size_t offset = niin_sym_offset(dest); \ if (niin_is_lsa_peer(pe)) \ return niin_atomicCAS((TYPE*)niin_get_peer_ptr(offset, pe), cond, value); \ + TYPE previous; \ + if (niin_gpunetio_atomic_compare_swap_try(atomicContext, \ + offset, cond, value, pe, &previous)) \ + return previous; \ NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_compare_swap (network)", (TYPE)0); \ } @@ -144,11 +229,22 @@ NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_COMPARE_SWAP) #define NIIN_DEFINE_ATOMIC_SWAP(TYPENAME, TYPE) \ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_swap( \ TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_atomic_swap_try(atomicContext, offset, value, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_swap (proxy)", (TYPE)0); \ + } \ if (pe == nvshmem_my_pe()) \ return niin_atomicExch(dest, value); \ - size_t offset = niin_sym_offset(dest); \ if (niin_is_lsa_peer(pe)) \ return niin_atomicExch((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + TYPE previous; \ + if (niin_gpunetio_atomic_swap_try(atomicContext, offset, \ + value, pe, &previous)) \ + return previous; \ NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_swap (network)", (TYPE)0); \ } @@ -161,11 +257,22 @@ NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_SWAP) #define NIIN_DEFINE_ATOMIC_FETCH(TYPENAME, TYPE) \ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_fetch( \ const TYPE* dest, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_try(atomicContext, offset, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch (proxy)", (TYPE)0); \ + } \ if (pe == nvshmem_my_pe()) \ return niin_atomicAdd((TYPE*)dest, (TYPE)0); \ - size_t offset = niin_sym_offset(dest); \ if (niin_is_lsa_peer(pe)) \ return niin_atomicAdd((TYPE*)niin_get_peer_ptr(offset, pe), (TYPE)0); \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_try(atomicContext, offset, \ + pe, &previous)) \ + return previous; \ NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch (network)", (TYPE)0); \ } @@ -178,7 +285,25 @@ NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_FETCH) #define NIIN_DEFINE_ATOMIC_SET(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmem_##TYPENAME##_atomic_set( \ TYPE* dest, TYPE value, int pe) { \ - (void)nvshmem_##TYPENAME##_atomic_swap(dest, value, pe); \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + if (niin_gpunetio_atomic_set_try(atomicContext, offset, value, pe)) return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_set (proxy)"); \ + return; \ + } \ + if (pe == nvshmem_my_pe()) { \ + (void)niin_atomicExch(dest, value); \ + return; \ + } \ + if (niin_is_lsa_peer(pe)) { \ + (void)niin_atomicExch((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + return; \ + } \ + if (niin_gpunetio_atomic_set_try(atomicContext, offset, \ + value, pe)) \ + return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_set (network)"); \ } NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_SET) @@ -190,7 +315,15 @@ NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_SET) #define NIIN_DEFINE_ATOMIC_INC(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmem_##TYPENAME##_atomic_inc( \ TYPE* dest, int pe) { \ - nvshmem_##TYPENAME##_atomic_add(dest, (TYPE)1, pe); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_world_is_lsa_only()) { \ + if (pe == niin_rank()) \ + niin_atomicAddNoReturn(dest, (TYPE)1); \ + else \ + niin_atomicAddSystemNoReturn((TYPE*)niin_get_peer_ptr_lsa_only(offset, pe), (TYPE)1); \ + return; \ + } \ + niin_atomic_add_slow(dest, (TYPE)1, pe, offset); \ } NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_INC) @@ -199,25 +332,52 @@ NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_INC) #define NIIN_DEFINE_ATOMIC_FETCH_INC(TYPENAME, TYPE) \ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_fetch_inc( \ TYPE* dest, int pe) { \ - return nvshmem_##TYPENAME##_atomic_fetch_add(dest, (TYPE)1, pe); \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_inc_try(atomicContext, offset, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_inc (proxy)", (TYPE)0); \ + } \ + if (pe == nvshmem_my_pe()) \ + return niin_atomicAdd(dest, (TYPE)1); \ + if (niin_is_lsa_peer(pe)) \ + return niin_atomicAddSystem((TYPE*)niin_get_peer_ptr(offset, pe), (TYPE)1); \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_inc_try(atomicContext, \ + offset, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_inc (network)", (TYPE)0); \ } NIIN_AMO_STANDARD_TYPES(NIIN_DEFINE_ATOMIC_FETCH_INC) #undef NIIN_DEFINE_ATOMIC_FETCH_INC // --------------------------------------------------------------------------- -// Bitwise atomics: and, or, xor, fetch_and, fetch_or, fetch_xor -// NVLink only -- network = NOT_IMPLEMENTED +// Bitwise atomics: and, or, xor, fetch_and, fetch_or, fetch_xor. The network +// case uses NIIN's optional direct GPUNetIO provider, which emits masked AMOs. // --------------------------------------------------------------------------- #define NIIN_DEFINE_ATOMIC_FETCH_AND(TYPENAME, TYPE) \ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_fetch_and( \ TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_and_try(atomicContext, offset, value, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_and (proxy)", (TYPE)0); \ + } \ if (pe == nvshmem_my_pe()) \ return niin_atomicAnd(dest, value); \ - size_t offset = niin_sym_offset(dest); \ if (niin_is_lsa_peer(pe)) \ return niin_atomicAnd((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_and_try(atomicContext, \ + offset, value, pe, &previous)) \ + return previous; \ NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_and (network)", (TYPE)0); \ } @@ -227,7 +387,25 @@ NIIN_AMO_BITWISE_TYPES(NIIN_DEFINE_ATOMIC_FETCH_AND) #define NIIN_DEFINE_ATOMIC_AND(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmem_##TYPENAME##_atomic_and( \ TYPE* dest, TYPE value, int pe) { \ - (void)nvshmem_##TYPENAME##_atomic_fetch_and(dest, value, pe); \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + if (niin_gpunetio_atomic_and_try(atomicContext, offset, value, pe)) return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_and (proxy)"); \ + return; \ + } \ + if (pe == nvshmem_my_pe()) { \ + (void)niin_atomicAnd(dest, value); \ + return; \ + } \ + if (niin_is_lsa_peer(pe)) { \ + (void)niin_atomicAnd((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + return; \ + } \ + if (niin_gpunetio_atomic_and_try(atomicContext, offset, \ + value, pe)) \ + return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_and (network)"); \ } NIIN_AMO_BITWISE_TYPES(NIIN_DEFINE_ATOMIC_AND) @@ -236,11 +414,22 @@ NIIN_AMO_BITWISE_TYPES(NIIN_DEFINE_ATOMIC_AND) #define NIIN_DEFINE_ATOMIC_FETCH_OR(TYPENAME, TYPE) \ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_fetch_or( \ TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_or_try(atomicContext, offset, value, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_or (proxy)", (TYPE)0); \ + } \ if (pe == nvshmem_my_pe()) \ return niin_atomicOr(dest, value); \ - size_t offset = niin_sym_offset(dest); \ if (niin_is_lsa_peer(pe)) \ return niin_atomicOr((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_or_try(atomicContext, \ + offset, value, pe, &previous)) \ + return previous; \ NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_or (network)", (TYPE)0); \ } @@ -250,7 +439,25 @@ NIIN_AMO_BITWISE_TYPES(NIIN_DEFINE_ATOMIC_FETCH_OR) #define NIIN_DEFINE_ATOMIC_OR(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmem_##TYPENAME##_atomic_or( \ TYPE* dest, TYPE value, int pe) { \ - (void)nvshmem_##TYPENAME##_atomic_fetch_or(dest, value, pe); \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + if (niin_gpunetio_atomic_or_try(atomicContext, offset, value, pe)) return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_or (proxy)"); \ + return; \ + } \ + if (pe == nvshmem_my_pe()) { \ + (void)niin_atomicOr(dest, value); \ + return; \ + } \ + if (niin_is_lsa_peer(pe)) { \ + (void)niin_atomicOr((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + return; \ + } \ + if (niin_gpunetio_atomic_or_try(atomicContext, offset, \ + value, pe)) \ + return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_or (network)"); \ } NIIN_AMO_BITWISE_TYPES(NIIN_DEFINE_ATOMIC_OR) @@ -259,11 +466,22 @@ NIIN_AMO_BITWISE_TYPES(NIIN_DEFINE_ATOMIC_OR) #define NIIN_DEFINE_ATOMIC_FETCH_XOR(TYPENAME, TYPE) \ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_fetch_xor( \ TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_xor_try(atomicContext, offset, value, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_xor (proxy)", (TYPE)0); \ + } \ if (pe == nvshmem_my_pe()) \ return niin_atomicXor(dest, value); \ - size_t offset = niin_sym_offset(dest); \ if (niin_is_lsa_peer(pe)) \ return niin_atomicXor((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + TYPE previous; \ + if (niin_gpunetio_atomic_fetch_xor_try(atomicContext, \ + offset, value, pe, &previous)) \ + return previous; \ NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch_xor (network)", (TYPE)0); \ } @@ -273,10 +491,172 @@ NIIN_AMO_BITWISE_TYPES(NIIN_DEFINE_ATOMIC_FETCH_XOR) #define NIIN_DEFINE_ATOMIC_XOR(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmem_##TYPENAME##_atomic_xor( \ TYPE* dest, TYPE value, int pe) { \ - (void)nvshmem_##TYPENAME##_atomic_fetch_xor(dest, value, pe); \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + if (niin_gpunetio_atomic_xor_try(atomicContext, offset, value, pe)) return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_xor (proxy)"); \ + return; \ + } \ + if (pe == nvshmem_my_pe()) { \ + (void)niin_atomicXor(dest, value); \ + return; \ + } \ + if (niin_is_lsa_peer(pe)) { \ + (void)niin_atomicXor((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + return; \ + } \ + if (niin_gpunetio_atomic_xor_try(atomicContext, offset, \ + value, pe)) \ + return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_xor (network)"); \ } NIIN_AMO_BITWISE_TYPES(NIIN_DEFINE_ATOMIC_XOR) #undef NIIN_DEFINE_ATOMIC_XOR +// --------------------------------------------------------------------------- +// Floating AMOs require the NIIN SRQ proxy for a remote target. GPUNetIO's +// native RC atomic WQEs are integral only. The float/double base operations +// are raw-bit fetch/swap/set; the six NVSHMEMX add/fetch-add spellings perform +// real floating-point addition at the target proxy. Do not route any of these +// through niin_atomicAdd: its generic implementation intentionally performs +// modulo integer arithmetic on raw bits for the integral AMO family. +// --------------------------------------------------------------------------- + +template +__device__ __forceinline__ T niin_atomicFloatingAdd(T* addr, T value) { + if constexpr (std::is_same::value) { + return atomicAdd(addr, value); + } else if constexpr (std::is_same::value) { + return atomicAdd(addr, value); + } else { + static_assert(std::is_same::value, + "NIIN floating AMO add supports float, double, and __half only"); + return atomicAdd(addr, value); + } +} + +// atomicOr(x, 0) reads the original 32/64-bit representation without changing +// it. Unlike atomicAdd(x, 0), it preserves NaN payloads and signed zero. +template +__device__ __forceinline__ T niin_atomicFloatingFetch(const T* addr) { + static_assert(std::is_same::value || std::is_same::value, + "NIIN floating raw fetch supports float/double only"); + return niin_atomicOr(const_cast(addr), T{}); +} + +#define NIIN_DEFINE_FLOATING_ATOMIC_SWAP(TYPENAME, TYPE) \ +__device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_swap( \ + TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_proxy_atomic_try(atomicContext, \ + NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP, offset, value, TYPE{}, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_swap (proxy)", (TYPE)0); \ + } \ + if (pe == nvshmem_my_pe()) return niin_atomicExch(dest, value); \ + if (niin_is_lsa_peer(pe)) \ + return niin_atomicExch((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_swap (network)", (TYPE)0); \ +} + +NIIN_DEFINE_FLOATING_ATOMIC_SWAP(float, float) +NIIN_DEFINE_FLOATING_ATOMIC_SWAP(double, double) +#undef NIIN_DEFINE_FLOATING_ATOMIC_SWAP + +#define NIIN_DEFINE_FLOATING_ATOMIC_FETCH(TYPENAME, TYPE) \ +__device__ __forceinline__ TYPE nvshmem_##TYPENAME##_atomic_fetch( \ + const TYPE* dest, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_proxy_atomic_try(atomicContext, \ + NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH, offset, TYPE{}, TYPE{}, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch (proxy)", (TYPE)0); \ + } \ + if (pe == nvshmem_my_pe()) return niin_atomicFloatingFetch(dest); \ + if (niin_is_lsa_peer(pe)) \ + return niin_atomicFloatingFetch((TYPE*)niin_get_peer_ptr(offset, pe)); \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_atomic_fetch (network)", (TYPE)0); \ +} + +NIIN_DEFINE_FLOATING_ATOMIC_FETCH(float, float) +NIIN_DEFINE_FLOATING_ATOMIC_FETCH(double, double) +#undef NIIN_DEFINE_FLOATING_ATOMIC_FETCH + +#define NIIN_DEFINE_FLOATING_ATOMIC_SET(TYPENAME, TYPE) \ +__device__ __forceinline__ void nvshmem_##TYPENAME##_atomic_set( \ + TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + if (niin_gpunetio_proxy_atomic_try(atomicContext, \ + NIIN_GPUNETIO_PROXY_ATOMIC_OP_SET, offset, value, TYPE{}, pe)) return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_set (proxy)"); \ + return; \ + } \ + if (pe == nvshmem_my_pe()) { (void)niin_atomicExch(dest, value); return; } \ + if (niin_is_lsa_peer(pe)) { \ + (void)niin_atomicExch((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + return; \ + } \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmem_" #TYPENAME "_atomic_set (network)"); \ +} + +NIIN_DEFINE_FLOATING_ATOMIC_SET(float, float) +NIIN_DEFINE_FLOATING_ATOMIC_SET(double, double) +#undef NIIN_DEFINE_FLOATING_ATOMIC_SET + +#define NIIN_DEFINE_EXTENDED_ATOMIC_FETCH_ADD(TYPENAME, TYPE) \ +__device__ __forceinline__ TYPE nvshmemx_##TYPENAME##_atomic_fetch_add( \ + TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + TYPE previous; \ + if (niin_gpunetio_proxy_atomic_try(atomicContext, \ + NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD, offset, value, TYPE{}, pe, &previous)) \ + return previous; \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmemx_" #TYPENAME "_atomic_fetch_add (proxy)", (TYPE)0); \ + } \ + if (pe == nvshmem_my_pe()) return niin_atomicFloatingAdd(dest, value); \ + if (niin_is_lsa_peer(pe)) \ + return niin_atomicFloatingAdd((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + NIIN_NOT_IMPLEMENTED_RETURN("nvshmemx_" #TYPENAME "_atomic_fetch_add (network)", (TYPE)0); \ +} + +#define NIIN_DEFINE_EXTENDED_ATOMIC_ADD(TYPENAME, TYPE) \ +__device__ __forceinline__ void nvshmemx_##TYPENAME##_atomic_add( \ + TYPE* dest, TYPE value, int pe) { \ + const niinGpunetioAtomicContext* atomicContext = niin_gpunetio_atomic_context(); \ + size_t offset = niin_sym_offset(dest); \ + if (niin_gpunetio_proxy_atomic_active(atomicContext)) { \ + if (niin_gpunetio_proxy_atomic_try(atomicContext, \ + NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD, offset, value, TYPE{}, pe)) return; \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmemx_" #TYPENAME "_atomic_add (proxy)"); \ + return; \ + } \ + if (pe == nvshmem_my_pe()) { (void)niin_atomicFloatingAdd(dest, value); return; } \ + if (niin_is_lsa_peer(pe)) { \ + (void)niin_atomicFloatingAdd((TYPE*)niin_get_peer_ptr(offset, pe), value); \ + return; \ + } \ + NIIN_NOT_IMPLEMENTED_VOID("nvshmemx_" #TYPENAME "_atomic_add (network)"); \ +} + +NIIN_DEFINE_EXTENDED_ATOMIC_FETCH_ADD(half, __half) +NIIN_DEFINE_EXTENDED_ATOMIC_FETCH_ADD(float, float) +NIIN_DEFINE_EXTENDED_ATOMIC_FETCH_ADD(double, double) +NIIN_DEFINE_EXTENDED_ATOMIC_ADD(half, __half) +NIIN_DEFINE_EXTENDED_ATOMIC_ADD(float, float) +NIIN_DEFINE_EXTENDED_ATOMIC_ADD(double, double) +#undef NIIN_DEFINE_EXTENDED_ATOMIC_FETCH_ADD +#undef NIIN_DEFINE_EXTENDED_ATOMIC_ADD + #endif // NIIN_ATOMICS_H_ diff --git a/projects/rccl/contrib/niin/include/niin/collectives.h b/projects/rccl/contrib/niin/include/niin/collectives.h index 6f6f90f7a55f..7e85b2e4943d 100644 --- a/projects/rccl/contrib/niin/include/niin/collectives.h +++ b/projects/rccl/contrib/niin/include/niin/collectives.h @@ -23,9 +23,35 @@ __device__ __forceinline__ void niin_device_barrier_all() { bar.sync(ncclCoopThread{}, cuda::memory_order_acq_rel); } else { // Multi-node — need GIN barrier for cross-node sync - ncclGin gin(comm, niin_gin_context_index()); - ncclBarrierSession bar(ncclCoopThread{}, ncclTeamTagWorld{}, gin, 0); - bar.sync(ncclCoopThread{}, cuda::memory_order_acq_rel, ncclGinFenceLevel::Relaxed); + // RMA operations are distributed across GIN contexts by CTA. The + // barrier must therefore fence all of them, not only this CTA's context. + ncclGinAllContexts gin(comm); + ncclGinBarrierSession bar(ncclCoopThread{}, gin, ncclTeamTagWorld{}, 0); + // A barrier must make prior remote puts visible after it returns. Ask + // NCCL's existing GIN barrier to fence both the producer and consumer + // sides instead of using its synchronization-only mode. + bar.sync(ncclCoopThread{}, cuda::memory_order_acq_rel, + ncclGinFenceLevel::Put | ncclGinFenceLevel::Get); + } +} + +// The block-scoped NVSHMEMX barriers have a stronger participation contract: +// every thread in one matching CTA per PE enters the operation. Use NCCL's +// CTA cooperative form for those wrappers rather than letting one thread use +// the scalar barrier form. In particular, the CTA form owns the required +// intra-CTA rendezvous while advancing the shared NCCL barrier session. +__device__ __forceinline__ void niin_device_barrier_all_cta() { + ncclDevComm const& comm = niin_comm(); + ncclCoopCta coop; + if (comm.lsaSize == comm.nRanks || !niin_has_gin()) { + ncclLsaBarrierSession bar( + coop, comm, ncclTeamLsa(comm), comm.lsaBarrier, 0); + bar.sync(coop, cuda::memory_order_acq_rel); + } else { + ncclGinAllContexts gin(comm); + ncclGinBarrierSession bar(coop, gin, ncclTeamTagWorld{}, 0); + bar.sync(coop, cuda::memory_order_acq_rel, + ncclGinFenceLevel::Put | ncclGinFenceLevel::Get); } } @@ -46,6 +72,31 @@ __host__ __device__ __forceinline__ void nvshmem_sync_all() { nvshmem_barrier_all(); } +// --------------------------------------------------------------------------- +// Block-scoped extended barriers. +// +// NVSHMEMX requires every thread in the CTA to participate in the block +// variants. One representative thread performs NIIN's existing global +// barrier/sync while the CTA-level barriers make the operation collective at +// block scope. Barrier adds the system-scope release/acquire ordering that +// sync intentionally does not provide. +// --------------------------------------------------------------------------- +#define NIIN_HAS_NVSHMEMX_BLOCK_COLLECTIVES 1 + +__device__ __forceinline__ void nvshmemx_sync_all_block() { + __syncthreads(); + niin_device_barrier_all_cta(); + __syncthreads(); +} + +__device__ __forceinline__ void nvshmemx_barrier_all_block() { + cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); + __syncthreads(); + niin_device_barrier_all_cta(); + __syncthreads(); + cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); +} + __host__ __device__ __forceinline__ void nvshmem_barrier(nvshmem_team_t team) { if (team == NVSHMEM_TEAM_WORLD) { nvshmem_barrier_all(); diff --git a/projects/rccl/contrib/niin/include/niin/config.h b/projects/rccl/contrib/niin/include/niin/config.h index bba2d3151474..5c61448eabe9 100644 --- a/projects/rccl/contrib/niin/include/niin/config.h +++ b/projects/rccl/contrib/niin/include/niin/config.h @@ -52,4 +52,12 @@ #error "NIIN_ON_NOT_IMPLEMENTED must be NIIN_TRAP, NIIN_NOOP, or NIIN_WARN" #endif +// Network (GIN) transport helpers are defined in headers but deliberately not +// inlined: a caller that only reaches NVLink peers should not carry a network +// descriptor build in its register footprint. `inline` keeps one definition per +// program under RDC; `__noinline__` keeps it out of the caller's frame. +#ifndef NIIN_NOINLINE_DEVICE +#define NIIN_NOINLINE_DEVICE inline __device__ __noinline__ +#endif + #endif // NIIN_CONFIG_H_ diff --git a/projects/rccl/contrib/niin/include/niin/context.h b/projects/rccl/contrib/niin/include/niin/context.h index 5047a80e10f5..e8fc54960ad1 100644 --- a/projects/rccl/contrib/niin/include/niin/context.h +++ b/projects/rccl/contrib/niin/include/niin/context.h @@ -10,58 +10,199 @@ #include "niin/config.h" #include "niin/types.h" - -// The niinContext holds all state needed by device-side NVSHMEM API functions. -// The NVSHMEM host API publishes this pointer with cudaMemcpyToSymbol during -// initialization; low-level NIIN users can still populate and pass it manually. -struct niinContext { - ncclDevComm const* comm; // NCCL device communicator - ncclWindow_t heapWindow; // The single symmetric heap window - void* heapBase; // Local base pointer of the heap - size_t heapSize; // Size of the symmetric heap - int ginContextIndex; // Which GIN context index to use - bool peerNativeAtomic; // True if peer GPUs support native system-scope atomics - bool forceSeparatePutSignal; // Force put+fence+signal instead of fused put_signal -}; +#include "niin/context_abi.h" // Device-side global context pointer used by the header-only device API. -__device__ niinContext* niin_g_ctx; +// Relocatable device code lets `inline` coalesce this header-defined variable +// across consumer translation units. Whole-program CUDA compilation cannot use +// an externally linked inline device variable, so a single-TU consumer gets an +// internal definition instead. +#if defined(__CUDACC_RDC__) +inline __device__ niinContext* niin_g_ctx; +inline __constant__ niinContext niin_g_ctx_constant; +#else +static __device__ niinContext* niin_g_ctx; +static __constant__ niinContext niin_g_ctx_constant; +#endif + +#ifndef NIIN_USE_DEVICE_CONTEXT_POINTER +#define NIIN_USE_DEVICE_CONTEXT_POINTER 0 +#endif + +__device__ __forceinline__ niinContext const& niin_ctx() { +#if NIIN_USE_DEVICE_CONTEXT_POINTER + return *niin_g_ctx; +#else + return niin_g_ctx_constant; +#endif +} // --------------------------------------------------------------------------- // Internal helpers for accessing context fields // --------------------------------------------------------------------------- __device__ __forceinline__ ncclDevComm const& niin_comm() { +#if NIIN_USE_DEVICE_CONTEXT_POINTER return *niin_g_ctx->comm; +#else + return niin_g_ctx_constant.commValue; +#endif } __device__ __forceinline__ ncclWindow_t niin_heap_window() { - return niin_g_ctx->heapWindow; + return niin_ctx().heapWindow; } __device__ __forceinline__ void* niin_heap_base() { - return niin_g_ctx->heapBase; + return niin_ctx().heapBase; } __device__ __forceinline__ size_t niin_heap_size() { - return niin_g_ctx->heapSize; + return niin_ctx().heapSize; +} + +__device__ __forceinline__ void** niin_peer_heap_base_p2p() { + return niin_ctx().peerHeapBaseP2p; +} + +__device__ __forceinline__ int niin_rank() { + return niin_ctx().rank; +} + +__device__ __forceinline__ int niin_n_ranks() { + return niin_ctx().nRanks; +} + +__device__ __forceinline__ int niin_lsa_rank() { + return niin_ctx().lsaRank; +} + +__device__ __forceinline__ int niin_lsa_size() { + return niin_ctx().lsaSize; +} + +// Number of GIN contexts on this comm. Each context owns one QP per peer, and +// NCCL derives connectionId from the context index, so consecutive indices also +// spread across NICs. NCCL rounds the requested count up to a multiple of the +// connection count, so read the count back from the devComm rather than trusting +// what NIIN asked for at init. +__device__ __forceinline__ uint32_t niin_gin_context_count() { + uint32_t n = niin_comm().ginContextCount; + return n > 0 ? n : 1; } -__device__ __forceinline__ int niin_gin_context_index() { - return niin_g_ctx->ginContextIndex; +// Rotating cursor behind niin_gin_next_context(). Header-static, so each +// translation unit rotates its own copy; the sequence only has to spread +// operations over contexts, not be globally ordered. +static __device__ unsigned int niin_gin_context_cursor = 0; + +// Take the next GIN context for one network operation, round-robin over the +// available contexts, the way NVSHMEM's IBGDA and IBRC transports pick a QP. +// +// Selection is per operation, so a thread's consecutive puts can land on +// different QPs. Puts on different QPs are unordered against each other at the +// receiving NIC, so every operation that has to stay ordered with a later flush +// or signal reuses the context returned here rather than calling this again, +// and nvshmem_fence()/nvshmem_quiet() drain all contexts. +__device__ __forceinline__ int niin_gin_next_context() { + uint32_t n = niin_gin_context_count(); + if (n == 1) return 0; + uint32_t ticket = atomicAdd(&niin_gin_context_cursor, 1u); + return (int)((n & (n - 1)) == 0 ? (ticket & (n - 1)) : (ticket % n)); +} + +// Resolve an operation's context: a caller that already reserved one passes it +// through so the whole operation stays on a single QP. +__device__ __forceinline__ int niin_gin_context_or_next(int ctx) { + return ctx >= 0 ? ctx : niin_gin_next_context(); +} + +// Context reserved for collectives. GIN barriers exchange indexed signals whose +// storage and shadow counters are per-context, so a barrier only converges if +// every PE drives it from the same context index. +__device__ __forceinline__ int niin_gin_collective_context_index() { + return 0; +} + +__device__ __forceinline__ const struct niinGpunetioAtomicContext* niin_gpunetio_atomic_context() { + return niin_ctx().gpunetioAtomicContext; } __device__ __forceinline__ bool niin_has_gin() { - return niin_comm().ginConnectionCount > 0; + return niin_ctx().ginConnectionCount > 0; +} + +__device__ __forceinline__ bool niin_world_is_lsa_only() { + return niin_ctx().worldIsLsaOnly; +} + +__device__ __forceinline__ size_t niin_linear_block_id() { + return (size_t)blockIdx.x + (size_t)blockIdx.y * gridDim.x + + (size_t)blockIdx.z * gridDim.x * gridDim.y; +} + +__device__ __forceinline__ void niin_note_lsa_store_pending() { + niinContext const& ctx = niin_ctx(); + unsigned int* pending = ctx.lsaStorePending; + size_t blockId = niin_linear_block_id(); + if (pending != nullptr && blockId < ctx.lsaStorePendingLen) pending[blockId] = 1u; +} + +__device__ __forceinline__ bool niin_consume_lsa_store_pending() { + niinContext const& ctx = niin_ctx(); + unsigned int* pending = ctx.lsaStorePending; + size_t blockId = niin_linear_block_id(); + if (pending == nullptr || blockId >= ctx.lsaStorePendingLen) return true; + if (*(volatile unsigned int*)&pending[blockId] == 0u) return false; + pending[blockId] = 0u; + return true; +} + +__device__ __forceinline__ void niin_clear_lsa_store_pending() { + niinContext const& ctx = niin_ctx(); + unsigned int* pending = ctx.lsaStorePending; + size_t blockId = niin_linear_block_id(); + if (pending != nullptr && blockId < ctx.lsaStorePendingLen) pending[blockId] = 0u; +} + +__device__ __forceinline__ void niin_note_gin_op_pending() { + unsigned int* pending = niin_ctx().ginPendingOps; + if (pending != nullptr) atomicAdd(pending, 1u); +} + +__device__ __forceinline__ void niin_note_gin_op_complete() { + unsigned int* pending = niin_ctx().ginPendingOps; + if (pending == nullptr) return; + unsigned int old = atomicAdd(pending, 0u); + while (old != 0u) { + unsigned int prev = atomicCAS(pending, old, old - 1u); + if (prev == old) return; + old = prev; + } +} + +__device__ __forceinline__ bool niin_has_gin_ops_pending() { + unsigned int* pending = niin_ctx().ginPendingOps; + if (pending == nullptr) return niin_has_gin(); + return *(volatile unsigned int*)pending != 0u; +} + +__device__ __forceinline__ unsigned int niin_take_gin_ops_pending() { + unsigned int* pending = niin_ctx().ginPendingOps; + if (pending == nullptr) return niin_has_gin() ? 1u : 0u; + unsigned int old = *(volatile unsigned int*)pending; + if (old == 0u) return 0u; + return atomicExch(pending, 0u); } // Compute byte offset of a symmetric pointer relative to the heap base. __device__ __forceinline__ size_t niin_sym_offset(const void* symPtr) { - return (size_t)((const char*)symPtr - (const char*)niin_g_ctx->heapBase); + return (size_t)((const char*)symPtr - (const char*)niin_ctx().heapBase); } // Check whether pe is reachable via NVLink (i.e., is in our LSA team). __device__ __forceinline__ bool niin_is_lsa_peer(int pe) { + if (niin_world_is_lsa_only()) return true; ncclDevComm const& c = niin_comm(); if (c.lsaSize <= 1) return false; ncclTeam lsa = ncclTeamLsa(c); @@ -71,16 +212,37 @@ __device__ __forceinline__ bool niin_is_lsa_peer(int pe) { // Check whether peer GPUs support native system-scope atomics. __device__ __forceinline__ bool niin_peer_native_atomic() { - return niin_g_ctx->peerNativeAtomic; + return niin_ctx().peerNativeAtomic; } __device__ __forceinline__ bool niin_force_separate_put_signal() { - return niin_g_ctx->forceSeparatePutSignal; + return niin_ctx().forceSeparatePutSignal; +} + +// TMA policy selected at initialization (NVSHMEM_TMA_POLICY). +__device__ __forceinline__ int niin_tma_policy() { + return niin_ctx().tmaPolicy; } // Get a peer pointer for an LSA peer at the given symmetric offset. +__device__ __forceinline__ char* niin_get_peer_heap_base_p2p(int pe) { + void** peerHeapBaseP2p = niin_peer_heap_base_p2p(); + if (peerHeapBaseP2p != nullptr) { + auto base = (char*)__ldg((const unsigned long long*)peerHeapBaseP2p + pe); + if (base != nullptr) return base; + } + return nullptr; +} + __device__ __forceinline__ void* niin_get_peer_ptr(size_t offset, int pe) { + char* base = niin_get_peer_heap_base_p2p(pe); + if (base != nullptr) return base + offset; return ncclGetPeerPointer(niin_heap_window(), offset, pe); } +__device__ __forceinline__ void* niin_get_peer_ptr_lsa_only(size_t offset, int pe) { + auto base = (char*)__ldg((const unsigned long long*)niin_peer_heap_base_p2p() + pe); + return base + offset; +} + #endif // NIIN_CONTEXT_H_ diff --git a/projects/rccl/contrib/niin/include/niin/context_abi.h b/projects/rccl/contrib/niin/include/niin/context_abi.h new file mode 100644 index 000000000000..216fa38b82a9 --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/context_abi.h @@ -0,0 +1,49 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_CONTEXT_ABI_H_ +#define NIIN_CONTEXT_ABI_H_ + +// Keep the layout consumed by host-side binders separate from device helper +// functions. The direct GPUNetIO atomic provider needs this ABI to update its +// one optional pointer, but must not pull in GIN/LSA helper implementations or +// NCCL transport internals. +#include + +#include "niin/gpunetio/context.h" + +struct niinContext { + ncclDevComm const* comm; // NCCL device communicator + ncclDevComm commValue; // Constant-context copy of the communicator + ncclWindow_t heapWindow; // The single symmetric heap window + void* heapBase; // Local base pointer of the heap + size_t heapSize; // Size of the symmetric heap + void** peerHeapBaseP2p; // Per-world-rank LSA heap base, null for non-LSA peers + int rank; // WORLD rank cached for LSA fast paths + int nRanks; // WORLD size cached for LSA fast paths + int lsaRank; // LSA rank cached for LSA fast paths + int lsaSize; // LSA size cached for LSA fast paths + unsigned int ginConnectionCount; // Nonzero when GIN is provisioned + bool worldIsLsaOnly; // True when TEAM_WORLD is one LSA domain + int nodeRank; // Rank among PEs on this physical node + int nodeSize; // Number of PEs on this physical node + // Optional, NIIN-owned AMO provider (native GPUNetIO direct WQEs or the + // separate raw-verbs/SRQ proxy-all service). This stays separate from the GIN + // contexts NIIN rotates through: RMA and signaling always continue on GIN. + const struct niinGpunetioAtomicContext* gpunetioAtomicContext; + bool peerNativeAtomic; // True if peer GPUs support native system-scope atomics + bool forceSeparatePutSignal; // Force put+fence+signal instead of fused put_signal + unsigned int* ginPendingOps; // Device scalar: unquieted GIN operations + unsigned int* lsaStorePending; // Per-CTA flag: local/LSA stores needing quiet fence + size_t lsaStorePendingLen; // Number of entries in lsaStorePending + int tmaPolicy; // nvshmemx_tma_policy_t for the LSA put/get paths + uintptr_t* tmaSmemBases; // Per-CTA shared-memory base registered by give_smem + size_t tmaSmemBasesLen; // Number of entries in tmaSmemBases + size_t* tmaSmemSize; // Shared-memory size given by every CTA (single scalar) +}; + +#endif // NIIN_CONTEXT_ABI_H_ diff --git a/projects/rccl/contrib/niin/include/niin/gpunetio/atomics.h b/projects/rccl/contrib/niin/include/niin/gpunetio/atomics.h new file mode 100644 index 000000000000..65ac1a2d6998 --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/gpunetio/atomics.h @@ -0,0 +1,562 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_GPUNETIO_ATOMICS_H_ +#define NIIN_GPUNETIO_ATOMICS_H_ + +#include "niin/gpunetio/device.h" +#include "niin/gpunetio/proxy_atomics/device.h" + +#include +#include + +// All currently native NIIN integer AMOs. The SRQ fallback owns operations +// which cannot be represented by GPUNetIO atomics (for example float add). +enum class niinGpunetioAtomicOp : uint8_t { + FetchAdd, + Add, + CompareSwap, + Swap, + Fetch, + Set, + Inc, + FetchInc, + FetchAnd, + And, + FetchOr, + Or, + FetchXor, + Xor, +}; + +// Keep the direct-QP opcode namespace private to that path, but map it +// explicitly to NIIN's proxy wire enum when a bound ProxyAll provider owns +// the whole AMO linearization domain. Do not rely on matching enum ordinals. +__host__ __device__ constexpr enum niinGpunetioProxyAtomicOp niin_gpunetio_proxy_op( + enum niinGpunetioAtomicOp op) { + switch (op) { + case niinGpunetioAtomicOp::FetchAdd: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD; + case niinGpunetioAtomicOp::Add: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD; + case niinGpunetioAtomicOp::CompareSwap: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_COMPARE_SWAP; + case niinGpunetioAtomicOp::Swap: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP; + case niinGpunetioAtomicOp::Fetch: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH; + case niinGpunetioAtomicOp::Set: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_SET; + case niinGpunetioAtomicOp::Inc: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_INC; + case niinGpunetioAtomicOp::FetchInc: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_INC; + case niinGpunetioAtomicOp::FetchAnd: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_AND; + case niinGpunetioAtomicOp::And: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_AND; + case niinGpunetioAtomicOp::FetchOr: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_OR; + case niinGpunetioAtomicOp::Or: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_OR; + case niinGpunetioAtomicOp::FetchXor: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_XOR; + case niinGpunetioAtomicOp::Xor: return NIIN_GPUNETIO_PROXY_ATOMIC_OP_XOR; + } + return NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH; +} + +#if NIIN_GPUNETIO_HAS_DEVICE_API + +namespace niin { +namespace gpunetio { +namespace detail { + +template +__device__ __forceinline__ uint64_t atomicBits(T value) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "GPUNetIO atomics require 4- or 8-byte operands"); + uint64_t bits = 0; + memcpy(&bits, &value, sizeof(T)); + return bits; +} + +template +__device__ __forceinline__ T atomicValue(uint64_t bits) { + T value; + memcpy(&value, &bits, sizeof(T)); + return value; +} + +template +__device__ __forceinline__ bool getResponseSlot(const niinGpunetioAtomicContext* context, int pe, + size_t* slotOffset) { + // One local result slot belongs to every destination PE. The endpoint lock + // below holds that slot from WQE construction through completion, so callers + // do not need to allocate or manage response-buffer tickets themselves. + if (context == nullptr || context->responseBase == nullptr || context->endpointLocks == nullptr || + context->nPes <= 0 || context->responseStride < sizeof(uint64_t)) + return false; + + const size_t nPes = static_cast(context->nPes); + if (pe < 0 || pe >= context->nPes || nPes > context->responseBytes / context->responseStride) return false; + + const size_t offset = static_cast(pe) * context->responseStride; + if (offset > context->responseBytes || sizeof(T) > context->responseBytes - offset) return false; + + const uintptr_t base = reinterpret_cast(context->responseBase); + const uintptr_t address = base + offset; + if (address < base || (address & (alignof(T) - 1)) != 0) return false; + + *slotOffset = offset; + return true; +} + +template +__device__ __forceinline__ bool directEndpointAvailable(const niinGpunetioAtomicContext* context, int pe) { + constexpr uint32_t commonFlags = NIIN_GPUNETIO_ATOMIC_CONTEXT_READY | + NIIN_GPUNETIO_ATOMIC_CONTEXT_DIRECT | + NIIN_GPUNETIO_ATOMIC_CONTEXT_EXTENDED; + constexpr uint32_t endpointFlags = NIIN_GPUNETIO_ATOMIC_ENDPOINT_CONNECTED | + NIIN_GPUNETIO_ATOMIC_ENDPOINT_EXTENDED; + if (context == nullptr || context->version != NIIN_GPUNETIO_ATOMIC_CONTEXT_VERSION || + (context->flags & commonFlags) != commonFlags || context->endpoints == nullptr || pe < 0 || + pe >= context->nPes) + return false; + + const niinGpunetioAtomicEndpoint& endpoint = context->endpoints[pe]; + uint32_t requiredEndpointFlags = endpointFlags | + (sizeof(T) == 4 ? NIIN_GPUNETIO_ATOMIC_ENDPOINT_ATOMIC_4B + : NIIN_GPUNETIO_ATOMIC_ENDPOINT_ATOMIC_8B); + return endpoint.deviceQp != nullptr && (endpoint.flags & requiredEndpointFlags) == requiredEndpointFlags; +} + +template +__device__ __forceinline__ bool atomicConsumesTwoWqebbs(niinGpunetioAtomicOp op) { + // A masked 64-bit compare-and-swap WQE occupies two WQEBBs. The set/swap + // and bitwise AND/OR operations use that encoding. It has no control + // segment in the second WQEBB, so a final signaled NOP supplies the CQE that + // makes the whole chain visible to the caller. + if constexpr (sizeof(T) != 8) { + return false; + } else { + return op == niinGpunetioAtomicOp::Swap || op == niinGpunetioAtomicOp::Set || + op == niinGpunetioAtomicOp::FetchAnd || op == niinGpunetioAtomicOp::And || + op == niinGpunetioAtomicOp::FetchOr || op == niinGpunetioAtomicOp::Or; + } +} + +__device__ __forceinline__ bool validCstScratch(const niinGpunetioAtomicContext* context) { + return context != nullptr && context->cstScratchOffset >= sizeof(uint64_t) && + context->responseStride >= sizeof(uint8_t) && + context->cstScratchOffset <= context->responseStride - sizeof(uint8_t); +} + +__device__ __forceinline__ bool atomicReturnsPrevious(niinGpunetioAtomicOp op) { + switch (op) { + case niinGpunetioAtomicOp::FetchAdd: + case niinGpunetioAtomicOp::CompareSwap: + case niinGpunetioAtomicOp::Swap: + case niinGpunetioAtomicOp::Fetch: + case niinGpunetioAtomicOp::FetchInc: + case niinGpunetioAtomicOp::FetchAnd: + case niinGpunetioAtomicOp::FetchOr: + case niinGpunetioAtomicOp::FetchXor: + return true; + case niinGpunetioAtomicOp::Add: + case niinGpunetioAtomicOp::Set: + case niinGpunetioAtomicOp::Inc: + case niinGpunetioAtomicOp::And: + case niinGpunetioAtomicOp::Or: + case niinGpunetioAtomicOp::Xor: + return false; + } + return false; +} + +__device__ __forceinline__ bool lockEndpoint(int* endpointLock) { + // `0` is available, `1` is held, and a negative value permanently poisons + // the endpoint after a CQ error. The latter prevents a second AMO from + // submitting to a QP whose first error has already been observed. + for (;;) { + const int previous = atomicCAS(endpointLock, 0, 1); + if (previous == 0) return true; + if (previous < 0) return false; + } +} + +__device__ __forceinline__ void unlockEndpoint(int* endpointLock) { + atomicExch(endpointLock, 0); +} + +__device__ __forceinline__ void failEndpoint(int* endpointLock) { + atomicExch(endpointLock, -1); +} + +__device__ __forceinline__ bool isNativeOp(niinGpunetioAtomicOp op) { + switch (op) { + case niinGpunetioAtomicOp::FetchAdd: + case niinGpunetioAtomicOp::Add: + case niinGpunetioAtomicOp::CompareSwap: + case niinGpunetioAtomicOp::Swap: + case niinGpunetioAtomicOp::Fetch: + case niinGpunetioAtomicOp::Set: + case niinGpunetioAtomicOp::Inc: + case niinGpunetioAtomicOp::FetchInc: + case niinGpunetioAtomicOp::FetchAnd: + case niinGpunetioAtomicOp::And: + case niinGpunetioAtomicOp::FetchOr: + case niinGpunetioAtomicOp::Or: + case niinGpunetioAtomicOp::FetchXor: + case niinGpunetioAtomicOp::Xor: + return true; + } + return false; +} + +template +__device__ __forceinline__ void prepareAtomicWqe( + struct doca_gpu_dev_verbs_qp* qp, struct doca_gpu_dev_verbs_wqe* wqe0, + struct doca_gpu_dev_verbs_wqe* wqe1, uint16_t wqeIndex, + enum doca_gpu_dev_verbs_wqe_ctrl_flags ctrlFlags, uint64_t remoteAddress, uint32_t remoteRkey, + uint64_t localAddress, uint32_t localLkey, niinGpunetioAtomicOp op, T operand, T compare) { + const uint64_t operandBits = atomicBits(operand); + const uint64_t compareBits = atomicBits(compare); + const uint64_t mask = sizeof(T) == 4 ? uint64_t(UINT32_MAX) : UINT64_MAX; + + if constexpr (sizeof(T) == 4) { + switch (op) { + case niinGpunetioAtomicOp::FetchAdd: + case niinGpunetioAtomicOp::Add: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_FA, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, operandBits, 0, 0, 0, 0, 0); + break; + case niinGpunetioAtomicOp::Inc: + case niinGpunetioAtomicOp::FetchInc: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_FA, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 1, 0, 0, 0, 0, 0); + break; + case niinGpunetioAtomicOp::Fetch: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_FA, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, 0, 0, 0, 0); + break; + case niinGpunetioAtomicOp::FetchXor: + case niinGpunetioAtomicOp::Xor: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_FA, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, operandBits, mask, 0, 0, 0, 0); + break; + case niinGpunetioAtomicOp::Swap: + case niinGpunetioAtomicOp::Set: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_CS, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, operandBits, 0, mask, 0); + break; + case niinGpunetioAtomicOp::FetchAnd: + case niinGpunetioAtomicOp::And: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_CS, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, operandBits, 0, ~operandBits & mask, 0); + break; + case niinGpunetioAtomicOp::FetchOr: + case niinGpunetioAtomicOp::Or: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_CS, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, operandBits, 0, operandBits, 0); + break; + case niinGpunetioAtomicOp::CompareSwap: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_CS, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, operandBits, compareBits, mask, mask); + break; + } + } else { + switch (op) { + case niinGpunetioAtomicOp::FetchAdd: + case niinGpunetioAtomicOp::Add: + // The normal 64-bit fetch-add WQE is preferable when its exact + // semantics are sufficient; the masked form remains available below + // for increment, fetch, and xor. + doca_gpu_dev_verbs_wqe_prepare_atomic(qp, wqe0, wqeIndex, + DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_FA, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 8, + operandBits, 0); + break; + case niinGpunetioAtomicOp::CompareSwap: + doca_gpu_dev_verbs_wqe_prepare_atomic(qp, wqe0, wqeIndex, + DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_CS, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 8, + compareBits, operandBits); + break; + case niinGpunetioAtomicOp::Inc: + case niinGpunetioAtomicOp::FetchInc: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_FA, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 1, 0, 0, 0, 0, 0); + break; + case niinGpunetioAtomicOp::Fetch: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_FA, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, 0, 0, 0, 0); + break; + case niinGpunetioAtomicOp::FetchXor: + case niinGpunetioAtomicOp::Xor: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_FA, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, operandBits, mask, 0, 0, 0, 0); + break; + case niinGpunetioAtomicOp::Swap: + case niinGpunetioAtomicOp::Set: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_CS, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, operandBits, 0, mask, 0); + break; + case niinGpunetioAtomicOp::FetchAnd: + case niinGpunetioAtomicOp::And: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_CS, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, operandBits, 0, ~operandBits, 0); + break; + case niinGpunetioAtomicOp::FetchOr: + case niinGpunetioAtomicOp::Or: + doca_gpu_dev_verbs_wqe_prepare_atomic_ext( + qp, wqe0, wqe1, wqeIndex, DOCA_GPUNETIO_IB_MLX5_OPCODE_ATOMIC_MASKED_CS, ctrlFlags, + remoteAddress, remoteRkey, localAddress, localLkey, 0, 0, operandBits, 0, operandBits, 0); + break; + } + } +} + +template +__device__ __forceinline__ T readAtomicResult(const niinGpunetioAtomicContext* context, void* address) { + uint64_t bits; + if constexpr (sizeof(T) == 4) { + // RC atomics place 32-bit responses in network byte order. + bits = doca_gpu_dev_verbs_bswap32(*reinterpret_cast(address)); + } else { + bits = *reinterpret_cast(address); + if ((context->flags & NIIN_GPUNETIO_ATOMIC_CONTEXT_8B_RESULT_HOST_ENDIAN) == 0) + bits = doca_gpu_dev_verbs_bswap64(bits); + } + return atomicValue(bits); +} + +} // namespace detail +} // namespace gpunetio +} // namespace niin + +// Try the native GPUNetIO path. It returns false without issuing a WQE when +// the independent NIIN provider is absent, not ready, lacks the required +// capability, or its per-PE response slot is invalid. The context owns one +// response slot and one lock per destination PE, so callers do not need to +// manage response-buffer tickets. `previous` is valid only for a fetching +// operation and may be null when the caller wants to discard that value. +template +__device__ __forceinline__ bool niin_gpunetio_atomic_try( + const niinGpunetioAtomicContext* context, niinGpunetioAtomicOp op, size_t remoteOffset, T operand, + T compare, int pe, T* previous = nullptr) { + static_assert(std::is_integral::value, "native GPUNetIO atomics support integral types only"); + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "native GPUNetIO atomics require 4- or 8-byte operands"); + + // ProxyAll is deliberately checked before direct endpoint availability. + // A CPU RMW service cannot be mixed with a NIC AMO against the same heap + // location, even for self/LSA destinations selected by the public adapter. + if (niin_gpunetio_proxy_atomic_active(context)) + return niin_gpunetio_proxy_atomic_try(context, niin_gpunetio_proxy_op(op), remoteOffset, operand, + compare, pe, previous); + + using namespace niin::gpunetio::detail; + if (!directEndpointAvailable(context, pe) || !isNativeOp(op)) + return false; + + const bool returnsPrevious = atomicReturnsPrevious(op); + if (previous != nullptr && !returnsPrevious) return false; + + size_t responseSlotOffset = 0; + if (!getResponseSlot(context, pe, &responseSlotOffset)) return false; + + const niinGpunetioAtomicEndpoint& endpoint = context->endpoints[pe]; + if (remoteOffset > endpoint.remoteHeapBytes || sizeof(T) > endpoint.remoteHeapBytes - remoteOffset) return false; + // Mellanox RC AMOs require naturally aligned operands. Do this validation + // before taking the per-PE lock or reserving a WQE so a bad user pointer + // cannot submit an invalid operation and poison the dedicated QP. + if ((remoteOffset & (alignof(T) - 1)) != 0) return false; + + struct doca_gpu_dev_verbs_qp* qp = reinterpret_cast(endpoint.deviceQp); + const uint64_t remoteAddress = endpoint.remoteHeapBase + remoteOffset; + if (remoteAddress < endpoint.remoteHeapBase || (remoteAddress & (alignof(T) - 1)) != 0) return false; + + void* localResult = static_cast(context->responseBase) + responseSlotOffset; + const bool atomicUsesTwoWqebbs = atomicConsumesTwoWqebbs(op); + // Match NVSHMEM's GPUNetIO completion rule. A fetching AMO needs a fenced + // DUMP unless the provider has established that its completion mode makes + // the response visible without one. A 64-bit masked CS always needs a + // terminal WQE because its atomic encoding spans two WQEBBs. + const bool needsCst = returnsPrevious && + (context->flags & NIIN_GPUNETIO_ATOMIC_CONTEXT_MAY_SKIP_CST) == 0; + if (needsCst && !validCstScratch(context)) return false; + const bool needsTerminalWqe = atomicUsesTwoWqebbs || needsCst; + const uint32_t wqeCount = 1 + (atomicUsesTwoWqebbs ? 1 : 0) + (needsTerminalWqe ? 1 : 0); + + // The one-slot-per-PE ring makes this an intentionally serialized path per + // destination. Hold the lock through CQ completion so another caller cannot + // reuse either the response location or this dedicated QP prematurely. + int* endpointLock = context->endpointLocks + pe; + if (!lockEndpoint(endpointLock)) return false; + const uint64_t firstWqe = + doca_gpu_dev_verbs_reserve_wq_slots(qp, wqeCount); + const uint64_t completionWqe = firstWqe + wqeCount - 1; + const uint16_t atomicWqeIndex = static_cast(firstWqe); + + struct doca_gpu_dev_verbs_wqe* wqe0 = doca_gpu_dev_verbs_get_wqe_ptr(qp, atomicWqeIndex); + struct doca_gpu_dev_verbs_wqe* wqe1 = atomicUsesTwoWqebbs + ? doca_gpu_dev_verbs_get_wqe_ptr( + qp, static_cast(firstWqe + 1)) + : nullptr; + enum doca_gpu_dev_verbs_wqe_ctrl_flags atomicCtrl = + needsTerminalWqe ? DOCA_GPUNETIO_IB_MLX5_WQE_CTRL_CQ_ERROR_UPDATE + : DOCA_GPUNETIO_IB_MLX5_WQE_CTRL_CQ_UPDATE; + + prepareAtomicWqe(qp, wqe0, wqe1, atomicWqeIndex, atomicCtrl, remoteAddress, endpoint.remoteHeapRkey, + reinterpret_cast(localResult), context->responseLkey, op, operand, compare); + + if (needsTerminalWqe) { + struct doca_gpu_dev_verbs_wqe* completion = + doca_gpu_dev_verbs_get_wqe_ptr(qp, static_cast(completionWqe)); + if (needsCst) { + void* cstScratch = static_cast(context->responseBase) + responseSlotOffset + + context->cstScratchOffset; + doca_gpu_dev_verbs_wqe_prepare_dump(qp, completion, static_cast(completionWqe), + DOCA_GPUNETIO_IB_MLX5_WQE_CTRL_CQ_UPDATE_FENCE, + reinterpret_cast(cstScratch), context->responseLkey, + sizeof(uint8_t)); + } else { + doca_gpu_dev_verbs_wqe_prepare_nop(qp, completion, static_cast(completionWqe), + DOCA_GPUNETIO_IB_MLX5_WQE_CTRL_CQ_UPDATE); + } + } + + doca_gpu_dev_verbs_mark_wqes_ready(qp, firstWqe, + completionWqe); + // CPU_PROXY requires its producer index to be explicitly published after + // ringing the GPU-to-host proxy doorbell. GPUNetIO's default submission + // intentionally omits that update, which is correct only for direct GPU + // doorbells. The host records the handler choice in this NIIN-owned flag. + const uint32_t submitOptions = + (context->flags & NIIN_GPUNETIO_ATOMIC_CONTEXT_CPU_PROXY_DOORBELL) != 0 + ? DOCA_GPUNETIO_VERBS_GPU_CODE_OPT_CPU_PROXY_UPDATE_PI + : DOCA_GPUNETIO_VERBS_GPU_CODE_OPT_DEFAULT; + doca_gpu_dev_verbs_submit( + qp, completionWqe + 1, submitOptions); + const int completionStatus = + doca_gpu_dev_verbs_poll_cq_at( + qp, completionWqe); + if (completionStatus != 0) { + failEndpoint(endpointLock); + return false; + } + cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); + + if (previous != nullptr) *previous = readAtomicResult(context, localResult); + unlockEndpoint(endpointLock); + return true; +} + +#else // NIIN_GPUNETIO_HAS_DEVICE_API + +// Keep callers compileable in non-GPUNetIO builds. The public NIIN layer can +// then select its existing fail-closed behavior or a future proxy fallback. +template +__device__ __forceinline__ bool niin_gpunetio_atomic_try( + const niinGpunetioAtomicContext* context, niinGpunetioAtomicOp op, size_t remoteOffset, T operand, + T compare, int pe, T* previous = nullptr) { + // The proxy device dispatcher has no GPUNetIO header dependency. This lets + // a ProxyAll client compile its CUDA AMO call sites without enabling the + // direct WQE path, while an unbound context retains fail-closed behavior. + return niin_gpunetio_proxy_atomic_try(context, niin_gpunetio_proxy_op(op), remoteOffset, operand, + compare, pe, previous); +} + +#endif // NIIN_GPUNETIO_HAS_DEVICE_API + +// Thin typed helpers keep the eventual public NIIN atomics header free of +// GPUNetIO opcode knowledge. The context owns the response ring, so no +// caller-visible result slot is part of this interface. +template +__device__ __forceinline__ bool niin_gpunetio_atomic_fetch_add_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe, T* previous) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::FetchAdd, offset, value, T{}, pe, previous); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_add_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::Add, offset, value, T{}, pe); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_compare_swap_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T compare, T value, int pe, T* previous) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::CompareSwap, offset, value, compare, pe, previous); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_swap_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe, T* previous) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::Swap, offset, value, T{}, pe, previous); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_fetch_try( + const niinGpunetioAtomicContext* ctx, size_t offset, int pe, T* previous) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::Fetch, offset, T{}, T{}, pe, previous); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_set_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::Set, offset, value, T{}, pe); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_inc_try( + const niinGpunetioAtomicContext* ctx, size_t offset, int pe) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::Inc, offset, T{}, T{}, pe); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_fetch_inc_try( + const niinGpunetioAtomicContext* ctx, size_t offset, int pe, T* previous) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::FetchInc, offset, T{}, T{}, pe, previous); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_fetch_and_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe, T* previous) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::FetchAnd, offset, value, T{}, pe, previous); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_and_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::And, offset, value, T{}, pe); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_fetch_or_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe, T* previous) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::FetchOr, offset, value, T{}, pe, previous); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_or_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::Or, offset, value, T{}, pe); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_fetch_xor_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe, T* previous) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::FetchXor, offset, value, T{}, pe, previous); +} + +template +__device__ __forceinline__ bool niin_gpunetio_atomic_xor_try( + const niinGpunetioAtomicContext* ctx, size_t offset, T value, int pe) { + return niin_gpunetio_atomic_try(ctx, niinGpunetioAtomicOp::Xor, offset, value, T{}, pe); +} + +#endif // NIIN_GPUNETIO_ATOMICS_H_ diff --git a/projects/rccl/contrib/niin/include/niin/gpunetio/context.h b/projects/rccl/contrib/niin/include/niin/gpunetio/context.h new file mode 100644 index 000000000000..f584c8bbd62f --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/gpunetio/context.h @@ -0,0 +1,106 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_GPUNETIO_CONTEXT_H_ +#define NIIN_GPUNETIO_CONTEXT_H_ + +#include +#include + +// This is deliberately a NIIN-owned ABI. The host implementation creates +// the QPs, registers the heap and response ring, exchanges endpoint metadata, +// and copies one of these contexts to device memory. Nothing in this ABI +// relies on a GIN context or a NCCL transport-private object. +#define NIIN_GPUNETIO_ATOMIC_CONTEXT_VERSION 3u + +enum niinGpunetioAtomicContextFlags : uint32_t { + // The host has completed bootstrap and every endpoint below is safe to use. + NIIN_GPUNETIO_ATOMIC_CONTEXT_READY = 1u << 0, + // The endpoint QPs may be submitted by GPU code. + NIIN_GPUNETIO_ATOMIC_CONTEXT_DIRECT = 1u << 1, + // The endpoint QPs were transitioned with 4/8-byte masked atomic support. + NIIN_GPUNETIO_ATOMIC_CONTEXT_EXTENDED = 1u << 2, + // A host/SRQ fallback is attached. Native code does not dereference it. + NIIN_GPUNETIO_ATOMIC_CONTEXT_PROXY = 1u << 3, + // The provider has verified that the CQ completion makes atomic response + // data visible without a terminal fenced DUMP. Leave this clear by default + // for the conservative completion path. + NIIN_GPUNETIO_ATOMIC_CONTEXT_MAY_SKIP_CST = 1u << 4, + // The selected HCA reports 8-byte atomic responses in host/device byte + // order. Without this bit, the direct path converts 8-byte fetch results + // from network byte order just as it always does for 4-byte results. + NIIN_GPUNETIO_ATOMIC_CONTEXT_8B_RESULT_HOST_ENDIAN = 1u << 5, + // Reserved for a future supported GPUNetIO CPU doorbell configuration. This + // is only a doorbell transport detail (not the future SRQ software-atomic + // fallback). Current direct-provider setup rejects CPU-doorbell QPs. + NIIN_GPUNETIO_ATOMIC_CONTEXT_CPU_PROXY_DOORBELL = 1u << 6, +}; + +enum niinGpunetioAtomicEndpointFlags : uint32_t { + // `deviceQp` is a connected `doca_gpu_dev_verbs_qp*` for this destination. + NIIN_GPUNETIO_ATOMIC_ENDPOINT_CONNECTED = 1u << 0, + NIIN_GPUNETIO_ATOMIC_ENDPOINT_ATOMIC_4B = 1u << 1, + NIIN_GPUNETIO_ATOMIC_ENDPOINT_ATOMIC_8B = 1u << 2, + NIIN_GPUNETIO_ATOMIC_ENDPOINT_EXTENDED = 1u << 3, +}; + +// Bootstrap metadata for one destination PE. `deviceQp` is intentionally a +// void pointer here so this stable NIIN context stays independent of the +// vendor device-header layout. The device adapter casts it only after the +// runtime capability checks pass. Key fields use the byte order required by +// the GPUNetIO device header, not the host ibverbs value. When +// `DOCA_GPUNETIO_VERBS_MKEY_SWAPPED=1`, the host provider stores +// `htobe32(ibv_mr->rkey)` here. +struct niinGpunetioAtomicEndpoint { + void* deviceQp; + uint64_t remoteHeapBase; + size_t remoteHeapBytes; + uint32_t remoteHeapRkey; + uint32_t flags; +}; + +// Device-resident state for NIIN's GPUNetIO atomics provider. +// +// `responseBase` / `responseBytes` identify a GPU allocation registered with +// `responseLkey`. It is a one-slot-per-destination-PE response ring: slot +// `pe` starts at `responseBase + pe * responseStride`. The first eight bytes +// of every slot are reserved for the atomic result, so the same context can +// serve both 4- and 8-byte operations. `endpointLocks[pe]` serializes that +// slot and its dedicated QP; the array is GPU memory, has `nPes` zeroed +// 32-bit entries, and is owned by the provider for the context lifetime. +// +// `cstScratchOffset` is relative to *each* response slot. It names a +// provider-reserved byte used only by a terminal fenced DUMP on systems that +// cannot skip the consistency step. It must be inside `responseStride` and +// outside the first eight result bytes. Keeping it per PE avoids unrelated +// QPs contending on one DUMP address. +// +// `responseLkey` uses the same device-header byte-order contract as +// `remoteHeapRkey`: with the pinned GPUNetIO headers it is +// `htobe32(ibv_mr->lkey)`. A provider built against a GPUNetIO release with +// `DOCA_GPUNETIO_VERBS_MKEY_SWAPPED=0` must instead preserve the host-order +// ibverbs key, because the device WQE builders perform the conversion there. +// `proxyContext` is owned by the optional NIIN SRQ fallback. The front-end +// dispatcher checks it before attempting a direct WQE, but the direct WQE +// construction code itself never dereferences it. +struct niinGpunetioAtomicContext { + uint32_t version; + uint32_t flags; + int nPes; + int localPe; + const struct niinGpunetioAtomicEndpoint* endpoints; + void* responseBase; + size_t responseBytes; + size_t responseStride; + size_t cstScratchOffset; + uint32_t responseLkey; + uint32_t reserved; + int* endpointLocks; + void* proxyContext; +}; + +#endif // NIIN_GPUNETIO_CONTEXT_H_ diff --git a/projects/rccl/contrib/niin/include/niin/gpunetio/device.h b/projects/rccl/contrib/niin/include/niin/gpunetio/device.h new file mode 100644 index 000000000000..8a437ded7d1b --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/gpunetio/device.h @@ -0,0 +1,37 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_GPUNETIO_DEVICE_H_ +#define NIIN_GPUNETIO_DEVICE_H_ + +#include "niin/gpunetio/context.h" + +// The device side of the NIIN GPUNetIO backend is header-only and always +// compiled into CUDA consumers: NCCL ships the GPUNetIO sources and NIIN's +// build links their include tree into its own include/ directory, so +// `-I$NIIN_HOME/include` is all a consumer needs. NIIN_GPUNETIO_ENABLE=0 opts +// out and leaves network AMOs unimplemented. +#if defined(__CUDACC__) && (!defined(NIIN_GPUNETIO_ENABLE) || NIIN_GPUNETIO_ENABLE) +#if defined(__has_include) && !__has_include() +#if !__has_include() +#error "GPUNetIO device headers not found. Build NIIN from an NCCL source checkout so include/gpunetio exists, or compile with -DNIIN_GPUNETIO_ENABLE=0." +#endif +// Some GPUNetIO SDK layouts put the public headers directly on the include +// path. Keep the NIIN source a customer of either published include layout. +#include +#else +#include +#endif +#if !defined(DOCA_GPUNETIO_VERSION_MAJOR) || DOCA_GPUNETIO_VERSION_MAJOR != 4 +#error "NIIN direct atomics require the GPUNetIO 4.x device ABI used by NCCL GIN" +#endif +#define NIIN_GPUNETIO_HAS_DEVICE_API 1 +#else +#define NIIN_GPUNETIO_HAS_DEVICE_API 0 +#endif + +#endif // NIIN_GPUNETIO_DEVICE_H_ diff --git a/projects/rccl/contrib/niin/include/niin/gpunetio/host.h b/projects/rccl/contrib/niin/include/niin/gpunetio/host.h new file mode 100644 index 000000000000..7a72f4a4b436 --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/gpunetio/host.h @@ -0,0 +1,107 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_GPUNETIO_HOST_H_ +#define NIIN_GPUNETIO_HOST_H_ + +#include +#include + +#include + +// The optional SRQ/GDRCopy proxy-all provider has a separate implementation +// and runtime dependency. Expose its host API only when the CMake target was +// built with that implementation; otherwise these declarations would promise +// symbols that the direct-only archive deliberately does not contain. +#if defined(NIIN_GPUNETIO_SRQ_PROXY_ATOMICS_ENABLE) && \ + NIIN_GPUNETIO_SRQ_PROXY_ATOMICS_ENABLE +#include "niin/gpunetio/proxy_atomics/host.h" +#endif + +struct niinContext; +struct niinGpunetioAtomicHostContext; + +// NIIN brings the provider up itself during nvshmem_init(), so these entry +// points are declared weak: an application that does not link the provider +// archive still links, and NIIN sees a null symbol and leaves network AMOs +// fail-closed exactly as before. +#ifndef NIIN_GPUNETIO_WEAK +#define NIIN_GPUNETIO_WEAK __attribute__((weak)) +#endif + +// The host provider deliberately has a small, independent configuration +// surface. It must not select, inspect, or modify a GIN context: existing +// NIIN RMA and signaling continue to use GIN exactly as before. +enum niinGpunetioAtomicNicHandler : int { + // Let GPUNetIO select GPU doorbells where available. Initialization fails + // collectively if this provider revision instead selects CPU proxy + // doorbells, which the direct atomic path has not validated. + NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_AUTO = 0, + // Require GPU SM doorbells. Initialization fails rather than falling back. + NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_GPU_SM_DB = 1, + // Request GPUNetIO's CPU doorbell proxy. This is currently rejected during + // setup rather than permitting a potentially non-terminating device AMO. + NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_CPU_PROXY = 2, +}; + +struct niinGpunetioAtomicOptions { + // HCA name, for example "mlx5_0". nullptr selects NIIN_GPUNETIO_IB_DEV, + // then the first active verbs device. + const char* ibDevice; + // One-based verbs port. Zero selects the first active port on the HCA. + int ibPort; + // GID index used for GRH/RoCE addressing. -1 uses + // NIIN_GPUNETIO_GID_INDEX, then index 0. + int gidIndex; + // Local PKey table index for the provider's private RC QPs. The caller is + // responsible for selecting matching PKeys on every PE; zero is the + // conventional full-membership/default table entry. + uint16_t pkeyIndex; + // Service level for the private RC QPs (0-15). Zero is the default fabric + // service level. + uint8_t serviceLevel; + uint8_t reserved0; + // GPUNetIO SQ depth per destination PE. It must be a power of two. + uint32_t sqDepth; + enum niinGpunetioAtomicNicHandler nicHandler; +}; + +#define NIIN_GPUNETIO_ATOMIC_OPTIONS_INITIALIZER \ + { nullptr, 0, -1, 0, 0, 0, 256, NIIN_GPUNETIO_ATOMIC_NIC_HANDLER_AUTO } + +// Create a NIIN-owned, atomic-only GPUNetIO provider. Every PE in `comm` +// must call this routine once, outside ncclGroupStart/End. The implementation +// performs a public NCCL all-gather only to exchange this provider's QP and MR +// metadata; it does not use GIN for that exchange or for data movement. +// +// `heapBase`/`heapBytes` must name NIIN's existing symmetric heap. NIIN +// registers it directly with this provider only for remote atomic access; no +// GPUNetIO put/get/RMA operation is exposed or implemented here. +NIIN_GPUNETIO_WEAK ncclResult_t niinGpunetioAtomicInit( + ncclComm_t comm, void* heapBase, size_t heapBytes, + const struct niinGpunetioAtomicOptions* options, + struct niinGpunetioAtomicHostContext** out); + +// Bind a ready provider to a context that has already been initialized by +// niinCommit(). This copies only NIIN's private atomic context pointer into +// `deviceContext`; it leaves its ncclDevComm, GIN context, window, and all +// existing RMA/signaling state untouched. +NIIN_GPUNETIO_WEAK ncclResult_t niinGpunetioAtomicBind(struct niinGpunetioAtomicHostContext* provider, + struct niinContext* deviceContext); + +// Make one bounded forward-progress pass for a future supported GPUNetIO CPU +// doorbell configuration. Current direct-provider initialization rejects that +// configuration, so this is retained only as a diagnostics/future-extension +// hook. +NIIN_GPUNETIO_WEAK ncclResult_t niinGpunetioAtomicProgress(struct niinGpunetioAtomicHostContext* provider); + +// Destroy the NIIN-owned QPs, private registrations, response ring, and +// device export. Call after all kernels using the provider have completed and +// before niinFinalize() destroys the device context / NCCL communicator. +NIIN_GPUNETIO_WEAK ncclResult_t niinGpunetioAtomicFinalize(struct niinGpunetioAtomicHostContext* provider); + +#endif // NIIN_GPUNETIO_HOST_H_ diff --git a/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/api.h b/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/api.h new file mode 100644 index 000000000000..e6b783faf12c --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/api.h @@ -0,0 +1,132 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_GPUNETIO_PROXY_ATOMICS_API_H_ +#define NIIN_GPUNETIO_PROXY_ATOMICS_API_H_ + +#include "niin/gpunetio/proxy_atomics/protocol.h" + +#include + +#include +#include +#include +#include +#include + +#if defined(__CUDACC__) +#define NIIN_GPUNETIO_PROXY_HOST_DEVICE __host__ __device__ +#else +#define NIIN_GPUNETIO_PROXY_HOST_DEVICE +#endif + +namespace niin { +namespace gpunetio { +namespace proxy { + +// Map public C++ values to fixed-width protocol representations. Integral +// aliases (including ptrdiff_t) are represented only by their actual width, +// never by the spelling used in an application header. +template +struct AtomicValueTraits { + static constexpr bool kIsIntegral = std::is_integral::value; + static constexpr bool kSupported = kIsIntegral && (sizeof(T) == 4 || sizeof(T) == 8); + static constexpr uint8_t kWireType = + !kIsIntegral ? UINT8_MAX + : (sizeof(T) == 4 ? NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL32 + : (sizeof(T) == 8 ? NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL64 + : UINT8_MAX)); + static constexpr size_t kBytes = kSupported ? sizeof(T) : 0; + static constexpr bool kHasBaseRawOps = kSupported; + static constexpr bool kHasAddOps = kSupported; +}; + +template <> +struct AtomicValueTraits { + static constexpr bool kIsIntegral = false; + static constexpr bool kSupported = true; + static constexpr uint8_t kWireType = NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT32; + static constexpr size_t kBytes = sizeof(float); + static constexpr bool kHasBaseRawOps = true; + static constexpr bool kHasAddOps = true; +}; + +template <> +struct AtomicValueTraits { + static constexpr bool kIsIntegral = false; + static constexpr bool kSupported = true; + static constexpr uint8_t kWireType = NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT64; + static constexpr size_t kBytes = sizeof(double); + static constexpr bool kHasBaseRawOps = true; + static constexpr bool kHasAddOps = true; +}; + +template <> +struct AtomicValueTraits<__half> { + static constexpr bool kIsIntegral = false; + static constexpr bool kSupported = true; + static constexpr uint8_t kWireType = NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT16; + static constexpr size_t kBytes = sizeof(__half); + static constexpr bool kHasBaseRawOps = false; + static constexpr bool kHasAddOps = true; +}; + +static_assert(sizeof(float) == 4 && sizeof(double) == 8, + "NIIN proxy requires binary32/binary64 storage sizes"); +static_assert(std::numeric_limits::is_iec559 && std::numeric_limits::is_iec559, + "NIIN proxy requires IEC 559 float/double semantics"); +static_assert(sizeof(__half) == 2, "NIIN proxy requires binary16 storage"); +static_assert(sizeof(ptrdiff_t) == 4 || sizeof(ptrdiff_t) == 8, + "NIIN proxy supports only 32- or 64-bit ptrdiff_t"); + +// This intentionally narrows the generic transport grammar to NVSHMEM's +// public surface: floating values support raw fetch/swap/set and the six +// nvshmemx add/fetch-add extensions; half is extension-only. +template +NIIN_GPUNETIO_PROXY_HOST_DEVICE constexpr bool isPublicAtomicOperation(uint8_t op) { + using Traits = AtomicValueTraits; + if constexpr (!Traits::kSupported) { + return false; + } else if constexpr (Traits::kIsIntegral) { + return niinGpunetioProxyAtomicOpSupportsType(op, Traits::kWireType); + } else if constexpr (std::is_same::value) { + return op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD; + } else { + return op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_SET; + } +} + +template +NIIN_GPUNETIO_PROXY_HOST_DEVICE inline uint64_t atomicValueBits(T value) { + static_assert(AtomicValueTraits::kSupported, + "NIIN proxy AMOs require 4/8-byte integral, float, double, or __half values"); + uint64_t bits = 0; + memcpy(&bits, &value, sizeof(T)); + return bits; +} + +template +NIIN_GPUNETIO_PROXY_HOST_DEVICE inline T atomicValueFromBits(uint64_t bits) { + static_assert(AtomicValueTraits::kSupported, + "NIIN proxy AMOs require 4/8-byte integral, float, double, or __half values"); + T value; + memcpy(&value, &bits, sizeof(T)); + return value; +} + +} // namespace proxy +} // namespace gpunetio +} // namespace niin + +#undef NIIN_GPUNETIO_PROXY_HOST_DEVICE + +#endif // NIIN_GPUNETIO_PROXY_ATOMICS_API_H_ diff --git a/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/device.h b/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/device.h new file mode 100644 index 000000000000..d1e8b2f4fb51 --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/device.h @@ -0,0 +1,213 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_GPUNETIO_PROXY_ATOMICS_DEVICE_H_ +#define NIIN_GPUNETIO_PROXY_ATOMICS_DEVICE_H_ + +#include "niin/gpunetio/context.h" +#include "niin/gpunetio/proxy_atomics/api.h" + +#if defined(__CUDACC__) +#include +#endif + +// One source slot has a GPU-published request and a target-published +// completion. The allocation is cudaHostAllocMapped memory: `slots` in the +// device context is its GPU alias, while the NIIN proxy threads retain the CPU +// alias. Only the ticket fields are concurrently accessed by CPU/NIC and GPU; +// their acquire/release protocol makes the surrounding body visible. +struct alignas(64) niinGpunetioProxyAtomicSlot { + struct niinGpunetioProxyAtomicRequest request; + struct niinGpunetioProxyAtomicCompletion completion; +}; + +// This device ABI stays separate from the direct GPUNetIO WQE context. It is +// deliberately made of NIIN-owned fixed-layout state; no GIN, GPUNetIO, or +// ibverbs object is exposed to device code. +#define NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_VERSION 2u + +enum niinGpunetioProxyAtomicDeviceContextFlags : uint32_t { + NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_READY = 1u << 0, + NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_PROXY_ALL = 1u << 1, +}; + +struct niinGpunetioProxyAtomicDeviceContext { + uint32_t version; + uint32_t flags; + int nPes; + int localPe; + size_t heapBytes; + uint32_t slotsPerPe; + uint32_t reserved; + struct niinGpunetioProxyAtomicSlot* slots; + int* endpointLocks; + // CUDA-mapped host memory. The progress thread publishes a nonzero value + // before it stops on an unrecoverable transport error, allowing a GPU + // waiter to fail closed rather than spin forever on its completion ticket. + uint32_t* progressFailure; +}; + +#if defined(__CUDACC__) + +namespace niin { +namespace gpunetio { +namespace proxy { +namespace detail { + +__device__ __forceinline__ bool proxyContextAvailable( + const struct niinGpunetioAtomicContext* atomicContext, + const struct niinGpunetioProxyAtomicDeviceContext** proxyContext) { + constexpr uint32_t requiredAtomicFlags = NIIN_GPUNETIO_ATOMIC_CONTEXT_READY | + NIIN_GPUNETIO_ATOMIC_CONTEXT_PROXY; + if (atomicContext == nullptr || atomicContext->version != NIIN_GPUNETIO_ATOMIC_CONTEXT_VERSION || + (atomicContext->flags & requiredAtomicFlags) != requiredAtomicFlags || + atomicContext->proxyContext == nullptr) + return false; + const auto* context = + static_cast(atomicContext->proxyContext); + constexpr uint32_t requiredProxyFlags = NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_READY | + NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_PROXY_ALL; + if (context->version != NIIN_GPUNETIO_PROXY_ATOMIC_DEVICE_CONTEXT_VERSION || + (context->flags & requiredProxyFlags) != requiredProxyFlags || context->nPes <= 0 || + context->slotsPerPe == 0 || context->slots == nullptr || context->endpointLocks == nullptr || + context->progressFailure == nullptr) + return false; + *proxyContext = context; + return true; +} + +__device__ __forceinline__ bool lockEndpoint(int* lock) { + for (;;) { + const int old = atomicCAS(lock, 0, 1); + if (old == 0) return true; + if (old < 0) return false; + } +} + +__device__ __forceinline__ void unlockEndpoint(int* lock) { atomicExch(lock, 0); } + +__device__ __forceinline__ void failEndpoint(int* lock) { atomicExch(lock, -1); } + +template +__device__ __forceinline__ bool issue(const struct niinGpunetioAtomicContext* atomicContext, + enum niinGpunetioProxyAtomicOp op, size_t remoteOffset, + T operand, T compare, int pe, T* previous) { + using Traits = AtomicValueTraits; + static_assert(Traits::kSupported, + "NIIN proxy AMOs require 4/8-byte integral, float, double, or __half values"); + if (!isPublicAtomicOperation(static_cast(op))) return false; + + const struct niinGpunetioProxyAtomicDeviceContext* proxyContext = nullptr; + if (!proxyContextAvailable(atomicContext, &proxyContext) || pe < 0 || pe >= proxyContext->nPes || + remoteOffset > proxyContext->heapBytes || Traits::kBytes > proxyContext->heapBytes - remoteOffset || + (remoteOffset & (Traits::kBytes - 1)) != 0) + return false; + + cuda::atomic_ref progressFailure( + *proxyContext->progressFailure); + if (progressFailure.load(cuda::memory_order_acquire) != 0) return false; + + // The first implementation intentionally leases one slot per destination. + // The wire carries sourceSlot and supports a future ticketed ring, but using + // one locked slot first provides strict completion ordering without a GPU + // allocator or an ABA-sensitive free list. + const uint64_t slotIndex = static_cast(pe) * proxyContext->slotsPerPe; + const uint64_t slotCount = static_cast(proxyContext->nPes) * proxyContext->slotsPerPe; + if (slotIndex >= slotCount) return false; + struct niinGpunetioProxyAtomicSlot* slot = proxyContext->slots + slotIndex; + int* endpointLock = proxyContext->endpointLocks + pe; + if (!lockEndpoint(endpointLock)) return false; + if (progressFailure.load(cuda::memory_order_acquire) != 0) { + failEndpoint(endpointLock); + return false; + } + + // The slot cannot be reused until the prior caller consumed completion and + // released this lock, so a nonzero monotonically incremented ticket is + // enough to distinguish this request from every earlier use. + cuda::atomic_ref requestTicket(slot->request.ticket); + cuda::atomic_ref completionTicket(slot->completion.ticket); + const uint64_t ticket = niinGpunetioProxyAtomicNextTicket( + requestTicket.load(cuda::memory_order_relaxed)); + + slot->request.magic = NIIN_GPUNETIO_PROXY_ATOMIC_PROTOCOL_MAGIC; + slot->request.version = NIIN_GPUNETIO_PROXY_ATOMIC_PROTOCOL_VERSION; + slot->request.op = static_cast(op); + slot->request.type = Traits::kWireType; + slot->request.sourcePe = static_cast(proxyContext->localPe); + slot->request.targetPe = static_cast(pe); + slot->request.sourceSlot = static_cast(slotIndex); + slot->request.flags = NIIN_GPUNETIO_PROXY_ATOMIC_REQUEST_FLAG_NONE; + slot->request.remoteOffset = remoteOffset; + slot->request.operandBits = atomicValueBits(operand); + slot->request.compareBits = atomicValueBits(compare); + slot->completion.resultBits = 0; + slot->completion.status = NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_INTERNAL_ERROR; + slot->completion.reserved = 0; + completionTicket.store(NIIN_GPUNETIO_PROXY_ATOMIC_UNPUBLISHED_TICKET, + cuda::memory_order_relaxed); + + // Ticket is physically last in the request. A system-scope release makes + // every preceding request field visible before the host proxy observes it. + requestTicket.store(ticket, cuda::memory_order_release); + + while (completionTicket.load(cuda::memory_order_acquire) != ticket) { + if (progressFailure.load(cuda::memory_order_acquire) != 0) { + failEndpoint(endpointLock); + return false; + } + } + + const uint32_t status = slot->completion.status; + if (status != NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_SUCCESS) { + // A non-success completion means a potentially fatal transport/target + // condition. Poison only this destination endpoint; a later AMO cannot + // overwrite its slot while the caller is reporting failure. + failEndpoint(endpointLock); + return false; + } + if (previous != nullptr && niinGpunetioProxyAtomicOpReturnsPrevious(static_cast(op))) + *previous = atomicValueFromBits(slot->completion.resultBits); + unlockEndpoint(endpointLock); + return true; +} + +} // namespace detail +} // namespace proxy +} // namespace gpunetio +} // namespace niin + +template +__device__ __forceinline__ bool niin_gpunetio_proxy_atomic_try( + const struct niinGpunetioAtomicContext* context, enum niinGpunetioProxyAtomicOp op, + size_t remoteOffset, T operand, T compare, int pe, T* previous = nullptr) { + return niin::gpunetio::proxy::detail::issue(context, op, remoteOffset, operand, compare, pe, + previous); +} + +__device__ __forceinline__ bool niin_gpunetio_proxy_atomic_active( + const struct niinGpunetioAtomicContext* context) { + const struct niinGpunetioProxyAtomicDeviceContext* proxyContext = nullptr; + return niin::gpunetio::proxy::detail::proxyContextAvailable(context, &proxyContext); +} + +#else + +template +inline bool niin_gpunetio_proxy_atomic_try(const struct niinGpunetioAtomicContext*, + enum niinGpunetioProxyAtomicOp, size_t, T, T, int, + T* = nullptr) { + return false; +} + +inline bool niin_gpunetio_proxy_atomic_active(const struct niinGpunetioAtomicContext*) { + return false; +} + +#endif // defined(__CUDACC__) + +#endif // NIIN_GPUNETIO_PROXY_ATOMICS_DEVICE_H_ diff --git a/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/host.h b/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/host.h new file mode 100644 index 000000000000..fd36abf07bf0 --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/host.h @@ -0,0 +1,81 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_GPUNETIO_PROXY_ATOMICS_HOST_H_ +#define NIIN_GPUNETIO_PROXY_ATOMICS_HOST_H_ + +#include "niin/gpunetio/proxy_atomics/protocol.h" + +#include +#include + +#include + +struct niinContext; +struct niinGpunetioProxyAtomicHostContext; + +// Configuration for NIIN's experimental raw-verbs/SRQ proxy. Its current +// ProxyAll protocol is a development prototype, not a supported replacement +// for the native direct provider: CPU BAR RMWs cannot be transparently ordered +// with a concurrently running target GPU kernel. Native GPUNetIO QP atomics +// are initialized separately with niinGpunetioAtomicInit(). +struct niinGpunetioProxyAtomicOptions { + enum niinGpunetioProxyAtomicExecutionMode mode; + const char* ibDevice; + int ibPort; + int gidIndex; + uint16_t pkeyIndex; + uint8_t serviceLevel; + uint8_t reserved0; + // The first implementation leases a single slot per destination. Keep this + // field explicit so an application cannot silently assume a ticketed ring; + // values other than one are rejected until the allocator is implemented. + uint32_t requestSlotsPerPe; + uint32_t srqDepth; + uint32_t sendQueueDepth; + // Optional dlopen path for libgdrapi. nullptr tries libgdrapi.so.2 then + // libgdrapi.so, retaining no link-time GDRCopy dependency for normal NIIN. + const char* gdrCopyLibrary; +}; + +#define NIIN_GPUNETIO_PROXY_ATOMIC_OPTIONS_INITIALIZER \ + { \ + NIIN_GPUNETIO_PROXY_ATOMIC_EXECUTION_PROXY_ALL, nullptr, 0, -1, 0, 0, 0, \ + 1, 256, 256, nullptr \ + } + +// Collectively create the NIIN-only proxy plane: +// +// - a GDRCopy CPU mapping of the existing symmetric heap on each PE; +// - mapped host-memory request/completion slots visible to GPU and CPU; +// - one raw ibverbs RC SEND QP per remote PE plus a target-owned SRQ; +// - a NIIN host progress thread that serializes target RMW operations. +// +// The setup uses public NCCL only for bootstrap metadata. It does not inspect, +// borrow, or modify GIN state, GPUNetIO objects, or NCCL-core resources. +ncclResult_t niinGpunetioProxyAtomicInit( + ncclComm_t comm, void* heapBase, size_t heapBytes, + const struct niinGpunetioProxyAtomicOptions* options, + struct niinGpunetioProxyAtomicHostContext** out); + +// Bind the proxy-all device context after niinCommit(). A context can be bound +// to either this proxy provider or the native direct provider, not both. +ncclResult_t niinGpunetioProxyAtomicBind(struct niinGpunetioProxyAtomicHostContext* provider, + struct niinContext* deviceContext); + +// Make bounded progress when the application owns host progress scheduling. +// Init also starts one NIIN-owned progress thread, so callers normally do not +// need this routine; it is useful for diagnostics and test harnesses. +ncclResult_t niinGpunetioProxyAtomicProgress(struct niinGpunetioProxyAtomicHostContext* provider); + +// Collectively destroy the proxy only after all PEs have completed every +// kernel that can issue AMOs. Finalize performs a private-NCCL quiesce before +// releasing RC QPs, the SRQ, and GDRCopy mappings; every PE in the communicator +// must call it in the same lifecycle phase. +ncclResult_t niinGpunetioProxyAtomicFinalize(struct niinGpunetioProxyAtomicHostContext* provider); + +#endif // NIIN_GPUNETIO_PROXY_ATOMICS_HOST_H_ diff --git a/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/protocol.h b/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/protocol.h new file mode 100644 index 000000000000..b8d9e7b8c866 --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/gpunetio/proxy_atomics/protocol.h @@ -0,0 +1,225 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef NIIN_GPUNETIO_PROXY_ATOMICS_PROTOCOL_H_ +#define NIIN_GPUNETIO_PROXY_ATOMICS_PROTOCOL_H_ + +// Wire ABI for NIIN's optional software-atomic sidecar. +// +// This is deliberately independent of GPUNetIO and GIN. A GPU producer +// publishes a request into CUDA-mapped host memory; a NIIN-owned host proxy +// SENDs it on a raw-verbs RC QP to a target-owned SRQ. The target proxy +// performs a CPU-mapped RMW and writes a completion body followed by its +// ticket to the source completion ring. + +#include +#include +#include + +#if defined(__CUDACC__) +#define NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE __host__ __device__ +#else +#define NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE +#endif + +#define NIIN_GPUNETIO_PROXY_ATOMIC_PROTOCOL_MAGIC UINT32_C(0x4e495041) // "NIPA" +#define NIIN_GPUNETIO_PROXY_ATOMIC_PROTOCOL_VERSION UINT16_C(1) +#define NIIN_GPUNETIO_PROXY_ATOMIC_UNPUBLISHED_TICKET UINT64_C(0) + +// The two modes are mutually exclusive. ProxyAll routes self, LSA, and +// network AMOs through the CPU RMW service, avoiding a split linearization +// domain between CPU RMW and NIC AMOs. +enum niinGpunetioProxyAtomicExecutionMode : uint32_t { + NIIN_GPUNETIO_PROXY_ATOMIC_EXECUTION_NATIVE_ONLY = 0, + NIIN_GPUNETIO_PROXY_ATOMIC_EXECUTION_PROXY_ALL = 1, +}; + +enum niinGpunetioProxyAtomicOp : uint8_t { + NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD = 0, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_ADD = 1, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_COMPARE_SWAP = 2, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP = 3, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH = 4, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_SET = 5, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_INC = 6, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_INC = 7, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_AND = 8, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_AND = 9, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_OR = 10, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_OR = 11, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_XOR = 12, + NIIN_GPUNETIO_PROXY_ATOMIC_OP_XOR = 13, +}; + +// Integral values are width-only: signed and unsigned aliases have identical +// raw representations. Floating values use IEEE raw bits. +enum niinGpunetioProxyAtomicType : uint8_t { + NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL32 = 0, + NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL64 = 1, + NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT16 = 2, + NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT32 = 3, + NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT64 = 4, +}; + +enum niinGpunetioProxyAtomicCompletionStatus : uint32_t { + NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_SUCCESS = 0, + NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_BAD_REQUEST = 1, + NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_BAD_ADDRESS = 2, + NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_UNSUPPORTED = 3, + NIIN_GPUNETIO_PROXY_ATOMIC_COMPLETION_INTERNAL_ERROR = 4, +}; + +enum niinGpunetioProxyAtomicRequestFlags : uint32_t { + NIIN_GPUNETIO_PROXY_ATOMIC_REQUEST_FLAG_NONE = 0, +}; + +// `ticket` is physically last and is published last by the GPU producer after +// a system-scope release. The target receives this fixed-width payload through +// its SRQ; it contains no host pointers, size_t values, or provider state. +struct alignas(uint64_t) niinGpunetioProxyAtomicRequest { + uint32_t magic; + uint16_t version; + uint8_t op; + uint8_t type; + uint32_t sourcePe; + uint32_t targetPe; + uint32_t sourceSlot; + uint32_t flags; + uint64_t remoteOffset; + uint64_t operandBits; + uint64_t compareBits; + uint64_t ticket; +}; + +// The target writes bytes [0, ticket) first, then ticket in a second ordered +// RC RDMA write. A GPU accepts a completion only after observing its ticket. +struct alignas(uint64_t) niinGpunetioProxyAtomicCompletion { + uint64_t resultBits; + uint32_t status; + uint32_t reserved; + uint64_t ticket; +}; + +static_assert(std::is_standard_layout::value, + "NIIN proxy request must remain a plain wire record"); +static_assert(std::is_trivially_copyable::value, + "NIIN proxy request must be sent without serialization"); +static_assert(sizeof(niinGpunetioProxyAtomicRequest) == 56, + "NIIN proxy request ABI changed"); +static_assert(offsetof(niinGpunetioProxyAtomicRequest, ticket) == 48, + "NIIN proxy request ticket must remain the final wire field"); +static_assert(std::is_standard_layout::value, + "NIIN proxy completion must remain a plain wire record"); +static_assert(std::is_trivially_copyable::value, + "NIIN proxy completion must be written without serialization"); +static_assert(sizeof(niinGpunetioProxyAtomicCompletion) == 24, + "NIIN proxy completion ABI changed"); +static_assert(offsetof(niinGpunetioProxyAtomicCompletion, ticket) == 16, + "NIIN proxy completion ticket must remain the final wire field"); + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr size_t +niinGpunetioProxyAtomicCompletionBodyBytes() { + return offsetof(niinGpunetioProxyAtomicCompletion, ticket); +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr bool +niinGpunetioProxyAtomicIsKnownOp(uint8_t op) { + return op <= NIIN_GPUNETIO_PROXY_ATOMIC_OP_XOR; +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr bool +niinGpunetioProxyAtomicIsKnownType(uint8_t type) { + return type <= NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT64; +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr bool +niinGpunetioProxyAtomicTypeIsIntegral(uint8_t type) { + return type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL32 || + type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL64; +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr bool +niinGpunetioProxyAtomicTypeIsFloating(uint8_t type) { + return type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT16 || + type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT32 || + type == NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT64; +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr size_t +niinGpunetioProxyAtomicElementBytes(uint8_t type) { + switch (type) { + case NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL32: + case NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT32: + return 4; + case NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_INTEGRAL64: + case NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT64: + return 8; + case NIIN_GPUNETIO_PROXY_ATOMIC_TYPE_FLOAT16: + return 2; + default: + return 0; + } +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr bool +niinGpunetioProxyAtomicOpReturnsPrevious(uint8_t op) { + switch (op) { + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_ADD: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_COMPARE_SWAP: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_SWAP: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_INC: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_AND: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_OR: + case NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_XOR: + return true; + default: + return false; + } +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr bool +niinGpunetioProxyAtomicOpIsBitwise(uint8_t op) { + return op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_AND || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_AND || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_OR || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_OR || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_XOR || + op == NIIN_GPUNETIO_PROXY_ATOMIC_OP_XOR; +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr bool +niinGpunetioProxyAtomicOpSupportsType(uint8_t op, uint8_t type) { + if (!niinGpunetioProxyAtomicIsKnownOp(op) || !niinGpunetioProxyAtomicIsKnownType(type)) + return false; + if (niinGpunetioProxyAtomicTypeIsIntegral(type)) return true; + // The wire can represent raw float/double/half fetch/swap/set and floating + // add/fetch-add. The public adapter narrows this to actual NVSHMEM spellings. + return !niinGpunetioProxyAtomicOpIsBitwise(op) && + op != NIIN_GPUNETIO_PROXY_ATOMIC_OP_INC && + op != NIIN_GPUNETIO_PROXY_ATOMIC_OP_FETCH_INC && + op != NIIN_GPUNETIO_PROXY_ATOMIC_OP_COMPARE_SWAP; +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr bool niinGpunetioProxyAtomicRequestIsWellFormed( + const niinGpunetioProxyAtomicRequest& request) { + const size_t elementBytes = niinGpunetioProxyAtomicElementBytes(request.type); + return request.magic == NIIN_GPUNETIO_PROXY_ATOMIC_PROTOCOL_MAGIC && + request.version == NIIN_GPUNETIO_PROXY_ATOMIC_PROTOCOL_VERSION && + request.flags == NIIN_GPUNETIO_PROXY_ATOMIC_REQUEST_FLAG_NONE && request.ticket != 0 && + elementBytes != 0 && (request.remoteOffset % elementBytes) == 0 && + niinGpunetioProxyAtomicOpSupportsType(request.op, request.type); +} + +NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE constexpr uint64_t +niinGpunetioProxyAtomicNextTicket(uint64_t previous) { + return previous == UINT64_MAX ? UINT64_C(1) : previous + UINT64_C(1); +} + +#undef NIIN_GPUNETIO_PROXY_PROTOCOL_HOST_DEVICE + +#endif // NIIN_GPUNETIO_PROXY_ATOMICS_PROTOCOL_H_ diff --git a/projects/rccl/contrib/niin/include/niin/host.h b/projects/rccl/contrib/niin/include/niin/host.h index baef0f027e6b..266b9bc4833a 100644 --- a/projects/rccl/contrib/niin/include/niin/host.h +++ b/projects/rccl/contrib/niin/include/niin/host.h @@ -11,6 +11,16 @@ #include #include #include +#include +#include +#include + +// Host helpers construct and copy the device context but do not need the +// device-side accessor definitions in niin/context.h. Include the stable ABI +// directly so this header remains self-contained for low-level users. +#include "niin/context_abi.h" + +#include "niin/context.h" #ifndef NIIN_CHECK_NCCL #define NIIN_CHECK_NCCL(cmd) do { \ @@ -42,11 +52,43 @@ struct niinContext_host { ncclWindow_t heapWindow; void* heapBase; size_t heapSize; - int ginContextIndex; + void** peerHeapBaseP2p; + int nodeRank; + int nodeSize; bool peerNativeAtomic; // Whether peer GPUs support native system-scope atomics bool forceSeparatePutSignal; // Force put+fence+signal instead of fused put_signal + unsigned int* ginPendingOps; // Device scalar tracking unquieted GIN ops + unsigned int* lsaStorePending; // Device per-CTA flags for local/LSA stores + size_t lsaStorePendingLen; // Entries in lsaStorePending + // TMA state, owned by niinTmaEnable/niinTmaDisable. niinInit clears these, so + // callers that manage a niinContext_host themselves get TMA off by default. + int tmaPolicy; // nvshmemx_tma_policy_t + uintptr_t* tmaSmemBases; // Device array of per-CTA registered smem bases + size_t tmaSmemBasesLen; // Entries in tmaSmemBases + size_t* tmaSmemSize; // Device scalar holding the given smem size }; +// Default number of GIN contexts to request. Each context is one QP per peer, +// so this is the number of QPs NIIN round-robins operations across. NCCL rounds +// it up to a multiple of the connection count, so the effective value can be +// higher on multi-NIC nodes. +#define NIIN_DEFAULT_NUM_QPS 4 + +// Read the QP count from NIIN_NUM_QPS. Anything unparseable or < 1 warns and +// falls back to the default. +inline int niinParseNumQps() { + const char* env = getenv("NIIN_NUM_QPS"); + if (env == nullptr || env[0] == '\0') return NIIN_DEFAULT_NUM_QPS; + + char* end = nullptr; + const long value = strtol(env, &end, 10); + if (end == env || *end != '\0' || value < 1 || value > INT_MAX) { + fprintf(stderr, "NIIN: NIIN_NUM_QPS must be a positive integer\n"); + return NIIN_DEFAULT_NUM_QPS; + } + return static_cast(value); +} + // niinInit: host-side initialization — phase 1 (NCCL collective calls). // // Registers heapBuf as a symmetric window and creates a device communicator @@ -60,9 +102,11 @@ struct niinContext_host { // aligned to NCCL_WIN_REQUIRED_ALIGNMENT and same size on all ranks) // heapSize - Size of heapBuf in bytes // hostCtx - [OUT] Host-side staging struct to populate -// ginContextIndex - Which GIN context index to use (default 0) // barrierCount - Number of barrier sessions to request (default 1) // ginSignalCount - Number of GIN signals to request (default 0) +// ginContextCount - Number of GIN contexts (QPs per peer) to provision. 0 +// takes the value from NIIN_NUM_QPS, else the default. +// Device operations round-robin over them per operation. // // Returns ncclSuccess on success. inline ncclResult_t niinInit(ncclComm_t comm, @@ -70,20 +114,34 @@ inline ncclResult_t niinInit(ncclComm_t comm, size_t heapSize, niinContext_host* hostCtx, bool enableGin = true, - int ginContextIndex = 0, int barrierCount = 1, - int ginSignalCount = 0) { + int ginSignalCount = 0, + int ginContextCount = 0, + int nodeRank = 0, + int nodeSize = 1) { // Register the symmetric heap as a window + // The NVSHMEM heap contains both RMA payloads and signal locations. GIN + // requires strict ordering for a window whose remote signal makes a prior + // payload visible to an independent consumer CTA. NIIN_CHECK_NCCL(ncclCommWindowRegister(comm, heapBuf, heapSize, &hostCtx->heapWindow, - NCCL_WIN_COLL_SYMMETRIC)); + NCCL_WIN_COLL_SYMMETRIC | + NCCL_WIN_STRICT_ORDERING)); // Create a device communicator with GIN resources ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; if (enableGin) { + // barrierCount provisions the hybrid LSA+GIN barrier. NIIN selects the + // cheaper pure-LSA path when every PE is local, so provision that handle + // as well. Without it, a same-node communicator has an empty + // devComm.lsaBarrier even though the device wrapper selects it. + reqs.lsaBarrierCount = barrierCount; reqs.barrierCount = barrierCount; + // Device barriers fence every GIN context, which uses the dedicated + // world-GIN barrier handle rather than the legacy hybrid handle. + reqs.worldGinBarrierCount = barrierCount; reqs.ginForceEnable = true; - reqs.ginContextCount = 1; + reqs.ginContextCount = ginContextCount > 0 ? ginContextCount : niinParseNumQps(); reqs.ginSignalCount = ginSignalCount + barrierCount; reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; } else { @@ -95,8 +153,170 @@ inline ncclResult_t niinInit(ncclComm_t comm, hostCtx->heapBase = heapBuf; hostCtx->heapSize = heapSize; - hostCtx->ginContextIndex = ginContextIndex; + hostCtx->peerHeapBaseP2p = nullptr; + hostCtx->nodeRank = nodeRank; + hostCtx->nodeSize = nodeSize; + hostCtx->ginPendingOps = nullptr; + hostCtx->lsaStorePending = nullptr; + hostCtx->lsaStorePendingLen = 0; + + // TMA is opt-in; niinTmaEnable() turns it on after this call. + hostCtx->tmaPolicy = NVSHMEMX_TMA_DISABLE; + hostCtx->tmaSmemBases = nullptr; + hostCtx->tmaSmemBasesLen = 0; + hostCtx->tmaSmemSize = nullptr; + + return ncclSuccess; +} + +inline void niinPendingStateDisable(niinContext_host* hostCtx) { + if (hostCtx->ginPendingOps != nullptr) cudaFree(hostCtx->ginPendingOps); + if (hostCtx->lsaStorePending != nullptr) cudaFree(hostCtx->lsaStorePending); + hostCtx->ginPendingOps = nullptr; + hostCtx->lsaStorePending = nullptr; + hostCtx->lsaStorePendingLen = 0; +} + +inline ncclResult_t niinPendingStateEnable(niinContext_host* hostCtx) { + if (hostCtx->devComm.ginConnectionCount == 0) { + if (hostCtx->ginPendingOps != nullptr) { + cudaError_t e = cudaFree(hostCtx->ginPendingOps); + if (e != cudaSuccess) { + fprintf(stderr, "NIIN: CUDA error %s freeing idle GIN counter at %s:%d\n", + cudaGetErrorString(e), __FILE__, __LINE__); + return ncclInternalError; + } + hostCtx->ginPendingOps = nullptr; + } + } else if (hostCtx->ginPendingOps == nullptr) { + cudaError_t e = cudaMalloc(&hostCtx->ginPendingOps, sizeof(unsigned int)); + if (e == cudaSuccess) e = cudaMemset(hostCtx->ginPendingOps, 0, sizeof(unsigned int)); + if (e != cudaSuccess) { + fprintf(stderr, "NIIN: CUDA error %s allocating pending GIN counter at %s:%d\n", + cudaGetErrorString(e), __FILE__, __LINE__); + niinPendingStateDisable(hostCtx); + return ncclInternalError; + } + } + + if (hostCtx->lsaStorePending == nullptr) { + size_t pendingBytes = (size_t)NIIN_TMA_MAX_BLOCKS * sizeof(unsigned int); + cudaError_t e = cudaMalloc(&hostCtx->lsaStorePending, pendingBytes); + if (e == cudaSuccess) e = cudaMemset(hostCtx->lsaStorePending, 0, pendingBytes); + if (e != cudaSuccess) { + fprintf(stderr, "NIIN: CUDA error %s allocating LSA pending table at %s:%d\n", + cudaGetErrorString(e), __FILE__, __LINE__); + niinPendingStateDisable(hostCtx); + return ncclInternalError; + } + hostCtx->lsaStorePendingLen = NIIN_TMA_MAX_BLOCKS; + } + + return ncclSuccess; +} + +// niinTmaDisable: release the tables allocated by niinTmaEnable. Safe to call +// when TMA was never enabled. +inline void niinTmaDisable(niinContext_host* hostCtx) { + if (hostCtx->tmaSmemBases != nullptr) cudaFree(hostCtx->tmaSmemBases); + if (hostCtx->tmaSmemSize != nullptr) cudaFree(hostCtx->tmaSmemSize); + hostCtx->tmaSmemBases = nullptr; + hostCtx->tmaSmemSize = nullptr; + hostCtx->tmaSmemBasesLen = 0; + hostCtx->tmaPolicy = NVSHMEMX_TMA_DISABLE; +} + +// niinTmaEnable: allocate the per-CTA TMA registration tables. +// +// Call between niinInit() and niinCommit(). Once enabled, kernels that call +// nvshmemx_give_smem() route their LSA put/get traffic through cp.async.bulk. +// Requires compute capability 9.0 or newer; the caller is responsible for +// checking that (nvshmem_init() does so from NVSHMEM_TMA_POLICY). +// +// Arguments: +// hostCtx - Host-side staging struct populated by niinInit +// policy - NVSHMEMX_TMA_ENABLE or NVSHMEMX_TMA_FORCE; DISABLE is a no-op +// +// Returns ncclSuccess on success; leaves TMA off on failure. +inline ncclResult_t niinTmaEnable(niinContext_host* hostCtx, nvshmemx_tma_policy_t policy) { + if (policy == NVSHMEMX_TMA_DISABLE) return ncclSuccess; + + // niinInit() left these null; clear them again so a partial allocation below + // is never mistaken for a live table. + hostCtx->tmaSmemBases = nullptr; + hostCtx->tmaSmemSize = nullptr; + + size_t basesBytes = (size_t)NIIN_TMA_MAX_BLOCKS * sizeof(uintptr_t); + cudaError_t e = cudaMalloc(&hostCtx->tmaSmemBases, basesBytes); + if (e == cudaSuccess) e = cudaMemset(hostCtx->tmaSmemBases, 0, basesBytes); + if (e == cudaSuccess) e = cudaMalloc(&hostCtx->tmaSmemSize, sizeof(size_t)); + if (e == cudaSuccess) e = cudaMemset(hostCtx->tmaSmemSize, 0, sizeof(size_t)); + if (e != cudaSuccess) { + fprintf(stderr, "NIIN: CUDA error %s allocating TMA tables at %s:%d\n", + cudaGetErrorString(e), __FILE__, __LINE__); + niinTmaDisable(hostCtx); + return ncclInternalError; + } + + hostCtx->tmaSmemBasesLen = NIIN_TMA_MAX_BLOCKS; + hostCtx->tmaPolicy = policy; + return ncclSuccess; +} + +inline ncclResult_t niinPublishContextCache(const niinContext* ctx) { +#if NIIN_USE_DEVICE_CONTEXT_POINTER + (void)ctx; + return ncclSuccess; +#else + NIIN_CHECK_CUDA(cudaMemcpyToSymbol(niin_g_ctx_constant, ctx, sizeof(niinContext), + 0, cudaMemcpyHostToDevice)); + return ncclSuccess; +#endif +} +inline ncclResult_t niinRefreshContextCache(niinContext* devCtx) { +#if NIIN_USE_DEVICE_CONTEXT_POINTER + (void)devCtx; + return ncclSuccess; +#else + niinContext ctx; + NIIN_CHECK_CUDA(cudaMemcpy(&ctx, devCtx, sizeof(niinContext), cudaMemcpyDeviceToHost)); + return niinPublishContextCache(&ctx); +#endif +} + +inline ncclResult_t niinBuildPeerHeapBaseP2p(niinContext_host* hostCtx) { + const int nRanks = hostCtx->devComm.nRanks; + if (nRanks <= 0) return ncclSuccess; + + void** peerHeapBaseP2pHost = static_cast(calloc(static_cast(nRanks), sizeof(void*))); + if (peerHeapBaseP2pHost == nullptr) return ncclInternalError; + + ncclResult_t result = ncclSuccess; + for (int pe = 0; pe < nRanks; pe++) { + result = ncclGetPeerDevicePointer(hostCtx->heapWindow, 0, pe, &peerHeapBaseP2pHost[pe]); + if (result != ncclSuccess) break; + } + + void** peerHeapBaseP2p = nullptr; + if (result == ncclSuccess) { + cudaError_t e = cudaMalloc(&peerHeapBaseP2p, static_cast(nRanks) * sizeof(void*)); + if (e == cudaSuccess) { + e = cudaMemcpy(peerHeapBaseP2p, peerHeapBaseP2pHost, + static_cast(nRanks) * sizeof(void*), + cudaMemcpyHostToDevice); + } + if (e != cudaSuccess) { + fprintf(stderr, "NIIN: CUDA error %s building peer heap table at %s:%d\n", + cudaGetErrorString(e), __FILE__, __LINE__); + if (peerHeapBaseP2p != nullptr) cudaFree(peerHeapBaseP2p); + result = ncclInternalError; + } + } + + free(peerHeapBaseP2pHost); + if (result != ncclSuccess) return result; + hostCtx->peerHeapBaseP2p = peerHeapBaseP2p; return ncclSuccess; } @@ -110,8 +330,11 @@ inline ncclResult_t niinInit(ncclComm_t comm, // devCtx - [OUT] Device pointer to niinContext (must be cudaMalloc'd) // // Returns ncclSuccess on success. -inline ncclResult_t niinCommit(const niinContext_host* hostCtx, +inline ncclResult_t niinCommit(niinContext_host* hostCtx, niinContext* devCtx) { + NIIN_CHECK_NCCL(niinPendingStateEnable(hostCtx)); + NIIN_CHECK_NCCL(niinBuildPeerHeapBaseP2p(hostCtx)); + // Copy devComm to device memory ncclDevComm* d_devComm; NIIN_CHECK_CUDA(cudaMalloc(&d_devComm, sizeof(ncclDevComm))); @@ -121,14 +344,32 @@ inline ncclResult_t niinCommit(const niinContext_host* hostCtx, // Build and copy niinContext to device niinContext ctx; ctx.comm = d_devComm; + ctx.commValue = hostCtx->devComm; ctx.heapWindow = hostCtx->heapWindow; ctx.heapBase = hostCtx->heapBase; ctx.heapSize = hostCtx->heapSize; - ctx.ginContextIndex = hostCtx->ginContextIndex; + ctx.peerHeapBaseP2p = hostCtx->peerHeapBaseP2p; + ctx.rank = hostCtx->devComm.rank; + ctx.nRanks = hostCtx->devComm.nRanks; + ctx.lsaRank = hostCtx->devComm.lsaRank; + ctx.lsaSize = hostCtx->devComm.lsaSize; + ctx.ginConnectionCount = hostCtx->devComm.ginConnectionCount; + ctx.worldIsLsaOnly = hostCtx->devComm.lsaSize == hostCtx->devComm.nRanks; + ctx.nodeRank = hostCtx->nodeRank; + ctx.nodeSize = hostCtx->nodeSize; + ctx.gpunetioAtomicContext = nullptr; ctx.peerNativeAtomic = hostCtx->peerNativeAtomic; ctx.forceSeparatePutSignal = hostCtx->forceSeparatePutSignal; + ctx.ginPendingOps = hostCtx->ginPendingOps; + ctx.lsaStorePending = hostCtx->lsaStorePending; + ctx.lsaStorePendingLen = hostCtx->lsaStorePendingLen; + ctx.tmaPolicy = hostCtx->tmaPolicy; + ctx.tmaSmemBases = hostCtx->tmaSmemBases; + ctx.tmaSmemBasesLen = hostCtx->tmaSmemBasesLen; + ctx.tmaSmemSize = hostCtx->tmaSmemSize; NIIN_CHECK_CUDA(cudaMemcpy(devCtx, &ctx, sizeof(niinContext), cudaMemcpyHostToDevice)); + NIIN_CHECK_NCCL(niinPublishContextCache(&ctx)); return ncclSuccess; } @@ -159,6 +400,9 @@ inline ncclResult_t niinFinalize(ncclComm_t comm, niinContext* devCtx) { // Free the device-side devComm copy cudaFree((void*)ctx.comm); + cudaFree(ctx.peerHeapBaseP2p); + cudaFree(ctx.ginPendingOps); + cudaFree(ctx.lsaStorePending); // Deregister the window (best-effort) ncclCommWindowDeregister(comm, ctx.heapWindow); diff --git a/projects/rccl/contrib/niin/include/niin/nvshmem_host.h b/projects/rccl/contrib/niin/include/niin/nvshmem_host.h index b4fe539f76f2..7f0548224883 100644 --- a/projects/rccl/contrib/niin/include/niin/nvshmem_host.h +++ b/projects/rccl/contrib/niin/include/niin/nvshmem_host.h @@ -11,11 +11,12 @@ // barrier, finalize) implemented on top of NCCL. Uses a global singleton // to match NVSHMEM's implicit-global-state programming model. // -// For multi-PE operation, launch with mpirun and call: -// nvshmemx_init_attr(NVSHMEMX_INIT_WITH_MPI_COMM, &attr) -// or use NVSHMEMX_INIT_WITH_UNIQUEID after distributing a unique ID with the -// application's bootstrap mechanism. nvshmem_init() can also auto-detect -// rank/size metadata from supported launchers. +// For multi-PE operation, nvshmem_init() auto-detects supported launcher +// rank/size metadata and distributes an NCCL unique ID through a small record +// in a shared bootstrap directory. Alternatively, launch with MPI and call +// nvshmemx_init_attr(NVSHMEMX_INIT_WITH_MPI_COMM, &attr), or use +// NVSHMEMX_INIT_WITH_UNIQUEID after distributing a unique ID with the +// application's bootstrap mechanism. // // For single-PE operation (testing), just call nvshmem_init(). @@ -24,11 +25,20 @@ #include #include +#include +#include #include #include #include +#include +#include +#include // strcasecmp, for NVSHMEM_TMA_POLICY parsing +#include +#include +#include #include "niin/host.h" +#include "niin/sync.h" #ifdef NIIN_HAS_MPI #include @@ -48,6 +58,16 @@ namespace niin { namespace detail { +// Publish the device context from device code rather than through the CUDA +// runtime's host-side symbol lookup. The latter is not reliable for an +// externally-owned RDC device symbol in every consumer module: a successful +// NIIN host initialization can otherwise leave the consumer kernel's +// niin_g_ctx null. This kernel and all device API wrappers resolve the same +// device-link symbol. +static __global__ void publishDeviceContext(niinContext* ctx) { + niin_g_ctx = ctx; +} + // Free-list heap block struct HeapBlock { size_t offset; @@ -67,6 +87,8 @@ struct GlobalState { int nRanks; int lsaRank; int lsaSize; + int nodeRank; + int nodeSize; int cudaDev; void* heapBase; size_t heapSize; @@ -80,6 +102,7 @@ struct GlobalState { bool hostRmaAvail; // Whether host RMA (ncclPutSignal) is available bool ginAvail; // Whether this communicator supports GIN device resources void* barrierScratch; // Scratch buffer for host-side barrier (separate from heap) + struct niinGpunetioAtomicHostContext* gpunetioAtomics; // Network AMO provider, when enabled }; inline GlobalState& state() { @@ -121,6 +144,101 @@ inline bool parseForceSeparatePutSignal() { return false; } +// Parse NVSHMEM_TMA_POLICY, matching NVSHMEM's DISABLE/ENABLE/FORCE spelling. +// Unrecognized values fall back to DISABLE with a diagnostic, as NVSHMEM does. +inline nvshmemx_tma_policy_t parseTmaPolicy() { + const char* env = getenv("NVSHMEM_TMA_POLICY"); + if (env == nullptr) return NVSHMEMX_TMA_DISABLE; + if (strcasecmp(env, "ENABLE") == 0) return NVSHMEMX_TMA_ENABLE; + if (strcasecmp(env, "FORCE") == 0) return NVSHMEMX_TMA_FORCE; + if (strcasecmp(env, "DISABLE") == 0) return NVSHMEMX_TMA_DISABLE; + fprintf(stderr, "NIIN: invalid NVSHMEM_TMA_POLICY value \"%s\"; using DISABLE\n", env); + return NVSHMEMX_TMA_DISABLE; +} + +// Resolve the requested TMA policy against the current device's compute +// capability. TMA needs sm_90 or newer: FORCE reports failure there, ENABLE +// degrades to DISABLE with a warning. Returns false when init should fail. +inline bool resolveTmaPolicy(nvshmemx_tma_policy_t* policy) { + if (*policy == NVSHMEMX_TMA_DISABLE) return true; + + int dev = 0; + cudaGetDevice(&dev); + int capMajor = 0, capMinor = 0; + cudaDeviceGetAttribute(&capMajor, cudaDevAttrComputeCapabilityMajor, dev); + cudaDeviceGetAttribute(&capMinor, cudaDevAttrComputeCapabilityMinor, dev); + if (capMajor >= 9) return true; + + if (*policy == NVSHMEMX_TMA_FORCE) { + fprintf(stderr, "NIIN: NVSHMEM_TMA_POLICY=FORCE requires sm_90 or newer; " + "device %d is sm_%d%d\n", dev, capMajor, capMinor); + return false; + } + fprintf(stderr, "NIIN: NVSHMEM_TMA_POLICY=ENABLE requires sm_90 or newer; " + "device %d is sm_%d%d, disabling TMA\n", dev, capMajor, capMinor); + *policy = NVSHMEMX_TMA_DISABLE; + return true; +} + +// NIIN_GPUNETIO_ATOMICS selects how hard NIIN tries to bring up the network +// AMO provider: 0 disables it, 1 requires it and reports why it did not come +// up, and the default enables it whenever the provider is linked and the +// fabric supports it. +inline int parseGpunetioAtomicsMode() { + const char* env = getenv("NIIN_GPUNETIO_ATOMICS"); + if (env == nullptr || env[0] == '\0') return -1; + if (strcmp(env, "0") == 0) return 0; + if (strcmp(env, "1") == 0) return 1; + fprintf(stderr, "NIIN: NIIN_GPUNETIO_ATOMICS must be 0 or 1\n"); + return -1; +} + +// Bring up the network atomic provider. Its initialization is collective and +// all-gathers QP metadata internally, so every PE has to agree before any of +// them enters it: a PE that bailed out early while the others proceeded would +// hang them. Agree first, then initialize. +inline void enableGpunetioAtomics(GlobalState& s) { + s.gpunetioAtomics = nullptr; + const int mode = parseGpunetioAtomicsMode(); + if (mode == 0) return; + + const bool linked = (niinGpunetioAtomicInit != nullptr && niinGpunetioAtomicBind != nullptr && + niinGpunetioAtomicFinalize != nullptr); + // Network AMOs only mean something when some peer is off this LSA domain. + int local = (linked && s.nRanks > s.lsaSize) ? 1 : 0; + + int agreed = local; + if (s.nRanks > 1) { + int* dev = static_cast(s.barrierScratch); + if (dev == nullptr) return; + if (cudaMemcpy(dev, &local, sizeof(int), cudaMemcpyHostToDevice) != cudaSuccess) return; + if (ncclAllReduce(dev, dev, 1, ncclInt, ncclMin, s.comm, s.stream) != ncclSuccess) return; + if (cudaStreamSynchronize(s.stream) != cudaSuccess) return; + if (cudaMemcpy(&agreed, dev, sizeof(int), cudaMemcpyDeviceToHost) != cudaSuccess) return; + } + if (agreed == 0) { + if (mode == 1 && s.rank == 0) { + fprintf(stderr, "NIIN: NIIN_GPUNETIO_ATOMICS=1 but the provider is %s\n", + linked ? "unusable on this job's topology" : "not linked into this application"); + } + return; + } + + niinGpunetioAtomicOptions options = NIIN_GPUNETIO_ATOMIC_OPTIONS_INITIALIZER; + struct niinGpunetioAtomicHostContext* provider = nullptr; + ncclResult_t r = niinGpunetioAtomicInit(s.comm, s.heapBase, s.heapSize, &options, &provider); + if (r == ncclSuccess) r = niinGpunetioAtomicBind(provider, s.devCtx); + if (r == ncclSuccess) r = niinRefreshContextCache(s.devCtx); + if (r != ncclSuccess) { + if (provider != nullptr) niinGpunetioAtomicFinalize(provider); + if (mode == 1 && s.rank == 0) + fprintf(stderr, "NIIN: network atomic provider unavailable (%s); network AMOs stay unimplemented\n", + ncclGetErrorString(r)); + return; + } + s.gpunetioAtomics = provider; +} + // Detect rank/nRanks from common MPI/PMI environment variables. // Returns true if detection succeeded. inline bool detectRankFromEnv(int* rank, int* nRanks) { @@ -143,9 +261,171 @@ inline bool detectRankFromEnv(int* rank, int* nRanks) { return false; } -// Detect which local GPU to use. Tries LOCAL_RANK-style env vars, falls back -// to rank % deviceCount. -inline int detectLocalDevice(int rank) { +// Generate a launcher-scoped path in a shared filesystem. A caller can set +// NIIN_BOOTSTRAP_DIR and/or NIIN_BOOTSTRAP_ID to override the defaults. The +// HOME fallback is appropriate for the common Slurm configuration where home +// directories are shared between allocated nodes. +inline bool bootstrapPath(char* path, size_t pathSize) { + char directory[PATH_MAX]; + const char* configuredDirectory = getenv("NIIN_BOOTSTRAP_DIR"); + if (configuredDirectory != nullptr && configuredDirectory[0] != '\0') { + if (snprintf(directory, sizeof(directory), "%s", configuredDirectory) >= + static_cast(sizeof(directory))) { + fprintf(stderr, "NIIN: NIIN_BOOTSTRAP_DIR is too long\n"); + return false; + } + } else { + const char* home = getenv("HOME"); + if (home == nullptr || home[0] == '\0' || + snprintf(directory, sizeof(directory), "%s/.niin", home) >= + static_cast(sizeof(directory))) { + fprintf(stderr, "NIIN: set NIIN_BOOTSTRAP_DIR to a shared directory\n"); + return false; + } + } + + if (mkdir(directory, 0700) != 0 && errno != EEXIST) { + fprintf(stderr, "NIIN: cannot create bootstrap directory %s: %s\n", + directory, strerror(errno)); + return false; + } + + char generatedId[128]; + const char* id = getenv("NIIN_BOOTSTRAP_ID"); + if (id == nullptr || id[0] == '\0') { + const char* jobId = getenv("SLURM_JOB_ID"); + if (jobId == nullptr || jobId[0] == '\0') jobId = getenv("PMI_JOBID"); + if (jobId == nullptr || jobId[0] == '\0') jobId = getenv("PMIX_NAMESPACE"); + if (jobId == nullptr || jobId[0] == '\0') { + fprintf(stderr, "NIIN: set NIIN_BOOTSTRAP_ID when the launcher has no job ID\n"); + return false; + } + const char* stepId = getenv("SLURM_STEP_ID"); + if (snprintf(generatedId, sizeof(generatedId), "%s.%s", jobId, + stepId != nullptr ? stepId : "0") >= static_cast(sizeof(generatedId))) { + fprintf(stderr, "NIIN: launcher bootstrap ID is too long\n"); + return false; + } + id = generatedId; + } + + for (const char* c = id; *c != '\0'; ++c) { + const bool safe = (*c >= 'a' && *c <= 'z') || (*c >= 'A' && *c <= 'Z') || + (*c >= '0' && *c <= '9') || *c == '.' || *c == '_' || *c == '-'; + if (!safe) { + fprintf(stderr, "NIIN: NIIN bootstrap ID contains an unsafe character\n"); + return false; + } + } + + if (snprintf(path, pathSize, "%s/niin-bootstrap-%s", directory, id) >= + static_cast(pathSize)) { + fprintf(stderr, "NIIN: bootstrap path is too long\n"); + return false; + } + return true; +} + +inline bool writeBootstrapRecord(int fd, const void* buffer, size_t size) { + const char* cursor = static_cast(buffer); + while (size > 0) { + ssize_t written = write(fd, cursor, size); + if (written < 0) { + if (errno == EINTR) continue; + return false; + } + cursor += written; + size -= static_cast(written); + } + return true; +} + +inline bool readBootstrapRecord(int fd, void* buffer, size_t size) { + char* cursor = static_cast(buffer); + while (size > 0) { + ssize_t readCount = read(fd, cursor, size); + if (readCount < 0) { + if (errno == EINTR) continue; + return false; + } + if (readCount == 0) return false; + cursor += readCount; + size -= static_cast(readCount); + } + return true; +} + +struct BootstrapRecord { + uint64_t magic; + int nRanks; + ncclUniqueId id; +}; + +constexpr uint64_t NIIN_BOOTSTRAP_MAGIC = UINT64_C(0x4e49494e424f4f54); + +// Distribute an NCCL unique ID without requiring the application to initialize +// MPI. Rank zero publishes one small record atomically; other ranks wait for +// it and validate the expected world size before joining the communicator. +inline int bootstrapUniqueId(int rank, int nRanks, ncclUniqueId* id, + char* path, size_t pathSize) { + if (!bootstrapPath(path, pathSize)) return -1; + + if (rank == 0) { + ncclResult_t result = ncclGetUniqueId(id); + if (result != ncclSuccess) { + fprintf(stderr, "NIIN: ncclGetUniqueId failed: %s\n", ncclGetErrorString(result)); + return -1; + } + + const int fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (fd < 0) { + fprintf(stderr, "NIIN: cannot create bootstrap record %s: %s\n", path, strerror(errno)); + return -1; + } + const BootstrapRecord record = {NIIN_BOOTSTRAP_MAGIC, nRanks, *id}; + const bool written = writeBootstrapRecord(fd, &record, sizeof(record)) && fsync(fd) == 0; + close(fd); + if (!written) { + fprintf(stderr, "NIIN: cannot write bootstrap record %s: %s\n", path, strerror(errno)); + unlink(path); + return -1; + } + return 0; + } + + constexpr int kRetries = 6000; // 60 seconds at 10 ms per attempt. + for (int attempt = 0; attempt < kRetries; ++attempt) { + const int fd = open(path, O_RDONLY); + if (fd < 0) { + if (errno == ENOENT) { + usleep(10000); + continue; + } + fprintf(stderr, "NIIN: cannot open bootstrap record %s: %s\n", path, strerror(errno)); + return -1; + } + BootstrapRecord record = {}; + const bool readOk = readBootstrapRecord(fd, &record, sizeof(record)); + close(fd); + if (!readOk) { + usleep(10000); + continue; + } + if (record.magic != NIIN_BOOTSTRAP_MAGIC || record.nRanks != nRanks) { + fprintf(stderr, "NIIN: bootstrap record %s does not match this job\n", path); + return -1; + } + *id = record.id; + return 0; + } + + fprintf(stderr, "NIIN: timed out waiting for bootstrap record %s\n", path); + return -1; +} + +// Detect this PE's rank on its physical node. Tries LOCAL_RANK-style env +// vars, falling back to rank % deviceCount. +inline int detectLocalRank(int rank) { const char* localRankEnvs[] = { "OMPI_COMM_WORLD_LOCAL_RANK", "MPI_LOCALRANKID", @@ -161,6 +441,26 @@ inline int detectLocalDevice(int rank) { return nDevs > 0 ? rank % nDevs : 0; } +inline int detectLocalDevice(int rank) { + return detectLocalRank(rank); +} + +inline int detectNodeSize() { + const char* localSizeEnvs[] = { + "OMPI_COMM_WORLD_LOCAL_SIZE", + "MPI_LOCALNRANKS", + "SLURM_NTASKS_PER_NODE", + }; + for (const char* env : localSizeEnvs) { + const char* value = getenv(env); + if (value == nullptr || value[0] == '\0') continue; + char* end = nullptr; + const long size = strtol(value, &end, 10); + if (end != value && size > 0 && size <= INT_MAX) return static_cast(size); + } + return 1; +} + // Common init logic shared by nvshmem_init() and nvshmemx_init_attr() inline int initCommon(ncclComm_t comm) { auto& s = state(); @@ -228,26 +528,61 @@ inline int initCommon(ncclComm_t comm) { } s.hostCtx.forceSeparatePutSignal = parseForceSeparatePutSignal(); + // SHMEM TEAM WORLD is exactly one LSA domain on single-node jobs. Do not + // force GIN resources into that communicator: all WORLD peers are reachable + // by LSA, and device collectives/quiet have pure-LSA specializations. + const bool needGin = s.ginAvail && s.nRanks > s.nodeSize; + // Two-phase init ncclGroupStart(); - r = niinInit(comm, s.heapBase, s.heapSize, &s.hostCtx, s.ginAvail); + r = niinInit(comm, s.heapBase, s.heapSize, &s.hostCtx, needGin, + /*barrierCount=*/1, /*ginSignalCount=*/0, + /*ginContextCount=*/0, s.nodeRank, s.nodeSize); ncclGroupEnd(); if (r != ncclSuccess) return -1; + // TMA registration tables, if the policy asks for them. niinInit left the + // context with TMA off, so a DISABLE policy needs nothing further. + { + nvshmemx_tma_policy_t policy = parseTmaPolicy(); + if (!resolveTmaPolicy(&policy)) return -1; + if (niinTmaEnable(&s.hostCtx, policy) != ncclSuccess) return -1; + } + r = niinCommit(&s.hostCtx, s.devCtx); if (r != ncclSuccess) return -1; - // Set the __device__ global so kernels don't need manual context setup. - // niin_g_ctx is declared in niin/context.h. - niinContext* devCtxPtr = s.devCtx; - cudaMemcpyToSymbol(niin_g_ctx, &devCtxPtr, sizeof(niinContext*)); + // Publish the context used by every device-side NIIN wrapper. Do this in a + // device kernel so the update resolves through the same RDC symbol as a + // consumer kernel, including consumers that own the external context + // symbol. Check completion here: proceeding with a null context turns a + // setup failure into an unrelated illegal memory access in the consumer. + publishDeviceContext<<<1, 1>>>(s.devCtx); + cudaError_t cudaResult = cudaGetLastError(); + if (cudaResult != cudaSuccess) { + fprintf(stderr, "NIIN: device-context publication launch failed: %s\n", + cudaGetErrorString(cudaResult)); + return -1; + } + cudaResult = cudaDeviceSynchronize(); + if (cudaResult != cudaSuccess) { + fprintf(stderr, "NIIN: device-context publication failed: %s\n", + cudaGetErrorString(cudaResult)); + return -1; + } // Extract LSA info from the devComm s.lsaRank = s.hostCtx.devComm.lsaRank; s.lsaSize = s.hostCtx.devComm.lsaSize; // Initialize predefined teams - niin::teams::initPredefined(s.rank, s.nRanks, s.lsaRank, s.lsaSize); + niin::teams::initPredefined(s.rank, s.nRanks, s.lsaRank, s.lsaSize, + s.nodeRank, s.nodeSize); + + // Network AMOs are on by default wherever the provider can come up. A + // failure here leaves them fail-closed, exactly as before the provider + // existed, rather than failing initialization. + enableGpunetioAtomics(s); s.initialized = true; return 0; @@ -262,7 +597,6 @@ inline int initCommon(ncclComm_t comm) { // --------------------------------------------------------------------------- #define NIIN_KERNEL_INIT() \ do { \ - extern __device__ niinContext* niin_g_ctx; \ if (threadIdx.x == 0) \ niin_g_ctx = niin::detail::state().devCtx; \ __syncthreads(); \ @@ -271,8 +605,12 @@ inline int initCommon(ncclComm_t comm) { // This doesn't work directly because devCtx is a host variable. The actual // approach: pass devCtx as a kernel argument. We provide a helper: __device__ inline void niin_set_context(niinContext* ctx) { - if (threadIdx.x == 0) niin_g_ctx = ctx; + if (threadIdx.x == 0) { + niin_g_ctx = ctx; + cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_device); + } __syncthreads(); + cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_device); } // --------------------------------------------------------------------------- @@ -280,9 +618,9 @@ __device__ inline void niin_set_context(niinContext* ctx) { // --------------------------------------------------------------------------- // nvshmem_init: initialize NIIN with auto-detected rank/size. -// Multi-PE: detects rank from supported launcher metadata and bootstraps -// ncclUniqueId via MPI_Bcast when NIIN_HAS_MPI is available. Single-PE -// fallback otherwise. +// Multi-PE: detects rank from supported launcher metadata and bootstraps the +// NCCL unique ID internally. This preserves the ordinary NVSHMEM contract: +// an application can call nvshmem_init() directly without initializing MPI. inline int nvshmem_init(void) { auto& s = niin::detail::state(); if (s.initialized) { @@ -293,7 +631,9 @@ inline int nvshmem_init(void) { int rank = 0, nRanks = 1; bool multiPE = niin::detail::detectRankFromEnv(&rank, &nRanks); - int localDev = niin::detail::detectLocalDevice(rank); + s.nodeRank = niin::detail::detectLocalRank(rank); + s.nodeSize = niin::detail::detectNodeSize(); + int localDev = s.nodeRank; cudaSetDevice(localDev); s.cudaDev = localDev; @@ -303,23 +643,28 @@ inline int nvshmem_init(void) { ncclComm_t comm; if (multiPE && nRanks > 1) { -#ifdef NIIN_HAS_MPI - // Use MPI for ncclUniqueId distribution ncclUniqueId id; - if (rank == 0) ncclGetUniqueId(&id); - MPI_Bcast(&id, sizeof(id), MPI_BYTE, 0, MPI_COMM_WORLD); - ncclCommInitRankConfig(&comm, nRanks, id, rank, &config); -#else - fprintf(stderr, "NIIN: multi-PE detected (rank=%d nRanks=%d) but NIIN_HAS_MPI not defined. " - "Use nvshmemx_init_attr() with MPI or compile with -DNIIN_HAS_MPI.\n", - rank, nRanks); - return -1; -#endif + char bootstrapFile[PATH_MAX]; + if (niin::detail::bootstrapUniqueId(rank, nRanks, &id, bootstrapFile, + sizeof(bootstrapFile)) != 0) { + return -1; + } + const ncclResult_t result = ncclCommInitRankConfig(&comm, nRanks, id, rank, &config); + if (rank == 0) unlink(bootstrapFile); + if (result != ncclSuccess) { + fprintf(stderr, "NIIN: ncclCommInitRankConfig failed: %s\n", ncclGetErrorString(result)); + return -1; + } } else { // Single-PE mode ncclUniqueId id; - ncclGetUniqueId(&id); - ncclCommInitRankConfig(&comm, 1, id, 0, &config); + ncclResult_t result = ncclGetUniqueId(&id); + if (result == ncclSuccess) result = ncclCommInitRankConfig(&comm, 1, id, 0, &config); + if (result != ncclSuccess) { + fprintf(stderr, "NIIN: single-PE communicator initialization failed: %s\n", + ncclGetErrorString(result)); + return -1; + } } return niin::detail::initCommon(comm); @@ -333,10 +678,18 @@ inline void nvshmem_finalize(void) { cudaSetDevice(s.cudaDev); cudaDeviceSynchronize(); + // Tear the atomic provider down while its device context and the NCCL + // communicator it all-gathered over are both still alive. + if (s.gpunetioAtomics != nullptr) { + niinGpunetioAtomicFinalize(s.gpunetioAtomics); + s.gpunetioAtomics = nullptr; + } + ncclGroupStart(); niinFinalize(s.comm, s.devCtx); ncclGroupEnd(); + niinTmaDisable(&s.hostCtx); cudaFree(s.devCtx); cudaFree(s.barrierScratch); ncclMemFree(s.heapBase); @@ -417,7 +770,9 @@ inline int nvshmemx_init_attr(unsigned int flags, nvshmemx_init_attr_t* attr) { int rank = attr->args.uid_args.myrank; int nRanks = attr->args.uid_args.nranks; - int localDev = niin::detail::detectLocalDevice(rank); + s.nodeRank = niin::detail::detectLocalRank(rank); + s.nodeSize = niin::detail::detectNodeSize(); + int localDev = s.nodeRank; cudaSetDevice(localDev); s.cudaDev = localDev; @@ -439,7 +794,9 @@ inline int nvshmemx_init_attr(unsigned int flags, nvshmemx_init_attr_t* attr) { MPI_Comm_rank(mpi_comm, &rank); MPI_Comm_size(mpi_comm, &nRanks); - int localDev = niin::detail::detectLocalDevice(rank); + s.nodeRank = niin::detail::detectLocalRank(rank); + s.nodeSize = niin::detail::detectNodeSize(); + int localDev = s.nodeRank; cudaSetDevice(localDev); s.cudaDev = localDev; @@ -641,7 +998,14 @@ inline void niin_host_fence() { } inline void niin_host_quiet() { + auto& s = niin::detail::state(); + // Wait for in-flight kernels to retire first -- they may still be issuing puts. niin_host_fence(); + if (!s.initialized || s.hostCtx.devComm.ginConnectionCount == 0) return; + // A put can still be in flight on a GIN context after the kernel that issued it + // has retired, so cudaDeviceSynchronize() alone does not complete it. + niin_quiet_kernel<><<<1, 1, 0, s.stream>>>(); + cudaStreamSynchronize(s.stream); } // --------------------------------------------------------------------------- diff --git a/projects/rccl/contrib/niin/include/niin/query.h b/projects/rccl/contrib/niin/include/niin/query.h index 3b4c1992bfa9..75b66276e367 100644 --- a/projects/rccl/contrib/niin/include/niin/query.h +++ b/projects/rccl/contrib/niin/include/niin/query.h @@ -12,18 +12,18 @@ // Device-only query helpers (used by device-side nvshmem_* implementations) __device__ __forceinline__ int niin_device_my_pe() { - return niin_comm().rank; + return niin_rank(); } __device__ __forceinline__ int niin_device_n_pes() { - return niin_comm().nRanks; + return niin_n_ranks(); } // nvshmem_my_pe: returns this PE's rank in TEAM_WORLD // __host__ __device__ — on device reads from niin_g_ctx, on host reads from global state __host__ __device__ __forceinline__ int nvshmem_my_pe() { #ifdef __CUDA_ARCH__ - return niin_comm().rank; + return niin_rank(); #else // Host implementation provided by nvshmem_host.h via niin::detail::state() // We can't call it here (circular include), so return from the extern state. @@ -34,7 +34,7 @@ __host__ __device__ __forceinline__ int nvshmem_my_pe() { __host__ __device__ __forceinline__ int nvshmem_n_pes() { #ifdef __CUDA_ARCH__ - return niin_comm().nRanks; + return niin_n_ranks(); #else extern int niin_host_n_pes(); return niin_host_n_pes(); @@ -62,14 +62,13 @@ __host__ __device__ __forceinline__ void* nvshmem_ptr(void* ptr, int pe) { // Team queries — all predefined teams supported on device __host__ __device__ __forceinline__ int nvshmem_team_my_pe(nvshmem_team_t team) { #ifdef __CUDA_ARCH__ - ncclDevComm const& c = niin_comm(); switch (team) { - case NVSHMEM_TEAM_WORLD: return c.rank; - case NVSHMEM_TEAM_SHARED: return c.lsaRank; - case NVSHMEMX_TEAM_NODE: return c.lsaRank; - case NVSHMEMX_TEAM_SAME_MYPE_NODE: return c.rank / c.lsaSize; // rail rank + case NVSHMEM_TEAM_WORLD: return niin_rank(); + case NVSHMEM_TEAM_SHARED: return niin_ctx().nodeRank; + case NVSHMEMX_TEAM_NODE: return niin_ctx().nodeRank; + case NVSHMEMX_TEAM_SAME_MYPE_NODE: return niin_rank() / niin_ctx().nodeSize; case NVSHMEMI_TEAM_SAME_GPU: return 0; // always rank 0 (size=1) - case NVSHMEMI_TEAM_GPU_LEADERS: return c.rank; // same as WORLD + case NVSHMEMI_TEAM_GPU_LEADERS: return niin_rank(); // same as WORLD case NVSHMEM_TEAM_INVALID: return -1; default: return -1; // unknown team } @@ -81,14 +80,13 @@ __host__ __device__ __forceinline__ int nvshmem_team_my_pe(nvshmem_team_t team) __host__ __device__ __forceinline__ int nvshmem_team_n_pes(nvshmem_team_t team) { #ifdef __CUDA_ARCH__ - ncclDevComm const& c = niin_comm(); switch (team) { - case NVSHMEM_TEAM_WORLD: return c.nRanks; - case NVSHMEM_TEAM_SHARED: return c.lsaSize; - case NVSHMEMX_TEAM_NODE: return c.lsaSize; - case NVSHMEMX_TEAM_SAME_MYPE_NODE: return (c.nRanks + c.lsaSize - 1) / c.lsaSize; // num nodes + case NVSHMEM_TEAM_WORLD: return niin_n_ranks(); + case NVSHMEM_TEAM_SHARED: return niin_ctx().nodeSize; + case NVSHMEMX_TEAM_NODE: return niin_ctx().nodeSize; + case NVSHMEMX_TEAM_SAME_MYPE_NODE: return (niin_n_ranks() + niin_ctx().nodeSize - 1) / niin_ctx().nodeSize; case NVSHMEMI_TEAM_SAME_GPU: return 1; - case NVSHMEMI_TEAM_GPU_LEADERS: return c.nRanks; // same as WORLD + case NVSHMEMI_TEAM_GPU_LEADERS: return niin_n_ranks(); // same as WORLD case NVSHMEM_TEAM_INVALID: return -1; default: return -1; // unknown team } diff --git a/projects/rccl/contrib/niin/include/niin/rma.h b/projects/rccl/contrib/niin/include/niin/rma.h index 9cb68478fc23..66d8803f921f 100644 --- a/projects/rccl/contrib/niin/include/niin/rma.h +++ b/projects/rccl/contrib/niin/include/niin/rma.h @@ -9,17 +9,23 @@ #define NIIN_RMA_H_ #include "niin/context.h" +#include "niin/tma.h" // --------------------------------------------------------------------------- // Internal helpers // --------------------------------------------------------------------------- // Single-thread vectorized memcpy for LSA peer transfers. +// When this CTA has lent NIIN shared memory (nvshmemx_give_smem) the transfer +// goes out over TMA instead; niin_tma_try_copy reports -1 whenever TMA cannot +// meet its alignment/size constraints, and we fall through to load/store. // Uses int4 (16-byte) loads/stores when src and dst have matching alignment // mod 16 (so a single head adjustment aligns both). Falls back to byte // copy when alignments differ. __device__ __forceinline__ void niin_memcpy_to_peer( void* __restrict__ dst, const void* __restrict__ src, size_t bytes) { + if (niin_tma_try_copy(dst, src, bytes) == 0) return; + char* d = static_cast(dst); const char* s = static_cast(src); uintptr_t da = reinterpret_cast(d); @@ -51,9 +57,12 @@ __device__ __forceinline__ void niin_memcpy_to_peer( for (size_t i = 0; i < bytes; i++) d[i] = s[i]; } -// Single-thread vectorized memcpy from an LSA peer. +// Single-thread vectorized memcpy from an LSA peer. Takes the TMA path on the +// same terms as niin_memcpy_to_peer. __device__ __forceinline__ void niin_memcpy_from_peer( void* __restrict__ dst, const void* __restrict__ src, size_t bytes) { + if (niin_tma_try_copy(dst, src, bytes) == 0) return; + char* d = static_cast(dst); const char* s = static_cast(src); uintptr_t da = reinterpret_cast(d); @@ -82,14 +91,15 @@ __device__ __forceinline__ void niin_memcpy_from_peer( } // Helper: GIN put (network path for arbitrary data) -__device__ __forceinline__ void niin_gin_put(size_t dstOffset, const void* src, size_t bytes, int pe) { +NIIN_NOINLINE_DEVICE void niin_gin_put(size_t dstOffset, const void* src, size_t bytes, int pe, + int ctx = -1) { if (!niin_has_gin()) { NIIN_NOT_IMPLEMENTED_VOID("nvshmem_put (network without GIN)"); return; } ncclDevComm const& comm = niin_comm(); ncclTeam world = ncclTeamWorld(comm); - ncclGin gin(comm, niin_gin_context_index()); + ncclGin gin(comm, niin_gin_context_or_next(ctx)); // For GIN put, both src and dst must be in registered windows. // src is in our local heap, dst is at dstOffset in peer's heap. gin.put( @@ -97,39 +107,44 @@ __device__ __forceinline__ void niin_gin_put(size_t dstOffset, const void* src, niin_heap_window(), dstOffset, niin_heap_window(), niin_sym_offset(src), bytes ); + niin_note_gin_op_pending(); } // Helper: GIN get (network path for block data into the symmetric heap) -__device__ __forceinline__ void niin_gin_get(void* dest, size_t srcOffset, size_t bytes, int pe) { +NIIN_NOINLINE_DEVICE void niin_gin_get(void* dest, size_t srcOffset, size_t bytes, int pe, + int ctx = -1) { if (!niin_has_gin()) { NIIN_NOT_IMPLEMENTED_VOID("nvshmem_get (network without GIN)"); return; } ncclDevComm const& comm = niin_comm(); ncclTeam world = ncclTeamWorld(comm); - ncclGin gin(comm, niin_gin_context_index()); + ncclGin gin(comm, niin_gin_context_or_next(ctx)); gin.get( world, pe, niin_heap_window(), srcOffset, niin_heap_window(), niin_sym_offset(dest), bytes ); + niin_note_gin_op_pending(); gin.flush(ncclCoopThread{}); + niin_note_gin_op_complete(); } // Helper: GIN putValue (network path for small values <= 8 bytes) template -__device__ __forceinline__ void niin_gin_put_value(size_t dstOffset, T value, int pe) { +NIIN_NOINLINE_DEVICE void niin_gin_put_value(size_t dstOffset, T value, int pe, int ctx = -1) { if (!niin_has_gin()) { NIIN_NOT_IMPLEMENTED_VOID("nvshmem_p (network without GIN)"); return; } ncclDevComm const& comm = niin_comm(); ncclTeam world = ncclTeamWorld(comm); - ncclGin gin(comm, niin_gin_context_index()); + ncclGin gin(comm, niin_gin_context_or_next(ctx)); gin.putValue( world, pe, niin_heap_window(), dstOffset, value ); + niin_note_gin_op_pending(); } // --------------------------------------------------------------------------- @@ -140,10 +155,12 @@ __device__ __forceinline__ void nvshmem_##TYPENAME##_p(TYPE* dest, TYPE value, i size_t offset = niin_sym_offset(dest); \ if (pe == nvshmem_my_pe()) { \ *(volatile TYPE*)dest = value; \ + niin_note_lsa_store_pending(); \ return; \ } \ - if (niin_is_lsa_peer(pe)) { \ + if (niin_world_is_lsa_only() || niin_is_lsa_peer(pe)) { \ *(volatile TYPE*)niin_get_peer_ptr(offset, pe) = value; \ + niin_note_lsa_store_pending(); \ return; \ } \ if (sizeof(TYPE) <= 8) { \ @@ -165,7 +182,7 @@ __device__ __forceinline__ TYPE nvshmem_##TYPENAME##_g(const TYPE* src, int pe) return *(volatile const TYPE*)src; \ } \ size_t offset = niin_sym_offset(src); \ - if (niin_is_lsa_peer(pe)) { \ + if (niin_world_is_lsa_only() || niin_is_lsa_peer(pe)) { \ return *(volatile const TYPE*)niin_get_peer_ptr(offset, pe); \ } \ NIIN_NOT_IMPLEMENTED_RETURN("nvshmem_" #TYPENAME "_g (network)", (TYPE)0); \ @@ -184,10 +201,12 @@ __device__ __forceinline__ void nvshmem_##TYPENAME##_put(TYPE* dest, const TYPE* size_t offset = niin_sym_offset(dest); \ if (pe == nvshmem_my_pe()) { \ niin_memcpy_to_peer(dest, src, bytes); \ + niin_note_lsa_store_pending(); \ return; \ } \ - if (niin_is_lsa_peer(pe)) { \ + if (niin_world_is_lsa_only() || niin_is_lsa_peer(pe)) { \ niin_memcpy_to_peer(niin_get_peer_ptr(offset, pe), src, bytes); \ + niin_note_lsa_store_pending(); \ return; \ } \ niin_gin_put(offset, src, bytes, pe); \ @@ -205,7 +224,7 @@ __device__ __forceinline__ void nvshmem_##TYPENAME##_get(TYPE* dest, const TYPE* return; \ } \ size_t offset = niin_sym_offset(src); \ - if (niin_is_lsa_peer(pe)) { \ + if (niin_world_is_lsa_only() || niin_is_lsa_peer(pe)) { \ niin_memcpy_from_peer(dest, niin_get_peer_ptr(offset, pe), bytes); \ return; \ } \ @@ -223,10 +242,12 @@ __device__ __forceinline__ void nvshmem_putmem(void* dest, const void* src, size_t offset = niin_sym_offset(dest); if (pe == nvshmem_my_pe()) { niin_memcpy_to_peer(dest, src, bytes); + niin_note_lsa_store_pending(); return; } - if (niin_is_lsa_peer(pe)) { + if (niin_world_is_lsa_only() || niin_is_lsa_peer(pe)) { niin_memcpy_to_peer(niin_get_peer_ptr(offset, pe), src, bytes); + niin_note_lsa_store_pending(); return; } niin_gin_put(offset, src, bytes, pe); @@ -239,7 +260,7 @@ __device__ __forceinline__ void nvshmem_getmem(void* dest, const void* src, return; } size_t offset = niin_sym_offset(src); - if (niin_is_lsa_peer(pe)) { + if (niin_world_is_lsa_only() || niin_is_lsa_peer(pe)) { niin_memcpy_from_peer(dest, niin_get_peer_ptr(offset, pe), bytes); return; } diff --git a/projects/rccl/contrib/niin/include/niin/signaling.h b/projects/rccl/contrib/niin/include/niin/signaling.h index 83257a4d8ecf..11c893b0446b 100644 --- a/projects/rccl/contrib/niin/include/niin/signaling.h +++ b/projects/rccl/contrib/niin/include/niin/signaling.h @@ -10,84 +10,127 @@ #include "niin/context.h" -__device__ __forceinline__ void niin_gin_flush_thread() { +// Flush one GIN context. Callers pass the context the operation being completed +// was issued on; draining every context is nvshmem_quiet()'s job. +NIIN_NOINLINE_DEVICE void niin_gin_flush_thread(int ctx) { if (!niin_has_gin()) return; - ncclGin gin(niin_comm(), niin_gin_context_index()); + ncclGin gin(niin_comm(), ctx); gin.flush(ncclCoopThread{}); + niin_note_gin_op_complete(); } -__device__ __forceinline__ void niin_deliver_signal_remote_va( - size_t sigOffset, uint64_t signal, int sig_op, int pe) { +NIIN_NOINLINE_DEVICE void niin_deliver_signal_remote_va( + size_t sigOffset, uint64_t signal, int sig_op, int pe, int ctx = -1) { if (!niin_has_gin()) { NIIN_NOT_IMPLEMENTED_VOID("nvshmem_put_signal (network without GIN)"); return; } ncclDevComm const& comm = niin_comm(); ncclTeam world = ncclTeamWorld(comm); - ncclGin gin(comm, niin_gin_context_index()); + ncclGin gin(comm, niin_gin_context_or_next(ctx)); if (sig_op == NVSHMEM_SIGNAL_SET) { gin.putValue(world, pe, niin_heap_window(), sigOffset, signal); } else { gin.signal(world, pe, ncclGin_VASignalAdd{niin_heap_window(), sigOffset, signal}); } + niin_note_gin_op_pending(); gin.flush(ncclCoopThread{}); + niin_note_gin_op_complete(); } __device__ __forceinline__ void niin_deliver_signal( - uint64_t* sig_addr, uint64_t signal, int sig_op, int pe) { - size_t sigOffset = niin_sym_offset(sig_addr); - if (pe == nvshmem_my_pe()) { + uint64_t* sig_addr, uint64_t signal, int sig_op, int pe, int ctx = -1) { + if (pe == niin_rank()) { if (sig_op == NVSHMEM_SIGNAL_SET) atomicExch((unsigned long long*)sig_addr, signal); else atomicAdd((unsigned long long*)sig_addr, signal); - __threadfence_system(); + return; + } + size_t sigOffset = niin_sym_offset(sig_addr); + if (niin_world_is_lsa_only() && niin_peer_native_atomic()) { + uint64_t* peerSig = (uint64_t*)niin_get_peer_ptr_lsa_only(sigOffset, pe); + if (sig_op == NVSHMEM_SIGNAL_SET) { + atomicExch_system((unsigned long long*)peerSig, signal); + return; + } + atomicAdd_system((unsigned long long*)peerSig, signal); return; } if (niin_is_lsa_peer(pe) && niin_peer_native_atomic()) { uint64_t* peerSig = (uint64_t*)niin_get_peer_ptr(sigOffset, pe); - if (sig_op == NVSHMEM_SIGNAL_SET) atomicExch((unsigned long long*)peerSig, signal); - else atomicAdd((unsigned long long*)peerSig, signal); - __threadfence_system(); + if (sig_op == NVSHMEM_SIGNAL_SET) { + atomicExch_system((unsigned long long*)peerSig, signal); + return; + } + atomicAdd_system((unsigned long long*)peerSig, signal); return; } - niin_deliver_signal_remote_va(sigOffset, signal, sig_op, pe); + niin_deliver_signal_remote_va(sigOffset, signal, sig_op, pe, ctx); } __device__ __forceinline__ bool niin_use_separate_put_signal(int pe, int sig_op) { if (sig_op == NVSHMEM_SIGNAL_SET) return true; if (niin_force_separate_put_signal()) return true; + if (niin_world_is_lsa_only()) return !niin_peer_native_atomic(); if (!niin_is_lsa_peer(pe)) return true; - return niin_is_lsa_peer(pe) && !niin_peer_native_atomic(); + return !niin_peer_native_atomic(); } -__device__ __forceinline__ void niin_putmem_signal_impl( - void* dest, const void* src, size_t bytes, - uint64_t* sig_addr, uint64_t signal, int sig_op, int pe) { - size_t dstOffset = niin_sym_offset(dest); - if (pe == nvshmem_my_pe()) { - niin_memcpy_to_peer(dest, src, bytes); - __threadfence_system(); - niin_deliver_signal(sig_addr, signal, sig_op, pe); +__device__ __forceinline__ void niin_memcpy_to_peer_latency_path( + void* __restrict__ dst, const void* __restrict__ src, size_t bytes) { + if (bytes == 1) { + *(volatile unsigned char*)dst = *(volatile const unsigned char*)src; return; } - if (niin_is_lsa_peer(pe)) { - void* peerDst = niin_get_peer_ptr(dstOffset, pe); - niin_memcpy_to_peer(peerDst, src, bytes); - __threadfence_system(); - niin_deliver_signal(sig_addr, signal, sig_op, pe); + niin_memcpy_to_peer(dst, src, bytes); +} + +__device__ __forceinline__ void niin_deliver_signal_lsa_target( + uint64_t* target, uint64_t signal, int sig_op, bool self) { + if (sig_op == NVSHMEM_SIGNAL_SET) { + *(volatile uint64_t*)target = signal; return; } + if (self) atomicAdd((unsigned long long*)target, signal); + else atomicAdd_system((unsigned long long*)target, signal); +} - if (niin_use_separate_put_signal(pe, sig_op)) { - niin_gin_put(dstOffset, src, bytes, pe); - niin_gin_flush_thread(); - niin_deliver_signal(sig_addr, signal, sig_op, pe); - return; +__device__ __forceinline__ void niin_deliver_signal_lsa_fast( + uint64_t* sig_addr, uint64_t signal, int sig_op, int pe) { + const bool self = pe == niin_rank(); + uint64_t* target = sig_addr; + if (!self) { + size_t sigOffset = niin_sym_offset(sig_addr); + target = (uint64_t*)(niin_world_is_lsa_only() + ? niin_get_peer_ptr_lsa_only(sigOffset, pe) + : niin_get_peer_ptr(sigOffset, pe)); } + niin_deliver_signal_lsa_target(target, signal, sig_op, self); +} +// The GIN transport routines stay out of line. Inlining them expanded a full +// network descriptor build into every caller -- including callers that only +// ever touch an NVLink peer -- and the register pressure showed up in kernels +// that never leave the node. NVLink and self paths remain force-inlined: they +// are a peer-pointer copy plus a store, and a call would cost more than it +// saves. +NIIN_NOINLINE_DEVICE void niin_putmem_signal_remote_split( + size_t dstOffset, const void* src, size_t bytes, + uint64_t* sig_addr, uint64_t signal, int sig_op, int pe) { + // One context for the whole operation: the payload, the flush that orders it + // and the signal all have to travel the same QP to stay ordered. + const int ctx = niin_gin_next_context(); + niin_gin_put(dstOffset, src, bytes, pe, ctx); + niin_gin_flush_thread(ctx); + niin_deliver_signal_remote_va(niin_sym_offset(sig_addr), signal, sig_op, pe, ctx); +} + +NIIN_NOINLINE_DEVICE void niin_putmem_signal_remote_fused( + size_t dstOffset, const void* src, size_t bytes, + uint64_t* sig_addr, uint64_t signal, int pe) { size_t sigOffset = niin_sym_offset(sig_addr); ncclDevComm const& comm = niin_comm(); ncclTeam world = ncclTeamWorld(comm); - ncclGin gin(comm, niin_gin_context_index()); + ncclGin gin(comm, niin_gin_next_context()); ncclGin_VASignalAdd sigAction; sigAction.signalWindow = niin_heap_window(); sigAction.signalOffset = sigOffset; @@ -98,6 +141,60 @@ __device__ __forceinline__ void niin_putmem_signal_impl( niin_heap_window(), niin_sym_offset(src), bytes, sigAction ); + niin_note_gin_op_pending(); +} + +NIIN_NOINLINE_DEVICE void niin_putmem_signal_slow( + void* dest, const void* src, size_t bytes, + uint64_t* sig_addr, uint64_t signal, int sig_op, int pe) { + if (pe == niin_rank()) { + niin_memcpy_to_peer(dest, src, bytes); + __threadfence_system(); + niin_clear_lsa_store_pending(); + niin_deliver_signal(sig_addr, signal, sig_op, pe); + return; + } + + size_t dstOffset = niin_sym_offset(dest); + if (niin_world_is_lsa_only() || niin_is_lsa_peer(pe)) { + void* peerDst = niin_get_peer_ptr(dstOffset, pe); + niin_memcpy_to_peer(peerDst, src, bytes); + __threadfence_system(); + niin_clear_lsa_store_pending(); + niin_deliver_signal(sig_addr, signal, sig_op, pe); + return; + } + + if (sig_op == NVSHMEM_SIGNAL_SET || niin_force_separate_put_signal()) { + niin_putmem_signal_remote_split(dstOffset, src, bytes, sig_addr, signal, sig_op, pe); + return; + } + + niin_putmem_signal_remote_fused(dstOffset, src, bytes, sig_addr, signal, pe); +} + +__device__ __forceinline__ void niin_putmem_signal_impl( + void* dest, const void* src, size_t bytes, + uint64_t* sig_addr, uint64_t signal, int sig_op, int pe) { + const int me = niin_rank(); + const bool self = pe == me; + if (niin_world_is_lsa_only() && (self || niin_peer_native_atomic())) { + void* target = dest; + uint64_t* sigTarget = sig_addr; + if (!self) { + size_t dstOffset = niin_sym_offset(dest); + size_t sigOffset = niin_sym_offset(sig_addr); + target = niin_get_peer_ptr_lsa_only(dstOffset, pe); + sigTarget = (uint64_t*)niin_get_peer_ptr_lsa_only(sigOffset, pe); + } + niin_memcpy_to_peer_latency_path(target, src, bytes); + __threadfence_system(); + niin_clear_lsa_store_pending(); + niin_deliver_signal_lsa_target(sigTarget, signal, sig_op, self); + return; + } + + niin_putmem_signal_slow(dest, src, bytes, sig_addr, signal, sig_op, pe); } // --------------------------------------------------------------------------- @@ -170,7 +267,7 @@ NIIN_SIZED_RMA(NIIN_DEFINE_PUT_SIGNAL_SIZED_NBI) // nvshmem_signal_fetch: read a signal value atomically // --------------------------------------------------------------------------- __device__ __forceinline__ uint64_t nvshmem_signal_fetch(const uint64_t* sig_addr) { - return atomicAdd((unsigned long long*)sig_addr, 0ULL); + return *(volatile uint64_t*)sig_addr; } // --------------------------------------------------------------------------- @@ -178,12 +275,11 @@ __device__ __forceinline__ uint64_t nvshmem_signal_fetch(const uint64_t* sig_add // --------------------------------------------------------------------------- __device__ __forceinline__ uint64_t nvshmem_signal_wait_until( uint64_t* sig_addr, int cmp, uint64_t cmp_value) { + volatile uint64_t* v = (volatile uint64_t*)sig_addr; uint64_t val; do { - val = atomicAdd((unsigned long long*)sig_addr, 0ULL); + val = *v; } while (!niin_cmp_eval_u(cmp, val, cmp_value)); - // Acquire fence: data written before the signal must be visible - cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); return val; } diff --git a/projects/rccl/contrib/niin/include/niin/stream.h b/projects/rccl/contrib/niin/include/niin/stream.h index 44eca9cad62d..20e80e602666 100644 --- a/projects/rccl/contrib/niin/include/niin/stream.h +++ b/projects/rccl/contrib/niin/include/niin/stream.h @@ -226,8 +226,10 @@ __global__ void niin_stream_put_signal_complete_kernel(uint64_t* sig_addr, uint6 int sig_op, int pe) { if (threadIdx.x == 0) { __threadfence_system(); + // The payload kernels chose their own contexts, so complete all of them + // before the flag rather than one this thread never issued on. if (pe != nvshmem_my_pe() && !niin_is_lsa_peer(pe)) - niin_gin_flush_thread(); + niin_device_quiet(); niin_deliver_signal(sig_addr, signal, sig_op, pe); } } diff --git a/projects/rccl/contrib/niin/include/niin/sync.h b/projects/rccl/contrib/niin/include/niin/sync.h index cfd3b390715d..c4504c42c5c3 100644 --- a/projects/rccl/contrib/niin/include/niin/sync.h +++ b/projects/rccl/contrib/niin/include/niin/sync.h @@ -9,23 +9,96 @@ #define NIIN_SYNC_H_ #include "niin/context.h" +#include "niin/tma.h" static __device__ unsigned long long niin_test_any_cursor = 0; // --------------------------------------------------------------------------- // Device-side fence/quiet implementations // --------------------------------------------------------------------------- +// Complete every context. A thread cannot know which QPs carried its own +// operations once selection is per operation, and the on-stream and host paths +// complete work issued by other threads entirely. +__device__ __forceinline__ void niin_device_drain_all_contexts() { + ncclDevComm const& comm = niin_comm(); + if (niin_world_is_lsa_only() || !niin_has_gin()) return; + uint32_t nCtx = niin_gin_context_count(); + if (nCtx == 1) { // the common single-QP case: one flush, no loop + ncclGin gin(comm, 0); + gin.flush(ncclCoopThread{}); + return; + } + for (uint32_t ctx = 0; ctx < nCtx; ctx++) { + ncclGin gin(comm, (int)ctx); + gin.flush(ncclCoopThread{}); + } +} + +// fence orders operations; it does not have to complete them. On a single GIN +// context the transport already provides that ordering: every operation goes to +// the same RC QP per peer, and an RC QP processes its work queue in order, so +// consecutive puts to a PE land in order with no completion wait. A local +// memory fence is then all fence owes the caller. +// +// Completion is only needed when something outside that one queue can reorder +// against it: +// - several contexts, because operations rotate across QPs and two QPs are +// unordered against each other at the receiving NIC; +// - a bound GPUNetIO atomic sidecar, whose QPs are a separate network domain +// from GIN, so a pending put is not ordered against a following AMO. +// Either of those turns fence into the same all-contexts drain quiet does. +__device__ __forceinline__ bool niin_fence_needs_completion() { + if (niin_world_is_lsa_only() || !niin_has_gin()) return false; + if (niin_gin_context_count() > 1) return true; + return niin_gpunetio_atomic_context() != nullptr; +} + +NIIN_NOINLINE_DEVICE void niin_device_fence_slow() { + // Drain this thread's TMA bulk ops first. Unlike the context drain below this + // is not gated on connectivity: in a mixed topology, TMA puts to LSA peers + // are invisible to GIN, so a fence after a TMA put must order it either way. + const bool hadTma = niin_tma_smem_registered(); + niin_tma_drain_if_registered(); + const bool hadLsaStores = niin_consume_lsa_store_pending(); + const bool hadGinOps = !niin_world_is_lsa_only() && niin_has_gin_ops_pending(); + if (hadGinOps && niin_fence_needs_completion()) { + (void)niin_take_gin_ops_pending(); + niin_device_drain_all_contexts(); + } + if (hadTma || hadLsaStores || hadGinOps) __threadfence_system(); +} + __device__ __forceinline__ void niin_device_fence() { - __threadfence_system(); + if (niin_world_is_lsa_only() && niin_tma_policy() == NVSHMEMX_TMA_DISABLE) { + if (niin_consume_lsa_store_pending()) __threadfence_system(); + return; + } + niin_device_fence_slow(); +} + +NIIN_NOINLINE_DEVICE void niin_device_quiet_slow() { + const bool hadTma = niin_tma_smem_registered(); + niin_tma_drain_if_registered(); + const bool hadLsaStores = niin_consume_lsa_store_pending(); + const bool hadGinOps = !niin_world_is_lsa_only() && (niin_take_gin_ops_pending() != 0u); + if (hadGinOps) niin_device_drain_all_contexts(); + if (hadTma || hadLsaStores || hadGinOps) __threadfence_system(); } __device__ __forceinline__ void niin_device_quiet() { - ncclDevComm const& comm = niin_comm(); - if (comm.ginConnectionCount > 0) { - ncclGin gin(comm, niin_gin_context_index()); - gin.flush(ncclCoopThread{}); + if (niin_world_is_lsa_only() && niin_tma_policy() == NVSHMEMX_TMA_DISABLE) { + if (niin_consume_lsa_store_pending()) __threadfence_system(); + return; } - __threadfence_system(); + niin_device_quiet_slow(); +} + +// Kernel form of quiet for host-side paths. Templated so each translation unit +// including NIIN gets its own weak instantiation rather than a duplicate symbol +// at link time. +template +__global__ void niin_quiet_kernel() { + niin_device_quiet(); } // --------------------------------------------------------------------------- diff --git a/projects/rccl/contrib/niin/include/niin/teams.h b/projects/rccl/contrib/niin/include/niin/teams.h index abb0dd634121..f9d4284eeae5 100644 --- a/projects/rccl/contrib/niin/include/niin/teams.h +++ b/projects/rccl/contrib/niin/include/niin/teams.h @@ -49,24 +49,33 @@ inline TeamTable& table() { } // Called by niin_host_init_common after global state is set up -inline void initPredefined(int worldRank, int worldSize, int lsaRank, int lsaSize) { +inline void initPredefined(int worldRank, int worldSize, int lsaRank, int lsaSize, + int nodeRank = -1, int nodeSize = 0) { auto& t = table(); + // LSA is an NCCL topology domain; on systems such as NVL72 it can span + // multiple physical nodes. NVSHMEM's SHARED/NODE teams must instead be + // based on launcher-local PE topology. + if (nodeRank < 0 || nodeSize < 1) { + nodeRank = lsaRank; + nodeSize = lsaSize; + } + // TEAM_WORLD (id=0): all PEs t.teams[NVSHMEM_TEAM_WORLD] = {true, 0, 1, worldSize, worldRank}; - // TEAM_SHARED (id=1): same node (LSA team) - int lsaStart = worldRank - lsaRank; - t.teams[NVSHMEM_TEAM_SHARED] = {true, lsaStart, 1, lsaSize, lsaRank}; + // TEAM_SHARED (id=1): PEs on the same physical node. + int nodeStart = worldRank - nodeRank; + t.teams[NVSHMEM_TEAM_SHARED] = {true, nodeStart, 1, nodeSize, nodeRank}; // NVSHMEMX_TEAM_NODE (id=2): alias for SHARED - t.teams[NVSHMEMX_TEAM_NODE] = {true, lsaStart, 1, lsaSize, lsaRank}; + t.teams[NVSHMEMX_TEAM_NODE] = {true, nodeStart, 1, nodeSize, nodeRank}; // NVSHMEMX_TEAM_SAME_MYPE_NODE (id=3): same local rank across nodes (rail team) - // E.g., local rank 0 on all nodes. stride=lsaSize, start=lsaRank. - int railSize = (worldSize + lsaSize - 1) / lsaSize; // number of nodes - int railRank = worldRank / lsaSize; - t.teams[NVSHMEMX_TEAM_SAME_MYPE_NODE] = {true, lsaRank, lsaSize, railSize, railRank}; + // E.g., local rank 0 on all nodes. stride=nodeSize, start=nodeRank. + int railSize = (worldSize + nodeSize - 1) / nodeSize; // number of nodes + int railRank = worldRank / nodeSize; + t.teams[NVSHMEMX_TEAM_SAME_MYPE_NODE] = {true, nodeRank, nodeSize, railSize, railRank}; // NVSHMEMI_TEAM_SAME_GPU (id=4): PEs sharing same GPU // In NCCL's 1-PE-per-GPU model, this is always just this PE alone. diff --git a/projects/rccl/contrib/niin/include/niin/threadgroup.h b/projects/rccl/contrib/niin/include/niin/threadgroup.h index c6d4368c9cb9..1179a3f62b24 100644 --- a/projects/rccl/contrib/niin/include/niin/threadgroup.h +++ b/projects/rccl/contrib/niin/include/niin/threadgroup.h @@ -16,109 +16,329 @@ #include "niin/rma.h" #include "niin/signaling.h" +#include "niin/sync.h" // =========================================================================== // Internal: cooperative memcpy kernels using all threads in a group. // Uses int4 (16-byte) vectorized copies for NVLink bandwidth. // =========================================================================== -// Internal: cooperative vectorized memcpy with 4x unroll for ILP. -// Each unrolled access is offset by nthreads to maintain coalescing: -// iter 0: thread k accesses [k], [k+nthreads], [k+2*nthreads], [k+3*nthreads] -// All 4 accesses within an iteration are independently coalesced across the warp. -#define NIIN_COOP_UNROLL 4 - -__device__ __forceinline__ void niin_coop_copy( +NIIN_NOINLINE_DEVICE void niin_coop_copy_slow( void* __restrict__ dst, const void* __restrict__ src, size_t bytes, int tid, int nthreads) { char* d = static_cast(dst); const char* s = static_cast(src); - uintptr_t da = reinterpret_cast(d); - uintptr_t sa = reinterpret_cast(s); - - // Only use 16B vector path if src/dst have matching alignment mod 16, - // so a single head adjustment can align both simultaneously. - if (((da ^ sa) & 0xF) == 0) { - // Phase 1: head bytes — align both dst and src to 16 bytes. - size_t head = (16 - (da & 0xF)) & 0xF; - if (head > bytes) head = bytes; - for (size_t j = tid; j < head; j += nthreads) - d[j] = s[j]; - d += head; - s += head; - size_t remaining = bytes - head; - - // Phase 2: vectorized int4 with 4x unroll, all accesses coalesced. - // Both d and s are now 16-byte aligned. - size_t n16 = remaining / 16; - size_t stride = (size_t)nthreads; + + if ((reinterpret_cast(d) % 16) == 0 && + (reinterpret_cast(s) % 16) == 0) { + size_t n16 = bytes / 16; auto* d16 = reinterpret_cast(d); auto const* s16 = reinterpret_cast(s); - - size_t i = tid; - size_t unrolled_end = (n16 >= stride * NIIN_COOP_UNROLL) - ? n16 - stride * (NIIN_COOP_UNROLL - 1) : 0; - for (; i < unrolled_end; i += stride * NIIN_COOP_UNROLL) { - d16[i + stride * 0] = s16[i + stride * 0]; - d16[i + stride * 1] = s16[i + stride * 1]; - d16[i + stride * 2] = s16[i + stride * 2]; - d16[i + stride * 3] = s16[i + stride * 3]; - } - for (; i < n16; i += stride) + for (size_t i = static_cast(tid); i < n16; i += static_cast(nthreads)) { d16[i] = s16[i]; - + } d += n16 * 16; s += n16 * 16; - bytes = remaining - n16 * 16; + bytes -= n16 * 16; + if (bytes == 0) return; + } + + if ((reinterpret_cast(d) % 8) == 0 && + (reinterpret_cast(s) % 8) == 0) { + size_t n8 = bytes / 8; + auto* d8 = reinterpret_cast(d); + auto const* s8 = reinterpret_cast(s); + for (size_t i = static_cast(tid); i < n8; i += static_cast(nthreads)) { + d8[i] = s8[i]; + } + d += n8 * 8; + s += n8 * 8; + bytes -= n8 * 8; + if (bytes == 0) return; + } + + if ((reinterpret_cast(d) % 4) == 0 && + (reinterpret_cast(s) % 4) == 0) { + size_t n4 = bytes / 4; + auto* d4 = reinterpret_cast(d); + auto const* s4 = reinterpret_cast(s); + for (size_t i = static_cast(tid); i < n4; i += static_cast(nthreads)) { + d4[i] = s4[i]; + } + d += n4 * 4; + s += n4 * 4; + bytes -= n4 * 4; + if (bytes == 0) return; } - // Phase 3: tail bytes, or full byte copy if alignments didn't match. - for (size_t j = tid; j < bytes; j += nthreads) - d[j] = s[j]; + if ((reinterpret_cast(d) % 2) == 0 && + (reinterpret_cast(s) % 2) == 0) { + size_t n2 = bytes / 2; + auto* d2 = reinterpret_cast(d); + auto const* s2 = reinterpret_cast(s); + for (size_t i = static_cast(tid); i < n2; i += static_cast(nthreads)) { + d2[i] = s2[i]; + } + d += n2 * 2; + s += n2 * 2; + bytes -= n2 * 2; + if (bytes == 0) return; + } + + for (size_t i = static_cast(tid); i < bytes; i += static_cast(nthreads)) { + d[i] = s[i]; + } } -// Cooperative put: all threads copy src -> peer dest via NVLink +__device__ __forceinline__ void niin_coop_copy( + void* __restrict__ dst, const void* __restrict__ src, size_t bytes, + int tid, int nthreads) { + uintptr_t daddr = reinterpret_cast(dst); + uintptr_t saddr = reinterpret_cast(src); + if (((daddr | saddr | bytes) & 0xF) != 0) { + niin_coop_copy_slow(dst, src, bytes, tid, nthreads); + return; + } + + auto* d16 = reinterpret_cast(dst); + auto const* s16 = reinterpret_cast(src); + size_t n16 = bytes / 16; + for (size_t i = static_cast(tid); i < n16; i += static_cast(nthreads)) { + d16[i] = s16[i]; + } +} + +template __device__ __forceinline__ void niin_coop_put( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, int ctx = -1, bool markLsaPending = true); + +template +__device__ __forceinline__ void niin_coop_get( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, int ctx = -1); + +template +__device__ __forceinline__ void niin_coop_put_lsa_ldst( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, bool markLsaPending) { + void* target = dest; + if (pe != niin_device_my_pe()) { + target = niin_get_peer_ptr_lsa_only(niin_sym_offset(dest), pe); + } + niin_coop_copy(target, src, bytes, tid, nthreads); + if (markLsaPending && tid == 0) niin_note_lsa_store_pending(); +} + +template +__device__ __forceinline__ void niin_coop_get_lsa_ldst( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads) { + const void* source = src; + if (pe != niin_device_my_pe()) source = niin_get_peer_ptr_lsa_only(niin_sym_offset(src), pe); + niin_coop_copy(dest, source, bytes, tid, nthreads); +} + +template +__device__ __forceinline__ void niin_coop_put_lsa( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, bool markLsaPending) { + void* target = dest; + if (pe != niin_device_my_pe()) { + target = niin_get_peer_ptr(niin_sym_offset(dest), pe); + } + if (niin_tma_try_copy(target, src, bytes) == 0) { + if (markLsaPending && tid == 0) niin_note_lsa_store_pending(); + return; + } + niin_coop_copy(target, src, bytes, tid, nthreads); + if (markLsaPending && tid == 0) niin_note_lsa_store_pending(); +} + +template +__device__ __forceinline__ void niin_coop_get_lsa( void* dest, const void* src, size_t bytes, int pe, int tid, int nthreads) { - if (pe == niin_device_my_pe()) { - niin_coop_copy(dest, src, bytes, tid, nthreads); + const void* source = src; + if (pe != niin_device_my_pe()) source = niin_get_peer_ptr(niin_sym_offset(src), pe); + if (niin_tma_try_copy(dest, source, bytes) == 0) return; + niin_coop_copy(dest, source, bytes, tid, nthreads); +} + +template +NIIN_NOINLINE_DEVICE void niin_coop_put_mixed( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, int ctx, bool markLsaPending) { + if (pe == niin_device_my_pe() || niin_is_lsa_peer(pe)) { + niin_coop_put_lsa(dest, src, bytes, pe, tid, nthreads, markLsaPending); return; } - size_t offset = niin_sym_offset(dest); - if (niin_is_lsa_peer(pe)) { - void* peerDst = niin_get_peer_ptr(offset, pe); - niin_coop_copy(peerDst, src, bytes, tid, nthreads); - // Release fence at system scope to ensure all peer stores are visible - cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); + if (tid == 0) { + niin_gin_put(niin_sym_offset(dest), src, bytes, pe, ctx); + } +} + +template +NIIN_NOINLINE_DEVICE void niin_coop_get_mixed( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, int ctx) { + if (pe == niin_device_my_pe() || niin_is_lsa_peer(pe)) { + niin_coop_get_lsa(dest, src, bytes, pe, tid, nthreads); return; } - // Network: thread 0 does GIN put - if (tid == 0) - niin_gin_put(offset, src, bytes, pe); + if (tid == 0) { + niin_gin_get(dest, niin_sym_offset(src), bytes, pe, ctx); + } } -// Cooperative get: all threads copy peer src -> local dest via NVLink +// Cooperative put: all threads copy src -> peer dest via NVLink. +// SCOPE tells the TMA path whether the caller is a warp or a whole CTA; when +// this CTA has lent NIIN shared memory the copy goes out over cp.async.bulk +// instead of the cooperative int4 loop. niin_tma_try_copy returns -1 for every +// transfer TMA cannot take, and it does so uniformly across the threadgroup, +// so either all threads take the TMA path or none do. +template +__device__ __forceinline__ void niin_coop_put( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, int ctx, bool markLsaPending) { + if (niin_world_is_lsa_only()) { + niin_coop_put_lsa(dest, src, bytes, pe, tid, nthreads, markLsaPending); + return; + } + niin_coop_put_mixed(dest, src, bytes, pe, tid, nthreads, ctx, markLsaPending); +} + +// Cooperative get: all threads copy peer src -> local dest via NVLink. +// Takes the TMA path on the same terms as niin_coop_put; a destination in +// shared memory routes straight into smem instead of staging through it. +template __device__ __forceinline__ void niin_coop_get( void* dest, const void* src, size_t bytes, int pe, - int tid, int nthreads) { - if (pe == niin_device_my_pe()) { - niin_coop_copy(dest, src, bytes, tid, nthreads); + int tid, int nthreads, int ctx) { + if (niin_world_is_lsa_only()) { + niin_coop_get_lsa(dest, src, bytes, pe, tid, nthreads); return; } + niin_coop_get_mixed(dest, src, bytes, pe, tid, nthreads, ctx); +} - size_t offset = niin_sym_offset(src); - if (niin_is_lsa_peer(pe)) { - const void* peerSrc = niin_get_peer_ptr(offset, pe); - niin_coop_copy(dest, peerSrc, bytes, tid, nthreads); +template +NIIN_NOINLINE_DEVICE void niin_coop_put_generic_body( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, int ctx, bool markLsaPending) { + niin_coop_put(dest, src, bytes, pe, tid, nthreads, ctx, markLsaPending); +} + +template +NIIN_NOINLINE_DEVICE void niin_coop_get_generic_body( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads, int ctx) { + niin_coop_get(dest, src, bytes, pe, tid, nthreads, ctx); +} + +template +__device__ __forceinline__ void niin_threadgroup_sync() { + if constexpr (SCOPE == NIIN_TMA_WARP) { + __syncwarp(); + } else if constexpr (SCOPE == NIIN_TMA_BLOCK) { + __syncthreads(); + } +} + +template +__device__ __forceinline__ bool niin_threadgroup_is_leader() { + if constexpr (SCOPE == NIIN_TMA_THREAD) { + return true; + } else if constexpr (SCOPE == NIIN_TMA_WARP) { + return nccl::utility::lane() == 0; + } else { + return threadIdx.x == 0; + } +} + +template +__device__ __forceinline__ void niin_threadgroup_quiet() { + if constexpr (SCOPE == NIIN_TMA_THREAD) { + niin_device_quiet(); return; } - // Network: thread 0 does GIN get - if (tid == 0) - niin_gin_get(dest, offset, bytes, pe); + const bool hadTma = niin_tma_smem_registered(); + niin_tma_drain_if_registered(); + + if (niin_threadgroup_is_leader()) { + if (niin_world_is_lsa_only() && niin_tma_policy() == NVSHMEMX_TMA_DISABLE) { + if (niin_consume_lsa_store_pending()) __threadfence_system(); + } else { + const bool hadLsaStores = niin_consume_lsa_store_pending(); + const bool hadGinOps = !niin_world_is_lsa_only() && (niin_take_gin_ops_pending() != 0u); + if (hadGinOps) niin_device_drain_all_contexts(); + if (hadTma || hadLsaStores || hadGinOps) __threadfence_system(); + } + } + + niin_threadgroup_sync(); +} + +__device__ __forceinline__ void nvshmemx_qp_quiet( + int pe, nvshmemx_qp_handle_t* qp_handle, int num_qps) { + (void)pe; (void)qp_handle; (void)num_qps; + niin_threadgroup_quiet(); +} + +__device__ __forceinline__ void nvshmemx_qp_quiet_warp( + int pe, nvshmemx_qp_handle_t* qp_handle, int num_qps) { + (void)pe; (void)qp_handle; (void)num_qps; + niin_threadgroup_quiet(); +} + +__device__ __forceinline__ void nvshmemx_qp_quiet_block( + int pe, nvshmemx_qp_handle_t* qp_handle, int num_qps) { + (void)pe; (void)qp_handle; (void)num_qps; + niin_threadgroup_quiet(); +} + +__device__ __forceinline__ void niin_threadgroup_deliver_signal( + bool lsa, uint64_t* sig, uint64_t signal, int sig_op, int pe, int ctx) { + if (lsa) { + __threadfence_system(); + niin_clear_lsa_store_pending(); + if (pe == niin_rank() || niin_peer_native_atomic()) { + niin_deliver_signal_lsa_fast(sig, signal, sig_op, pe); + return; + } + } + niin_deliver_signal(sig, signal, sig_op, pe, ctx); +} + +// NBI calls are nonblocking with respect to RMA completion, but the warp/block +// NVSHMEM APIs are collective over the threadgroup. +template +__device__ __forceinline__ void niin_coop_put_nbi( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads) { + niin_threadgroup_sync(); + if (niin_world_is_lsa_only() && niin_tma_policy() == NVSHMEMX_TMA_DISABLE) { + niin_coop_put_lsa_ldst(dest, src, bytes, pe, tid, nthreads, true); + } else { + niin_coop_put_generic_body(dest, src, bytes, pe, tid, nthreads, -1, true); + } + niin_threadgroup_sync(); +} + +template +__device__ __forceinline__ void niin_coop_get_nbi( + void* dest, const void* src, size_t bytes, int pe, + int tid, int nthreads) { + niin_threadgroup_sync(); + if (niin_world_is_lsa_only() && niin_tma_policy() == NVSHMEMX_TMA_DISABLE) { + niin_coop_get_lsa_ldst(dest, src, bytes, pe, tid, nthreads); + } else { + niin_coop_get_generic_body(dest, src, bytes, pe, tid, nthreads, -1); + } + niin_threadgroup_sync(); } // =========================================================================== @@ -127,8 +347,8 @@ __device__ __forceinline__ void niin_coop_get( #define NIIN_DEFINE_PUT_WARP(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmemx_##TYPENAME##_put_warp( \ TYPE* dest, const TYPE* src, size_t nelems, int pe) { \ - niin_coop_put(dest, src, nelems * sizeof(TYPE), pe, \ - nccl::utility::lane(), 32); \ + niin_coop_put(dest, src, nelems * sizeof(TYPE), pe, \ + nccl::utility::lane(), 32); \ __syncwarp(); \ } @@ -138,8 +358,8 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_PUT_WARP) #define NIIN_DEFINE_PUT_BLOCK(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmemx_##TYPENAME##_put_block( \ TYPE* dest, const TYPE* src, size_t nelems, int pe) { \ - niin_coop_put(dest, src, nelems * sizeof(TYPE), pe, \ - threadIdx.x, blockDim.x); \ + niin_coop_put(dest, src, nelems * sizeof(TYPE), pe, \ + threadIdx.x, blockDim.x); \ __syncthreads(); \ } @@ -152,8 +372,8 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_PUT_BLOCK) #define NIIN_DEFINE_GET_WARP(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmemx_##TYPENAME##_get_warp( \ TYPE* dest, const TYPE* src, size_t nelems, int pe) { \ - niin_coop_get(dest, src, nelems * sizeof(TYPE), pe, \ - nccl::utility::lane(), 32); \ + niin_coop_get(dest, src, nelems * sizeof(TYPE), pe, \ + nccl::utility::lane(), 32); \ __syncwarp(); \ } @@ -163,8 +383,8 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_GET_WARP) #define NIIN_DEFINE_GET_BLOCK(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmemx_##TYPENAME##_get_block( \ TYPE* dest, const TYPE* src, size_t nelems, int pe) { \ - niin_coop_get(dest, src, nelems * sizeof(TYPE), pe, \ - threadIdx.x, blockDim.x); \ + niin_coop_get(dest, src, nelems * sizeof(TYPE), pe, \ + threadIdx.x, blockDim.x); \ __syncthreads(); \ } @@ -177,8 +397,8 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_GET_BLOCK) #define NIIN_DEFINE_PUT_SIZED_WARP(SIZE, NBYTES) \ __device__ __forceinline__ void nvshmemx_put##SIZE##_warp( \ void* dest, const void* src, size_t nelems, int pe) { \ - niin_coop_put(dest, src, nelems * NBYTES, pe, \ - nccl::utility::lane(), 32); \ + niin_coop_put(dest, src, nelems * NBYTES, pe, \ + nccl::utility::lane(), 32); \ __syncwarp(); \ } @@ -188,8 +408,8 @@ NIIN_SIZED_RMA(NIIN_DEFINE_PUT_SIZED_WARP) #define NIIN_DEFINE_PUT_SIZED_BLOCK(SIZE, NBYTES) \ __device__ __forceinline__ void nvshmemx_put##SIZE##_block( \ void* dest, const void* src, size_t nelems, int pe) { \ - niin_coop_put(dest, src, nelems * NBYTES, pe, \ - threadIdx.x, blockDim.x); \ + niin_coop_put(dest, src, nelems * NBYTES, pe, \ + threadIdx.x, blockDim.x); \ __syncthreads(); \ } @@ -199,8 +419,8 @@ NIIN_SIZED_RMA(NIIN_DEFINE_PUT_SIZED_BLOCK) #define NIIN_DEFINE_GET_SIZED_WARP(SIZE, NBYTES) \ __device__ __forceinline__ void nvshmemx_get##SIZE##_warp( \ void* dest, const void* src, size_t nelems, int pe) { \ - niin_coop_get(dest, src, nelems * NBYTES, pe, \ - nccl::utility::lane(), 32); \ + niin_coop_get(dest, src, nelems * NBYTES, pe, \ + nccl::utility::lane(), 32); \ __syncwarp(); \ } @@ -210,8 +430,8 @@ NIIN_SIZED_RMA(NIIN_DEFINE_GET_SIZED_WARP) #define NIIN_DEFINE_GET_SIZED_BLOCK(SIZE, NBYTES) \ __device__ __forceinline__ void nvshmemx_get##SIZE##_block( \ void* dest, const void* src, size_t nelems, int pe) { \ - niin_coop_get(dest, src, nelems * NBYTES, pe, \ - threadIdx.x, blockDim.x); \ + niin_coop_get(dest, src, nelems * NBYTES, pe, \ + threadIdx.x, blockDim.x); \ __syncthreads(); \ } @@ -222,19 +442,19 @@ NIIN_SIZED_RMA(NIIN_DEFINE_GET_SIZED_BLOCK) // putmem/getmem warp/block // =========================================================================== __device__ __forceinline__ void nvshmemx_putmem_warp(void* dest, const void* src, size_t bytes, int pe) { - niin_coop_put(dest, src, bytes, pe, nccl::utility::lane(), 32); + niin_coop_put(dest, src, bytes, pe, nccl::utility::lane(), 32); __syncwarp(); } __device__ __forceinline__ void nvshmemx_putmem_block(void* dest, const void* src, size_t bytes, int pe) { - niin_coop_put(dest, src, bytes, pe, threadIdx.x, blockDim.x); + niin_coop_put(dest, src, bytes, pe, threadIdx.x, blockDim.x); __syncthreads(); } __device__ __forceinline__ void nvshmemx_getmem_warp(void* dest, const void* src, size_t bytes, int pe) { - niin_coop_get(dest, src, bytes, pe, nccl::utility::lane(), 32); + niin_coop_get(dest, src, bytes, pe, nccl::utility::lane(), 32); __syncwarp(); } __device__ __forceinline__ void nvshmemx_getmem_block(void* dest, const void* src, size_t bytes, int pe) { - niin_coop_get(dest, src, bytes, pe, threadIdx.x, blockDim.x); + niin_coop_get(dest, src, bytes, pe, threadIdx.x, blockDim.x); __syncthreads(); } @@ -244,7 +464,8 @@ __device__ __forceinline__ void nvshmemx_getmem_block(void* dest, const void* sr #define NIIN_DEFINE_PUT_NBI_WARP(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmemx_##TYPENAME##_put_nbi_warp( \ TYPE* dest, const TYPE* src, size_t nelems, int pe) { \ - nvshmemx_##TYPENAME##_put_warp(dest, src, nelems, pe); \ + niin_coop_put_nbi(dest, src, nelems * sizeof(TYPE), pe, \ + nccl::utility::lane(), 32); \ } NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_PUT_NBI_WARP) @@ -253,7 +474,8 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_PUT_NBI_WARP) #define NIIN_DEFINE_PUT_NBI_BLOCK(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmemx_##TYPENAME##_put_nbi_block( \ TYPE* dest, const TYPE* src, size_t nelems, int pe) { \ - nvshmemx_##TYPENAME##_put_block(dest, src, nelems, pe); \ + niin_coop_put_nbi(dest, src, nelems * sizeof(TYPE), pe, \ + threadIdx.x, blockDim.x); \ } NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_PUT_NBI_BLOCK) @@ -262,7 +484,8 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_PUT_NBI_BLOCK) #define NIIN_DEFINE_GET_NBI_WARP(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmemx_##TYPENAME##_get_nbi_warp( \ TYPE* dest, const TYPE* src, size_t nelems, int pe) { \ - nvshmemx_##TYPENAME##_get_warp(dest, src, nelems, pe); \ + niin_coop_get_nbi(dest, src, nelems * sizeof(TYPE), pe, \ + nccl::utility::lane(), 32); \ } NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_GET_NBI_WARP) @@ -271,7 +494,8 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_GET_NBI_WARP) #define NIIN_DEFINE_GET_NBI_BLOCK(TYPENAME, TYPE) \ __device__ __forceinline__ void nvshmemx_##TYPENAME##_get_nbi_block( \ TYPE* dest, const TYPE* src, size_t nelems, int pe) { \ - nvshmemx_##TYPENAME##_get_block(dest, src, nelems, pe); \ + niin_coop_get_nbi(dest, src, nelems * sizeof(TYPE), pe, \ + threadIdx.x, blockDim.x); \ } NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_GET_NBI_BLOCK) @@ -279,23 +503,24 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_GET_NBI_BLOCK) // NBI putmem/getmem warp/block __device__ __forceinline__ void nvshmemx_putmem_nbi_warp(void* dest, const void* src, size_t bytes, int pe) { - nvshmemx_putmem_warp(dest, src, bytes, pe); + niin_coop_put_nbi(dest, src, bytes, pe, nccl::utility::lane(), 32); } __device__ __forceinline__ void nvshmemx_putmem_nbi_block(void* dest, const void* src, size_t bytes, int pe) { - nvshmemx_putmem_block(dest, src, bytes, pe); + niin_coop_put_nbi(dest, src, bytes, pe, threadIdx.x, blockDim.x); } __device__ __forceinline__ void nvshmemx_getmem_nbi_warp(void* dest, const void* src, size_t bytes, int pe) { - nvshmemx_getmem_warp(dest, src, bytes, pe); + niin_coop_get_nbi(dest, src, bytes, pe, nccl::utility::lane(), 32); } __device__ __forceinline__ void nvshmemx_getmem_nbi_block(void* dest, const void* src, size_t bytes, int pe) { - nvshmemx_getmem_block(dest, src, bytes, pe); + niin_coop_get_nbi(dest, src, bytes, pe, threadIdx.x, blockDim.x); } // NBI sized put/get warp/block #define NIIN_DEFINE_PUT_SIZED_NBI_WARP(SIZE, NBYTES) \ __device__ __forceinline__ void nvshmemx_put##SIZE##_nbi_warp( \ void* dest, const void* src, size_t nelems, int pe) { \ - nvshmemx_put##SIZE##_warp(dest, src, nelems, pe); \ + niin_coop_put_nbi(dest, src, nelems * NBYTES, pe, \ + nccl::utility::lane(), 32); \ } NIIN_SIZED_RMA(NIIN_DEFINE_PUT_SIZED_NBI_WARP) @@ -304,7 +529,8 @@ NIIN_SIZED_RMA(NIIN_DEFINE_PUT_SIZED_NBI_WARP) #define NIIN_DEFINE_PUT_SIZED_NBI_BLOCK(SIZE, NBYTES) \ __device__ __forceinline__ void nvshmemx_put##SIZE##_nbi_block( \ void* dest, const void* src, size_t nelems, int pe) { \ - nvshmemx_put##SIZE##_block(dest, src, nelems, pe); \ + niin_coop_put_nbi(dest, src, nelems * NBYTES, pe, \ + threadIdx.x, blockDim.x); \ } NIIN_SIZED_RMA(NIIN_DEFINE_PUT_SIZED_NBI_BLOCK) @@ -313,7 +539,8 @@ NIIN_SIZED_RMA(NIIN_DEFINE_PUT_SIZED_NBI_BLOCK) #define NIIN_DEFINE_GET_SIZED_NBI_WARP(SIZE, NBYTES) \ __device__ __forceinline__ void nvshmemx_get##SIZE##_nbi_warp( \ void* dest, const void* src, size_t nelems, int pe) { \ - nvshmemx_get##SIZE##_warp(dest, src, nelems, pe); \ + niin_coop_get_nbi(dest, src, nelems * NBYTES, pe, \ + nccl::utility::lane(), 32); \ } NIIN_SIZED_RMA(NIIN_DEFINE_GET_SIZED_NBI_WARP) @@ -322,7 +549,8 @@ NIIN_SIZED_RMA(NIIN_DEFINE_GET_SIZED_NBI_WARP) #define NIIN_DEFINE_GET_SIZED_NBI_BLOCK(SIZE, NBYTES) \ __device__ __forceinline__ void nvshmemx_get##SIZE##_nbi_block( \ void* dest, const void* src, size_t nelems, int pe) { \ - nvshmemx_get##SIZE##_block(dest, src, nelems, pe); \ + niin_coop_get_nbi(dest, src, nelems * NBYTES, pe, \ + threadIdx.x, blockDim.x); \ } NIIN_SIZED_RMA(NIIN_DEFINE_GET_SIZED_NBI_BLOCK) @@ -384,14 +612,19 @@ __device__ __forceinline__ void nvshmemx_##TYPENAME##_put_signal_warp( \ uint64_t* sig, uint64_t signal, int sig_op, int pe) { \ /* Re-converge the warp before reading src: callers often have only lane 0 */ \ /* wait on the inbound signal before forwarding the buffer. */ \ + const bool lsa = niin_world_is_lsa_only() || niin_is_lsa_peer(pe); \ + int ctx = -1; \ + if (nccl::utility::lane() == 0 && !lsa) ctx = niin_gin_next_context(); \ __syncwarp(); \ - cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); \ - niin_coop_put(dest, src, nelems * sizeof(TYPE), pe, \ - nccl::utility::lane(), 32); \ - if (nccl::utility::lane() == 0 && !niin_is_lsa_peer(pe)) niin_gin_flush_thread(); \ - cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); \ + if (!lsa) cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); \ + niin_coop_put(dest, src, nelems * sizeof(TYPE), pe, \ + nccl::utility::lane(), 32, ctx, false); \ + if (ctx >= 0) niin_gin_flush_thread(ctx); \ + if (!lsa) cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); \ __syncwarp(); \ - if (nccl::utility::lane() == 0) niin_deliver_signal(sig, signal, sig_op, pe); \ + if (nccl::utility::lane() == 0) { \ + niin_threadgroup_deliver_signal(lsa, sig, signal, sig_op, pe, ctx); \ + } \ __syncwarp(); \ } @@ -404,14 +637,19 @@ __device__ __forceinline__ void nvshmemx_##TYPENAME##_put_signal_block( \ uint64_t* sig, uint64_t signal, int sig_op, int pe) { \ /* Re-converge the CTA before reading src: callers often have only thread 0 */ \ /* wait on the inbound signal before forwarding the buffer. */ \ + const bool lsa = niin_world_is_lsa_only() || niin_is_lsa_peer(pe); \ + int ctx = -1; \ + if (threadIdx.x == 0 && !lsa) ctx = niin_gin_next_context(); \ __syncthreads(); \ - cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); \ - niin_coop_put(dest, src, nelems * sizeof(TYPE), pe, \ - threadIdx.x, blockDim.x); \ - if (threadIdx.x == 0 && !niin_is_lsa_peer(pe)) niin_gin_flush_thread(); \ - cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); \ + if (!lsa) cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); \ + niin_coop_put(dest, src, nelems * sizeof(TYPE), pe, \ + threadIdx.x, blockDim.x, ctx, false); \ + if (ctx >= 0) niin_gin_flush_thread(ctx); \ + if (!lsa) cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); \ __syncthreads(); \ - if (threadIdx.x == 0) niin_deliver_signal(sig, signal, sig_op, pe); \ + if (threadIdx.x == 0) { \ + niin_threadgroup_deliver_signal(lsa, sig, signal, sig_op, pe, ctx); \ + } \ __syncthreads(); \ } @@ -422,25 +660,35 @@ NIIN_STANDARD_RMA_TYPES(NIIN_DEFINE_PUT_SIGNAL_BLOCK) __device__ __forceinline__ void nvshmemx_putmem_signal_warp( void* dest, const void* src, size_t bytes, uint64_t* sig, uint64_t signal, int sig_op, int pe) { + const bool lsa = niin_world_is_lsa_only() || niin_is_lsa_peer(pe); + int ctx = -1; + if (nccl::utility::lane() == 0 && !lsa) ctx = niin_gin_next_context(); __syncwarp(); - cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); - niin_coop_put(dest, src, bytes, pe, nccl::utility::lane(), 32); - if (nccl::utility::lane() == 0 && !niin_is_lsa_peer(pe)) niin_gin_flush_thread(); - cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); + if (!lsa) cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); + niin_coop_put(dest, src, bytes, pe, nccl::utility::lane(), 32, ctx, false); + if (ctx >= 0) niin_gin_flush_thread(ctx); + if (!lsa) cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); __syncwarp(); - if (nccl::utility::lane() == 0) niin_deliver_signal(sig, signal, sig_op, pe); + if (nccl::utility::lane() == 0) { + niin_threadgroup_deliver_signal(lsa, sig, signal, sig_op, pe, ctx); + } __syncwarp(); } __device__ __forceinline__ void nvshmemx_putmem_signal_block( void* dest, const void* src, size_t bytes, uint64_t* sig, uint64_t signal, int sig_op, int pe) { + const bool lsa = niin_world_is_lsa_only() || niin_is_lsa_peer(pe); + int ctx = -1; + if (threadIdx.x == 0 && !lsa) ctx = niin_gin_next_context(); __syncthreads(); - cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); - niin_coop_put(dest, src, bytes, pe, threadIdx.x, blockDim.x); - if (threadIdx.x == 0 && !niin_is_lsa_peer(pe)) niin_gin_flush_thread(); - cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); + if (!lsa) cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); + niin_coop_put(dest, src, bytes, pe, threadIdx.x, blockDim.x, ctx, false); + if (ctx >= 0) niin_gin_flush_thread(ctx); + if (!lsa) cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); __syncthreads(); - if (threadIdx.x == 0) niin_deliver_signal(sig, signal, sig_op, pe); + if (threadIdx.x == 0) { + niin_threadgroup_deliver_signal(lsa, sig, signal, sig_op, pe, ctx); + } __syncthreads(); } diff --git a/projects/rccl/contrib/niin/include/niin/tma.h b/projects/rccl/contrib/niin/include/niin/tma.h new file mode 100644 index 000000000000..5811daf94432 --- /dev/null +++ b/projects/rccl/contrib/niin/include/niin/tma.h @@ -0,0 +1,701 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +// TMA (Tensor Memory Accelerator) backed RMA for NIIN. +// +// Mirrors NVSHMEM's TMA feature: an application hands a slice of its CTA shared +// memory to the runtime with nvshmemx_give_smem(), and NIIN's LSA (NVLink) +// put/get paths then move data with cp.async.bulk instead of vectorized +// load/store. Three routes exist, picked from where the local buffer lives: +// +// smem -> peer gmem direct cp.async.bulk, no staging (puts) +// peer gmem -> smem direct cp.async.bulk with an mbarrier (gets) +// gmem <-> peer gmem staged through the registered shared-memory tile +// +// Every route is opportunistic: alignment, size, registration, and CTA-shape +// constraints that TMA cannot meet make the helper return -1, and the caller +// falls back to the regular load/store path. That keeps the put/get contract +// unchanged regardless of whether TMA is available. +// +// TMA requires sm_90 or newer. On older architectures every helper here +// compiles to a no-op that returns -1. + +#ifndef NIIN_TMA_H_ +#define NIIN_TMA_H_ + +#include "niin/context.h" + +// Threadgroup scope for the TMA helpers. Mirrors NVSHMEM's threadgroup_t for +// the scopes NIIN's device API exposes. +enum niinTmaScope { + NIIN_TMA_THREAD = 0, + NIIN_TMA_WARP = 1, + NIIN_TMA_BLOCK = 2 +}; + +__host__ __device__ __forceinline__ constexpr bool niin_tma_is_16b_aligned(size_t value) { + return (value & (size_t)0xF) == 0; +} + +__host__ __device__ __forceinline__ constexpr size_t niin_tma_align_down_16(size_t value) { + return value & ~(size_t)0xF; +} + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + +// --------------------------------------------------------------------------- +// PTX primitives +// --------------------------------------------------------------------------- + +// Elect one leader among the active threads of the calling warp. Returns true +// for exactly one thread. Callers must be warp-converged. +__device__ __forceinline__ bool niin_tma_elect_warp() { + uint32_t isLeader; + asm volatile(R"({ +.reg .pred elect_p; +.reg .u32 elect_id; +elect.sync elect_id|elect_p, 0xffffffff; +selp.u32 %0, 1, 0, elect_p; +})" + : "=r"(isLeader)); + return isLeader != 0; +} + +// Elect exactly one thread across an entire CTA. Call from all threads. +// +// The __shfl_sync broadcast makes the warp id a compiler-visible warp-uniform +// value, which keeps the compiler from peeling the elect over active threads. +// A plain `threadIdx.x == 0` test does not give the compiler that information. +__device__ __forceinline__ bool niin_tma_block_is_elected() { + unsigned int tid = threadIdx.x + threadIdx.y * blockDim.x + + threadIdx.z * blockDim.x * blockDim.y; + unsigned int warpId = tid / warpSize; + unsigned int uniformWarpId = __shfl_sync(0xffffffff, warpId, 0); + return (uniformWarpId == 0) && niin_tma_elect_warp(); +} + +// Convert a generic pointer into a 32-bit shared-memory-space address. +__device__ __forceinline__ unsigned int niin_tma_cvta_to_shared(const void* ptr) { + unsigned int smemAddr; + asm(R"({ +.reg .u64 smem_u64; +cvta.to.shared.u64 smem_u64, %1; +cvt.u32.u64 %0, smem_u64; +})" + : "=r"(smemAddr) + : "l"((uint64_t)(uintptr_t)ptr)); + return smemAddr; +} + +__device__ __forceinline__ void niin_tma_bulk_shared_to_global( + void* gmemDst, unsigned int smemAddr, uint32_t bytes) { + asm volatile("cp.async.bulk.global.shared::cta.bulk_group [%0], [%1], %2;" + : + : "l"((uint64_t)(uintptr_t)gmemDst), "r"(smemAddr), "r"(bytes) + : "memory"); +} + +__device__ __forceinline__ void niin_tma_bulk_global_to_shared( + void* smemDst, const void* gmemSrc, uint32_t bytes, uint64_t* mbar) { + unsigned int dstAddr = niin_tma_cvta_to_shared(smemDst); + unsigned int mbarAddr = niin_tma_cvta_to_shared(mbar); + asm volatile( + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes " + "[%0], [%1], %2, [%3];" + : + : "r"(dstAddr), "l"((uint64_t)(uintptr_t)gmemSrc), "r"(bytes), "r"(mbarAddr) + : "memory"); +} + +__device__ __forceinline__ void niin_tma_bulk_commit_group() { + asm volatile("cp.async.bulk.commit_group;" ::: "memory"); +} + +// Wait until the bulk group has finished reading shared memory. The remote +// write may still be in flight; this only makes the source tile reusable. +__device__ __forceinline__ void niin_tma_bulk_wait_group_read_0() { + asm volatile("cp.async.bulk.wait_group.read 0;" ::: "memory"); +} + +// Wait for full completion: shared-memory read and global write both done. +__device__ __forceinline__ void niin_tma_bulk_wait_group_0() { + asm volatile("cp.async.bulk.wait_group 0;" ::: "memory"); +} + +// Order generic shared-memory stores against reads by the async proxy. +__device__ __forceinline__ void niin_tma_fence_proxy_async_shared_cta() { + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); +} + +// mbarrier helpers for pairing with an inbound (gmem -> smem) cp.async.bulk. +// The barrier is an 8-byte shared-memory object with an arrive count of 1, so +// these are safe as long as the calling thread is the only arriver. +__device__ __forceinline__ void niin_tma_mbarrier_init(uint64_t* mbar) { + unsigned int addr = niin_tma_cvta_to_shared(mbar); + asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;" ::"r"(addr)); +} + +__device__ __forceinline__ void niin_tma_mbarrier_arrive_expect_tx(uint64_t* mbar, + uint32_t bytes) { + unsigned int addr = niin_tma_cvta_to_shared(mbar); + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(addr), + "r"(bytes)); +} + +__device__ __forceinline__ void niin_tma_mbarrier_try_wait(uint64_t* mbar, int phase) { + unsigned int addr = niin_tma_cvta_to_shared(mbar); + asm volatile(R"({ +.reg .pred p; +niinWaitL_%=: +mbarrier.try_wait.parity.shared::cta.b64 p, [%0], %1; +@!p bra niinWaitL_%=; +})" ::"r"(addr), + "r"(phase) + : "memory"); +} + +// Manually add to a barrier's transaction counter, for barriers that are not +// released by a cp.async.bulk completion. +__device__ __forceinline__ void niin_tma_mbarrier_complete_tx(uint64_t* mbar, + uint32_t txCount) { + unsigned int addr = niin_tma_cvta_to_shared(mbar); + asm volatile("mbarrier.complete_tx.relaxed.cta.shared::cta.b64 [%0], %1;" ::"r"(addr), + "r"(txCount) + : "memory"); +} + +#else // pre-Hopper, or the host compilation pass + +// No-op stubs for the primitives an application may call around an +// smem-sourced put, so that code compiles unchanged on architectures where TMA +// is unavailable and every RMA takes the load/store path. +__device__ __forceinline__ void niin_tma_bulk_commit_group() {} +__device__ __forceinline__ void niin_tma_bulk_wait_group_read_0() {} +__device__ __forceinline__ void niin_tma_bulk_wait_group_0() {} +__device__ __forceinline__ void niin_tma_fence_proxy_async_shared_cta() {} + +#endif // __CUDA_ARCH__ >= 900 + +// --------------------------------------------------------------------------- +// Per-CTA shared-memory registration +// --------------------------------------------------------------------------- + +// These three stay outside any __CUDA_ARCH__ wrapper. nvcc parses __global__ +// and __device__ bodies during the host pass too, so a helper that only exists +// when __CUDA_ARCH__ is defined cannot be named from device code at all. Guard +// the body, never the declaration. + +// Linear CTA id within the grid, used to index the registration table. +__device__ __forceinline__ size_t niin_tma_block_id() { +#ifdef __CUDA_ARCH__ + return (size_t)blockIdx.x + (size_t)blockIdx.y * gridDim.x + + (size_t)blockIdx.z * gridDim.x * gridDim.y; +#else + return 0; +#endif +} + +// True when this CTA has registered shared memory and TMA is not disabled. +// Gates the TMA dispatch in put, get, fence, and quiet. +__device__ __forceinline__ bool niin_tma_smem_registered() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (niin_tma_policy() == NVSHMEMX_TMA_DISABLE) return false; + niinContext const& ctx = niin_ctx(); + uintptr_t* bases = ctx.tmaSmemBases; + size_t blockId = niin_tma_block_id(); + return bases != nullptr && blockId < ctx.tmaSmemBasesLen && bases[blockId] != 0; +#else + return false; +#endif +} + +// Shared-memory size every CTA passed to give_smem (the API requires it to be +// uniform across the grid, so a single scalar covers the whole table). +__device__ __forceinline__ size_t niin_tma_smem_size() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + size_t* p = niin_ctx().tmaSmemSize; + return p != nullptr ? *p : 0; +#else + return 0; +#endif +} + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + +// Barrier slots live in the static carve at the base of the registered buffer: +// NIIN_TMA_NUM_BARRIER_SLOTS slots of 16 bytes, each holding one 8-byte +// mbarrier plus padding so the slot stride matches cp.async.bulk alignment. +// +// Slot assignment (must not collide between concurrently running TMA paths): +// 0 single-issuer staging mbarrier (thread and warp scope) +// 0, 1 block staging ready_bar[0], ready_bar[1] +// 2, 3 block staging done_bar[0], done_bar[1] +// 4 direct gmem -> smem get mbarrier +// 5..31 reserved +__device__ __forceinline__ uint64_t* niin_tma_barrier_slot(uintptr_t smemBase, int slot) { + return reinterpret_cast(smemBase + (uintptr_t)slot * 16); +} + +__device__ __forceinline__ char* niin_tma_data_buffer(uintptr_t smemBase) { + return reinterpret_cast(smemBase + (uintptr_t)NIIN_SMEM_DATA_REGION_OFFSET); +} + +// Elect the single thread that issues a transfer for the given scope. Thread +// scope has no election: every caller issues its own transfer. +template +__device__ __forceinline__ bool niin_tma_is_leader() { + if constexpr (SCOPE == NIIN_TMA_THREAD) return true; + else if constexpr (SCOPE == NIIN_TMA_WARP) return niin_tma_elect_warp(); + else return niin_tma_block_is_elected(); +} + +// Re-converge the threadgroup after the leader has issued its transfer. +template +__device__ __forceinline__ void niin_tma_group_sync() { + if constexpr (SCOPE == NIIN_TMA_BLOCK) __syncthreads(); + else if constexpr (SCOPE == NIIN_TMA_WARP) __syncwarp(); +} + +#endif // __CUDA_ARCH__ >= 900 + +// --------------------------------------------------------------------------- +// nvshmemx_give_smem / nvshmemx_release_smem +// --------------------------------------------------------------------------- + +// nvshmemx_give_smem: lend a block of shared memory to NIIN for TMA transfers. +// +// Call from every thread of every CTA in the grid, once per kernel launch, +// before any TMA-backed put or get, and follow it with __syncthreads(). All +// CTAs must pass the same size. A CTA that skips the call falls back to +// load/store for every RMA it issues in that kernel. +// +// The __syncthreads() is required, not advisory: only one elected thread +// publishes the registration, and a warp- or block-scoped RMA needs every +// thread in the group to agree on whether TMA is in play. +// +// Every CTA that calls give_smem must call nvshmemx_release_smem() before the +// kernel returns. Without that, the registration outlives the launch and a +// later kernel whose CTAs reuse the same block ids would take the TMA path +// through a stale shared-memory pointer. +// +// TMA routing additionally requires 16-byte aligned source and destination +// pointers and a transfer size that is a multiple of 16 bytes. Transfers that +// miss those constraints keep the normal put/get contract by falling back to +// load/store. Block-scoped puts staged from a global-memory source need at +// least two full warps in the CTA; smaller CTAs fall back as well. +// +// Grids larger than NIIN_TMA_MAX_BLOCKS CTAs are supported, but CTAs with a +// linear block id at or beyond that bound cannot register. +// +// smem: shared-memory pointer, 16-byte aligned +// size: bytes, at least nvshmemx_ask_smem(NVSHMEMX_SMEM_BARRIERS_ONLY) +__device__ __forceinline__ void nvshmemx_give_smem(void* smem, size_t size) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (niin_tma_policy() == NVSHMEMX_TMA_DISABLE) return; + if (smem == nullptr || size == 0) return; + // A buffer that is not 16-byte aligned cannot host the mbarriers or serve as + // a cp.async.bulk tile. Decline the registration rather than corrupt data; + // this CTA then uses load/store. + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)smem)) return; + + niinContext const& ctx = niin_ctx(); + uintptr_t* bases = ctx.tmaSmemBases; + size_t blockId = niin_tma_block_id(); + if (bases == nullptr || blockId >= ctx.tmaSmemBasesLen) { + // Grid is larger than the registration table; this CTA cannot use TMA. + return; + } + // The leading bytes are reserved for mbarriers. A buffer too small to hold + // them cannot be registered, so this CTA falls back to load/store. + if (size < (size_t)NIIN_SMEM_DATA_REGION_OFFSET) return; + + if (niin_tma_block_is_elected()) { + bases[blockId] = (uintptr_t)smem; + size_t* sizeSlot = ctx.tmaSmemSize; + if (sizeSlot != nullptr) *sizeSlot = size; + } +#else + (void)smem; (void)size; +#endif +} + +// nvshmemx_release_smem: drop this CTA's shared-memory registration. +// +// Call from every CTA that called nvshmemx_give_smem(), before the kernel +// returns. Call from all threads; only the elected leader writes. Follow with +// __syncthreads() if other threads must observe the cleared state. +__device__ __forceinline__ void nvshmemx_release_smem() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (niin_tma_policy() == NVSHMEMX_TMA_DISABLE) return; + niinContext const& ctx = niin_ctx(); + uintptr_t* bases = ctx.tmaSmemBases; + size_t blockId = niin_tma_block_id(); + if (bases != nullptr && blockId < ctx.tmaSmemBasesLen) { + if (niin_tma_block_is_elected()) bases[blockId] = 0; + } +#endif +} + +// --------------------------------------------------------------------------- +// Transfer implementations +// --------------------------------------------------------------------------- + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + +// Local shared memory -> local or peer global memory, no staging. +// +// The caller must have issued fence.proxy.async.shared::cta after writing the +// source tile and before calling this, so the async proxy sees those stores. +// One fence covers a run of puts as long as the source is not rewritten in +// between. This matches NVSHMEM's contract for smem-sourced TMA puts. +// +// BLOCKING waits for the remote write to complete before returning; otherwise +// the transfer is left in flight for a later fence/quiet to drain. +// +// Returns 0 when the transfer was issued, -1 to fall back. +template +__device__ __forceinline__ int niin_tma_copy_shared_global(void* gmemDst, const void* smemSrc, + size_t bytes) { + if (bytes == 0) return 0; + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)gmemDst)) return -1; + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)smemSrc)) return -1; + if (!niin_tma_is_16b_aligned(bytes)) return -1; + if (bytes > (size_t)UINT32_MAX) return -1; + + unsigned int smemAddr = niin_tma_cvta_to_shared(smemSrc); + + // One elected thread issues the whole transfer as a single bulk op: that + // amortizes TMA setup better than splitting the buffer across warps. + if (niin_tma_is_leader()) { + niin_tma_bulk_shared_to_global(gmemDst, smemAddr, (uint32_t)bytes); + niin_tma_bulk_commit_group(); + if constexpr (BLOCKING) { + niin_tma_bulk_wait_group_0(); + __threadfence_system(); + } + } + + niin_tma_group_sync(); + return 0; +} + +// Local or peer global memory -> local shared memory, no staging. Used by gets +// whose destination is shared memory. Always blocking: the mbarrier wait means +// the data has landed when this returns. +// +// Returns 0 when the transfer completed, -1 to fall back. +template +__device__ __forceinline__ int niin_tma_copy_global_shared(void* smemDst, const void* gmemSrc, + size_t bytes) { + if (bytes == 0) return 0; + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)smemDst)) return -1; + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)gmemSrc)) return -1; + if (!niin_tma_is_16b_aligned(bytes)) return -1; + if (bytes > (size_t)UINT32_MAX) return -1; + + uintptr_t base = niin_ctx().tmaSmemBases[niin_tma_block_id()]; + size_t smemSize = niin_tma_smem_size(); + constexpr size_t kReserve = (size_t)NIIN_SMEM_DATA_REGION_OFFSET; + if (base == 0 || smemSize <= kReserve) return -1; + + // Refuse a destination that would land on the reserved mbarrier region. + uintptr_t dstStart = (uintptr_t)smemDst; + uintptr_t dstEnd = dstStart + bytes; + if (dstStart < base + kReserve && dstEnd > base) return -1; + + uint64_t* mbar = niin_tma_barrier_slot(base, 4); + if (niin_tma_is_leader()) { + niin_tma_mbarrier_init(mbar); + niin_tma_fence_proxy_async_shared_cta(); + niin_tma_mbarrier_arrive_expect_tx(mbar, (uint32_t)bytes); + niin_tma_bulk_global_to_shared(smemDst, gmemSrc, (uint32_t)bytes, mbar); + niin_tma_mbarrier_try_wait(mbar, 0); + } + + niin_tma_group_sync(); + return 0; +} + +// Single-issuer global -> global copy, staged through this CTA's registered +// tile. Used for thread and warp scope. +// +// The issuer pulls each chunk into the tile with an inbound cp.async.bulk that +// releases an mbarrier, then pushes the same tile out with an outbound +// cp.async.bulk. Before reusing the tile it waits for the outbound op to finish +// reading shared memory. Blocking calls also wait for the remote write. +// +// Because the staging tile and mbarrier are per-CTA state, at most one thread +// in a CTA may have a staged transfer in flight at a time. Two threads of the +// same CTA calling a thread-scoped put concurrently would corrupt each other's +// staging. NIIN inherits that constraint from NVSHMEM. +// +// Returns 0 when the transfer was issued, -1 to fall back. +template +__device__ __forceinline__ int niin_tma_copy_global_global_single(void* gmemDst, + const void* gmemSrc, + size_t bytes) { + static_assert(SCOPE == NIIN_TMA_THREAD || SCOPE == NIIN_TMA_WARP, + "single-issuer staging is only for thread or warp scope"); + if (bytes == 0) return 0; + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)gmemDst)) return -1; + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)gmemSrc)) return -1; + if (!niin_tma_is_16b_aligned(bytes)) return -1; + + uintptr_t base = niin_ctx().tmaSmemBases[niin_tma_block_id()]; + size_t smemSize = niin_tma_smem_size(); + constexpr size_t kReserve = (size_t)NIIN_SMEM_DATA_REGION_OFFSET; + if (base == 0 || smemSize <= kReserve) return -1; + + size_t tileSize = niin_tma_align_down_16(smemSize - kReserve); + if (tileSize == 0) return -1; + // CTA shared memory is far below 4 GiB, so bulk byte counts fit in 32 bits. + const uint32_t tile = (uint32_t)tileSize; + + if (niin_tma_is_leader()) { + uint64_t* mbar = niin_tma_barrier_slot(base, 0); + char* dataBuf = niin_tma_data_buffer(base); + const char* src = (const char*)gmemSrc; + char* dst = (char*)gmemDst; + size_t remaining = bytes; + + niin_tma_mbarrier_init(mbar); + // Make the mbarrier init visible to the async proxy before the first bulk op. + niin_tma_fence_proxy_async_shared_cta(); + int phase = 0; + + while (remaining > 0) { + uint32_t chunk = remaining < (size_t)tile ? (uint32_t)remaining : tile; + + niin_tma_mbarrier_arrive_expect_tx(mbar, chunk); + niin_tma_bulk_global_to_shared(dataBuf, src, chunk, mbar); + niin_tma_mbarrier_try_wait(mbar, phase); + phase ^= 1; + + // Inbound and outbound both run on the async proxy and the mbarrier + // release orders them, so no fence is needed in between. + niin_tma_bulk_shared_to_global(dst, niin_tma_cvta_to_shared(dataBuf), chunk); + niin_tma_bulk_commit_group(); + + remaining -= chunk; + src += chunk; + dst += chunk; + + if (remaining > 0) niin_tma_bulk_wait_group_read_0(); + } + if constexpr (BLOCKING) { + niin_tma_bulk_wait_group_0(); + __threadfence_system(); + } + } + + niin_tma_group_sync(); + return 0; +} + +// Block-scoped, warp-specialized, double-buffered global -> global copy. +// +// Warp 0 lane 0 loads, warp 1 lane 0 stores, so the inbound TMA of tile N+1 +// overlaps the outbound TMA of tile N. Needs at least two full warps. +// +// Shared layout: [reserved barriers][buf0: tile][buf1: tile], tile being +// (size - NIIN_SMEM_DATA_REGION_OFFSET) / 2 rounded down to 16 bytes. +// +// ready_bar[i] load warp signals "buf[i] filled" through the cp.async.bulk +// complete_tx; the store warp waits on it. +// done_bar[i] store warp signals "outbound read of buf[i] finished, safe to +// refill"; the load warp waits on it. +// +// The buffer alternates every iteration (slot = i & 1) and the barrier parity +// flips every full pipe cycle of two iterations (phase = (i >> 1) & 1), so the +// hot loop needs no __syncthreads(). +// +// Returns 0 when the transfer was issued, -1 to fall back. +template +__device__ __forceinline__ int niin_tma_copy_global_global_block(void* gmemDst, + const void* gmemSrc, + size_t bytes) { + if (bytes == 0) return 0; + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)gmemDst)) return -1; + if (!niin_tma_is_16b_aligned((size_t)(uintptr_t)gmemSrc)) return -1; + if (!niin_tma_is_16b_aligned(bytes)) return -1; + + uintptr_t base = niin_ctx().tmaSmemBases[niin_tma_block_id()]; + size_t smemSize = niin_tma_smem_size(); + constexpr size_t kReserve = (size_t)NIIN_SMEM_DATA_REGION_OFFSET; + if (base == 0 || smemSize <= kReserve) return -1; + + size_t tileSize = niin_tma_align_down_16((smemSize - kReserve) / 2); + if (tileSize == 0) return -1; + const uint32_t tile = (uint32_t)tileSize; + + unsigned int blockThreads = blockDim.x * blockDim.y * blockDim.z; + if (blockThreads < 2u * (unsigned int)warpSize) return -1; + + unsigned int tid = threadIdx.x + threadIdx.y * blockDim.x + + threadIdx.z * blockDim.x * blockDim.y; + // CUDA forms warps from the linear CTA rank, so these are warp 0 lane 0 and + // warp 1 lane 0 for any CTA shape with at least two full warps. + bool isLoad = (tid == 0); + bool isStore = (tid == (unsigned int)warpSize); + + uint64_t* ready0 = niin_tma_barrier_slot(base, 0); + uint64_t* ready1 = niin_tma_barrier_slot(base, 1); + uint64_t* done0 = niin_tma_barrier_slot(base, 2); + uint64_t* done1 = niin_tma_barrier_slot(base, 3); + char* buf0 = niin_tma_data_buffer(base); + char* buf1 = buf0 + tile; + + // Init all four barriers once, then fence so the async proxy sees the init + // before any cp.async.bulk arrives on them. + if (isLoad) { + niin_tma_mbarrier_init(ready0); + niin_tma_mbarrier_init(ready1); + niin_tma_mbarrier_init(done0); + niin_tma_mbarrier_init(done1); + niin_tma_fence_proxy_async_shared_cta(); + } + __syncthreads(); + + if (isLoad) { + const char* src = (const char*)gmemSrc; + size_t remaining = bytes; + for (size_t i = 0; remaining > 0; i++) { + int slot = (int)(i & 1); + int phase = (int)((i >> 1) & 1); + uint32_t chunk = remaining < (size_t)tile ? (uint32_t)remaining : tile; + uint64_t* ready = slot ? ready1 : ready0; + uint64_t* done = slot ? done1 : done0; + char* buf = slot ? buf1 : buf0; + + // Each slot's done barrier is fresh for the first two iterations, so the + // wait is skipped there; after that we wait for the store warp's flip. + if (i >= 2) niin_tma_mbarrier_try_wait(done, phase ^ 1); + niin_tma_bulk_global_to_shared(buf, src, chunk, ready); + niin_tma_mbarrier_arrive_expect_tx(ready, chunk); + + src += chunk; + remaining -= chunk; + } + } else if (isStore) { + unsigned int dataAddr0 = niin_tma_cvta_to_shared(buf0); + unsigned int dataAddr1 = niin_tma_cvta_to_shared(buf1); + char* dst = (char*)gmemDst; + size_t remaining = bytes; + for (size_t i = 0; remaining > 0; i++) { + int slot = (int)(i & 1); + int phase = (int)((i >> 1) & 1); + uint32_t chunk = remaining < (size_t)tile ? (uint32_t)remaining : tile; + uint64_t* ready = slot ? ready1 : ready0; + uint64_t* done = slot ? done1 : done0; + unsigned int dataAddr = slot ? dataAddr1 : dataAddr0; + + niin_tma_mbarrier_try_wait(ready, phase); + niin_tma_bulk_shared_to_global(dst, dataAddr, chunk); + niin_tma_bulk_commit_group(); + niin_tma_bulk_wait_group_read_0(); + niin_tma_mbarrier_arrive_expect_tx(done, 1); + niin_tma_mbarrier_complete_tx(done, 1); + + dst += chunk; + remaining -= chunk; + } + } + + if constexpr (BLOCKING) { + if (isStore) { + niin_tma_bulk_wait_group_0(); + __threadfence_system(); + } + } + __syncthreads(); + return 0; +} + +// Route a global -> global copy by scope. +template +__device__ __forceinline__ int niin_tma_copy_global_global(void* gmemDst, const void* gmemSrc, + size_t bytes) { + if constexpr (SCOPE == NIIN_TMA_BLOCK) { + return niin_tma_copy_global_global_block(gmemDst, gmemSrc, bytes); + } else { + return niin_tma_copy_global_global_single(gmemDst, gmemSrc, bytes); + } +} + +#endif // __CUDA_ARCH__ >= 900 + +// --------------------------------------------------------------------------- +// Dispatch used by the LSA put/get paths +// --------------------------------------------------------------------------- + +// Try to move `bytes` between two already-resolved pointers with TMA, choosing +// the route from where the local buffer lives. `dst` and `src` must both be +// directly addressable (local memory or an LSA peer pointer). +// +// BLOCKING mirrors NVSHMEM's split: a blocking put waits for the transfer to +// complete before returning, while an nbi put leaves it in flight for a later +// fence/quiet to drain. Gets are always completion-bound, so a get-shaped +// route ignores BLOCKING. +// +// Returns 0 when TMA handled the copy, -1 when the caller must fall back. All +// the reasons to fall back are uniform across a threadgroup, so a warp- or +// block-scoped caller either takes the TMA path with every thread or none. +template +NIIN_NOINLINE_DEVICE int niin_tma_try_copy_enabled(void* dst, const void* src, size_t bytes) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (!niin_tma_smem_registered()) return -1; + + // A local-memory (stack) buffer is a legal RMA source but is not addressable + // by the TMA engine, which only reads global and shared space. Decline it. + if (__isLocal(src) || __isLocal(dst)) return -1; + + bool srcShared = __isShared(src); + bool dstShared = __isShared(dst); + // A shared-to-shared copy is not an RMA route; leave it to load/store. + if (srcShared && dstShared) return -1; + if (srcShared) return niin_tma_copy_shared_global(dst, src, bytes); + // gmem -> smem is a get landing in shared memory: the mbarrier wait is the + // transfer, so there is no non-blocking form of it. + if (dstShared) return niin_tma_copy_global_shared(dst, src, bytes); + return niin_tma_copy_global_global(dst, src, bytes); +#else + (void)dst; (void)src; (void)bytes; + return -1; +#endif +} + +template +__device__ __forceinline__ int niin_tma_try_copy(void* dst, const void* src, size_t bytes) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (niin_tma_policy() == NVSHMEMX_TMA_DISABLE) return -1; + return niin_tma_try_copy_enabled(dst, src, bytes); +#else + (void)dst; (void)src; (void)bytes; + return -1; +#endif +} + +// Drain every bulk op this thread has issued. Used by fence and quiet so that +// TMA-initiated transfers are ordered and complete alongside the load/store and +// GIN paths. A no-op when this CTA has no registered shared memory. +NIIN_NOINLINE_DEVICE void niin_tma_drain_registered_slow() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (niin_tma_smem_registered()) { + niin_tma_bulk_commit_group(); + niin_tma_bulk_wait_group_0(); + } +#endif +} + +__device__ __forceinline__ void niin_tma_drain_if_registered() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (niin_tma_policy() == NVSHMEMX_TMA_DISABLE) return; + niin_tma_drain_registered_slow(); +#endif +} + +#endif // NIIN_TMA_H_ diff --git a/projects/rccl/contrib/niin/include/niin/types.h b/projects/rccl/contrib/niin/include/niin/types.h index e6b8b7f6e130..a702d6f309da 100644 --- a/projects/rccl/contrib/niin/include/niin/types.h +++ b/projects/rccl/contrib/niin/include/niin/types.h @@ -33,6 +33,76 @@ typedef int nvshmem_team_t; #define NVSHMEMI_TEAM_GPU_LEADERS 5 // One PE per GPU (=WORLD in NCCL) #define NVSHMEM_TEAM_INVALID (-1) +// NVSHMEM qpair handles. NIIN currently ignores explicit qpair selection for +// LSA and drains all GIN contexts for quiet, but exposes the types/constants so +// code using nvshmemx_qp_quiet_* can build unchanged. +typedef int nvshmemx_qp_handle_t; +#define NVSHMEMX_QP_HOST 0 +#define NVSHMEMX_QP_DEFAULT 1 +#define NVSHMEMX_QP_ANY 0x7fffffff +#define NVSHMEMX_QP_ALL 0x7fffffff + +// --------------------------------------------------------------------------- +// TMA (Tensor Memory Accelerator) shared-memory registration +// +// NIIN mirrors NVSHMEM's TMA API: applications hand a slice of their CTA's +// shared memory to the runtime with nvshmemx_give_smem(), and the LSA (NVLink) +// put/get paths then route through cp.async.bulk instead of vectorized +// load/store. Sizing comes from nvshmemx_ask_smem(). +// --------------------------------------------------------------------------- + +// Static carve at the base of every nvshmemx_give_smem() buffer, reserved for +// NIIN internal use (mbarriers used to track async-proxy completion). User data +// for TMA staging starts at smem + NIIN_SMEM_DATA_REGION_OFFSET. +// Layout: 32 slots of 16 bytes each; slot 0 is at offset 0. +#define NIIN_TMA_BARRIER_REGION_BYTES 512 +#define NIIN_TMA_NUM_BARRIER_SLOTS 32 +#define NIIN_SMEM_DATA_REGION_OFFSET NIIN_TMA_BARRIER_REGION_BYTES + +// CTAs with a linear block id at or beyond this bound cannot register shared +// memory and fall back to the vectorized load/store path. +#define NIIN_TMA_MAX_BLOCKS 4096 + +typedef enum { + NVSHMEMX_TMA_DISABLE = 0, // Do not use TMA for transfers (default) + NVSHMEMX_TMA_ENABLE = 1, // Use TMA when smem is given and the arch supports it + NVSHMEMX_TMA_FORCE = 2, // Require TMA; initialization fails if unavailable + NVSHMEMX_TMA_POLICY_MAX = 0x7fffffff +} nvshmemx_tma_policy_t; + +typedef enum { + NVSHMEMX_SMEM_RECOMMENDED = 0, + NVSHMEMX_SMEM_MINIMUM = 1, + NVSHMEMX_SMEM_BARRIERS_ONLY = 2, + NVSHMEMX_SMEM_AMOUNT_MAX = 0x7fffffff +} nvshmemx_smem_amount_t; + +// nvshmemx_ask_smem: shared memory (in bytes) NIIN wants for TMA transfers. +// +// Available on host and device. On the host the return value sizes the dynamic +// shared memory of a kernel launch; on the device it sizes the buffer passed to +// nvshmemx_give_smem(). Every returned value includes +// NIIN_SMEM_DATA_REGION_OFFSET at the base of the buffer; TMA data tiles live +// in the remainder. +// +// NVSHMEMX_SMEM_RECOMMENDED - 64 KiB. The largest single cp.async.bulk +// transfer on Hopper/Blackwell, sized so the +// gmem->gmem staging path gets two full tiles. +// NVSHMEMX_SMEM_MINIMUM - 32 KiB. Enough for single-buffered staging +// and for smem->gmem puts. +// NVSHMEMX_SMEM_BARRIERS_ONLY - Barriers only, no data tile. The gmem->gmem +// staging path stays off, but smem->gmem puts +// still work when the application manages its +// own staging buffer. +__host__ __device__ __forceinline__ int nvshmemx_ask_smem(nvshmemx_smem_amount_t flag) { + switch (flag) { + case NVSHMEMX_SMEM_RECOMMENDED: return 65536; // 64 KiB + case NVSHMEMX_SMEM_MINIMUM: return 32768; // 32 KiB + case NVSHMEMX_SMEM_BARRIERS_ONLY: return NIIN_SMEM_DATA_REGION_OFFSET; + default: return 65536; + } +} + // X-macro for standard RMA types (matching NVSHMEM's type set) // Format: X(TYPENAME, TYPE) #define NIIN_STANDARD_RMA_TYPES(X) \ @@ -72,7 +142,8 @@ typedef int nvshmem_team_t; X(int64, int64_t) \ X(uint32, uint32_t) \ X(uint64, uint64_t) \ - X(size, size_t) + X(size, size_t) \ + X(ptrdiff, ptrdiff_t) // Types valid for bitwise AMOs (and, or, xor) #define NIIN_AMO_BITWISE_TYPES(X) \ diff --git a/projects/rccl/contrib/niin/include/nvshmem.h b/projects/rccl/contrib/niin/include/nvshmem.h index 37d685aeb24a..c2f4e6277bb2 100644 --- a/projects/rccl/contrib/niin/include/nvshmem.h +++ b/projects/rccl/contrib/niin/include/nvshmem.h @@ -19,9 +19,10 @@ // 5. All threads: use nvshmem_* functions as normal // 6. Host: nvshmem_finalize() // -// Unsupported operations (scalar network gets, most collectives, network -// atomics) invoke NIIN_NOT_IMPLEMENTED behavior, controlled by -// NIIN_ON_NOT_IMPLEMENTED. +// Unsupported operations (scalar network gets and most collectives) invoke +// NIIN_NOT_IMPLEMENTED behavior, controlled by NIIN_ON_NOT_IMPLEMENTED. +// Network AMOs retain that behavior unless an application explicitly binds +// NIIN's optional atomic-only GPUNetIO provider. #ifndef NVSHMEM_H_NIIN_ #define NVSHMEM_H_NIIN_ @@ -35,6 +36,7 @@ #include "niin/types.h" #include "niin/context.h" #include "niin/query.h" +#include "niin/tma.h" #include "niin/rma.h" #include "niin/signaling.h" #include "niin/sync.h" @@ -43,6 +45,10 @@ // Host-side helpers (niinInit, niinCommit, niinFinalize — low-level API) #include "niin/host.h" +// Optional NIIN-owned atomic providers. Applications opt in by linking the +// separate library. Its proxy host API is exposed through this header only +// when that library was built with NIIN_GPUNETIO_SRQ_PROXY_ATOMICS_ENABLE. +#include "niin/gpunetio/host.h" // Team management (split, translate, destroy) #include "niin/teams.h" diff --git a/projects/rccl/contrib/niin/include/nvshmemx.h b/projects/rccl/contrib/niin/include/nvshmemx.h index 6d7af63a9611..a6ad343bbc2b 100644 --- a/projects/rccl/contrib/niin/include/nvshmemx.h +++ b/projects/rccl/contrib/niin/include/nvshmemx.h @@ -61,11 +61,67 @@ inline int nvshmemx_quiet_on_stream(cudaStream_t stream) { return cudaGetLastError() == cudaSuccess ? 0 : -1; } -inline int nvshmemx_barrier_all_on_stream(cudaStream_t stream) { +namespace niin { +namespace detail { + +// NIIN has one NCCL communicator today: the world communicator. Do not use +// it for a subset team: ranks outside that team would not participate and the +// collective could hang. Per-team NCCL communicators will be needed before +// supporting stream collectives on non-WORLD teams. +inline bool host_stream_collective_team_is_supported(nvshmem_team_t team) { + return state().initialized && team == NVSHMEM_TEAM_WORLD; +} + +inline int host_stream_collective_status(ncclResult_t result) { + return result == ncclSuccess ? 0 : -1; +} + +} // namespace detail +} // namespace niin + +inline void nvshmemx_barrier_all_on_stream(cudaStream_t stream) { auto& s = niin::detail::state(); - if (!s.initialized) return -1; - ncclAllReduce(s.barrierScratch, s.barrierScratch, 1, ncclInt32, ncclSum, s.comm, stream); - return 0; + if (!s.initialized) return; + (void)ncclAllReduce(s.barrierScratch, s.barrierScratch, 1, ncclInt32, ncclSum, + s.comm, stream); +} + +inline void nvshmemx_sync_all_on_stream(cudaStream_t stream) { + nvshmemx_barrier_all_on_stream(stream); +} + +// --------------------------------------------------------------------------- +// Stream-ordered host collectives. +// +// These map directly to the world NCCL communicator and therefore preserve +// ordering with prior and subsequent work on the supplied CUDA stream. They +// deliberately do not synchronize the stream. +// --------------------------------------------------------------------------- +#define NIIN_DEFINE_REDUCE_ON_STREAM(TYPENAME, TYPE, NCCL_TYPE, OPNAME, NCCL_OP) \ +inline int nvshmemx_##TYPENAME##_##OPNAME##_reduce_on_stream( \ + nvshmem_team_t team, TYPE* dest, const TYPE* src, size_t nreduce, \ + cudaStream_t stream) { \ + if (!niin::detail::host_stream_collective_team_is_supported(team)) return -1; \ + auto& s = niin::detail::state(); \ + return niin::detail::host_stream_collective_status( \ + ncclAllReduce(src, dest, nreduce, NCCL_TYPE, nccl##NCCL_OP, s.comm, stream)); \ +} + +NIIN_DEFINE_REDUCE_ON_STREAM(int32, int32_t, ncclInt32, sum, Sum) +NIIN_DEFINE_REDUCE_ON_STREAM(int32, int32_t, ncclInt32, min, Min) +NIIN_DEFINE_REDUCE_ON_STREAM(int32, int32_t, ncclInt32, max, Max) +NIIN_DEFINE_REDUCE_ON_STREAM(int64, int64_t, ncclInt64, sum, Sum) +NIIN_DEFINE_REDUCE_ON_STREAM(int64, int64_t, ncclInt64, min, Min) +NIIN_DEFINE_REDUCE_ON_STREAM(int64, int64_t, ncclInt64, max, Max) +#undef NIIN_DEFINE_REDUCE_ON_STREAM + +inline int nvshmemx_alltoallmem_on_stream(nvshmem_team_t team, void* dest, + const void* src, size_t nelems, + cudaStream_t stream) { + if (!niin::detail::host_stream_collective_team_is_supported(team)) return -1; + auto& s = niin::detail::state(); + return niin::detail::host_stream_collective_status( + ncclAlltoAll(src, dest, nelems, ncclChar, s.comm, stream)); } // --------------------------------------------------------------------------- diff --git a/projects/rccl/contrib/niin/perftest/Makefile b/projects/rccl/contrib/niin/perftest/Makefile index f99c60324897..91868858dc0d 100644 --- a/projects/rccl/contrib/niin/perftest/Makefile +++ b/projects/rccl/contrib/niin/perftest/Makefile @@ -5,10 +5,26 @@ # mpirun -np 2 ./build/shmem_put_latency # # All tests require exactly 2 PEs (mpirun -np 2). +# +# shmem_atomic_bw links the NIIN GPUNetIO atomic provider, which `make -C ..` +# builds by default; it is skipped when that archive is absent. Match the +# provider's GPUNETIO_CUDA_ARCHITECTURES to the ARCH used here (89-real and +# sm_89 on L40S). NCCL_HOME ?= $(shell cd ../../.. && pwd) MPI_HOME ?= /usr/lib/x86_64-linux-gnu/openmpi ARCH ?= sm_89 +# Set NCCL_SOURCE_ROOT when NCCL_HOME is a build/install prefix rather than the +# matching source checkout that carries the vendored GPUNetIO sources. +NCCL_SOURCE_ROOT ?= $(NCCL_HOME) +GPUNETIO_HOME ?= $(NCCL_SOURCE_ROOT)/src/transport/net_ib/gdaki/doca-gpunetio +GPUNETIO_ATOMICS_LIB ?= ../lib/libniin_gpunetio_atomics.a +IBVERBS_LIBRARY ?= -libverbs + +# An absolute verbs library needs a matching runtime search path as well. +ifneq ($(filter /%,$(IBVERBS_LIBRARY)),) +IBVERBS_RPATH := -Xlinker -rpath,$(dir $(IBVERBS_LIBRARY)) +endif NVCC := nvcc NVCUFLAGS := --expt-relaxed-constexpr -std=c++17 -arch=$(ARCH) -DNIIN_HAS_MPI @@ -22,10 +38,41 @@ SRCDIR := device/pt-to-pt TESTS := shmem_put_latency shmem_put_bw shmem_p_latency shmem_get_latency shmem_put_signal_latency TARGETS := $(addprefix $(BUILDDIR)/,$(TESTS)) +ATOMIC_TARGET := $(BUILDDIR)/shmem_atomic_bw + +.PHONY: all atomic atomic-if-available clean + +all: $(TARGETS) atomic-if-available + +# The provider archive comes from `make -C ..`. Skip the test rather than fail +# the whole perftest build when NIIN was built without it. +atomic-if-available: + @if [ -f "$(GPUNETIO_ATOMICS_LIB)" ] && [ -d "$(GPUNETIO_HOME)" ]; then \ + $(MAKE) atomic; \ + else \ + echo "Skipping shmem_atomic_bw: no GPUNetIO atomic provider at $(GPUNETIO_ATOMICS_LIB)"; \ + fi + +# Link the provider archive after the CUDA object so its private GPUNetIO, +# ibverbs, CUDA-driver, dl, and pthread dependencies resolve normally. +atomic: $(ATOMIC_TARGET) -.PHONY: all clean +$(GPUNETIO_ATOMICS_LIB): + @if [ ! -f "$@" ]; then \ + echo "GPUNetIO atomic provider archive not found: $@"; \ + echo "Build it first with: make -C .. gpunetio-atomics"; \ + exit 1; \ + fi -all: $(TARGETS) +$(ATOMIC_TARGET): $(SRCDIR)/shmem_atomic_bw.cu common/perftest.h $(GPUNETIO_ATOMICS_LIB) | $(BUILDDIR) + @if [ ! -d "$(GPUNETIO_HOME)" ]; then \ + echo "GPUNetIO sources not found: $(GPUNETIO_HOME)"; \ + echo "Pass GPUNETIO_HOME=/path/to/unmodified/gpunetio-open"; \ + exit 1; \ + fi + $(NVCC) $(NVCUFLAGS) $(INCLUDES) $< -o $@ \ + -Xlinker --whole-archive $(GPUNETIO_ATOMICS_LIB) -Xlinker --no-whole-archive \ + $(LDFLAGS) $(IBVERBS_LIBRARY) $(IBVERBS_RPATH) -ldl -lpthread $(BUILDDIR)/%: $(SRCDIR)/%.cu common/perftest.h | $(BUILDDIR) $(NVCC) $(NVCUFLAGS) $(INCLUDES) $< -o $@ $(LDFLAGS) diff --git a/projects/rccl/contrib/niin/perftest/common/perftest.h b/projects/rccl/contrib/niin/perftest/common/perftest.h index 17f2af4ed480..ea31a3dfb966 100644 --- a/projects/rccl/contrib/niin/perftest/common/perftest.h +++ b/projects/rccl/contrib/niin/perftest/common/perftest.h @@ -19,6 +19,7 @@ #include #include #include +#include // strcasecmp #include "nvshmem.h" #include "nvshmemx.h" @@ -50,6 +51,26 @@ static size_t warmup_iters = 50; static size_t threads_per_block = 256; static size_t num_blocks = 4; static size_t max_size_log = 0; +// Used only by atomic perftests. Keep the upstream spelling so scripts can +// select the same operation with -a/--atomic_op. +static const char* atomic_op = "inc"; + +// --------------------------------------------------------------------------- +// TMA +// --------------------------------------------------------------------------- + +// Dynamic shared memory to request per CTA so kernels can register it with +// nvshmemx_give_smem(). Returns 0 unless NVSHMEM_TMA_POLICY asks for TMA, so a +// default run allocates no extra shared memory and keeps its usual occupancy. +// +// NVSHMEMX_SMEM_MINIMUM (32 KB) stays under the 48 KB cap on dynamic shared +// memory, so no cudaFuncSetAttribute opt-in is needed. +static size_t tma_smem_bytes() { + const char* env = getenv("NVSHMEM_TMA_POLICY"); + if (env == nullptr) return 0; + if (strcasecmp(env, "ENABLE") != 0 && strcasecmp(env, "FORCE") != 0) return 0; + return (size_t)nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM); +} // --------------------------------------------------------------------------- // CLI argument parsing (matching NVSHMEM perftest conventions) @@ -63,11 +84,12 @@ static void read_args(int argc, char** argv) { {"warmup", required_argument, 0, 'w'}, {"threads", required_argument, 0, 't'}, {"blocks", required_argument, 0, 'n'}, + {"atomic_op", required_argument, 0, 'a'}, {"help", no_argument, 0, 'h'}, {0, 0, 0, 0} }; int c, option_index = 0; - while ((c = getopt_long(argc, argv, "b:e:f:i:w:t:n:h", long_options, &option_index)) != -1) { + while ((c = getopt_long(argc, argv, "b:e:f:i:w:t:n:a:h", long_options, &option_index)) != -1) { switch (c) { case 'b': min_size = atol(optarg); break; case 'e': max_size = atol(optarg); break; @@ -76,6 +98,7 @@ static void read_args(int argc, char** argv) { case 'w': warmup_iters = atol(optarg); break; case 't': threads_per_block = atol(optarg); break; case 'n': num_blocks = atol(optarg); break; + case 'a': atomic_op = optarg; break; case 'h': printf("Usage: [options]\n" " -b, --min_size Minimum message size (default: 4)\n" @@ -84,11 +107,16 @@ static void read_args(int argc, char** argv) { " -i, --iters Iterations (default: 200)\n" " -w, --warmup Warmup iterations (default: 50)\n" " -t, --threads Threads per block (default: 256)\n" - " -n, --blocks Number of blocks (default: 4)\n"); + " -n, --blocks Number of blocks (default: 4)\n" + " -a, --atomic_op Atomic operation (atomic tests only)\n"); exit(0); default: break; } } + if (step_factor < 2) { + fprintf(stderr, "--step must be at least 2\n"); + exit(1); + } // Compute log2 of max_size for table allocation size_t tmp = max_size; max_size_log = 0; diff --git a/projects/rccl/contrib/niin/perftest/device/pt-to-pt/shmem_atomic_bw.cu b/projects/rccl/contrib/niin/perftest/device/pt-to-pt/shmem_atomic_bw.cu new file mode 100644 index 000000000000..8925f2eadd6d --- /dev/null +++ b/projects/rccl/contrib/niin/perftest/device/pt-to-pt/shmem_atomic_bw.cu @@ -0,0 +1,361 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +// GPUNetIO remote-atomic bandwidth. This follows NVSHMEM's +// shmem_atomic_bw operation/size matrix, but deliberately issues from one +// CUDA thread. The current NIIN direct provider owns one QP and one response +// slot per destination PE, and serializes the slot until CQ completion. A +// wider grid would therefore measure lock-spin contention rather than useful +// independent AMO throughput. + +#include "perftest.h" + +#include "niin/gpunetio/host.h" + +#include +#include +#include +#include + +namespace { + +constexpr double kBytesPerGb = 1000.0 * 1000.0 * 1000.0; +constexpr double kOperationsPerM = 1000.0 * 1000.0; +constexpr double kMillisecondsPerSecond = 1000.0; + +enum class AtomicOp : int { + Inc, + FetchInc, + Set, + Add, + FetchAdd, + And, + FetchAnd, + Or, + FetchOr, + Xor, + FetchXor, + Swap, + CompareSwap, +}; + +const char* atomicOpName(AtomicOp op) { + switch (op) { + case AtomicOp::Inc: return "inc"; + case AtomicOp::FetchInc: return "fetch_inc"; + case AtomicOp::Set: return "set"; + case AtomicOp::Add: return "add"; + case AtomicOp::FetchAdd: return "fetch_add"; + case AtomicOp::And: return "and"; + case AtomicOp::FetchAnd: return "fetch_and"; + case AtomicOp::Or: return "or"; + case AtomicOp::FetchOr: return "fetch_or"; + case AtomicOp::Xor: return "xor"; + case AtomicOp::FetchXor: return "fetch_xor"; + case AtomicOp::Swap: return "swap"; + case AtomicOp::CompareSwap: return "compare_swap"; + } + return "unknown"; +} + +bool parseAtomicOp(const char* name, AtomicOp* op) { + static constexpr AtomicOp kOps[] = { + AtomicOp::Inc, AtomicOp::FetchInc, AtomicOp::Set, + AtomicOp::Add, AtomicOp::FetchAdd, AtomicOp::And, + AtomicOp::FetchAnd, AtomicOp::Or, AtomicOp::FetchOr, + AtomicOp::Xor, AtomicOp::FetchXor, AtomicOp::Swap, + AtomicOp::CompareSwap, + }; + for (AtomicOp candidate : kOps) { + if (std::strcmp(name, atomicOpName(candidate)) == 0) { + *op = candidate; + return true; + } + } + return false; +} + +[[noreturn]] void providerCheck(ncclResult_t result, const char* what) { + std::fprintf(stderr, "shmem_atomic_bw: %s: %s\n", what, ncclGetErrorString(result)); +#ifdef NIIN_HAS_MPI + MPI_Abort(MPI_COMM_WORLD, 1); +#endif + std::exit(1); +} + +template +__global__ void atomicBwKernel(uint64_t* data, size_t elements, int peer, int iterations) { + if (blockIdx.x != 0 || threadIdx.x != 0) return; + + // Preserve returned values in a register and materialize it once. This + // keeps fetch AMOs observable without adding a device-memory operation per + // AMO to the timed path. + uint64_t consumed = 0; + for (int iteration = 0; iteration < iterations; ++iteration) { + const uint64_t setValue = static_cast(iteration) + 1; + for (size_t element = 0; element < elements; ++element) { + uint64_t* destination = data + element; + if constexpr (Op == AtomicOp::Inc) { + nvshmem_uint64_atomic_inc(destination, peer); + } else if constexpr (Op == AtomicOp::FetchInc) { + consumed ^= nvshmem_uint64_atomic_fetch_inc(destination, peer); + } else if constexpr (Op == AtomicOp::Set) { + nvshmem_uint64_atomic_set(destination, setValue, peer); + } else if constexpr (Op == AtomicOp::Add) { + nvshmem_uint64_atomic_add(destination, UINT64_C(1), peer); + } else if constexpr (Op == AtomicOp::FetchAdd) { + consumed ^= nvshmem_uint64_atomic_fetch_add(destination, UINT64_C(1), peer); + } else if constexpr (Op == AtomicOp::And) { + const unsigned int shift = static_cast(iteration < 63 ? iteration + 1 : 63); + nvshmem_uint64_atomic_and(destination, data[element] << shift, peer); + } else if constexpr (Op == AtomicOp::FetchAnd) { + const unsigned int shift = static_cast(iteration < 63 ? iteration + 1 : 63); + consumed ^= nvshmem_uint64_atomic_fetch_and(destination, data[element] << shift, peer); + } else if constexpr (Op == AtomicOp::Or) { + const unsigned int shift = static_cast(iteration < 63 ? iteration : 63); + nvshmem_uint64_atomic_or(destination, data[element] << shift, peer); + } else if constexpr (Op == AtomicOp::FetchOr) { + const unsigned int shift = static_cast(iteration < 63 ? iteration : 63); + consumed ^= nvshmem_uint64_atomic_fetch_or(destination, data[element] << shift, peer); + } else if constexpr (Op == AtomicOp::Xor) { + nvshmem_uint64_atomic_xor(destination, UINT64_C(1), peer); + } else if constexpr (Op == AtomicOp::FetchXor) { + consumed ^= nvshmem_uint64_atomic_fetch_xor(destination, UINT64_C(1), peer); + } else if constexpr (Op == AtomicOp::Swap) { + consumed ^= nvshmem_uint64_atomic_swap(destination, setValue, peer); + } else if constexpr (Op == AtomicOp::CompareSwap) { + consumed ^= nvshmem_uint64_atomic_compare_swap(destination, static_cast(iteration), + setValue, peer); + } + } + } + + // Every direct AMO has already waited for CQ completion on return, so no + // nvshmem_quiet() is needed here. This local store makes `consumed` live + // until after the final AMO without touching the remote target. + data[0] = consumed; +} + +template +void launchAtomicKernel(uint64_t* data, size_t elements, int peer, int iterations) { + atomicBwKernel<<<1, 1>>>(data, elements, peer, iterations); +} + +void launchAtomicKernel(AtomicOp op, uint64_t* data, size_t elements, int peer, int iterations) { + switch (op) { + case AtomicOp::Inc: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::FetchInc: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::Set: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::Add: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::FetchAdd: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::And: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::FetchAnd: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::Or: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::FetchOr: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::Xor: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::FetchXor: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::Swap: return launchAtomicKernel(data, elements, peer, iterations); + case AtomicOp::CompareSwap: + return launchAtomicKernel(data, elements, peer, iterations); + } +} + +void initializeData(uint64_t* data, size_t bytes, AtomicOp op) { + const int byteValue = (op == AtomicOp::And || op == AtomicOp::FetchAnd || op == AtomicOp::Or || + op == AtomicOp::FetchOr) + ? 0xff + : 0; + CUDA_CHECK(cudaMemset(data, byteValue, bytes)); + CUDA_CHECK(cudaDeviceSynchronize()); +} + +void printAtomicBwTable(const char* operation, uint64_t* sizes, double* bandwidth, + double* amoRate, int entries) { + std::printf("\nshmem_atomic_%s (GPUNetIO direct, serialized endpoint)\n", operation); + std::printf("%-16s %12s %12s\n", "size (Bytes)", "BW (GB/s)", "MAMO/s"); + std::printf("%-16s %12s %12s\n", "----------------", "------------", "------------"); + for (int i = 0; i < entries; ++i) { + std::printf("%-16lu %12.3f %12.3f\n", static_cast(sizes[i]), bandwidth[i], + amoRate[i]); + } + std::printf("\n"); + std::fflush(stdout); +} + +bool helpRequested(int argc, char* argv[]) { + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "-h") == 0 || std::strcmp(argv[i], "--help") == 0) return true; + } + return false; +} + +void printAtomicUsage() { + std::printf( + "Usage: shmem_atomic_bw [options]\n" + " -b, --min_size Minimum payload size (default: 8)\n" + " -e, --max_size Maximum payload size (default: 65536)\n" + " -f, --step Size step factor (default: 2)\n" + " -i, --iters Timed iterations (default: 10)\n" + " -w, --warmup Warmup iterations (default: 10)\n" + " -a, --atomic_op inc, fetch_inc, set, add, fetch_add, and, fetch_and,\n" + " or, fetch_or, xor, fetch_xor, swap, compare_swap (default: inc)\n" + " -t, --threads Must be 1 (one serialized direct endpoint issuer)\n" + " -n, --blocks Must be 1 (one serialized direct endpoint issuer)\n"); +} + +} // namespace + +int main(int argc, char* argv[]) { + // A serialized request/completion endpoint cannot use the generic RMA + // defaults (4 MiB x 200 iterations) responsibly. Match the upstream AMO + // test's compact sweep; explicit command-line arguments still override + // every one of these values in read_args(). + min_size = sizeof(uint64_t); + max_size = 64 * 1024; + iters = 10; + warmup_iters = 10; + num_blocks = 1; + threads_per_block = 1; + if (helpRequested(argc, argv)) { + printAtomicUsage(); + return 0; + } + read_args(argc, argv); + + AtomicOp op; + if (!parseAtomicOp(atomic_op, &op)) { + std::fprintf(stderr, + "Unsupported --atomic_op '%s'. Supported operations: inc, fetch_inc, set, add, " + "fetch_add, and, fetch_and, or, fetch_or, xor, fetch_xor, swap, compare_swap\n", + atomic_op); + return 1; + } + if (min_size < sizeof(uint64_t) || min_size > max_size) { + std::fprintf(stderr, "shmem_atomic_bw requires 8 <= min_size <= max_size\n"); + return 1; + } + if (iters == 0 || iters > static_cast(INT_MAX) || + warmup_iters > static_cast(INT_MAX)) { + std::fprintf(stderr, "shmem_atomic_bw requires 1 <= iters and 32-bit iteration counts\n"); + return 1; + } + if (num_blocks != 1 || threads_per_block != 1) { + std::fprintf(stderr, + "shmem_atomic_bw requires --blocks 1 and --threads 1: the current direct provider " + "serializes each destination endpoint through one response slot\n"); + return 1; + } + + init_wrapper(&argc, &argv); + + const int mype = nvshmem_my_pe(); + const int npes = nvshmem_n_pes(); + if (npes != 2) { + if (mype == 0) std::fprintf(stderr, "This test requires exactly 2 PEs\n"); + finalize_wrapper(); + return 1; + } + + // Public atomic APIs deliberately choose CUDA atomics for LSA peers. Make + // this a two-node test so the measured operation is the bound GPUNetIO QP. + auto& state = niin::detail::state(); + if (state.lsaSize != 1) { + if (mype == 0) { + std::fprintf(stderr, + "shmem_atomic_bw requires the two PEs on distinct LSA domains; " + "same-node peers use CUDA atomics instead of GPUNetIO\n"); + } + finalize_wrapper(); + return 1; + } + + // nvshmem_init() brings the provider up and binds it; this test only needs + // to confirm it is actually there before reporting network AMO numbers. + if (state.gpunetioAtomics == nullptr) { + if (mype == 0) { + std::fprintf(stderr, + "shmem_atomic_bw: no network atomic provider was bound. Link the NIIN " + "GPUNetIO provider and leave NIIN_GPUNETIO_ATOMICS unset (or =1 to see why)\n"); + } + finalize_wrapper(); + return 1; + } + + uint64_t* data = static_cast(nvshmem_malloc(max_size)); + if (data == nullptr) { + if (mype == 0) std::fprintf(stderr, "shmem_atomic_bw: nvshmem_malloc(%zu) failed\n", max_size); + finalize_wrapper(); + return 1; + } + + void** tables = nullptr; + alloc_tables(&tables, 3, static_cast(max_size_log + 1)); + uint64_t* sizes = static_cast(tables[0]); + double* bandwidth = static_cast(tables[1]); + double* amoRate = static_cast(tables[2]); + + cudaEvent_t start = nullptr; + cudaEvent_t stop = nullptr; + CUDA_CHECK(cudaEventCreate(&start)); + CUDA_CHECK(cudaEventCreate(&stop)); + + const int peer = 1; + int entries = 0; + for (size_t requestedBytes = min_size; requestedBytes <= max_size;) { + const size_t elements = requestedBytes / sizeof(uint64_t); + const size_t effectiveBytes = elements * sizeof(uint64_t); + if (elements != 0) { + sizes[entries] = effectiveBytes; + + // Every PE initializes its local target before the source PE issues a + // new phase. The host barriers prevent a target reset from racing a + // completed remote AMO phase. + initializeData(data, effectiveBytes, op); + nvshmem_barrier_all(); + if (mype == 0) { + launchAtomicKernel(op, data, elements, peer, static_cast(warmup_iters)); + CUDA_CHECK(cudaGetLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + } + nvshmem_barrier_all(); + + initializeData(data, effectiveBytes, op); + nvshmem_barrier_all(); + if (mype == 0) { + CUDA_CHECK(cudaEventRecord(start)); + launchAtomicKernel(op, data, elements, peer, static_cast(iters)); + CUDA_CHECK(cudaGetLastError()); + CUDA_CHECK(cudaEventRecord(stop)); + CUDA_CHECK(cudaEventSynchronize(stop)); + + float milliseconds = 0.0f; + CUDA_CHECK(cudaEventElapsedTime(&milliseconds, start, stop)); + const double seconds = milliseconds / kMillisecondsPerSecond; + const double operations = static_cast(elements) * static_cast(iters); + bandwidth[entries] = static_cast(effectiveBytes) * static_cast(iters) / + seconds / kBytesPerGb; + amoRate[entries] = operations / seconds / kOperationsPerM; + } + nvshmem_barrier_all(); + ++entries; + } + + if (requestedBytes > max_size / step_factor) break; + requestedBytes *= step_factor; + } + + if (mype == 0) printAtomicBwTable(atomicOpName(op), sizes, bandwidth, amoRate, entries); + + CUDA_CHECK(cudaDeviceSynchronize()); + nvshmem_barrier_all(); + nvshmem_free(data); + CUDA_CHECK(cudaEventDestroy(start)); + CUDA_CHECK(cudaEventDestroy(stop)); + free_tables(tables, 3); + finalize_wrapper(); + return 0; +} diff --git a/projects/rccl/contrib/niin/perftest/device/pt-to-pt/shmem_put_bw.cu b/projects/rccl/contrib/niin/perftest/device/pt-to-pt/shmem_put_bw.cu index 6585a7e9445f..8ef5e781785b 100644 --- a/projects/rccl/contrib/niin/perftest/device/pt-to-pt/shmem_put_bw.cu +++ b/projects/rccl/contrib/niin/perftest/device/pt-to-pt/shmem_put_bw.cu @@ -8,17 +8,29 @@ // Put bandwidth: measures NVLink store bandwidth using cooperative block puts. // Each block copies its chunk of data iter times with no inter-block sync. // Only a single __threadfence_system at the end to ensure visibility. +// +// Both kernels lend their dynamic shared memory to NIIN, so running with +// NVSHMEM_TMA_POLICY=ENABLE on sm_90+ measures the TMA path instead of +// vectorized stores. Without that variable tma_smem_bytes() is 0, give_smem is +// inert, and the run is unchanged. #include "perftest.h" #define B_TO_GB (1000.0 * 1000.0 * 1000.0) #define MS_TO_S 1000.0 +// int4 element type pins the dynamic shared memory base to 16-byte alignment, +// which nvshmemx_give_smem() requires. +extern __shared__ int4 bw_smem[]; + // --- Kernels --- // Block-cooperative put: each block handles len/nblocks elements. // No inter-block sync — all blocks blast data independently. -__global__ void bw_block(double* data_d, size_t len, int pe, int iter) { +__global__ void bw_block(double* data_d, size_t len, int pe, int iter, size_t smem_bytes) { + nvshmemx_give_smem(bw_smem, smem_bytes); + __syncthreads(); + int peer = !pe; int bid = blockIdx.x; int nblocks = gridDim.x; @@ -35,15 +47,22 @@ __global__ void bw_block(double* data_d, size_t len, int pe, int iter) { __syncthreads(); if (threadIdx.x == 0) nvshmem_quiet(); __syncthreads(); + + nvshmemx_release_smem(); + __syncthreads(); } // Thread-scope put: one thread per CTA puts the CTA's chunk. -__global__ void bw_thread(double* data_d, size_t len, int pe, int iter) { +__global__ void bw_thread(double* data_d, size_t len, int pe, int iter, size_t smem_bytes) { + nvshmemx_give_smem(bw_smem, smem_bytes); + __syncthreads(); + int peer = !pe; int bid = blockIdx.x; int nblocks = gridDim.x; size_t per_block = len / nblocks; + // One issuer per CTA: the thread-scoped TMA staging tile is per-CTA state. if (threadIdx.x == 0) { for (int i = 0; i < iter; i++) { nvshmem_double_put_nbi(data_d + bid * per_block, @@ -52,6 +71,10 @@ __global__ void bw_thread(double* data_d, size_t len, int pe, int iter) { } nvshmem_quiet(); } + + __syncthreads(); + nvshmemx_release_smem(); + __syncthreads(); } // --- Host --- @@ -80,6 +103,8 @@ int main(int argc, char* argv[]) { uint64_t* h_size = (uint64_t*)h_tables[0]; double* h_bw = (double*)h_tables[1]; + size_t smem_bytes = tma_smem_bytes(); + cudaEvent_t start, stop; CUDA_CHECK(cudaEventCreate(&start)); CUDA_CHECK(cudaEventCreate(&stop)); @@ -95,10 +120,10 @@ int main(int argc, char* argv[]) { h_size[idx] = size; if (!mype) { - bw_block<<>>(data_d, nelems, mype, skip); + bw_block<<>>(data_d, nelems, mype, skip, smem_bytes); CUDA_CHECK(cudaDeviceSynchronize()); CUDA_CHECK(cudaEventRecord(start)); - bw_block<<>>(data_d, nelems, mype, iter); + bw_block<<>>(data_d, nelems, mype, iter, smem_bytes); CUDA_CHECK(cudaEventRecord(stop)); CUDA_CHECK(cudaEventSynchronize(stop)); float ms; @@ -120,10 +145,10 @@ int main(int argc, char* argv[]) { h_size[idx] = size; if (!mype) { - bw_thread<<>>(data_d, nelems, mype, skip); + bw_thread<<>>(data_d, nelems, mype, skip, smem_bytes); CUDA_CHECK(cudaDeviceSynchronize()); CUDA_CHECK(cudaEventRecord(start)); - bw_thread<<>>(data_d, nelems, mype, iter); + bw_thread<<>>(data_d, nelems, mype, iter, smem_bytes); CUDA_CHECK(cudaEventRecord(stop)); CUDA_CHECK(cudaEventSynchronize(stop)); float ms; diff --git a/projects/rccl/contrib/niin/test/TESTING.md b/projects/rccl/contrib/niin/test/TESTING.md index c2bef87938b0..00a5e0c519bd 100644 --- a/projects/rccl/contrib/niin/test/TESTING.md +++ b/projects/rccl/contrib/niin/test/TESTING.md @@ -53,6 +53,7 @@ bash contrib/niin/test/run_all.sh --all | `HCA` | Auto-detected (first active mlx5) | IB HCA for loopback mode | | `GPU_PAIR` | `0,1` | Comma-separated GPU IDs to use | | `NP` | `2` | Number of MPI processes | +| `NIIN_TEST_BLOCK_COLLECTIVES` | unset | Set to `1` only on a validated topology to run the block barrier remote-put visibility test; public block API compilation is always covered | | `RESULTS_DIR` | `contrib/niin/test/results/` | Generated output directory for CSV files; ignored by git | ## Choosing the Right GPU Pair @@ -69,6 +70,14 @@ Look for `NODE` or `NV12`/`NV18` connections (not `SYS`). GPU_PAIR=0,3 bash contrib/niin/test/run_all.sh --p2p ``` +To include the `nvshmemx_sync_all_block()` / `nvshmemx_barrier_all_block()` +runtime visibility check, use a validated NVLink or GIN configuration: + +```bash +NIIN_TEST_BLOCK_COLLECTIVES=1 GPU_PAIR=0,3 \ + bash contrib/niin/test/run_all.sh --p2p --skip-perf +``` + **For IB loopback**: Any GPU pair works. The script disables P2P and routes through the HCA. Cross-NUMA is fine: ```bash GPU_PAIR=0,2 bash contrib/niin/test/run_all.sh --ib diff --git a/projects/rccl/contrib/niin/test/compat/utils.h b/projects/rccl/contrib/niin/test/compat/utils.h index de7cc270cd38..cf20b3f37489 100644 --- a/projects/rccl/contrib/niin/test/compat/utils.h +++ b/projects/rccl/contrib/niin/test/compat/utils.h @@ -149,7 +149,7 @@ __device__ __forceinline__ void niin_gin_barrier_all() { int pe = niin_device_my_pe(); int npes = niin_device_n_pes(); ncclDevComm const& comm = niin_comm(); - ncclGin gin(comm, niin_gin_context_index()); + ncclGin gin(comm, niin_gin_collective_context_index()); ncclTeam world = ncclTeamWorld(comm); // Use a dedicated signal in the heap for barrier (last 64 bytes) @@ -172,6 +172,7 @@ __device__ __forceinline__ void niin_gin_barrier_all() { *bar_sig = epoch + 1; } +#ifndef NIIN_HAS_NVSHMEMX_BLOCK_COLLECTIVES __device__ __forceinline__ void nvshmemx_barrier_all_block() { cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); __syncthreads(); @@ -184,6 +185,7 @@ __device__ __forceinline__ void nvshmemx_barrier_all_block() { __syncthreads(); cuda::atomic_thread_fence(cuda::memory_order_acquire, cuda::thread_scope_system); } +#endif __device__ __forceinline__ void nvshmemx_barrier_all_warp() { cuda::atomic_thread_fence(cuda::memory_order_release, cuda::thread_scope_system); diff --git a/projects/rccl/contrib/niin/test/host_collectives.cu b/projects/rccl/contrib/niin/test/host_collectives.cu new file mode 100644 index 000000000000..ad601b136c8c --- /dev/null +++ b/projects/rccl/contrib/niin/test/host_collectives.cu @@ -0,0 +1,386 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +// Focused smoke test for NIIN's public host collective APIs. It intentionally +// includes no compat shim, so the extended API declarations must come from the +// NIIN public headers. + +#include "nvshmem.h" +#include "nvshmemx.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// Populate the device sources on the same stream as the host NCCL collectives. +// This catches a collective implementation that fails to honor prior work on +// its caller-provided stream. +__global__ void populate_collective_sources(int32_t* src32, int64_t* src64, + unsigned char* alltoallSrc, int rank, + int nRanks) { + if (threadIdx.x != 0 || blockIdx.x != 0) return; + + constexpr size_t kReduceCount = 7; + constexpr size_t kAlltoallBytes = 13; + for (size_t i = 0; i < kReduceCount; ++i) { + src32[i] = rank + 1 + static_cast(i); + src64[i] = static_cast(100 * (rank + 1) + static_cast(i)); + } + for (int peer = 0; peer < nRanks; ++peer) { + for (size_t byte = 0; byte < kAlltoallBytes; ++byte) { + alltoallSrc[static_cast(peer) * kAlltoallBytes + byte] = + static_cast((31 * rank + 7 * peer + byte) % 127); + } + } +} + +// Every thread of this 3D CTA calls both public block APIs. The remote put +// before barrier verifies the barrier's release/acquire visibility guarantee. +__global__ void exercise_block_collective_apis(int32_t* blockStatus, + int* blockFailure, int rank, + int nRanks) { + nvshmemx_sync_all_block(); + + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + nvshmem_int32_p(blockStatus, rank + 1, (rank + 1) % nRanks); + } + + nvshmemx_barrier_all_block(); + + if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + const int expected = (rank + nRanks - 1) % nRanks + 1; + if (*blockStatus != expected) atomicExch(blockFailure, 1); + } +} + +namespace { + +using StreamCollective = void (*)(cudaStream_t); + +bool check_cuda(cudaError_t result, const char* operation, int rank) { + if (result == cudaSuccess) return true; + std::fprintf(stderr, "PE %d: %s failed: %s\n", rank, operation, + cudaGetErrorString(result)); + return false; +} + +bool check_mpi(int result, const char* operation, int rank) { + if (result == MPI_SUCCESS) return true; + std::fprintf(stderr, "PE %d: %s failed\n", rank, operation); + return false; +} + +bool check_niin(int result, const char* operation, int rank) { + if (result == 0) return true; + std::fprintf(stderr, "PE %d: %s failed\n", rank, operation); + return false; +} + +bool should_exercise_block_collectives() { + const char* setting = std::getenv("NIIN_TEST_BLOCK_COLLECTIVES"); + return setting != nullptr && std::strcmp(setting, "0") != 0; +} + +// Best-effort check for an actual collective: an event after a real NCCL +// collective cannot complete before every PE enters it. The worker thread +// avoids a host-side NCCL preconnect stall from preventing its PE from +// releasing PE 0 to enter the matching collective. +bool verify_stream_collective(cudaStream_t stream, StreamCollective collective, + const char* name, int rank, int nRanks) { + constexpr auto kEarlyCompletionWindow = std::chrono::milliseconds(100); + constexpr auto kRootArrivalDelay = std::chrono::milliseconds(250); + bool ok = true; + cudaEvent_t completion = nullptr; + + if (rank == 0) { + for (int peer = 1; peer < nRanks; ++peer) { + int released = 0; + ok &= check_mpi(MPI_Recv(&released, 1, MPI_INT, peer, 0, MPI_COMM_WORLD, + MPI_STATUS_IGNORE), + "wait for stream collective check", rank); + } + std::this_thread::sleep_for(kRootArrivalDelay); + collective(stream); + } else { + ok &= check_cuda(cudaEventCreateWithFlags(&completion, cudaEventDisableTiming), + "cudaEventCreateWithFlags", rank); + + std::atomic workerStarted{false}; + std::atomic workerReturned{false}; + std::atomic eventRecordResult{cudaSuccess}; + std::thread launcher([&] { + workerStarted.store(true, std::memory_order_release); + collective(stream); + if (completion != nullptr) { + eventRecordResult.store(cudaEventRecord(completion, stream), + std::memory_order_release); + } + workerReturned.store(true, std::memory_order_release); + }); + + while (!workerStarted.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + + int released = 1; + ok &= check_mpi(MPI_Send(&released, 1, MPI_INT, 0, 0, MPI_COMM_WORLD), + "release stream collective", rank); + + if (completion != nullptr) { + const auto deadline = std::chrono::steady_clock::now() + kEarlyCompletionWindow; + while (std::chrono::steady_clock::now() < deadline) { + if (!workerReturned.load(std::memory_order_acquire)) { + std::this_thread::yield(); + continue; + } + const cudaError_t recordResult = eventRecordResult.load(std::memory_order_acquire); + if (recordResult != cudaSuccess) { + ok &= check_cuda(recordResult, "cudaEventRecord", rank); + break; + } + const cudaError_t result = cudaEventQuery(completion); + if (result == cudaErrorNotReady) { + std::this_thread::yield(); + continue; + } + if (result == cudaSuccess) { + std::fprintf(stderr, + "PE %d: %s completed before PE 0 entered the collective\n", + rank, name); + } else { + ok &= check_cuda(result, "cudaEventQuery", rank); + } + ok = false; + break; + } + } + launcher.join(); + } + + ok &= check_cuda(cudaStreamSynchronize(stream), name, rank); + if (completion != nullptr) cudaEventDestroy(completion); + + int localFailure = ok ? 0 : 1; + int anyFailure = 0; + ok &= check_mpi(MPI_Allreduce(&localFailure, &anyFailure, 1, MPI_INT, MPI_MAX, + MPI_COMM_WORLD), + "combine stream collective result", rank); + return ok && anyFailure == 0; +} + +} // namespace + +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + + int rank = 0; + int nRanks = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nRanks); + if (nRanks < 2) { + if (rank == 0) std::fprintf(stderr, "host_collectives requires at least two PEs\n"); + MPI_Finalize(); + return 1; + } + + MPI_Comm mpiComm = MPI_COMM_WORLD; + nvshmemx_init_attr_t attr = NVSHMEMX_INIT_ATTR_INITIALIZER; + attr.mpi_comm = &mpiComm; + if (!check_niin(nvshmemx_init_attr(NVSHMEMX_INIT_WITH_MPI_COMM, &attr), + "nvshmemx_init_attr", rank)) { + // NCCL communicator initialization is collective. Do not strand a peer + // that is still inside it if one PE sees a local initialization failure. + MPI_Abort(MPI_COMM_WORLD, 1); + return 1; + } + + bool ok = true; + cudaStream_t stream = nullptr; + int* blockFailure = nullptr; + ok &= check_cuda(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), + "cudaStreamCreateWithFlags", rank); + ok &= check_cuda(cudaMalloc(&blockFailure, sizeof(*blockFailure)), "cudaMalloc", rank); + + constexpr size_t kReduceCount = 7; + constexpr size_t kAlltoallBytes = 13; + int32_t* src32 = static_cast(nvshmem_malloc(kReduceCount * sizeof(int32_t))); + int32_t* sum32 = static_cast(nvshmem_malloc(kReduceCount * sizeof(int32_t))); + int32_t* min32 = static_cast(nvshmem_malloc(kReduceCount * sizeof(int32_t))); + int32_t* max32 = static_cast(nvshmem_malloc(kReduceCount * sizeof(int32_t))); + int64_t* src64 = static_cast(nvshmem_malloc(kReduceCount * sizeof(int64_t))); + int64_t* sum64 = static_cast(nvshmem_malloc(kReduceCount * sizeof(int64_t))); + int64_t* min64 = static_cast(nvshmem_malloc(kReduceCount * sizeof(int64_t))); + int64_t* max64 = static_cast(nvshmem_malloc(kReduceCount * sizeof(int64_t))); + unsigned char* alltoallSrc = static_cast( + nvshmem_malloc(static_cast(nRanks) * kAlltoallBytes)); + unsigned char* alltoallDst = static_cast( + nvshmem_malloc(static_cast(nRanks) * kAlltoallBytes)); + int32_t* blockStatus = static_cast(nvshmem_malloc(sizeof(*blockStatus))); + + if (!blockFailure || !src32 || !sum32 || !min32 || !max32 || !src64 || !sum64 || + !min64 || !max64 || !alltoallSrc || !alltoallDst || !blockStatus) { + std::fprintf(stderr, "PE %d: test allocation failed\n", rank); + ok = false; + } + + // A failed setup on one PE must keep every PE out of the NCCL collectives. + // Otherwise the successful PEs could enter a world collective alone. + int localSetupFailure = ok ? 0 : 1; + int anySetupFailure = 0; + MPI_Allreduce(&localSetupFailure, &anySetupFailure, 1, MPI_INT, MPI_MAX, + MPI_COMM_WORLD); + if (anySetupFailure) ok = false; + + if (ok) { + // Keep sync and barrier in isolated phases: neither can be masked by a + // later collective, and a no-op has a completion event that fires early. + ok &= verify_stream_collective(stream, nvshmemx_sync_all_on_stream, + "nvshmemx_sync_all_on_stream", rank, nRanks); + ok &= verify_stream_collective(stream, nvshmemx_barrier_all_on_stream, + "nvshmemx_barrier_all_on_stream", rank, nRanks); + + if (should_exercise_block_collectives()) { + // Exercise both public block APIs with every thread of a 3D CTA. The + // barrier phase also verifies a predecessor PE's remote put is visible. + ok &= check_cuda(cudaMemsetAsync(blockStatus, 0, sizeof(*blockStatus), stream), + "clear block status", rank); + ok &= check_cuda(cudaMemsetAsync(blockFailure, 0, sizeof(*blockFailure), stream), + "clear block failure", rank); + ok &= check_cuda(cudaStreamSynchronize(stream), "synchronize block setup", rank); + ok &= check_mpi(MPI_Barrier(MPI_COMM_WORLD), "synchronize block setup PEs", rank); + exercise_block_collective_apis<<>>( + blockStatus, blockFailure, rank, nRanks); + ok &= check_cuda(cudaGetLastError(), "launch block collectives", rank); + int hBlockFailure = 0; + ok &= check_cuda(cudaMemcpyAsync(&hBlockFailure, blockFailure, sizeof(hBlockFailure), + cudaMemcpyDeviceToHost, stream), + "copy block collective result", rank); + ok &= check_cuda(cudaStreamSynchronize(stream), "block collectives", rank); + if (hBlockFailure != 0) { + std::fprintf(stderr, "PE %d: block barrier did not make remote put visible\n", rank); + ok = false; + } + } else if (rank == 0) { + std::fprintf(stderr, + "host_collectives: block runtime test skipped; set " + "NIIN_TEST_BLOCK_COLLECTIVES=1 on a validated topology\n"); + } + + // The producer kernel, host NCCL collectives, and result copies all share + // one stream. This verifies direct NCCL calls honor stream ordering. + populate_collective_sources<<<1, 1, 0, stream>>>(src32, src64, alltoallSrc, rank, + nRanks); + ok &= check_cuda(cudaGetLastError(), "launch collective source producer", rank); + ok &= check_niin(nvshmemx_int32_sum_reduce_on_stream( + NVSHMEM_TEAM_WORLD, sum32, src32, kReduceCount, stream), + "int32 sum reduce", rank); + ok &= check_niin(nvshmemx_int32_min_reduce_on_stream( + NVSHMEM_TEAM_WORLD, min32, src32, kReduceCount, stream), + "int32 min reduce", rank); + ok &= check_niin(nvshmemx_int32_max_reduce_on_stream( + NVSHMEM_TEAM_WORLD, max32, src32, kReduceCount, stream), + "int32 max reduce", rank); + ok &= check_niin(nvshmemx_int64_sum_reduce_on_stream( + NVSHMEM_TEAM_WORLD, sum64, src64, kReduceCount, stream), + "int64 sum reduce", rank); + ok &= check_niin(nvshmemx_int64_min_reduce_on_stream( + NVSHMEM_TEAM_WORLD, min64, src64, kReduceCount, stream), + "int64 min reduce", rank); + ok &= check_niin(nvshmemx_int64_max_reduce_on_stream( + NVSHMEM_TEAM_WORLD, max64, src64, kReduceCount, stream), + "int64 max reduce", rank); + ok &= check_niin(nvshmemx_alltoallmem_on_stream(NVSHMEM_TEAM_WORLD, alltoallDst, + alltoallSrc, kAlltoallBytes, stream), + "alltoallmem", rank); + + std::vector hSum32(kReduceCount); + std::vector hMin32(kReduceCount); + std::vector hMax32(kReduceCount); + std::vector hSum64(kReduceCount); + std::vector hMin64(kReduceCount); + std::vector hMax64(kReduceCount); + std::vector hAlltoallDst(static_cast(nRanks) * kAlltoallBytes); + ok &= check_cuda(cudaMemcpyAsync(hSum32.data(), sum32, kReduceCount * sizeof(int32_t), + cudaMemcpyDeviceToHost, stream), + "copy int32 sum", rank); + ok &= check_cuda(cudaMemcpyAsync(hMin32.data(), min32, kReduceCount * sizeof(int32_t), + cudaMemcpyDeviceToHost, stream), + "copy int32 min", rank); + ok &= check_cuda(cudaMemcpyAsync(hMax32.data(), max32, kReduceCount * sizeof(int32_t), + cudaMemcpyDeviceToHost, stream), + "copy int32 max", rank); + ok &= check_cuda(cudaMemcpyAsync(hSum64.data(), sum64, kReduceCount * sizeof(int64_t), + cudaMemcpyDeviceToHost, stream), + "copy int64 sum", rank); + ok &= check_cuda(cudaMemcpyAsync(hMin64.data(), min64, kReduceCount * sizeof(int64_t), + cudaMemcpyDeviceToHost, stream), + "copy int64 min", rank); + ok &= check_cuda(cudaMemcpyAsync(hMax64.data(), max64, kReduceCount * sizeof(int64_t), + cudaMemcpyDeviceToHost, stream), + "copy int64 max", rank); + ok &= check_cuda(cudaMemcpyAsync(hAlltoallDst.data(), alltoallDst, + hAlltoallDst.size(), cudaMemcpyDeviceToHost, stream), + "copy alltoall result", rank); + ok &= check_cuda(cudaStreamSynchronize(stream), "collective result copies", rank); + + for (size_t i = 0; i < kReduceCount; ++i) { + const int32_t expectedSum32 = + nRanks * (nRanks + 1) / 2 + nRanks * static_cast(i); + const int32_t expectedMin32 = 1 + static_cast(i); + const int32_t expectedMax32 = nRanks + static_cast(i); + const int64_t expectedSum64 = static_cast(100) * nRanks * (nRanks + 1) / 2 + + nRanks * static_cast(i); + const int64_t expectedMin64 = 100 + static_cast(i); + const int64_t expectedMax64 = static_cast(100 * nRanks) + i; + if (hSum32[i] != expectedSum32 || hMin32[i] != expectedMin32 || + hMax32[i] != expectedMax32 || hSum64[i] != expectedSum64 || + hMin64[i] != expectedMin64 || hMax64[i] != expectedMax64) { + std::fprintf(stderr, "PE %d: reduction result mismatch at element %zu\n", rank, i); + ok = false; + } + } + for (int source = 0; source < nRanks; ++source) { + for (size_t byte = 0; byte < kAlltoallBytes; ++byte) { + const unsigned char expected = + static_cast((31 * source + 7 * rank + byte) % 127); + if (hAlltoallDst[static_cast(source) * kAlltoallBytes + byte] != expected) { + std::fprintf(stderr, "PE %d: alltoall result mismatch from PE %d at byte %zu\n", + rank, source, byte); + ok = false; + } + } + } + } + + nvshmem_free(blockStatus); + nvshmem_free(alltoallDst); + nvshmem_free(alltoallSrc); + nvshmem_free(max64); + nvshmem_free(min64); + nvshmem_free(sum64); + nvshmem_free(src64); + nvshmem_free(max32); + nvshmem_free(min32); + nvshmem_free(sum32); + nvshmem_free(src32); + if (blockFailure != nullptr) cudaFree(blockFailure); + if (stream != nullptr) cudaStreamDestroy(stream); + nvshmem_finalize(); + + int localFailure = ok ? 0 : 1; + int globalFailure = 0; + MPI_Allreduce(&localFailure, &globalFailure, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + MPI_Finalize(); + return globalFailure; +} diff --git a/projects/rccl/contrib/niin/test/niin_test.cu b/projects/rccl/contrib/niin/test/niin_test.cu index 8ecaac37f9fe..2227fe68be9d 100644 --- a/projects/rccl/contrib/niin/test/niin_test.cu +++ b/projects/rccl/contrib/niin/test/niin_test.cu @@ -1033,6 +1033,234 @@ __global__ void test_misaligned_block_get(niinContext* ctx, TestResult* result, // ============================================================================ // Main: run all tests +// ============================================================================ +// Test 17h: TMA-backed RMA (nvshmemx_give_smem / release_smem) +// +// Every kernel here works whether or not TMA is active: below sm_90, or with +// NVSHMEM_TMA_POLICY unset, give_smem is a no-op and the same transfers take +// the vectorized load/store path. What the tests assert is that the data lands +// correctly either way. +// ============================================================================ +#define NIIN_TEST_TMA_SRC 327680 // 64 KB source +#define NIIN_TEST_TMA_DST 393216 // 64 KB block-put destination +#define NIIN_TEST_TMA_TDST 458752 // 8 KB thread-scoped put destination +#define NIIN_TEST_TMA_SDST 466944 // 4 KB smem-sourced put destination +#define NIIN_TEST_TMA_UDST 471040 // unaligned fallback destination + +// int4 element type pins the dynamic shared memory base to 16-byte alignment, +// which nvshmemx_give_smem() requires. +extern __shared__ int4 niinTestSmem[]; + +__device__ __forceinline__ char* niinTestSmemBase() { + return reinterpret_cast(niinTestSmem); +} + +__global__ void test_tma_setup(niinContext* ctx) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + int pe = nvshmem_my_pe(); + char* heap = (char*)niin_heap_base(); + int* src = (int*)(heap + NIIN_TEST_TMA_SRC); + for (int i = threadIdx.x; i < 16384; i += blockDim.x) src[i] = pe * 900000 + i; + + // Zero the regions our peer will write into, so a dropped transfer shows up. + int* dst = (int*)(heap + NIIN_TEST_TMA_DST); + for (int i = threadIdx.x; i < 16384; i += blockDim.x) dst[i] = 0; + int* tdst = (int*)(heap + NIIN_TEST_TMA_TDST); + for (int i = threadIdx.x; i < 2048; i += blockDim.x) tdst[i] = 0; + int* sdst = (int*)(heap + NIIN_TEST_TMA_SDST); + for (int i = threadIdx.x; i < 1024; i += blockDim.x) sdst[i] = 0; + char* udst = heap + NIIN_TEST_TMA_UDST + 3; + for (int i = threadIdx.x; i < 1000; i += blockDim.x) udst[i] = 0; +} + +// Block-scoped 64 KB put: the double-buffered, warp-specialized staging path. +__global__ void test_tma_put_block(niinContext* ctx, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + nvshmemx_give_smem(niinTestSmemBase(), nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM)); + __syncthreads(); + + int pe = niin_device_my_pe(); + int peer = (pe + 1) % npes; + int* src = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_SRC); + int* dst = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_DST); + nvshmemx_int_put_block(dst, src, 16384, peer); + nvshmem_quiet(); + + __syncthreads(); + nvshmemx_release_smem(); + __syncthreads(); +} + +__global__ void test_tma_put_block_verify(niinContext* ctx, TestResult* result, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + if (threadIdx.x == 0) { + int sender = (nvshmem_my_pe() - 1 + npes) % npes; + int* dst = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_DST); + bool ok = true; + for (int i = 0; i < 16384; i++) { + if (dst[i] != sender * 900000 + i) { ok = false; break; } + } + if (ok) recordPass(result); + else recordFail(result, "TMA block put wrong data"); + } +} + +// Thread-scoped 8 KB put: the single-issuer staging path. Only one thread in +// the CTA may have a staged transfer in flight, so only thread 0 issues. +__global__ void test_tma_put_thread(niinContext* ctx, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + nvshmemx_give_smem(niinTestSmemBase(), nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM)); + __syncthreads(); + + if (threadIdx.x == 0) { + int pe = niin_device_my_pe(); + int peer = (pe + 1) % npes; + int* src = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_SRC); + int* dst = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_TDST); + nvshmem_int_put(dst, src, 2048, peer); + nvshmem_quiet(); + } + __syncthreads(); + nvshmemx_release_smem(); + __syncthreads(); +} + +__global__ void test_tma_put_thread_verify(niinContext* ctx, TestResult* result, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + if (threadIdx.x == 0) { + int sender = (nvshmem_my_pe() - 1 + npes) % npes; + int* dst = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_TDST); + bool ok = true; + for (int i = 0; i < 2048; i++) { + if (dst[i] != sender * 900000 + i) { ok = false; break; } + } + if (ok) recordPass(result); + else recordFail(result, "TMA thread put wrong data"); + } +} + +// Put sourced from shared memory: no staging, straight smem -> peer gmem. +__global__ void test_tma_put_from_smem(niinContext* ctx, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + nvshmemx_give_smem(niinTestSmemBase(), nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM)); + __syncthreads(); + + int pe = niin_device_my_pe(); + int peer = (pe + 1) % npes; + int* stage = (int*)(niinTestSmemBase() + NIIN_SMEM_DATA_REGION_OFFSET); + for (int i = threadIdx.x; i < 1024; i += blockDim.x) stage[i] = pe * 700000 + i; + __syncthreads(); + // Publish the generic shared-memory stores to the async proxy before TMA + // reads them. NIIN inherits this caller obligation from NVSHMEM. + niin_tma_fence_proxy_async_shared_cta(); + + int* dst = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_SDST); + nvshmemx_int_put_block(dst, stage, 1024, peer); + nvshmem_quiet(); + + __syncthreads(); + nvshmemx_release_smem(); + __syncthreads(); +} + +__global__ void test_tma_put_from_smem_verify(niinContext* ctx, TestResult* result, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + if (threadIdx.x == 0) { + int sender = (nvshmem_my_pe() - 1 + npes) % npes; + int* dst = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_SDST); + bool ok = true; + for (int i = 0; i < 1024; i++) { + if (dst[i] != sender * 700000 + i) { ok = false; break; } + } + if (ok) recordPass(result); + else recordFail(result, "TMA smem-sourced put wrong data"); + } +} + +// Get landing directly in shared memory: peer gmem -> smem, no staging. +__global__ void test_tma_get_to_smem(niinContext* ctx, TestResult* result, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + nvshmemx_give_smem(niinTestSmemBase(), nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM)); + __syncthreads(); + + int pe = niin_device_my_pe(); + int peer = (pe + 1) % npes; + int* remoteSrc = (int*)((char*)niin_heap_base() + NIIN_TEST_TMA_SRC); + int* landing = (int*)(niinTestSmemBase() + NIIN_SMEM_DATA_REGION_OFFSET); + nvshmemx_int_get_block(landing, remoteSrc, 1024, peer); + __syncthreads(); + + if (threadIdx.x == 0) { + bool ok = true; + for (int i = 0; i < 1024; i++) { + if (landing[i] != peer * 900000 + i) { ok = false; break; } + } + if (ok) recordPass(result); + else recordFail(result, "TMA get into smem wrong data"); + } + __syncthreads(); + nvshmemx_release_smem(); + __syncthreads(); +} + +// Unaligned transfer while smem is registered: TMA declines it and the copy +// must still land correctly through the load/store fallback. +__global__ void test_tma_fallback(niinContext* ctx, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + nvshmemx_give_smem(niinTestSmemBase(), nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM)); + __syncthreads(); + + int pe = niin_device_my_pe(); + int peer = (pe + 1) % npes; + char* src = (char*)niin_heap_base() + NIIN_TEST_TMA_SRC + 3; + char* dst = (char*)niin_heap_base() + NIIN_TEST_TMA_UDST + 3; + nvshmemx_putmem_block(dst, src, 1000, peer); + nvshmem_quiet(); + + __syncthreads(); + nvshmemx_release_smem(); + __syncthreads(); +} + +__global__ void test_tma_fallback_verify(niinContext* ctx, TestResult* result, int npes) { + if (threadIdx.x == 0) niin_g_ctx = ctx; + __syncthreads(); + + if (threadIdx.x == 0) { + int sender = (nvshmem_my_pe() - 1 + npes) % npes; + const char* dst = (const char*)niin_heap_base() + NIIN_TEST_TMA_UDST + 3; + bool ok = true; + for (int i = 0; i < 1000; i++) { + // The sender's source region holds int-sized values sender*900000 + idx, + // read back here one byte at a time starting 3 bytes in. + int byteOff = 3 + i; + int word = sender * 900000 + (byteOff / 4); + char want = (char)((word >> (8 * (byteOff % 4))) & 0xFF); + if (dst[i] != want) { ok = false; break; } + } + if (ok) recordPass(result); + else recordFail(result, "TMA fallback (unaligned) wrong data"); + } +} + // ============================================================================ void printResult(const char* name, TestResult& r) { @@ -1078,15 +1306,33 @@ int main() { } // Step 3: Initialize NIIN (two-phase: NCCL collectives, then device copy) - niinContext_host hostCtxs[2]; + niinContext_host hostCtxs[2] = {}; NCCLCHECK(ncclGroupStart()); for (int i = 0; i < nPes; i++) { CUDACHECK(cudaSetDevice(i)); + // niinInit takes enableGin before the GIN tuning args; passing only three + // trailing values silently bound 0 to enableGin and disabled GIN entirely. NCCLCHECK(niinInit(comms[i], heapBufs[i], heapSize, &hostCtxs[i], - /*ginContextIndex=*/0, /*barrierCount=*/1, /*ginSignalCount=*/0)); + /*enableGin=*/true, /*barrierCount=*/1, + /*ginSignalCount=*/0)); } NCCLCHECK(ncclGroupEnd()); + // TMA needs sm_90 or newer. Turn it on where available so the TMA tests + // exercise cp.async.bulk; elsewhere give_smem is inert and those same tests + // cover the load/store fallback. + int capMajor = 0; + CUDACHECK(cudaDeviceGetAttribute(&capMajor, cudaDevAttrComputeCapabilityMajor, 0)); + bool tmaEnabled = capMajor >= 9; + if (tmaEnabled) { + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + NCCLCHECK(niinTmaEnable(&hostCtxs[i], NVSHMEMX_TMA_ENABLE)); + } + } + printf("TMA: %s\n\n", tmaEnabled ? "enabled (sm_90+)" + : "unavailable below sm_90 — TMA tests cover the fallback"); + // Phase 2: copy to device (after groupEnd so NCCL results are finalized) for (int i = 0; i < nPes; i++) { CUDACHECK(cudaSetDevice(i)); @@ -1470,6 +1716,102 @@ int main() { syncAll(); reportTest("Misaligned block get"); + // Test 17h: TMA-backed RMA. give_smem needs the whole CTA, and the + // double-buffered staging path needs at least two warps, so launch 128 + // threads with NVSHMEMX_SMEM_MINIMUM (32 KB) of dynamic shared memory. + // 32 KB stays under the 48 KB static cap, so no cudaFuncSetAttribute opt-in + // is needed; NVSHMEMX_SMEM_RECOMMENDED (64 KB) would require one. + { + const int tmaSmem = nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM); + + clearResults(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_setup<<<1, 128, 0, streams[i]>>>(ctxs[i]); + } + syncAll(); + + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_put_block<<<1, 128, tmaSmem, streams[i]>>>(ctxs[i], nPes); + } + syncAll(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_put_block_verify<<<1, 1, 0, streams[i]>>>(ctxs[i], results[i], nPes); + } + syncAll(); + reportTest("TMA block put (64KB)"); + + clearResults(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_put_thread<<<1, 128, tmaSmem, streams[i]>>>(ctxs[i], nPes); + } + syncAll(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_put_thread_verify<<<1, 1, 0, streams[i]>>>(ctxs[i], results[i], nPes); + } + syncAll(); + reportTest("TMA thread put (8KB)"); + + clearResults(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_put_from_smem<<<1, 128, tmaSmem, streams[i]>>>(ctxs[i], nPes); + } + syncAll(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_put_from_smem_verify<<<1, 1, 0, streams[i]>>>(ctxs[i], results[i], nPes); + } + syncAll(); + reportTest("TMA put from shared memory"); + + clearResults(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_get_to_smem<<<1, 128, tmaSmem, streams[i]>>>(ctxs[i], results[i], nPes); + } + syncAll(); + reportTest("TMA get into shared memory"); + + clearResults(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_fallback<<<1, 128, tmaSmem, streams[i]>>>(ctxs[i], nPes); + } + syncAll(); + for (int i = 0; i < nPes; i++) { + CUDACHECK(cudaSetDevice(i)); + test_tma_fallback_verify<<<1, 1, 0, streams[i]>>>(ctxs[i], results[i], nPes); + } + syncAll(); + reportTest("TMA unaligned fallback"); + } + + // Test 17i: nvshmemx_ask_smem on the host + { + TestResult askResult = {}; + int recommended = nvshmemx_ask_smem(NVSHMEMX_SMEM_RECOMMENDED); + int minimum = nvshmemx_ask_smem(NVSHMEMX_SMEM_MINIMUM); + int barriers = nvshmemx_ask_smem(NVSHMEMX_SMEM_BARRIERS_ONLY); + + if (recommended > minimum && minimum > barriers) askResult.passed++; + else { askResult.failed++; snprintf(askResult.failMsg, 256, + "ask_smem not ordered: %d %d %d", recommended, minimum, barriers); } + + if (barriers == NIIN_SMEM_DATA_REGION_OFFSET) askResult.passed++; + else { askResult.failed++; snprintf(askResult.failMsg, 256, + "ask_smem(BARRIERS_ONLY) got %d expected %d", + barriers, NIIN_SMEM_DATA_REGION_OFFSET); } + + printResult("nvshmemx_ask_smem [host]", askResult); + totalPassed += askResult.passed; + totalFailed += askResult.failed; + } + // Test 18: Team operations (host-side) { TestResult teamResult = {}; @@ -1577,6 +1919,7 @@ int main() { for (int i = 0; i < nPes; i++) { CUDACHECK(cudaSetDevice(i)); niinFinalize(comms[i], ctxs[i]); + niinTmaDisable(&hostCtxs[i]); } for (int i = 0; i < nPes; i++) { diff --git a/projects/rccl/contrib/niin/test/run_all.sh b/projects/rccl/contrib/niin/test/run_all.sh index af62f3666c1f..85f39b6859f9 100755 --- a/projects/rccl/contrib/niin/test/run_all.sh +++ b/projects/rccl/contrib/niin/test/run_all.sh @@ -38,6 +38,9 @@ RUN_IB=false BUILD_ONLY=false SKIP_PERF=false SKIP_FUNC=false +HOST_COLLECTIVES_NAME="host_collectives" +HOST_COLLECTIVES_BUILD_FAILED=0 +HOST_COLLECTIVES_RUNTIME_FAILED=0 # Auto-detect arch if [ -z "${ARCH:-}" ]; then @@ -89,7 +92,8 @@ NVCC_FLAGS=" -Xlinker -rpath,$NCCL_HOME/build/lib -Xlinker -rpath,$MPI_HOME/lib" -MPI_RUN="mpirun -np $NP --allow-run-as-root" +MPI_RUN="$MPI_HOME/bin/mpirun -np $NP --allow-run-as-root" +TEST_LD_LIBRARY_PATH="$NCCL_HOME/build/lib:$MPI_HOME/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" BUILD_DIR="/tmp/niin_test_suite" # =========================================================================== @@ -157,6 +161,7 @@ mkdir -p "$BUILD_DIR" # Discover NVSHMEM tests NVSHMEM_TESTS=() +NIIN_TESTS=("$SCRIPT_DIR/host_collectives.cu") if [ -d "$NVSHMEM_HOME/test" ]; then for f in "$NVSHMEM_HOME"/test/device/query/*.cu \ "$NVSHMEM_HOME"/test/device/sync/*.cu; do @@ -173,7 +178,7 @@ fi # Build functional tests BUILT_FUNC=() echo "test,build_status,build_error" > "$RESULTS_DIR/build_results.csv" -for src in "${NVSHMEM_TESTS[@]}"; do +for src in "${NIIN_TESTS[@]}" "${NVSHMEM_TESTS[@]}"; do name=$(basename ${src%.cu}) output=$(nvcc "$src" -o "$BUILD_DIR/$name" $NVCC_FLAGS 2>&1) if [ $? -eq 0 ]; then @@ -184,10 +189,19 @@ for src in "${NVSHMEM_TESTS[@]}"; do echo -n "x" err=$(echo "$output" | head -1 | tr ',' ';' | cut -c1-200) echo "$name,FAIL,$err" >> "$RESULTS_DIR/build_results.csv" + if [ "$name" = "$HOST_COLLECTIVES_NAME" ]; then + HOST_COLLECTIVES_BUILD_FAILED=1 + fi fi done echo -echo "Built ${#BUILT_FUNC[@]} of ${#NVSHMEM_TESTS[@]} functional tests" +TOTAL_FUNC_TESTS=$((${#NIIN_TESTS[@]} + ${#NVSHMEM_TESTS[@]})) +echo "Built ${#BUILT_FUNC[@]} of ${TOTAL_FUNC_TESTS} functional tests" + +if [ "$HOST_COLLECTIVES_BUILD_FAILED" -ne 0 ]; then + echo "ERROR: required $HOST_COLLECTIVES_NAME build failed" + exit 1 +fi # Build perftests make -C "$NIIN_DIR/perftest" \ @@ -200,6 +214,11 @@ if $BUILD_ONLY; then exit 0 fi +if ! $SKIP_FUNC && [ "$NP" -lt 2 ]; then + echo "ERROR: $HOST_COLLECTIVES_NAME requires NP >= 2" + exit 1 +fi + # =========================================================================== # Run functional tests # =========================================================================== @@ -210,7 +229,8 @@ run_func_tests() { local pass=0 xpass=0 xfail=0 fail=0 for name in "${BUILT_FUNC[@]}"; do - output=$(timeout $TIMEOUT_FUNC env CUDA_VISIBLE_DEVICES=$GPU_PAIR $env_prefix \ + output=$(timeout $TIMEOUT_FUNC env CUDA_VISIBLE_DEVICES=$GPU_PAIR \ + LD_LIBRARY_PATH="$TEST_LD_LIBRARY_PATH" $env_prefix \ $MPI_RUN "$BUILD_DIR/$name" 2>&1) rc=$? errors=$(echo "$output" | grep -i "error\|incorrect\|FAIL\|trap\|Segmentation" | \ @@ -242,6 +262,10 @@ run_func_tests() { fail=$((fail+1)) fi fi + + if [ "$name" = "$HOST_COLLECTIVES_NAME" ] && { [ $rc -ne 0 ] || [ -n "$errors" ]; }; then + HOST_COLLECTIVES_RUNTIME_FAILED=1 + fi done echo @@ -261,7 +285,8 @@ run_perf_tests() { # Put latency if [ -f "$perf_dir/shmem_put_latency" ]; then echo " put_latency..." - output=$(timeout $TIMEOUT_PERF env CUDA_VISIBLE_DEVICES=$GPU_PAIR $env_prefix \ + output=$(timeout $TIMEOUT_PERF env CUDA_VISIBLE_DEVICES=$GPU_PAIR \ + LD_LIBRARY_PATH="$TEST_LD_LIBRARY_PATH" $env_prefix \ $MPI_RUN "$perf_dir/shmem_put_latency" -b 4 -e 65536 -i 200 -w 50 2>&1) scope="" echo "$output" | while IFS= read -r line; do @@ -281,7 +306,8 @@ run_perf_tests() { local heap_env="" if [ "$mode" = "ib" ]; then heap_env="NVSHMEM_SYMMETRIC_SIZE=1073741824"; fi output=$(timeout $TIMEOUT_PERF env CUDA_VISIBLE_DEVICES=$GPU_PAIR \ - NVSHMEM_SYMMETRIC_SIZE=1073741824 $env_prefix \ + NVSHMEM_SYMMETRIC_SIZE=1073741824 \ + LD_LIBRARY_PATH="$TEST_LD_LIBRARY_PATH" $env_prefix \ $MPI_RUN "$perf_dir/shmem_put_bw" -b 1024 -e 67108864 -i 50 -w 10 -n 16 -t 256 2>&1) scope="" echo "$output" | while IFS= read -r line; do @@ -298,7 +324,8 @@ run_perf_tests() { # Get latency if [ -f "$perf_dir/shmem_get_latency" ]; then echo " get_latency..." - output=$(timeout $TIMEOUT_PERF env CUDA_VISIBLE_DEVICES=$GPU_PAIR $env_prefix \ + output=$(timeout $TIMEOUT_PERF env CUDA_VISIBLE_DEVICES=$GPU_PAIR \ + LD_LIBRARY_PATH="$TEST_LD_LIBRARY_PATH" $env_prefix \ $MPI_RUN "$perf_dir/shmem_get_latency" -b 4 -e 65536 -i 200 -w 50 2>&1) scope="" echo "$output" | while IFS= read -r line; do @@ -315,7 +342,8 @@ run_perf_tests() { # Put signal latency if [ -f "$perf_dir/shmem_put_signal_latency" ]; then echo " put_signal_latency..." - output=$(timeout $TIMEOUT_PERF env CUDA_VISIBLE_DEVICES=$GPU_PAIR $env_prefix \ + output=$(timeout $TIMEOUT_PERF env CUDA_VISIBLE_DEVICES=$GPU_PAIR \ + LD_LIBRARY_PATH="$TEST_LD_LIBRARY_PATH" $env_prefix \ $MPI_RUN "$perf_dir/shmem_put_signal_latency" -b 4 -e 65536 -i 200 -w 50 2>&1) scope="" echo "$output" | while IFS= read -r line; do @@ -395,3 +423,8 @@ echo "Results in: $RESULTS_DIR/" ls -la "$RESULTS_DIR"/*.csv echo echo "=== Done ===" + +if [ "$HOST_COLLECTIVES_RUNTIME_FAILED" -ne 0 ]; then + echo "ERROR: required $HOST_COLLECTIVES_NAME runtime test failed" + exit 1 +fi diff --git a/projects/rccl/contrib/pace/csrc/bindings.cpp b/projects/rccl/contrib/pace/csrc/bindings.cpp index 8e018eeab5b1..beff0483c004 100644 --- a/projects/rccl/contrib/pace/csrc/bindings.cpp +++ b/projects/rccl/contrib/pace/csrc/bindings.cpp @@ -69,4 +69,4 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { .def("out_numel_alignment", &pace::RSComm::out_numel_alignment) .def("reduce_scatter", &pace::RSComm::ReduceScatter, py::call_guard()); -} \ No newline at end of file +} diff --git a/projects/rccl/contrib/pace/csrc/collective/ag/ag_zero_sm.cu b/projects/rccl/contrib/pace/csrc/collective/ag/ag_zero_sm.cu index 43d37a5cb1f1..0107a2a5c520 100644 --- a/projects/rccl/contrib/pace/csrc/collective/ag/ag_zero_sm.cu +++ b/projects/rccl/contrib/pace/csrc/collective/ag/ag_zero_sm.cu @@ -939,7 +939,7 @@ static void execute_ag_zero_sm_core(uint64_t* args_cpu, int num_tensors, ncclWindow_t gin_win, void *gin_buffer, void *signal_buffer, void **p2p_ptrs, int unroll, int nvl_ring, int rdma_ring, size_t base_slot, int send_times, int capture_round_n, int rank, int num_local_ranks, int num_ranks, CUstreamBatchMemOpParams *mparam, ncclDevComm dev_comm, cudaStream_t stream, cudaStream_t copy_stream, bool launch_cord_kernel, bool use_ring) { - + const bool is_single_node = (num_local_ranks == num_ranks); // Cord kernel (multi-node) drives RDMA signalling on `stream`, ahead of the @@ -1117,4 +1117,4 @@ static void execute_ag_zero_sm_without_graph(uint64_t* args_cpu, int num_tensors total_send_slots += send_times; } -} \ No newline at end of file +} diff --git a/projects/rccl/contrib/pace/csrc/collective/ag/agcomm.cpp b/projects/rccl/contrib/pace/csrc/collective/ag/agcomm.cpp index 002c289dcdd6..68b494e318cf 100644 --- a/projects/rccl/contrib/pace/csrc/collective/ag/agcomm.cpp +++ b/projects/rccl/contrib/pace/csrc/collective/ag/agcomm.cpp @@ -135,11 +135,11 @@ std::tuple, std::optional> AGComm::All stream_wait(launch_stream, compute_stream); } } - + // Handle output tensors: use provided tensors or allocate new ones // Output pointers are stored in args_cpu[4*i+2] std::optional result = std::nullopt; // Only set when output_tensors is not provided - + if (output_tensors.has_value()) { // Use provided output tensors - store their pointers in args_cpu const auto& out_tensors = output_tensors.value(); @@ -232,4 +232,4 @@ std::tuple, std::optional> AGComm::All return {std::move(result), event}; } -} \ No newline at end of file +} diff --git a/projects/rccl/contrib/pace/csrc/collective/rs/rscomm.cpp b/projects/rccl/contrib/pace/csrc/collective/rs/rscomm.cpp index a825af9cab6c..f6a4f088452d 100644 --- a/projects/rccl/contrib/pace/csrc/collective/rs/rscomm.cpp +++ b/projects/rccl/contrib/pace/csrc/collective/rs/rscomm.cpp @@ -171,4 +171,4 @@ std::tuple> RSComm::ReduceScatter(cons return {result, event}; } -} \ No newline at end of file +} diff --git a/projects/rccl/contrib/pace/csrc/collective/sg/sg.cu b/projects/rccl/contrib/pace/csrc/collective/sg/sg.cu index 0cb11658b09e..47af03301354 100644 --- a/projects/rccl/contrib/pace/csrc/collective/sg/sg.cu +++ b/projects/rccl/contrib/pace/csrc/collective/sg/sg.cu @@ -35,7 +35,7 @@ namespace pace { /** * sg.cu - Scatter-Gather (All-to-All) communication implementation - * + * * Design follows ag.cu single-stream pattern: * copy_stream: all CE ops — c1 (input→send buf), c2 (peer NVL send→output), * c3 (RDMA recv→output) — issued in order per slot, with c3 @@ -194,13 +194,13 @@ class sg_device_mem_layout { uint8_t *base_ptr; // Local GPU's buffer base pointer const size_t slot_bytes; const int nvl_ring, num_local_ranks, local_rank, node_id, num_nodes; - + public: - __device__ sg_device_mem_layout(void *gin_win_ptr, int nvl_ring_, int rdma_ring_, size_t slot_bytes_, + __device__ sg_device_mem_layout(void *gin_win_ptr, int nvl_ring_, int rdma_ring_, size_t slot_bytes_, int rank, int num_local_ranks_, int num_ranks) - : base_ptr(reinterpret_cast(gin_win_ptr)), + : base_ptr(reinterpret_cast(gin_win_ptr)), slot_bytes(slot_bytes_), - nvl_ring(nvl_ring_), + nvl_ring(nvl_ring_), num_local_ranks(num_local_ranks_), local_rank(rank % num_local_ranks_), node_id(rank / num_local_ranks_), @@ -213,25 +213,25 @@ public: const int dst_node = dst_rank / num_local_ranks; const int dest_local = dst_rank % num_local_ranks; return reinterpret_cast(base_ptr + blockIdx.x * block_send_area + - dst_node * nvl_ring * batch_slot_bytes + + dst_node * nvl_ring * batch_slot_bytes + batch_slot_bytes * (slot % nvl_ring) + slot_bytes * dest_local); } - + // c2: Read from NVL peer's send buffer (per-block region) __device__ __forceinline__ float4* peer_send_ptr(void **p2p_ptrs, const int& src_local, const size_t& slot) { const size_t batch_slot_bytes = slot_bytes * num_local_ranks; const size_t block_send_area = batch_slot_bytes * nvl_ring * num_nodes; - return reinterpret_cast(reinterpret_cast(p2p_ptrs[src_local]) + + return reinterpret_cast(reinterpret_cast(p2p_ptrs[src_local]) + blockIdx.x * block_send_area + node_id * nvl_ring * batch_slot_bytes + batch_slot_bytes * (slot % nvl_ring) + slot_bytes * local_rank); } - + // c3: Read from RDMA recv buffer (per-block region) - only used in multi-node __device__ __forceinline__ float4* rdma_recv_ptr(void **p2p_ptrs, const int& src_node, const int& nvl_peer, const size_t& slot, const int rdma_ring) { const size_t batch_slot_bytes = slot_bytes * num_local_ranks; const size_t block_send_area = batch_slot_bytes * nvl_ring * num_nodes; const size_t block_recv_area = batch_slot_bytes * rdma_ring * num_nodes; - return reinterpret_cast(reinterpret_cast(p2p_ptrs[nvl_peer]) + + return reinterpret_cast(reinterpret_cast(p2p_ptrs[nvl_peer]) + block_send_area * gridDim.x + blockIdx.x * block_recv_area + batch_slot_bytes * (rdma_ring * src_node + (slot % rdma_ring)) + slot_bytes * local_rank); } @@ -243,7 +243,7 @@ public: return blockIdx.x * block_send_area + dst_node * nvl_ring * batch_slot_bytes + batch_slot_bytes * (slot % nvl_ring); } - + __device__ __forceinline__ uint64_t rdma_recv_offset(const int& src_node, const size_t& slot, const int rdma_ring) { const size_t batch_slot_bytes = slot_bytes * num_local_ranks; const size_t block_send_area = batch_slot_bytes * nvl_ring * num_nodes; @@ -309,18 +309,18 @@ scattergather_kernel( __shared__ int prefix_f4[SG_MAX_TENSORS]; // Cumulative f4 count for input __shared__ int tensor_f4[SG_MAX_TENSORS]; // Per-tensor f4 count for input __shared__ uint64_t cached_args[SG_MAX_TENSORS * 8]; // Cached args from global memory - + DEVICE_ASSERT(nargs <= SG_MAX_TENSORS); - + // Load args into shared memory (8 uint64_t per tensor) // Each thread loads multiple elements to cover nargs * 8 elements for (int i = threadIdx.x; i < nargs * 8; i += blockDim.x) { cached_args[i] = args[i]; } - + // Sync after loading cached_args __syncthreads(); - + // Initialize tensor metadata from cached args (now in shared memory) // Format: [in_ptr, in_X, in_Y, in_Z, out_ptr, out_X, out_Y, out_Z] per tensor // Z is already in bytes, so f4_count = X * Z / 16 (where 16 = sizeof(float4)) @@ -335,18 +335,18 @@ scattergather_kernel( prefix_f4[threadIdx.x] = f4_count; // No extra alignment needed } __syncthreads(); - + // Compute prefix sums if ((threadIdx.x >> 5) == 0) { hillis_steele_sum(prefix_f4, nargs, threadIdx.x); } __syncthreads(); - + // Calculate work distribution const int total_f4 = prefix_f4[nargs - 1]; const size_t slot_bytes = rdma_unroll * blockDim.x * sizeof(float4); const int slot_f4 = static_cast(rdma_unroll * blockDim.x); - + int smf4start, smf4end; // Divide work in float8 (2×float4) units so smf4start is always even (32-byte aligned), // enabling the float8 vectorized path without misaligned address errors. @@ -359,10 +359,10 @@ scattergather_kernel( } else { get_work_range(total_f4, num_sms, sm_id, smf4start, smf4end); } - + const int unrolled_times = (smf4end - smf4start) / slot_f4; const int tailf4 = (smf4end - smf4start) - unrolled_times * slot_f4; - + // Total segments per dst_local: each dst_local processes seg_batch slots const int seg_batch = unrolled_times + (tailf4 > 0); @@ -388,7 +388,7 @@ scattergather_kernel( const uint64_t gstart_id = __ldg(reinterpret_cast(c1_gstart_ptr())); __syncthreads(); // ensure all setup (cached_args, prefix_f4) is visible before warp groups diverge - + // ===== 4-path signal layout (per-SM) ===== // Offsets in u64 from sig_ptr(peer). The gstart slot counter occupies // [2*Nlr*num_sms, 2*Nlr*num_sms + num_sms) (see c1_gstart_ptr above), so @@ -430,13 +430,13 @@ scattergather_kernel( auto c3_ack_ptr = [&](const int& N) -> uint64_t* { return reinterpret_cast(sig_ptr(local_rank)) + sig_base_c3ack + (size_t)sm_id * num_nodes + N; }; - + // GIN signal indices for multi-node auto rdma_ready_sig_w_index = [&]() { return static_cast(sm_id * num_nodes + node_id); }; auto rdma_ready_sig_r_index = [&](const int& src_node) { return static_cast(sm_id * num_nodes + src_node); }; auto rdma_picked_sig_w_index = [&]() { return static_cast(num_sms * num_nodes + sm_id * num_nodes + node_id); }; auto rdma_picked_sig_r_index = [&](const int& dst_node) { return static_cast(num_sms * num_nodes + sm_id * num_nodes + dst_node); }; - + // ===== Push data accessors (per-SM sliced) ===== // Send buf layout within each GPU's gin_win: [sm][dst_node][R slot][src_local]. // c1 NVL-store target: write to forwarder (dst_local)'s send buf, sub-slot = local_rank. @@ -831,7 +831,7 @@ scattergather_kernel( } } __syncthreads(); - + // Update global slot counter if (threadIdx.x == 0) { st_na_global(c1_gstart_ptr(), static_cast(gstart_id + seg_batch)); @@ -994,23 +994,23 @@ void scattergatherfunc(std::vector layouts, std::vector layouts, std::vector>>(gin_win, signal_buffer, unroll, nvl_ring, rdma_ring, total_send_slots, send_times, capture_round_n, rank, num_local_ranks, num_ranks, dev_comm); CUDACHECK(cudaGetLastError()); } - + // Track position for flat iteration int tidx = 0; - + const bool debug = false; // debug output off by default size_t common_bytes, most_blocks, most_massive, src_stride, dst_stride; @@ -1190,7 +1190,7 @@ void scattergatherfunc(std::vector layouts, std::vector(signal_buffer); - - + + // Launch kernel const size_t rdma_unroll = static_cast(unroll); int flat_mode = 3; diff --git a/projects/rccl/contrib/pace/csrc/collective/sg/sgcomm.cpp b/projects/rccl/contrib/pace/csrc/collective/sg/sgcomm.cpp index 788f6116643e..6a7869ddb61c 100644 --- a/projects/rccl/contrib/pace/csrc/collective/sg/sgcomm.cpp +++ b/projects/rccl/contrib/pace/csrc/collective/sg/sgcomm.cpp @@ -214,4 +214,4 @@ std::tuple, std::optional> SGComm::Scatt return {results, event}; } -} \ No newline at end of file +} diff --git a/projects/rccl/contrib/pace/csrc/collective/sg/tensor_layout.hpp b/projects/rccl/contrib/pace/csrc/collective/sg/tensor_layout.hpp index 20f50fdb1a1d..1586e1f31024 100644 --- a/projects/rccl/contrib/pace/csrc/collective/sg/tensor_layout.hpp +++ b/projects/rccl/contrib/pace/csrc/collective/sg/tensor_layout.hpp @@ -115,4 +115,4 @@ class TensorLayout { size_t total_bytes; }; -} \ No newline at end of file +} diff --git a/projects/rccl/contrib/pace/csrc/device/launch.cuh b/projects/rccl/contrib/pace/csrc/device/launch.cuh index 1ea0c871737b..cf9deaa65086 100644 --- a/projects/rccl/contrib/pace/csrc/device/launch.cuh +++ b/projects/rccl/contrib/pace/csrc/device/launch.cuh @@ -47,4 +47,4 @@ if (kUseTMA) {\ #endif #endif -#endif \ No newline at end of file +#endif diff --git a/projects/rccl/contrib/pace/csrc/device/sync.cu b/projects/rccl/contrib/pace/csrc/device/sync.cu index 84754217c939..a9c24b12b9eb 100644 --- a/projects/rccl/contrib/pace/csrc/device/sync.cu +++ b/projects/rccl/contrib/pace/csrc/device/sync.cu @@ -30,4 +30,4 @@ void first_sync_func(size_t counts, uint32_t nctxs, struct ncclDevComm devComm, CUDACHECK(cudaGetLastError()); } -} \ No newline at end of file +} diff --git a/projects/rccl/contrib/pace/csrc/runtime/commoncomm.cpp b/projects/rccl/contrib/pace/csrc/runtime/commoncomm.cpp index 6fd645de71db..557fe7ae8452 100644 --- a/projects/rccl/contrib/pace/csrc/runtime/commoncomm.cpp +++ b/projects/rccl/contrib/pace/csrc/runtime/commoncomm.cpp @@ -407,4 +407,4 @@ Comm::~Comm() { } } -} \ No newline at end of file +} diff --git a/projects/rccl/contrib/pace/csrc/runtime/gincomm.hpp b/projects/rccl/contrib/pace/csrc/runtime/gincomm.hpp index 2e4b6c5b367b..6391f331ac58 100644 --- a/projects/rccl/contrib/pace/csrc/runtime/gincomm.hpp +++ b/projects/rccl/contrib/pace/csrc/runtime/gincomm.hpp @@ -168,7 +168,7 @@ class SGComm { // variants (num_sms <= 1) from normal multi-SM kernels. All variants read // strided-S input directly. int num_sms() const { return comm.launch.num_sms; }; - + std::tuple, std::optional> ScatterGather(const std::vector& src, const int scatter_dim, const int gather_dim, std::optional& previous_event, const bool async, const std::optional>& out); }; @@ -202,4 +202,4 @@ class AGComm { // of this contrib. } -#endif \ No newline at end of file +#endif diff --git a/projects/rccl/contrib/pace/csrc/services/event.hpp b/projects/rccl/contrib/pace/csrc/services/event.hpp index ee664a29a9e0..63389eb0e6fe 100644 --- a/projects/rccl/contrib/pace/csrc/services/event.hpp +++ b/projects/rccl/contrib/pace/csrc/services/event.hpp @@ -53,4 +53,4 @@ void stream_wait(const at::cuda::CUDAStream& s, const EventHandle& event); } // namespace pace -#endif \ No newline at end of file +#endif diff --git a/projects/rccl/contrib/pace/pace/ag/__init__.py b/projects/rccl/contrib/pace/pace/ag/__init__.py index b92abeeae689..a8993795d6df 100644 --- a/projects/rccl/contrib/pace/pace/ag/__init__.py +++ b/projects/rccl/contrib/pace/pace/ag/__init__.py @@ -1,2 +1,2 @@ from .agcomm import AGComm -from ..utils import CommConfig \ No newline at end of file +from ..utils import CommConfig diff --git a/projects/rccl/contrib/pace/pace/ag/agcomm.py b/projects/rccl/contrib/pace/pace/ag/agcomm.py index fcac20e8e5a8..b84eca8ef592 100644 --- a/projects/rccl/contrib/pace/pace/ag/agcomm.py +++ b/projects/rccl/contrib/pace/pace/ag/agcomm.py @@ -70,17 +70,17 @@ def all_gather(self, tensor: Union[torch.Tensor, List[torch.Tensor]], out: Optio storage_type, comm_type = None, None if input_dtype_mapping is not None: storage_type, comm_type = input_dtype_mapping - + assert (storage_type == None) == (comm_type == None) if storage_type == comm_type: storage_type = None comm_type = None - + if storage_type is not None: assert self.config.num_sms > 0, "Mapping can only be used with num_sms > 0" assert storage_type in [torch.float32], f'storage_type ({storage_type}) must be in [torch.float32]' assert comm_type in [torch.float32, torch.bfloat16], f'comm_type ({comm_type}) must be in [torch.float32, torch.bfloat16]' - + assert isinstance(tensor, list) or isinstance(tensor, torch.Tensor) list_input = isinstance(tensor, list) if list_input: @@ -94,7 +94,7 @@ def all_gather(self, tensor: Union[torch.Tensor, List[torch.Tensor]], out: Optio if storage_type is not None: for t in tensor: assert t.dtype == storage_type, f"tensor dtype ({t.dtype}) must match mapping requested dtype ({storage_type})" - + # Validate and normalize output tensors if provided output_list = None # The list form passed to C++ single_output = False # Track if user provided a single tensor @@ -107,7 +107,7 @@ def all_gather(self, tensor: Union[torch.Tensor, List[torch.Tensor]], out: Optio else: assert isinstance(out, list), "out must be a Tensor or a list of tensors" output_list = out - + assert len(output_list) == len(tensor), f"output list length ({len(output_list)}) must match input tensor count ({len(tensor)})" for i, (inp, o) in enumerate(zip(tensor, output_list)): expected_numel = inp.numel() * self.num_ranks @@ -127,59 +127,59 @@ def all_gather(self, tensor: Union[torch.Tensor, List[torch.Tensor]], out: Optio def get_split_tensors(self, tensors : Union[torch.Tensor, List[torch.Tensor]], output: torch.Tensor): """ Split output tensor into per-tensor, per-rank slices. - + Output layout (tight, no padding): [tensor0_allgather | tensor1_allgather | ...] Each tensor_i_allgather = [rank0_data | rank1_data | ... | rankN_data] - + Returns: outs: List[List[Tensor]] where outs[i][j] is tensor i's data from rank j """ if isinstance(tensors, torch.Tensor): tensors = [tensors] - + # Get the number of elements for each tensor tensor_numels = [t.numel() for t in tensors] - + # Calculate cumulative offsets for each tensor in output (in elements) tensor_offsets = [0] offset = 0 for numel in tensor_numels: offset += numel * self.num_ranks tensor_offsets.append(offset) - + # outs[i][j] = tensor i's data from rank j - outs = [[output[tensor_offsets[i] + j * tensor_numels[i]:tensor_offsets[i] + j * tensor_numels[i] + tensor_numels[i]] - for j in range(self.num_ranks)] + outs = [[output[tensor_offsets[i] + j * tensor_numels[i]:tensor_offsets[i] + j * tensor_numels[i] + tensor_numels[i]] + for j in range(self.num_ranks)] for i in range(len(tensors))] return outs - + def get_gathered_tensors(self, tensors : Union[torch.Tensor, List[torch.Tensor]], output: torch.Tensor): """ Get complete allgather result for each tensor (keeping num_ranks data contiguous). - + Output layout (tight, no padding): [tensor0_allgather | tensor1_allgather | ...] Each tensor_i_allgather = [rank0_data | rank1_data | ... | rankN_data] - + Returns: outs: List[Tensor] where outs[i] is tensor i's complete allgather result """ if isinstance(tensors, torch.Tensor): tensors = [tensors] - + # Get the number of elements for each tensor tensor_numels = [t.numel() for t in tensors] - + # Calculate cumulative offsets for each tensor in output (in elements) tensor_offsets = [0] offset = 0 for numel in tensor_numels: offset += numel * self.num_ranks tensor_offsets.append(offset) - + # outs[i] = tensor i's complete allgather result (numel * num_ranks elements) outs = [output[tensor_offsets[i]:tensor_offsets[i + 1]] for i in range(len(tensors))] return outs - + def get_comm_slots(self, tensors : Union[torch.Tensor, List[torch.Tensor]]): if isinstance(tensors, torch.Tensor): tensors = [tensors] @@ -188,4 +188,4 @@ def get_comm_slots(self, tensors : Union[torch.Tensor, List[torch.Tensor]]): aligned_sizes = [(s + align - 1) & ~(align - 1) for s in tsizes] total_size = sum(aligned_sizes) slot_bytes = align * 1024 * self.config.slot_unroll - return (total_size + slot_bytes - 1) // slot_bytes \ No newline at end of file + return (total_size + slot_bytes - 1) // slot_bytes diff --git a/projects/rccl/contrib/pace/pace/rs/__init__.py b/projects/rccl/contrib/pace/pace/rs/__init__.py index 491acf09ac7b..4d140f04c70e 100644 --- a/projects/rccl/contrib/pace/pace/rs/__init__.py +++ b/projects/rccl/contrib/pace/pace/rs/__init__.py @@ -1,2 +1,2 @@ from .rscomm import RSComm -from ..utils import CommConfig \ No newline at end of file +from ..utils import CommConfig diff --git a/projects/rccl/contrib/pace/pace/rs/rscomm.py b/projects/rccl/contrib/pace/pace/rs/rscomm.py index d4f912e76f5f..852c51b98681 100644 --- a/projects/rccl/contrib/pace/pace/rs/rscomm.py +++ b/projects/rccl/contrib/pace/pace/rs/rscomm.py @@ -42,7 +42,7 @@ def get_recommended_config(num_local_ranks: int, num_ranks: int) -> CommConfig: return CommConfig(24, 2, 2, 32) else: return CommConfig(slot_unroll=128, nvl_ring_size=4, rdma_ring_size=0, num_sms=32) - + def out_numel_alignment(self) -> int: return self.runtime.out_numel_alignment() @@ -139,4 +139,4 @@ def get_split_tensors(self, tensors : Union[torch.Tensor, List[torch.Tensor]], o cnt += (numel + alignment - 1) // alignment * alignment else: cnt += numel - return outs \ No newline at end of file + return outs diff --git a/projects/rccl/contrib/pace/pace/sg/sgcomm.py b/projects/rccl/contrib/pace/pace/sg/sgcomm.py index e6e08189650b..d67648d62ff7 100644 --- a/projects/rccl/contrib/pace/pace/sg/sgcomm.py +++ b/projects/rccl/contrib/pace/pace/sg/sgcomm.py @@ -51,7 +51,7 @@ def get_recommended_config(num_local_ranks: int, num_ranks: int) -> CommConfig: if is_blackwell: return CommConfig(16, 4, 2, num_sms) return CommConfig(4, 4, 2, num_sms) - + def scatter_gather(self, tensor: Union[torch.Tensor, List[torch.Tensor]], scatter_dim = 0, gather_dim = 1, out: Optional[Union[torch.Tensor, List[torch.Tensor]]] = None, previous_event: Optional[EventOverlap] = None, async_finish: bool = False): """ Perform scatter-gather on different dimensions @@ -153,7 +153,7 @@ def get_split_tensors(self, tensors : Union[torch.Tensor, List[torch.Tensor]], o prefix_sizes.append(psize) outs = [output[prefix_sizes[i]:prefix_sizes[i] + tensors[i].numel()] for i in range(len(tensors))] return outs - + def get_comm_slots(self, tensors : Union[torch.Tensor, List[torch.Tensor]]): if isinstance(tensors, torch.Tensor): tensors = [tensors] @@ -163,6 +163,6 @@ def get_comm_slots(self, tensors : Union[torch.Tensor, List[torch.Tensor]]): total_size = sum(aligned_sizes) slot_bytes = align * 1024 * self.config.slot_unroll return (total_size + slot_bytes - 1) // slot_bytes - + def is_ring_mode(self): - return self.runtime.is_ring_mode() \ No newline at end of file + return self.runtime.is_ring_mode() diff --git a/projects/rccl/contrib/pace/pace/utils.py b/projects/rccl/contrib/pace/pace/utils.py index b25ff8dd500d..850cb811a5ec 100644 --- a/projects/rccl/contrib/pace/pace/utils.py +++ b/projects/rccl/contrib/pace/pace/utils.py @@ -102,7 +102,7 @@ def host_wait(self) -> None: assert self.event is not None self.event.host_wait() self.extra_tensors = None - + def host_query(self) -> bool: assert self.event is not None sync = self.event.host_query() diff --git a/projects/rccl/contrib/pace/test/test_ag.py b/projects/rccl/contrib/pace/test/test_ag.py index c240c87866bc..f43321c87ef3 100644 --- a/projects/rccl/contrib/pace/test/test_ag.py +++ b/projects/rccl/contrib/pace/test/test_ag.py @@ -44,8 +44,8 @@ def check_func(r : torch.Tensor, ans : Union[torch.Tensor, List[torch.Tensor]], print(f'[rank {rank}]: get_gathered_tensors mismatch') torch.save((r, ans, gathered_r), f'ag_gathered_{rank}.h5') raise RuntimeError('get_gathered_tensors check failed') - - + + def test_ag(input_tensor, rand_input): fix_answer = torch.ones((input_tensor.numel() * num_ranks), dtype=input_tensor.dtype, device='cuda') rand_answer = torch.ones_like(fix_answer) @@ -54,7 +54,7 @@ def test_ag(input_tensor, rand_input): if not args.no_check: dist.all_gather_into_tensor(output_tensor=rand_answer, input_tensor=rand_input, group=group) dist.all_gather_into_tensor(output_tensor=fix_answer, input_tensor=input_tensor, group=group) - + output_t = torch.zeros_like(fix_answer) # normal @@ -98,7 +98,7 @@ def test_ag(input_tensor, rand_input): rs, _ = buffer.all_gather(batch_input) check_func(rs, batch_answer, in_ = batch_input) - + rs, event = buffer.all_gather(batch_input, async_finish=True) event.current_stream_wait() check_func(rs, batch_answer, in_ = batch_input) @@ -112,7 +112,7 @@ def test_ag(input_tensor, rand_input): assert rs is list_out, f'[rank {rank}]: list out= did not return the caller buffers' for i in range(len(batch_input)): assert torch.allclose(rs[i], batch_answer[i]), f'[rank {rank}]: list out= tensor {i} mismatch' - + dist.barrier() if rank == 0: print(f'[rank {rank}]: check passed') @@ -125,7 +125,7 @@ def test_ag(input_tensor, rand_input): if rank == 0 or True: print(f'[rank {rank}]: allgather bandwidth: {total_bytes / t / 1e9:.2f} GBps') dist.barrier() - + def test_func(): buffer.all_gather(input_tensor) @@ -170,7 +170,7 @@ def tfunc(bundle): print(f'[rank {rank}]: nccl_allgather bandwidth: {total_bytes / t / 1e9:.2f} GBps') dist.barrier() - + for seq_len in args.seq_len: if rank == 0: print(f'\n===== Testing seq_len={seq_len} =====') diff --git a/projects/rccl/contrib/pace/test/test_rs.py b/projects/rccl/contrib/pace/test/test_rs.py index e83a4b442225..6e060c9142db 100644 --- a/projects/rccl/contrib/pace/test/test_rs.py +++ b/projects/rccl/contrib/pace/test/test_rs.py @@ -172,7 +172,7 @@ def tfunc(bundle): dist.barrier() - + dtype = torch.float32 if args.dtype_size == 4 else torch.bfloat16 # Iterate over all sequence lengths dim0 = num_ranks if args.dim0 is None else args.dim0 diff --git a/projects/rccl/contrib/pace/test/test_sg.py b/projects/rccl/contrib/pace/test/test_sg.py index 1727669457c1..b8bb5a49c9b1 100644 --- a/projects/rccl/contrib/pace/test/test_sg.py +++ b/projects/rccl/contrib/pace/test/test_sg.py @@ -354,7 +354,7 @@ def check_strided(scatter_dim, gather_dim, dtype): answer_list = [all_to_all_baseline(t, scatter_dim, gather_dim, group=group) for t in qkv_list] all_inputs.append(qkv_list) all_answers.append(answer_list) - + default_stream = torch.cuda.current_stream() # Test 2: sync - each block launches QKV list on its own stream, wait immediately all_results = [] diff --git a/projects/rccl/contrib/pace/test/utils.py b/projects/rccl/contrib/pace/test/utils.py index b7f708b38b1c..f38cac90bb24 100644 --- a/projects/rccl/contrib/pace/test/utils.py +++ b/projects/rccl/contrib/pace/test/utils.py @@ -306,7 +306,7 @@ def dequantize_nvfp4_back_to_bfloat16(x_nvfp4: torch.Tensor, x_scales: torch.Ten if x_nvfp4.numel() == 0: ratio = int(x_nvfp4.dtype.itemsize * 8 / 4) return x_nvfp4.to(torch.bfloat16).view(x_nvfp4.size(0), x_nvfp4.size(1) * ratio) - NVFP4_TABLE = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1.0, -1.5, -2, -3, -4, -6], dtype=torch.float32, device=x_nvfp4.device) + NVFP4_TABLE = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1.0, -1.5, -2, -3, -4, -6], dtype=torch.float32, device=x_nvfp4.device) if use_ue8m0_for_nvfp4_sf: x_scales = x_scales.view(dtype=torch.int8).to(torch.int) << 23 x_scales = x_scales.view(dtype=torch.float) @@ -314,9 +314,9 @@ def dequantize_nvfp4_back_to_bfloat16(x_nvfp4: torch.Tensor, x_scales: torch.Ten x_scales = x_scales.view(dtype=torch.float8_e4m3fn).to(torch.float32) x_sf_scale = 1 / x_sf_scale x_scales = x_scales * x_sf_scale - + x_fp32 = int32_to_8floats_lookup(x_nvfp4, NVFP4_TABLE) - + x_fp32 = x_fp32.view(*x_fp32.shape[:-1], -1, 16) x_scales = x_scales.view(*x_scales.shape[:-1], -1, 1) x_fp32 = x_fp32 * x_scales @@ -345,4 +345,4 @@ def inplace_unique(x: torch.Tensor, num_slots: int): x[:, :valid_len] = sorted_bin_idx[:, :valid_len] def align_up(x, y): - return (x + y - 1) // y * y \ No newline at end of file + return (x + y - 1) // y * y diff --git a/projects/rccl/docs/api-reference/env-variables.rst b/projects/rccl/docs/api-reference/env-variables.rst index bc6fcb8387bf..1b730f2e265f 100644 --- a/projects/rccl/docs/api-reference/env-variables.rst +++ b/projects/rccl/docs/api-reference/env-variables.rst @@ -63,6 +63,12 @@ in the following table. Auto-detect is limited to gfx1250, the only architecture where the VMM path is validated. Use ``1`` to force it on elsewhere. + * - | ``NCCL_RMA_DISABLE`` + | Disables the RMA proxy, the network path for one-sided RMA. The proxy + is never connected and windows are not registered with it. + - | ``0``: RMA proxy enabled (default). + | ``1``: RMA proxy disabled. + * - | ``NCCL_MIN_CTAS`` | Minimum number of CTAs (channels) used for a collective. Overrides the ``minCTAs`` field of ``ncclConfig_t``. @@ -96,6 +102,15 @@ in the following table. - | ``0``: Disabled (default). | ``1``: Enabled. + * - | ``RCCL_HIERARCHICAL_LAZY_INIT`` + | Controls when a communicator of eight or more nodes builds the + sub-communicators that hierarchical AllGather uses. Ignored when + ``RCCL_HIERARCHICAL_REDUCE_SCATTER=1``. All ranks in a communicator + must use the same value. + - | ``0``: Build them during communicator initialization (default). + | ``1``: Build them on the first AllGather eligible for hierarchical + AllGather, outside graph capture. + Logging and debugging ===================== @@ -117,6 +132,7 @@ in the following table. | ``ERROR``: These messages report when a fatal condition has occurred in RCCL and the operation can't continue. | ``VERSION``: ``librccl`` version info is printed during the initialization phase. | ``WARN``: Prints warnings about unusual conditions that could lead to unexpected results. + | ``ATTN``: Prints ``WARN`` messages plus notices that need attention, such as a plugin named in ``NCCL_*_PLUGIN`` that could not be loaded or initialized. ``INFO`` also prints these notices. | ``INFO``: Prints standard logging messages about status and operations performed. | ``ABORT``: Unused. | ``TRACE``: Prints trace-level logging of function calls and parameters. Only active when ``librccl`` is built using ``ENABLE_TRACE``. @@ -152,7 +168,7 @@ in the following table. * - | ``NCCL_DEBUG_TIMESTAMP_LEVELS`` | The timestamp levels for ``NCCL_DEBUG``. - - | A set of ``NCCL_DEBUG`` levels can have a timestamp prepended set as a comma-separated list which can be inverted using the ``^`` prefix. The default set is ``WARN``. + - | A set of ``NCCL_DEBUG`` levels can have a timestamp prepended set as a comma-separated list which can be inverted using the ``^`` prefix. The default set is ``WARN`` and ``ATTN``. * - | ``NCCL_DEBUG_TIMESTAMP_FORMAT`` | The timestamp format for ``NCCL_DEBUG``. @@ -259,6 +275,23 @@ collected in the following table. CTA-policy gates in ``rcclUseAllGatherDirect`` still apply. | Any other value: Disabled. + * - | ``RCCL_HIERARCHICAL_ALLGATHER_MIN_BYTES_PER_RANK`` + | Sets the smallest AllGather, in bytes per rank (``sendcount`` x type + size), that can select hierarchical AllGather, so that small startup + AllGathers such as PyTorch DDP's 8-byte one stay on the default path. + ``rccl-tests`` reports the total size, this value times the number of + ranks. All ranks in a communicator must use the same value. + - | Bytes per rank + | Default: ``16`` + | ``0``: No lower bound. + + * - | ``RCCL_HIERARCHICAL_REDUCE_SCATTER_MIN_BYTES_PER_RANK`` + | Sets the smallest ReduceScatter, in bytes per rank (``recvcount`` x + type size), that can select hierarchical ReduceScatter. + - | Bytes per rank + | Default: ``16`` + | ``0``: No lower bound. + Network and topology ==================== @@ -454,6 +487,38 @@ in the following table. - | Remapping specification string | Used with Rome 4P2H topology + * - | ``NCCL_GIN_PLUGIN`` + | Selects external GIN (GPU-Initiated Networking) plugins. + - | Comma-separated list of paths or short names + | A short name is resolved against the ``librccl-gin`` prefix, so + ``example`` loads ``librccl-gin-example.so`` + | A plugin reporting the proxy device type is superseded by the built-in + GIN proxy. See :ref:`using-rccl-gin-plugin` + + * - | ``NCCL_GIN_ENABLE`` + | Controls whether any GIN backend is registered. RCCL-specific. + - | ``1``: Register GIN backends (default) + | ``0``: Register none, so GIN reports as unsupported + + * - | ``NCCL_GIN_TYPE`` + | Requires a specific GIN backend type, skipping all others. + - | ``-1``: Disables the generic filter, but does not auto-enable + the in-tree device backends + | ``2``: Proxy + | ``3``: GDAKI + | ``4``: GPI + | ``5``: EFA GDA + | ``6``: rocSHMEM GDA (required for that backend to initialize) + | ``7``: Anvil SDMA (also accepts unset) + + * - | ``NCCL_RMA_PLUGIN`` + | Selects external one-sided RMA plugins, which are also the backend + the built-in GIN proxy forwards to. + - | Comma-separated list of paths or short names + | A short name is resolved against the ``librccl-rma`` prefix, so + ``example`` loads ``librccl-rma-example.so`` + | See :ref:`using-rccl-rma-plugin` + Development and testing (advanced) ================================== @@ -559,6 +624,16 @@ variables are collected in the following table. - | ``0``: JSON output (default). | ``1``: Prometheus textfile output. + * - | ``NCCL_INSPECTOR_PROM_DUMP_STATS`` + | In Prometheus mode, also emits per-device ring-buffer counters: + | ``nccl_collectives_total``, ``nccl_collectives_dropped_total``, + | ``nccl_p2p_total`` and ``nccl_p2p_dropped_total``. They are + | cumulative, so ``rate(dropped) / rate(total)`` gives the fraction of + | operations lost. JSON output always carries the same counts in its + | ``dump_stats`` record. + - | ``0``: Disabled (default). + | ``1``: Enabled. + * - | ``NCCL_INSPECTOR_CLUSTER`` | Overrides the Prometheus ``cluster`` label. When unset, the Inspector | uses ``SLURM_CLUSTER_NAME``. Set this when that name is missing. @@ -607,12 +682,15 @@ variables are collected in the following table. * - | ``NCCL_INSPECTOR_DUMP_COLL_RING_SIZE`` | Per-communicator capacity of the ring buffer holding completed - | collectives waiting to be dumped. + | collectives waiting to be dumped. When it fills, the oldest entries + | are overwritten and counted as dropped, with a single warning per + | process. - | Integer number of entries (default: ``1024``). * - | ``NCCL_INSPECTOR_DUMP_P2P_RING_SIZE`` | Per-communicator capacity of the ring buffer holding completed - | point-to-point operations waiting to be dumped. + | point-to-point operations waiting to be dumped. Overflow is handled + | as for ``NCCL_INSPECTOR_DUMP_COLL_RING_SIZE``. - | Integer number of entries (default: ``1024``). * - | ``NCCL_INSPECTOR_COLL_POOL_SIZE`` @@ -633,6 +711,94 @@ variables are collected in the following table. - | ``0``: Fixed-size pools. | ``1``: Pools grow on demand (default). + * - | ``NCCL_INSPECTOR_OTEL_EXPORT`` + | Exports metrics over OTLP/HTTP instead of writing them to files. The + | Inspector emits one format only, and this setting takes precedence + | over ``NCCL_INSPECTOR_PROM_DUMP``. + - | ``0``: Disabled, so JSON or Prometheus files are written (default). + | ``1``: Enabled, posting OTLP/JSON to the endpoint below. + + * - | ``NCCL_INSPECTOR_OTEL_VERBOSE`` + | Selects per-collective metric points instead of aggregated ones. The + | per-collective form additionally reports + | ``nccl_collective_algobw_gbs`` and the per-device ring-buffer + | counters described under ``NCCL_INSPECTOR_PROM_DUMP_STATS``. + - | ``0``: Aggregated (default). + | ``1``: Per collective. + + * - | ``OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`` + | Destination for the OTLP metric export. Also accepts + | ``OTEL_EXPORTER_OTLP_ENDPOINT``. ``/v1/metrics`` is appended to + whichever variable is used when its URL carries no path. + - | Plaintext ``http://`` URL. ``https://`` is unsupported and disables + export, so use a local collector when the telemetry is sensitive. + | Default: ``http://localhost:4318/v1/metrics`` + + * - | ``OTEL_EXPORTER_OTLP_METRICS_HEADERS`` + | Extra headers attached to every OTLP request. Also accepts + ``OTEL_EXPORTER_OTLP_HEADERS``. + - | Comma-separated ``key=value`` list + | Sent verbatim over the plaintext connection above, so do not put a + credential here unless the collector is local. + +Profiler plugin +=============== + +An external profiler plugin loaded through ``NCCL_PROFILER_PLUGIN`` receives the +event types selected by ``NCCL_PROFILE_EVENT_MASK``. RCCL 2.31 raises the plugin +interface to v7, which adds two things a plugin can consume: + +* **Per-kernel barrier phases.** Enabling ``ncclProfileKernelPhase`` reports an + ``initial_sync``, ``compute`` and ``final_sync`` span per kernel channel, each + timed by the GPU globaltimer. A phase is a child of a kernel-channel event, so + RCCL enables ``ncclProfileKernelCh`` implicitly whenever the phase bit is set. + Only symmetric kernels emit phases, which requires buffers registered as + symmetric windows. Regular collective and point-to-point kernels report + kernel-channel events without phases. +* **Symmetric-kernel variant metadata.** Collective events carry the symmetric + kernel variant that ran and a flag marking whether the collective was served + symmetrically. The in-tree example surfaces these as the ``KernelVariant`` and + ``IsSymColl`` arguments of its ``COLL`` trace events. + +Both are v7-only: the v5 and v6 compatibility layers clear the phase bit, so a +plugin must export ``ncclProfiler_v7`` to receive them. + +.. list-table:: + :header-rows: 1 + :widths: 40,60 + + * - **Environment variable** + - **Values** + + * - | ``NCCL_PROFILER_PLUGIN`` + | Selects an external profiler plugin. + - | Path or short name + | A short name is resolved against the ``librccl-profiler`` prefix, so + ``example`` loads ``librccl-profiler-example.so`` + + * - | ``NCCL_PROFILE_EVENT_MASK`` + | Bitmask of the event types the plugin is offered. + - | ``1``: Group + | ``2``: Collective + | ``4``: Point-to-point + | ``8``: Proxy op + | ``16``: Proxy step + | ``32``: Proxy control + | ``64``: Kernel channel + | ``128``: Net plugin + | ``256``: Group API + | ``512``: Collective API + | ``1024``: Point-to-point API + | ``2048``: Kernel launch + | ``4096``: CE collective + | ``8192``: CE synchronization + | ``16384``: CE batch + | ``32768``: Kernel phase (v7, symmetric kernels only; implies kernel + channel) + | ``65536``: RCCL proxy diagnostics + | Combine by adding, so ``32771`` selects group, collective and kernel + phase. Default: ``0`` + Algorithm dispatch and tuning (gfx1250 / MI450) ================================================ diff --git a/projects/rccl/docs/contrib/CONTRIBUTING.md b/projects/rccl/docs/contrib/CONTRIBUTING.md index e6f58f04a344..d487c399b68f 100644 --- a/projects/rccl/docs/contrib/CONTRIBUTING.md +++ b/projects/rccl/docs/contrib/CONTRIBUTING.md @@ -40,7 +40,7 @@ guide is meant to carry. **Not a good fit:** - **Performance marketing** that compares NCCL with something else and draws conclusions that can be - misleading other users. + misleading other users. ## Making Your Contribution Successful diff --git a/projects/rccl/docs/dev_guide/nccl_coding_style.md b/projects/rccl/docs/dev_guide/nccl_coding_style.md index e6dae00fc6da..cf6ec963ac63 100644 --- a/projects/rccl/docs/dev_guide/nccl_coding_style.md +++ b/projects/rccl/docs/dev_guide/nccl_coding_style.md @@ -24,6 +24,14 @@ - All external function calls should be guarded with `CUDACHECK`, `SYSCHECK`, `PTHREADCHECK`, etc. - Use public helpers like `CommCheck` and `PtrCheck` to check function arguments. +## Logging +- Use `WARN()` only for a real error that is returned to the caller. It records `ncclLastError` and can enable additional INFO diagnostics. +- Use `ATTN()` for a non-fatal, actionable notice, such as a configuration fallback. It does not set `ncclLastError` or enable additional diagnostics. +- Use `INFO()` for normal debug information. Select the narrowest applicable subsystem flag. +- Use `INFO_LOC()` when INFO output needs the file, line, and function. `WARN()` and `ATTN()` already include this context. +- Do not add a function-name prefix to `WARN()` or `ATTN()` messages. The logging framework adds it automatically. +- When an error is intentionally ignored, use the appropriate `NOWARN` wrapper (which downgrades `WARN()` to `INFO()`) and add an `INFO()` or `INFO_LOC()` message only when the context is useful for debugging. + ## Memory Management - Always allocate memory through NCCL alloc functions, e.g. `ncclCalloc()` - Use appropriate memory fences for synchronization @@ -207,7 +215,7 @@ else { ```cpp for (int i = 0; i < n; ++i) maxPolicy |= policies[i]; -while (token) token = strtok(NULL, "|"); +while (token) token = strtok_r(NULL, "|", &savePtr); ``` * If loop statement cannot fit on a single-line, it must be enclosed in braces. diff --git a/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/README.md b/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/README.md new file mode 100644 index 000000000000..c3cec32a73da --- /dev/null +++ b/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/README.md @@ -0,0 +1,38 @@ + + +# NCCL Example: Device API AllGather with CFT counted + +This example shows a simple allgather implemented inside a CUDA kernel using +CFT counted puts and wait. + +Each rank owns one contiguous input slice. The kernel copies the local slice to +the local output slot, sends the same slice to every other rank with +`ncclCft::putCounted`, and then waits for the expected counted bytes from every +peer selected by a bit mask. + +## Build + +```shell +cd c +make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] +``` + +## Run + +```shell +./allgather_cft_counted +``` + +or with MPI: + +```shell +mpirun -np ./allgather_cft_counted +``` + +The example requires CFT-capable hardware and CUDA Toolkit support for CFT +fabric instructions. diff --git a/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/Makefile b/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/Makefile new file mode 100644 index 000000000000..120f091c7c9f --- /dev/null +++ b/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/Makefile @@ -0,0 +1,52 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# See LICENSE.txt for more license information +# + +include ../../../../../makefiles/common.mk +include ../../../../../makefiles/examples.mk + +TARGET = allgather_cft_counted + +COMMON_INC = ../../../common/include +COMMON_SRC = ../../../common/src + +INCLUDES += -I$(COMMON_INC) + +SOURCES = main.cu $(COMMON_SRC)/utils.cc +OBJECTS = $(SOURCES:.cu=.o) +OBJECTS := $(OBJECTS:.cc=.o) + +all: $(TARGET) + +$(TARGET): $(OBJECTS) +ifeq ($(MPI),1) + $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ +else + $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -lpthread -o $@ +endif + @echo "Built target $@" + +%.o: %.cu + $(NVCC) $(NVCUFLAGS) $(INCLUDES) -c $< -o $@ + +%.o: %.cc +ifeq ($(MPI),1) + $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ +else + $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ +endif + +test: $(TARGET) +ifeq ($(MPI),1) + $(MPIRUN) -np 2 ./$(TARGET) +else + ./$(TARGET) +endif + +clean: + rm -f $(OBJECTS) $(TARGET) + +.PHONY: all test clean diff --git a/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/README.md b/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/README.md new file mode 100644 index 000000000000..7e59dc7f4a61 --- /dev/null +++ b/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/README.md @@ -0,0 +1,20 @@ + + +# C: Device API AllGather with CFT counted + +Build from this directory: + +```shell +make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] +``` + +Run: + +```shell +./allgather_cft_counted +``` diff --git a/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/main.cu b/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/main.cu new file mode 100644 index 000000000000..dbd81ecd2e50 --- /dev/null +++ b/projects/rccl/docs/examples/06_device_api/05_allgather_cft_counted/c/main.cu @@ -0,0 +1,191 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#include "cuda_runtime.h" +#include "nccl.h" +#include "nccl_device.h" +#include "utils.h" +#include +#include +#include + +#define NCCL_DEVICE_CTA_COUNT 8 +#define NCCL_DEVICE_THREADS_PER_CTA 256 +#define ELEMENTS_PER_RANK 1024 +#define ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#define CFT_SMEM_OFFSET 0 +#define TMA_BARRIER_OFFSET ALIGN_UP(CFT_SMEM_OFFSET + sizeof(ncclCftSmem), alignof(cuda::barrier)) +#define TMA_BUFFER_OFFSET ALIGN_UP(TMA_BARRIER_OFFSET + sizeof(cuda::barrier), 16) +#define DYNAMIC_SMEM_BYTES (TMA_BUFFER_OFFSET + NCCL_DEVICE_THREADS_PER_CTA * sizeof(float)) + +__device__ __forceinline__ void red_release_sys_global_add_u64(uint64_t* addr, size_t val) { + asm volatile( + "red.release.sys.global.add.u64 [%0], %1;" + : + : "l"(addr), "l"(val) + : "memory"); +} + +__global__ void allGatherCftCountedKernel(ncclWindow_t sendWin, ncclWindow_t recvWin, size_t count, + ncclDevComm devComm) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && defined(CUDART_VERSION) && CUDART_VERSION >= 13030 + ncclCoopCta coop; + ncclTeam team = ncclTeamCft(devComm); + + int rank = team.rank; + int nRanks = team.nRanks; + size_t bytes = count * sizeof(float); + + const int globalTid = threadIdx.x + blockIdx.x * blockDim.x; + const int globalNthreads = blockDim.x * gridDim.x; + + float* send = (float*)ncclGetLocalPointer(sendWin, 0); + float* recv = (float*)ncclGetLocalPointer(recvWin, 0); + size_t ctaBytes = 0; + for (size_t offset = globalTid; offset < count; offset += globalNthreads) { + recv[rank * count + offset] = send[offset]; + size_t chunkElems = (count - offset) < (size_t)blockDim.x ? (count - offset) : (size_t)blockDim.x; + ctaBytes += chunkElems * sizeof(float); + } + + uintptr_t base = reinterpret_cast(recv); + uintptr_t end = base + sizeof(float) * count * nRanks; + uintptr_t counterAddr = ALIGN_UP(end, uintptr_t(256u)); + uint64_t* counterPtr = reinterpret_cast(counterAddr); + + coop.sync(); + if (coop.thread_rank() == 0) red_release_sys_global_add_u64(counterPtr, ctaBytes); + if (nRanks == 1) return; + + extern __shared__ __align__(16) unsigned char smemBytes[]; + ncclCftSmem* cftSmem = reinterpret_cast(smemBytes + CFT_SMEM_OFFSET); + cuda::barrier* mbar = + reinterpret_cast*>(smemBytes + TMA_BARRIER_OFFSET); + float* smem = reinterpret_cast(smemBytes + TMA_BUFFER_OFFSET); + ncclCft cft { coop, *cftSmem }; + + if (coop.thread_rank() == 0) init(mbar, 1); + + for (size_t base = blockIdx.x * blockDim.x; base < count; base += globalNthreads) { + uint32_t chunkElems = (uint32_t)((count - base) < (size_t)blockDim.x ? (count - base) : (size_t)blockDim.x); + uint32_t chunkBytes = chunkElems * sizeof(float); + if (coop.thread_rank() == 0) { + cuda::device::memcpy_async_tx(smem, send + base, cuda::aligned_size_t<16>(chunkBytes), *mbar); + cuda::barrier::arrival_token token = + cuda::device::barrier_arrive_tx(*mbar, 1, chunkBytes); + mbar->wait(std::move(token)); + } + coop.sync(); + + ncclCftLeId leId; + size_t leOffset; + for (int peer = 0; peer < nRanks; peer++) { + if (peer == rank) continue; + ncclGetCftLeInfo(recvWin, rank * count * sizeof(float), peer, team, devComm, &leId, &leOffset); + size_t dataOffset = leOffset + base * sizeof(float); + size_t counterOffset = ALIGN_UP(leOffset + ((nRanks - rank) * count * sizeof(float)), 256u); + cft.putCounted(coop, leId, dataOffset, counterOffset, smem, chunkBytes); + } + cft.submit(coop); + cft.flush(coop); + } + cft.waitCounted(coop, cuda::memory_order_acquire, ncclMemProxyType::Generic, counterPtr, bytes*nRanks, devComm.abortFlag); +#else + (void)sendWin; + (void)recvWin; + (void)count; + (void)devComm; +#endif +} + +void* allGatherCftCounted(int myRank, int totalRanks, int localDevice, int devicesPerRank) { + (void)devicesPerRank; + ncclComm_t comm; + ncclUniqueId id; + + if (myRank == 0) NCCLCHECK(ncclGetUniqueId(&id)); + util_broadcast(0, myRank, &id); + + CUDACHECK(cudaSetDevice(localDevice)); +#if !defined(CUDART_VERSION) || CUDART_VERSION < 13030 + printf("rank %d allgather_cft_counted: SKIPPED, requires CUDA Toolkit 13.3 or newer\n", myRank); + return NULL; +#endif + cudaDeviceProp prop; + CUDACHECK(cudaGetDeviceProperties(&prop, localDevice)); + if (prop.major * 10 + prop.minor < 100) { + printf("rank %d allgather_cft_counted: SKIPPED, requires compute capability 10.0 or newer\n", myRank); + return NULL; + } + NCCLCHECK(ncclCommInitRank(&comm, totalRanks, id, myRank)); + + size_t count = ELEMENTS_PER_RANK; + size_t counterBytes = /*counter*/8u + /*alignment*/255u; + size_t sendBytes = count * sizeof(float); + size_t recvBytes = count * totalRanks * sizeof(float); + + float* hSend = (float*)malloc(sendBytes); + float* hRecv = (float*)malloc(recvBytes); + for (size_t i = 0; i < count; i++) hSend[i] = (float)(myRank * 1000 + i); + + void* dSend; + void* dRecv; + ncclWindow_t sendWin; + ncclWindow_t recvWin; + NCCLCHECK(ncclMemAlloc(&dSend, sendBytes)); + NCCLCHECK(ncclMemAlloc(&dRecv, recvBytes + counterBytes)); + CUDACHECK(cudaMemcpy(dSend, hSend, sendBytes, cudaMemcpyHostToDevice)); + CUDACHECK(cudaMemset(dRecv, 0, recvBytes)); + CUDACHECK(cudaMemset(reinterpret_cast(dRecv) + recvBytes, 0, counterBytes)); + + NCCLCHECK(ncclCommWindowRegister(comm, dSend, sendBytes, &sendWin, NCCL_WIN_COLL_SYMMETRIC)); + NCCLCHECK(ncclCommWindowRegister(comm, dRecv, recvBytes + counterBytes, &recvWin, NCCL_WIN_COLL_SYMMETRIC|NCCL_WIN_CFT_COUNTED)); + + ncclDevComm devComm; + ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; + reqs.cftCaps = NCCL_CFT; + reqs.cftBarrierCount = NCCL_DEVICE_CTA_COUNT; + NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); + + cudaStream_t stream; + CUDACHECK(cudaStreamCreate(&stream)); + allGatherCftCountedKernel<<>>(sendWin, recvWin, count, devComm); + CUDACHECK(cudaStreamSynchronize(stream)); + + CUDACHECK(cudaMemcpy(hRecv, dRecv, recvBytes, cudaMemcpyDeviceToHost)); + bool ok = true; + for (int r = 0; r < totalRanks; r++) { + for (size_t i = 0; i < count; i++) { + float expected = (float)(r * 1000 + i); + float got = hRecv[r * count + i]; + if (got != expected) { + printf("rank %d mismatch at source rank %d element %zu: got %f expected %f\n", myRank, r, i, got, expected); + ok = false; + goto done_check; + } + } + } +done_check: + printf("rank %d allgather_cft_counted: %s\n", myRank, ok ? "PASSED" : "FAILED"); + + NCCLCHECK(ncclDevCommDestroy(comm, &devComm)); + NCCLCHECK(ncclCommWindowDeregister(comm, recvWin)); + NCCLCHECK(ncclCommWindowDeregister(comm, sendWin)); + NCCLCHECK(ncclMemFree(dRecv)); + NCCLCHECK(ncclMemFree(dSend)); + NCCLCHECK(ncclCommFinalize(comm)); + NCCLCHECK(ncclCommDestroy(comm)); + CUDACHECK(cudaStreamDestroy(stream)); + free(hRecv); + free(hSend); + return NULL; +} + +int main(int argc, char* argv[]) { + return run_example(argc, argv, allGatherCftCounted); +} diff --git a/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/README.md b/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/README.md new file mode 100644 index 000000000000..f1b114f0ae8a --- /dev/null +++ b/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/README.md @@ -0,0 +1,37 @@ + + +# NCCL Example: Device API AllGather with CFT multimem counted + +This example shows a simple allgather implemented inside a CUDA kernel using +CFT multimem counted puts and wait. + +Each rank owns one contiguous input slice. The kernel copies the local slice to +the local output slot, multicasts the same slice to all ranks with +`ncclCft::putMultimemCounted`, and then waits for the expected counted bytes. + +## Build + +```shell +cd c +make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] +``` + +## Run + +```shell +./allgather_cft_multimem_counted +``` + +or with MPI: + +```shell +mpirun -np ./allgather_cft_multimem_counted +``` + +The example requires CFT multimem-capable hardware and CUDA Toolkit support for +CFT fabric multimem instructions. diff --git a/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/Makefile b/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/Makefile new file mode 100644 index 000000000000..4ade468f93d6 --- /dev/null +++ b/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/Makefile @@ -0,0 +1,52 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# See LICENSE.txt for more license information +# + +include ../../../../../makefiles/common.mk +include ../../../../../makefiles/examples.mk + +TARGET = allgather_cft_multimem_counted + +COMMON_INC = ../../../common/include +COMMON_SRC = ../../../common/src + +INCLUDES += -I$(COMMON_INC) + +SOURCES = main.cu $(COMMON_SRC)/utils.cc +OBJECTS = $(SOURCES:.cu=.o) +OBJECTS := $(OBJECTS:.cc=.o) + +all: $(TARGET) + +$(TARGET): $(OBJECTS) +ifeq ($(MPI),1) + $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ +else + $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -lpthread -o $@ +endif + @echo "Built target $@" + +%.o: %.cu + $(NVCC) $(NVCUFLAGS) $(INCLUDES) -c $< -o $@ + +%.o: %.cc +ifeq ($(MPI),1) + $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ +else + $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ +endif + +test: $(TARGET) +ifeq ($(MPI),1) + $(MPIRUN) -np 2 ./$(TARGET) +else + ./$(TARGET) +endif + +clean: + rm -f $(OBJECTS) $(TARGET) + +.PHONY: all test clean diff --git a/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/README.md b/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/README.md new file mode 100644 index 000000000000..633ce82691d3 --- /dev/null +++ b/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/README.md @@ -0,0 +1,20 @@ + + +# C: Device API AllGather with CFT multimem counted + +Build from this directory: + +```shell +make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] +``` + +Run: + +```shell +./allgather_cft_multimem_counted +``` diff --git a/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/main.cu b/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/main.cu new file mode 100644 index 000000000000..6d0ecbc9c73a --- /dev/null +++ b/projects/rccl/docs/examples/06_device_api/06_allgather_cft_multimem_counted/c/main.cu @@ -0,0 +1,172 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#include "cuda_runtime.h" +#include "nccl.h" +#include "nccl_device.h" +#include "utils.h" +#include +#include +#include + +#define NCCL_DEVICE_CTA_COUNT 8 +#define NCCL_DEVICE_THREADS_PER_CTA 256 +#define ELEMENTS_PER_RANK 1024 +#define ALIGN_UP(x, align) (((x) + (align) - 1) & ~((align) - 1)) +#define CFT_SMEM_OFFSET 0 +#define TMA_BARRIER_OFFSET ALIGN_UP(CFT_SMEM_OFFSET + sizeof(ncclCftSmem), alignof(cuda::barrier)) +#define TMA_BUFFER_OFFSET ALIGN_UP(TMA_BARRIER_OFFSET + sizeof(cuda::barrier), 16) +#define DYNAMIC_SMEM_BYTES (TMA_BUFFER_OFFSET + NCCL_DEVICE_THREADS_PER_CTA * sizeof(float)) + +__global__ void allGatherCftMultimemCountedKernel(ncclWindow_t sendWin, ncclWindow_t recvWin, size_t count, + ncclDevComm devComm) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && defined(CUDART_VERSION) && CUDART_VERSION >= 13030 + ncclCoopCta coop; + ncclTeam team = ncclTeamCft(devComm); + + int rank = team.rank; + int nRanks = team.nRanks; + size_t bytes = count * sizeof(float); + + const int globalNthreads = blockDim.x * gridDim.x; + + float* send = (float*)ncclGetLocalPointer(sendWin, 0); + float* recv = (float*)ncclGetLocalPointer(recvWin, 0); + uintptr_t base = reinterpret_cast(recv); + uintptr_t end = base + sizeof(float) * count * nRanks; + uintptr_t counterAddr = ALIGN_UP(end, uintptr_t(256u)); + uint64_t* counterPtr = reinterpret_cast(counterAddr); + + ncclCftBarrierSession bar { coop, devComm, blockIdx.x }; + bar.sync(coop, cuda::memory_order_relaxed, ncclMemProxyType::Generic, ncclMemProxyType::Generic); + + extern __shared__ __align__(16) unsigned char smemBytes[]; + ncclCftSmem* cftSmem = reinterpret_cast(smemBytes + CFT_SMEM_OFFSET); + cuda::barrier* mbar = + reinterpret_cast*>(smemBytes + TMA_BARRIER_OFFSET); + float* smem = reinterpret_cast(smemBytes + TMA_BUFFER_OFFSET); + ncclCft cft { coop, *cftSmem }; + + if (coop.thread_rank() == 0) init(mbar, 1); + + for (size_t base = blockIdx.x * blockDim.x; base < count; base += globalNthreads) { + uint32_t chunkElems = (uint32_t)((count - base) < (size_t)blockDim.x ? (count - base) : (size_t)blockDim.x); + uint32_t chunkBytes = chunkElems * sizeof(float); + if (coop.thread_rank() == 0) { + cuda::device::memcpy_async_tx(smem, send + base, cuda::aligned_size_t<16>(chunkBytes), *mbar); + cuda::barrier::arrival_token token = + cuda::device::barrier_arrive_tx(*mbar, 1, chunkBytes); + mbar->wait(std::move(token)); + } + coop.sync(); + ncclCftLeId leId; + size_t leOffset; + ncclGetMultimemLeInfo(recvWin, rank * count * sizeof(float), devComm, &leId, &leOffset); + size_t dataOffset = leOffset + base * sizeof(float); + size_t counterOffset = ALIGN_UP(leOffset + (nRanks - rank) * count * sizeof(float), 256u); + cft.putMultimemCounted(coop, leId, dataOffset, counterOffset, smem, chunkBytes); + cft.submit(coop); + cft.flush(coop); + } + cft.waitCounted(coop, cuda::memory_order_acquire, ncclMemProxyType::Generic, counterPtr, bytes * nRanks, + devComm.abortFlag); +#else + (void)sendWin; + (void)recvWin; + (void)count; + (void)devComm; +#endif +} + +void* allGatherCftMultimemCounted(int myRank, int totalRanks, int localDevice, int devicesPerRank) { + (void)devicesPerRank; + ncclComm_t comm; + ncclUniqueId id; + + if (myRank == 0) NCCLCHECK(ncclGetUniqueId(&id)); + util_broadcast(0, myRank, &id); + + CUDACHECK(cudaSetDevice(localDevice)); +#if !defined(CUDART_VERSION) || CUDART_VERSION < 13030 + printf("rank %d allgather_cft_multimem_counted: SKIPPED, requires CUDA Toolkit 13.3 or newer\n", myRank); + return NULL; +#endif + cudaDeviceProp prop; + CUDACHECK(cudaGetDeviceProperties(&prop, localDevice)); + if (prop.major * 10 + prop.minor < 100) { + printf("rank %d allgather_cft_multimem_counted: SKIPPED, requires compute capability 10.0 or newer\n", myRank); + return NULL; + } + NCCLCHECK(ncclCommInitRank(&comm, totalRanks, id, myRank)); + + size_t count = ELEMENTS_PER_RANK; + size_t counterBytes = /*counter*/8u + /*alignment*/255u; + size_t sendBytes = count * sizeof(float); + size_t recvBytes = count * totalRanks * sizeof(float); + + float* hSend = (float*)malloc(sendBytes); + float* hRecv = (float*)malloc(recvBytes); + for (size_t i = 0; i < count; i++) hSend[i] = (float)(myRank * 1000 + i); + + void* dSend; + void* dRecv; + ncclWindow_t sendWin; + ncclWindow_t recvWin; + NCCLCHECK(ncclMemAlloc(&dSend, sendBytes)); + NCCLCHECK(ncclMemAlloc(&dRecv, recvBytes + counterBytes)); + CUDACHECK(cudaMemcpy(dSend, hSend, sendBytes, cudaMemcpyHostToDevice)); + CUDACHECK(cudaMemset(dRecv, 0, recvBytes)); + CUDACHECK(cudaMemset(reinterpret_cast(dRecv) + recvBytes, 0, counterBytes)); + + NCCLCHECK(ncclCommWindowRegister(comm, dSend, sendBytes, &sendWin, NCCL_WIN_COLL_SYMMETRIC)); + NCCLCHECK(ncclCommWindowRegister(comm, dRecv, recvBytes + counterBytes, &recvWin, NCCL_WIN_COLL_SYMMETRIC|NCCL_WIN_CFT_COUNTED)); + + ncclDevComm devComm; + ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; + reqs.cftCaps = NCCL_CFT | NCCL_CFT_MULTIMEM; + reqs.cftBarrierCount = NCCL_DEVICE_CTA_COUNT; + reqs.lsaMultimem = true; + NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); + + cudaStream_t stream; + CUDACHECK(cudaStreamCreate(&stream)); + allGatherCftMultimemCountedKernel<<>>(sendWin, recvWin, count, devComm); + CUDACHECK(cudaStreamSynchronize(stream)); + + CUDACHECK(cudaMemcpy(hRecv, dRecv, recvBytes, cudaMemcpyDeviceToHost)); + bool ok = true; + for (int r = 0; r < totalRanks; r++) { + for (size_t i = 0; i < count; i++) { + float expected = (float)(r * 1000 + i); + float got = hRecv[r * count + i]; + if (got != expected) { + printf("rank %d mismatch at source rank %d element %zu: got %f expected %f\n", myRank, r, i, got, expected); + ok = false; + goto done_check; + } + } + } +done_check: + printf("rank %d allgather_cft_multimem_counted: %s\n", myRank, ok ? "PASSED" : "FAILED"); + + NCCLCHECK(ncclDevCommDestroy(comm, &devComm)); + NCCLCHECK(ncclCommWindowDeregister(comm, recvWin)); + NCCLCHECK(ncclCommWindowDeregister(comm, sendWin)); + NCCLCHECK(ncclMemFree(dRecv)); + NCCLCHECK(ncclMemFree(dSend)); + NCCLCHECK(ncclCommFinalize(comm)); + NCCLCHECK(ncclCommDestroy(comm)); + CUDACHECK(cudaStreamDestroy(stream)); + free(hRecv); + free(hSend); + return NULL; +} + +int main(int argc, char* argv[]) { + return run_example(argc, argv, allGatherCftMultimemCounted); +} diff --git a/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/README.md b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/README.md new file mode 100644 index 000000000000..6e95dca9077c --- /dev/null +++ b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/README.md @@ -0,0 +1,59 @@ + + +# NCCL Example: GIN Ring Exchange + +This example compares GPU-Initiated Networking (GIN) put optimization patterns +in NCCL Device API kernels. It runs the same ring-exchange traffic and buffer +layout across different producer models and completion patterns so their costs +can be compared side by side. + +The example focuses on two choices: + +- How to choose the producer granularity. +- How to provide request-aggregation hints while retaining weak per-put signals. + +See the [category README](../README.md) for the shared overview of the +optimization dimensions, buffer layout, and timing interpretation. + +## Overview + +The benchmark compares CTA-cooperative puts against one put per producer thread +inside each CTA. It also compares the default request flags against an +aggregate-request pattern where every put carries a weak signal and non-final +puts hint to the backend that more requests are coming. + +`ncclGinOptFlagsAggregateRequests` is a hint to the backend, not a prescribed +doorbell operation. One way a backend can use the hint is to delay ringing its +NIC doorbell for hinted requests, then ring after a later unhinted request to +publish the batch with one notification. + +Each rank sends to `(rank + 1) % nRanks` and receives from +`(rank - 1 + nRanks) % nRanks`, so the benchmark is a ring exchange with one +outgoing peer and one incoming peer per rank. + +Speedups are reported relative to the baseline configuration: producer +`cta_coop` and completion `weak per-put signals`. + +## Variants + +- **C/CUDA**: `c/` (creates an `ncclGin` context and calls `context.put` and + `context.waitSignal`; also uses `ncclGinOptFlagsAggregateRequests`, + `ncclCoopCta`, and `ncclCoopThread`) + - How to build, run, and inspect the implementation: `c/README.md` + +Note: This example is C/CUDA-only because it demonstrates device-side GIN +posting from CUDA kernels. + +## Expected Output + +You should see: + +- GIN support checks for each rank +- The fixed workload and four implementation variants +- Rank-0 whole-batch timing, amortized per-put timing, speedup versus the + baseline, and validation results aggregated across all ranks diff --git a/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/Makefile b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/Makefile new file mode 100644 index 000000000000..6c47bf2771ee --- /dev/null +++ b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/Makefile @@ -0,0 +1,88 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# See LICENSE.txt for more license information +# + +# Include common build rules +include ../../../../../makefiles/common.mk +include ../../../../../makefiles/examples.mk + +# Target executable +TARGET = ring_exchange + +# Common utilities +COMMON_INC = ../../../common/include +COMMON_SRC = ../../../common/src + +# Build configuration +INCLUDES += -I$(COMMON_INC) + +# Source files +SOURCES = main.cu $(COMMON_SRC)/utils.cc +OBJECTS = $(SOURCES:.cu=.o) +OBJECTS := $(OBJECTS:.cc=.o) + +# Default target +all: $(TARGET) + +# Build executable +$(TARGET): $(OBJECTS) +ifeq ($(MPI),1) + $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ +else + $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -lpthread -o $@ +endif + @echo "Built target $@" + +# Compile CUDA files +main.o: kernels.cuh + +%.o: %.cu + $(NVCC) $(NVCUFLAGS) $(INCLUDES) -c $< -o $@ + +%.o: %.cc +ifeq ($(MPI),1) + $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ +else + $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ +endif + +# Test target +test: $(TARGET) + @echo "Testing $(TARGET)..." +ifeq ($(MPI),1) + @echo "Running with 2 processes" + $(MPIRUN) -np 2 ./$(TARGET) +else + @echo "Running with all available GPUs" + ./$(TARGET) +endif + +# Clean build artifacts +clean: + rm -f $(OBJECTS) $(TARGET) + +# Install target +install: $(TARGET) + @mkdir -p $(PREFIX)/bin + cp $(TARGET) $(PREFIX)/bin/ + +# Help +help: + @echo "NCCL Example: GIN Ring Exchange" + @echo "===========================" + @echo "" + @echo "This example compares CTA-cooperative and thread-per-put" + @echo "GIN put producer and doorbell optimization patterns." + @echo "The executable runs one fixed ring-exchange workload." + @echo "" + @echo "Targets:" + @echo " all - Build the example (default)" + @echo " test - Build and run the example" + @echo " clean - Remove build artifacts" + @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" + @echo " help - Show this help" + +.PHONY: all test clean install help diff --git a/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/README.md b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/README.md new file mode 100644 index 000000000000..e05a35a0a10e --- /dev/null +++ b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/README.md @@ -0,0 +1,328 @@ + + +# C: GIN Ring Exchange + +This is the C/CUDA implementation of the NCCL Device API GIN ring-exchange +example. + +The executable runs the optimization choices described in the example overview +on one fixed ring-exchange workload and buffer layout so the results are +comparable. Each rank sends to `(rank + 1) % nRanks` and receives from +`(rank - 1 + nRanks) % nRanks`, giving each rank one outgoing peer and one +incoming peer. + +The example is split by responsibility: `kernels.cuh` contains the workload +geometry and four case-specific producer kernels, while `main.cu` contains +buffer initialization and verification, NCCL setup, timing, and result +reporting. + +Each configuration has its own kernel and contains its GIN put sequence +directly. The measured kernels do not select completion behavior through a +runtime flag: the per-put kernels use the default flags for every put, while the +aggregate-request kernels mark non-final puts with +`ncclGinOptFlagsAggregateRequests`. + +The comparison spans two dimensions: + +- **Producer model**: CTA-cooperative puts with `ncclCoopCta` versus one + `ncclCoopThread` put per producer thread inside each CTA. +- **Request aggregation**: weak per-put signals without versus with + aggregate-request hints. + +## Configuration Cases + +The executable runs these numbered configurations: + +| Case | Producer | Completion | +| ---: | --- | --- | +| 1 | `cta_coop` | `weak per-put signals` | +| 2 | `cta_coop` | `weak AggregateRequests` | +| 3 | `thread_per_put` | `weak per-put signals` | +| 4 | `thread_per_put` | `weak AggregateRequests` | + +These rows map directly to `ctaCooperativePutKernel`, +`ctaCooperativePutAggregateRequestsKernel`, `threadPerPutKernel`, and +`threadPerPutAggregateRequestsKernel`, respectively. + +All printed speedups are relative to configuration case 1. That row is the +simple baseline where every put uses the default flags and signals individually. + +The host does not request a GIN context count. The kernels use the +communicator's default count and map each CTA to +`ctaIndex % devComm.ginContextCount`. Each CTA uses its own signal index, and +its selected context participates in the world GIN barrier and the measured +put/flush path. + +The communicator requirements set `ginStrongSignalsRequired = false` because +every payload put uses `ncclGin_WeakSignalInc`; the example does not mix strong +and weak payload signals. The GIN barriers remain part of kernel coordination +and are separate from the payload completion pattern being compared. + +## Producer Models + +### CTA-Cooperative + +The CTA-cooperative kernels loop over the puts and post each one with +`ncclCoopCta`. The puts are therefore issued serially within each CTA. The +cooperative leader performs the backend post, while the other threads +participate in the required cooperative synchronization but do not issue +independent puts. This makes the implementation straightforward, but it can be +slower because most threads are not performing useful posting work. + +### Thread per Put + +The thread-per-put kernels assign one put to each CUDA thread and post with +`ncclCoopThread`. This exposes the puts in a CTA in parallel and gives each +producer thread independent posting work. + +## Completion Patterns + +Both producer models run the same two request patterns. They move the same +payload bytes to the same destination offsets and attach a weak signal to every +put; the difference is whether non-final puts carry an aggregate-request hint. + +`ncclGinOptFlagsAggregateRequests` hints to the backend that more requests are +coming. The backend decides how to use that information. For example, a backend +may delay ringing its NIC doorbell for hinted requests, then ring after a later +unhinted request so several requests become visible to the networking engine +with one notification. + +### Weak Per-Put Signals + +- **Purpose**: Simple baseline with straightforward per-put completion. +- **Hint behavior**: Every put uses the default flags, so no request indicates + that more requests are coming. +- **Signal behavior**: Every put increments the CTA-owned signal with + `ncclGin_WeakSignalInc`. Each weak signal covers only the put it is attached + to. +- **Signal wait behavior**: The receiver waits for `PUTS_PER_CTA` signal + increments per CTA batch, because every put advances the CTA-owned signal. +- **Source pattern**: Attach a weak signal to every put. + +This path favors the simplest posting sequence and leaves request handling +entirely to the backend's default behavior. + +```cpp +context.put(world, dstRank, recvWin, offset, sendWin, offset, bytesPerPut, + ncclGin_WeakSignalInc{signalIndex}, ncclGin_None{}, coop); +``` + +The wait target advances by the number of puts in each CTA batch: + +```cpp +context.waitSignal(coop, signalIndex, signalBase + (batch + 1) * PUTS_PER_CTA); +``` + +### Weak Signals with Aggregate-Request Hints + +- **Purpose**: Tell the backend that related requests follow while retaining + weak completion for every put. +- **Hint behavior**: Non-final puts use `ncclGinOptFlagsAggregateRequests`; the + final put uses the default flags to end the hinted sequence. +- **Example backend behavior**: A backend may use the hint to defer doorbell + ringing for non-final puts and publish the batch when it processes the final + unhinted put. +- **Signal behavior**: Every put increments the CTA-owned signal with + `ncclGin_WeakSignalInc`, so each signal covers only its attached put. +- **Signal wait behavior**: The receiver waits for `PUTS_PER_CTA` signal + increments per CTA batch, confirming every put independently. +- **Source pattern**: Attach a weak signal to every put, mark the non-final puts + with the aggregate-request hint, and use the default flags on the final put. + +The non-final puts carry the aggregate-request hint while preserving their +individual weak signals: + +```cpp +context.put(world, dstRank, recvWin, offset, sendWin, offset, bytesPerPut, + ncclGin_WeakSignalInc{signalIndex}, ncclGin_None{}, coop, ncclGin_None{}, + cuda::thread_scope_thread, cuda::thread_scope_device, + ncclGinOptFlagsAggregateRequests); +``` + +The final put also carries a weak signal but omits the aggregate-request hint: + +```cpp +context.put(world, dstRank, recvWin, offset, sendWin, offset, bytesPerPut, + ncclGin_WeakSignalInc{signalIndex}, ncclGin_None{}, coop, + ncclGin_None{}, cuda::thread_scope_thread, cuda::thread_scope_device, + ncclGinOptFlagsDefault); +``` + +The wait target advances by the number of puts in each CTA batch: + +```cpp +context.waitSignal(coop, signalIndex, signalBase + (batch + 1) * PUTS_PER_CTA); +``` + +The CTA-cooperative kernel posts serially, so its final unhinted put naturally +follows all hinted puts. The thread-per-put kernel posts concurrently, so +nonzero threads post the hinted puts, the CTA synchronizes, and thread 0 posts +the final unhinted put. Thread 0 is only the designated final producer. + +## Buffer Layout + +The CTA-cooperative and thread-per-put producer models issue the same number of +puts per CTA. Each `ctaIndex` owns a contiguous buffer region, and each +`putIndex` inside that CTA identifies a contiguous put-sized region: + +For a small illustrative case with `putsPerCta = 4`, the rank-local send and +receive windows are laid out as: + +| `ctaIndex` | `putIndex = 0` | `putIndex = 1` | `putIndex = 2` | `putIndex = 3` | +| --- | --- | --- | --- | --- | +| `ctaIndex = 0` | `(0, 0)` | `(0, 1)` | `(0, 2)` | `(0, 3)` | +| `ctaIndex = 1` | `(1, 0)` | `(1, 1)` | `(1, 2)` | `(1, 3)` | +| `ctaIndex = 2` | `(2, 0)` | `(2, 1)` | `(2, 2)` | `(2, 3)` | +| `ctaIndex = 3` | `(3, 0)` | `(3, 1)` | `(3, 2)` | `(3, 3)` | + +Each table cell is one contiguous put-sized region containing `elemsPerPut` +`int` values. In memory, the table is flattened row-major: all `putIndex` +regions for `ctaIndex = 0` come first, then those for `ctaIndex = 1`, and so on. + +```text +idx = ((ctaIndex * putsPerCta) + putIndex) * elemsPerPut + elemIndex +``` + +In the CTA-cooperative kernel, `putIndex` is the loop index. In the +thread-per-put kernel, `putIndex` is `threadIdx.x`. Both kernels compute the +same byte offset: + +```cpp +const size_t bytesPerPut = ELEMS_PER_PUT * sizeof(int); +const size_t offset = ((size_t)ctaIndex * PUTS_PER_CTA + putIndex) * bytesPerPut; +``` + +The send buffer stores an `int` validation tag derived from +`(srcRank, ctaIndex, putIndex, elemIndex)` in every payload value. Receive +validation uses the same layout, so it catches an incorrect source rank, +`ctaIndex`, `putIndex`, or element placement. + +Because the validation tag is stored in a plain `int`, the example assumes a +bounded rank count. With the current constants, validation supports up to +`MAX_VALIDATION_RANKS` ranks, which is 4096. Larger jobs should use a wider +validation payload or a different tag encoding; the executable warns and exits +before running if `totalRanks` exceeds that limit. + +## Workload + +The example uses one fixed workload so the implementation comparison stays +easy to follow: 4 CTAs x 32 puts/CTA = 128 puts/batch, with 256 elements per put +(128 KiB per batch). + +| CTAs | Puts per CTA | CTA threads | Elements per put | Warmup iterations | Timed iterations | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 4 | 32 | 32 | 256 | 10 | 100 | + +## Measurement Scope + +The benchmark measures elapsed kernel time with CUDA events around the whole +timed run, after a separate warmup run. The timed interval includes GIN put +posting, receive-side signal waits, `context.flush`, CTA synchronization, and +the entry/exit GIN barriers spread across `TIMED_ITERS`. + +Validation results are reduced across all ranks outside the timed interval. +Rank 0 prints a case and the final result as `PASSED` only when every rank +validates its complete receive buffer; any rank that finds a mismatch prints +its local error before the aggregate result. + +The printed `us/batch` value is rank 0's average time for one complete batch. A +batch is one full pass over the rank-local buffer layout. It includes every +`ctaIndex` row and every `putIndex` column in that row: + +```text +CTA_COUNT * PUTS_PER_CTA +``` + +The workload has 4 CTAs and 32 puts per CTA, so one batch contains 128 logical +puts. The CTA-cooperative producer issues those 32 puts per CTA with +`ncclCoopCta`; the thread-per-put producer issues the same 32 puts with 32 +producer threads. The `us/batch` value is the time for that whole set of puts +plus the waits, flushes, and synchronization in the timed loop. + +The printed `us/put` column is an amortized value derived from the whole-batch +time: + +```text +us_per_batch / (CTA_COUNT * PUTS_PER_CTA) +``` + +It is an end-to-end amortized cost per logical put for comparing +implementations. It is not the isolated latency of a single `context.put` call. + +## Building and Running + +This C/CUDA variant can be built using either pthread or MPI for +parallelization. pthread is the default choice. To use MPI, set `MPI=1` at +build time and optionally provide `MPI_HOME`. + +### Build + +From this directory: + +```bash +make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] +``` + +### Run when compiled for pthreads + +From this directory: + +```bash +[NTHREADS=N] ./ring_exchange +``` + +### Run when compiled for MPI + +From this directory: + +```bash +mpirun -np ./ring_exchange +``` + +## Expected Output + +```text +Starting GIN Ring Exchange initialization +GIN Ring Exchange: ctas=4, puts_per_cta=32, elems_per_put=256 + +=== Comparing GIN ring-exchange implementations === +Timing columns: us/batch is whole-batch time; us/put is derived from us/batch / (ctas * puts_per_cta). +Speedups are relative to configuration case 1. + +Configuration cases: + Case Producer Completion + ---- -------------- ---------------------- + 1 cta_coop weak per-put signals (baseline) + 2 cta_coop weak AggregateRequests + 3 thread_per_put weak per-put signals + 4 thread_per_put weak AggregateRequests + + Case us/batch us/put speedup result + ---- ------------ ------------ ------------- -------- + 1 *.** *.** *.**x PASSED + 2 *.** *.** *.**x PASSED + 3 *.** *.** *.**x PASSED + 4 *.** *.** *.**x PASSED +GIN Ring Exchange result: PASSED +``` + +The launcher can print additional MPI or pthread setup lines before the example +output. Actual timings depend on GPU, NIC, topology, message size, rank count, +and GIN backend. + +## Common Issues and Solutions + +### Issue: Example reports that GIN is unsupported + +**Solution:** Run on a system where NCCL reports GIN support through +`ncclCommQueryProperties`. + +### Issue: Only one GPU or rank is available + +**Solution:** Run with at least two ranks, for example `NTHREADS=2` in pthread +mode or `mpirun -np 2` in MPI mode. diff --git a/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/kernels.cuh b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/kernels.cuh new file mode 100644 index 000000000000..db3032e82c9d --- /dev/null +++ b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/kernels.cuh @@ -0,0 +1,181 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef GIN_OPTIMIZATIONS_RING_EXCHANGE_KERNELS_CUH_ +#define GIN_OPTIMIZATIONS_RING_EXCHANGE_KERNELS_CUH_ + +#include "cuda_runtime.h" +#include "nccl.h" +#include "nccl_device.h" +#include + +#define CTA_COUNT 4 +#define PUTS_PER_CTA 32 +#define ELEMS_PER_PUT 256 + +/* + * NCCL Device API GIN Ring Exchange Kernels + * + * Every rank sends to (rank + 1) % nRanks and receives from + * (rank - 1 + nRanks) % nRanks. + * + * Compared dimensions: + * 1. Producer model: + * - CTA-cooperative: every logical put is posted with ncclCoopCta. + * - Thread-per-put: every producer thread posts one ncclCoopThread put. + * 2. Completion pattern: + * - Weak signals: every put increments a weak signal. + * - Weak AggregateRequests: every put increments a weak signal, while + * non-final puts hint to the backend that more requests are coming. + */ + +__global__ void ctaCooperativePutKernel(ncclWindow_t sendWin, ncclWindow_t recvWin, + int iterations, ncclDevComm devComm) { + ncclCoopCta coop = ncclCoopCta(); + ncclTeam world = ncclTeamWorld(devComm); + + const unsigned int ctaIndex = blockIdx.x; + const int contextIndex = ctaIndex % devComm.ginContextCount; + ncclGin context{devComm, contextIndex}; + + const ncclGinSignal_t signalIndex = ctaIndex; + const int dstRank = (devComm.rank + 1) % devComm.nRanks; + const auto signalBase = context.readSignal(signalIndex); + const size_t bytesPerPut = ELEMS_PER_PUT * sizeof(int); + + ncclGinBarrierSession bar{coop, context, ncclTeamTagWorld(), ctaIndex}; + bar.sync(coop, cuda::memory_order_acquire, ncclGinFenceLevel::None); + + for (int batch = 0; batch < iterations; ++batch) { + for (int putIndex = 0; putIndex < PUTS_PER_CTA; ++putIndex) { + const size_t offset = (ctaIndex * PUTS_PER_CTA + putIndex) * bytesPerPut; + context.put(world, dstRank, recvWin, offset, sendWin, offset, bytesPerPut, + ncclGin_WeakSignalInc{signalIndex}, ncclGin_None{}, coop, ncclGin_None{}, + cuda::thread_scope_thread, cuda::thread_scope_device, + ncclGinOptFlagsDefault); + } + + context.waitSignal(coop, signalIndex, signalBase + (batch + 1) * PUTS_PER_CTA); + if (coop.thread_rank() == 0) { + context.flush(ncclCoopThread{}); + } + coop.sync(); + } + + bar.sync(coop, cuda::memory_order_release, ncclGinFenceLevel::None); +} + +__global__ void ctaCooperativePutAggregateRequestsKernel(ncclWindow_t sendWin, ncclWindow_t recvWin, + int iterations, ncclDevComm devComm) { + ncclCoopCta coop = ncclCoopCta(); + ncclTeam world = ncclTeamWorld(devComm); + + const unsigned int ctaIndex = blockIdx.x; + const int contextIndex = ctaIndex % devComm.ginContextCount; + ncclGin context{devComm, contextIndex}; + + const ncclGinSignal_t signalIndex = ctaIndex; + const int dstRank = (devComm.rank + 1) % devComm.nRanks; + const auto signalBase = context.readSignal(signalIndex); + const size_t bytesPerPut = ELEMS_PER_PUT * sizeof(int); + + ncclGinBarrierSession bar{coop, context, ncclTeamTagWorld(), ctaIndex}; + bar.sync(coop, cuda::memory_order_acquire, ncclGinFenceLevel::None); + + for (int batch = 0; batch < iterations; ++batch) { + for (int putIndex = 0; putIndex < PUTS_PER_CTA; ++putIndex) { + const size_t offset = (ctaIndex * PUTS_PER_CTA + putIndex) * bytesPerPut; + const uint32_t optFlags = putIndex == PUTS_PER_CTA - 1 ? + ncclGinOptFlagsDefault : + ncclGinOptFlagsAggregateRequests; + context.put(world, dstRank, recvWin, offset, sendWin, offset, bytesPerPut, + ncclGin_WeakSignalInc{signalIndex}, ncclGin_None{}, coop, ncclGin_None{}, + cuda::thread_scope_thread, cuda::thread_scope_device, optFlags); + } + + context.waitSignal(coop, signalIndex, signalBase + (batch + 1) * PUTS_PER_CTA); + if (coop.thread_rank() == 0) { + context.flush(ncclCoopThread{}); + } + coop.sync(); + } + + bar.sync(coop, cuda::memory_order_release, ncclGinFenceLevel::None); +} + +__global__ void threadPerPutKernel(ncclWindow_t sendWin, ncclWindow_t recvWin, + int iterations, ncclDevComm devComm) { + ncclCoopCta coop = ncclCoopCta(); + ncclTeam world = ncclTeamWorld(devComm); + + const unsigned int ctaIndex = blockIdx.x; + const int contextIndex = ctaIndex % devComm.ginContextCount; + ncclGin context{devComm, contextIndex}; + + const ncclGinSignal_t signalIndex = ctaIndex; + const int dstRank = (devComm.rank + 1) % devComm.nRanks; + const auto signalBase = context.readSignal(signalIndex); + const int putIndex = threadIdx.x; + const size_t bytesPerPut = ELEMS_PER_PUT * sizeof(int); + const size_t offset = (ctaIndex * PUTS_PER_CTA + putIndex) * bytesPerPut; + + ncclGinBarrierSession bar{coop, context, ncclTeamTagWorld(), ctaIndex}; + bar.sync(coop, cuda::memory_order_acquire, ncclGinFenceLevel::None); + + for (int batch = 0; batch < iterations; ++batch) { + context.put(world, dstRank, recvWin, offset, sendWin, offset, bytesPerPut, + ncclGin_WeakSignalInc{signalIndex}, ncclGin_None{}, ncclCoopThread{}); + + context.waitSignal(coop, signalIndex, signalBase + (batch + 1) * PUTS_PER_CTA); + context.flush(coop); + } + + bar.sync(coop, cuda::memory_order_release, ncclGinFenceLevel::None); +} + +__global__ void threadPerPutAggregateRequestsKernel(ncclWindow_t sendWin, ncclWindow_t recvWin, + int iterations, ncclDevComm devComm) { + ncclCoopCta coop = ncclCoopCta(); + ncclTeam world = ncclTeamWorld(devComm); + + const unsigned int ctaIndex = blockIdx.x; + const int contextIndex = ctaIndex % devComm.ginContextCount; + ncclGin context{devComm, contextIndex}; + + const ncclGinSignal_t signalIndex = ctaIndex; + const int dstRank = (devComm.rank + 1) % devComm.nRanks; + const auto signalBase = context.readSignal(signalIndex); + const int putIndex = threadIdx.x; + const size_t bytesPerPut = ELEMS_PER_PUT * sizeof(int); + const size_t offset = (ctaIndex * PUTS_PER_CTA + putIndex) * bytesPerPut; + + ncclGinBarrierSession bar{coop, context, ncclTeamTagWorld(), ctaIndex}; + bar.sync(coop, cuda::memory_order_acquire, ncclGinFenceLevel::None); + + for (int batch = 0; batch < iterations; ++batch) { + if (putIndex != 0) { + context.put(world, dstRank, recvWin, offset, sendWin, offset, bytesPerPut, + ncclGin_WeakSignalInc{signalIndex}, ncclGin_None{}, ncclCoopThread{}, + ncclGin_None{}, cuda::thread_scope_thread, cuda::thread_scope_device, + ncclGinOptFlagsAggregateRequests); + } + + coop.sync(); + + if (putIndex == 0) { + context.put(world, dstRank, recvWin, offset, sendWin, offset, bytesPerPut, + ncclGin_WeakSignalInc{signalIndex}, ncclGin_None{}, ncclCoopThread{}); + } + + context.waitSignal(coop, signalIndex, signalBase + (batch + 1) * PUTS_PER_CTA); + context.flush(coop); + } + + bar.sync(coop, cuda::memory_order_release, ncclGinFenceLevel::None); +} + +#endif // GIN_OPTIMIZATIONS_RING_EXCHANGE_KERNELS_CUH_ diff --git a/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/main.cu b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/main.cu new file mode 100644 index 000000000000..c39b53ddb620 --- /dev/null +++ b/projects/rccl/docs/examples/09_gin_optimizations/01_ring_exchange/c/main.cu @@ -0,0 +1,309 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#include "cuda_runtime.h" +#include "kernels.cuh" +#include "nccl.h" +#include "nccl_device.h" +#include "utils.h" +#include +#include +#include +#include +#include + +#define WARMUP_ITERS 10 +#define TIMED_ITERS 100 +#define BUFFER_SIZE_BYTES ((size_t)CTA_COUNT * PUTS_PER_CTA * ELEMS_PER_PUT * sizeof(int)) +#define ELEMENT_VALUE_PERIOD 4096 +#define MAX_VALIDATION_RANKS \ + (INT_MAX / (CTA_COUNT * PUTS_PER_CTA * ELEMENT_VALUE_PERIOD) + 1) + +static std::atomic validationSuccess{true}; + +static void fillSendBuffer(int* sendBuff, int myRank) { + for (int ctaIndex = 0; ctaIndex < CTA_COUNT; ++ctaIndex) { + for (int putIndex = 0; putIndex < PUTS_PER_CTA; ++putIndex) { + for (size_t elem = 0; elem < ELEMS_PER_PUT; ++elem) { + const size_t idx = ((size_t)ctaIndex * PUTS_PER_CTA + putIndex) * ELEMS_PER_PUT + elem; + sendBuff[idx] = (((myRank * CTA_COUNT + ctaIndex) * PUTS_PER_CTA + putIndex) * + ELEMENT_VALUE_PERIOD) + (int)(elem % ELEMENT_VALUE_PERIOD); + } + } + } +} + +static bool verifyRecvBuffer(int myRank, int totalRanks, const int* recvBuff, const char* label) { + const int srcRank = (myRank - 1 + totalRanks) % totalRanks; + for (int ctaIndex = 0; ctaIndex < CTA_COUNT; ++ctaIndex) { + for (int putIndex = 0; putIndex < PUTS_PER_CTA; ++putIndex) { + for (size_t elem = 0; elem < ELEMS_PER_PUT; ++elem) { + const size_t idx = ((size_t)ctaIndex * PUTS_PER_CTA + putIndex) * ELEMS_PER_PUT + elem; + const int expected = (((srcRank * CTA_COUNT + ctaIndex) * PUTS_PER_CTA + putIndex) * + ELEMENT_VALUE_PERIOD) + (int)(elem % ELEMENT_VALUE_PERIOD); + if (recvBuff[idx] != expected) { + printf(" Rank %d %s mismatch at CTA %d put %d elem %zu: got %d, expected %d\n", + myRank, label, ctaIndex, putIndex, elem, recvBuff[idx], expected); + return false; + } + } + } + } + return true; +} + +static float runTimedKernel(int myRank, int caseIndex, ncclComm_t comm, + ncclWindow_t sendWin, ncclWindow_t recvWin, void* dRecvBuff, + int* hRecvBuff, ncclDevComm devComm, cudaStream_t stream, bool* success) { + CUDACHECK(cudaMemset(dRecvBuff, 0, BUFFER_SIZE_BYTES)); + switch (caseIndex) { + case 0: + ctaCooperativePutKernel<<>>( + sendWin, recvWin, WARMUP_ITERS, devComm); + break; + case 1: + ctaCooperativePutAggregateRequestsKernel<<>>( + sendWin, recvWin, WARMUP_ITERS, devComm); + break; + case 2: + threadPerPutKernel<<>>( + sendWin, recvWin, WARMUP_ITERS, devComm); + break; + case 3: + threadPerPutAggregateRequestsKernel<<>>( + sendWin, recvWin, WARMUP_ITERS, devComm); + break; + } + CUDACHECK(cudaGetLastError()); + CUDACHECK(cudaStreamSynchronize(stream)); + + CUDACHECK(cudaMemset(dRecvBuff, 0, BUFFER_SIZE_BYTES)); + + cudaEvent_t start, stop; + CUDACHECK(cudaEventCreate(&start)); + CUDACHECK(cudaEventCreate(&stop)); + + CUDACHECK(cudaEventRecord(start, stream)); + switch (caseIndex) { + case 0: + ctaCooperativePutKernel<<>>( + sendWin, recvWin, TIMED_ITERS, devComm); + break; + case 1: + ctaCooperativePutAggregateRequestsKernel<<>>( + sendWin, recvWin, TIMED_ITERS, devComm); + break; + case 2: + threadPerPutKernel<<>>( + sendWin, recvWin, TIMED_ITERS, devComm); + break; + case 3: + threadPerPutAggregateRequestsKernel<<>>( + sendWin, recvWin, TIMED_ITERS, devComm); + break; + } + CUDACHECK(cudaGetLastError()); + CUDACHECK(cudaEventRecord(stop, stream)); + CUDACHECK(cudaStreamSynchronize(stream)); + + float milliseconds = 0.0f; + CUDACHECK(cudaEventElapsedTime(&milliseconds, start, stop)); + + CUDACHECK(cudaMemcpy(hRecvBuff, dRecvBuff, BUFFER_SIZE_BYTES, cudaMemcpyDeviceToHost)); + const char* verifyLabels[4] = { + "cta_coop weak per-put signals", + "cta_coop weak AggregateRequests", + "thread_per_put weak per-put signals", + "thread_per_put weak AggregateRequests" + }; + *success = verifyRecvBuffer(myRank, devComm.nRanks, hRecvBuff, verifyLabels[caseIndex]); + + int passedRanks = *success ? 1 : 0; + CUDACHECK(cudaMemcpy(dRecvBuff, &passedRanks, sizeof(passedRanks), cudaMemcpyHostToDevice)); + NCCLCHECK(ncclAllReduce(dRecvBuff, dRecvBuff, 1, ncclInt, ncclSum, comm, stream)); + CUDACHECK(cudaStreamSynchronize(stream)); + CUDACHECK(cudaMemcpy(&passedRanks, dRecvBuff, sizeof(passedRanks), cudaMemcpyDeviceToHost)); + *success = passedRanks == devComm.nRanks; + + CUDACHECK(cudaEventDestroy(start)); + CUDACHECK(cudaEventDestroy(stop)); + return milliseconds; +} + +void* ringExchangeExample(int myRank, int totalRanks, int localDevice, int devicesPerRank) { + (void)devicesPerRank; + + if (totalRanks < 2) { + if (myRank == 0) { + printf("GIN Ring Exchange example requires at least 2 ranks.\n"); + } + return NULL; + } + if (totalRanks > MAX_VALIDATION_RANKS) { + if (myRank == 0) { + printf("WARNING: GIN Ring Exchange validation uses int payload tags and supports at most %d ranks; " + "got %d. Skipping the run to avoid validation tag overflow.\n", + MAX_VALIDATION_RANKS, totalRanks); + } + return NULL; + } + + ncclComm_t comm; + ncclUniqueId commId; + + if (myRank == 0) { + printf("Starting GIN Ring Exchange initialization\n"); + } + + if (myRank == 0) { + NCCLCHECK(ncclGetUniqueId(&commId)); + } + util_broadcast(0, myRank, &commId); + + CUDACHECK(cudaSetDevice(localDevice)); + printf(" Rank %d using GPU device %d\n", myRank, localDevice); + + NCCLCHECK(ncclCommInitRank(&comm, totalRanks, commId, myRank)); + + ncclCommProperties_t props = NCCL_COMM_PROPERTIES_INITIALIZER; + NCCLCHECK(ncclCommQueryProperties(comm, &props)); + if (!props.deviceApiSupport) { + printf("ERROR: rank %d communicator does not support Device API!\n", myRank); + NCCLCHECK(ncclCommFinalize(comm)); + NCCLCHECK(ncclCommDestroy(comm)); + return NULL; + } + if (props.ginType == NCCL_GIN_TYPE_NONE) { + printf("ERROR: rank %d communicator does not support GIN!\n", myRank); + NCCLCHECK(ncclCommFinalize(comm)); + NCCLCHECK(ncclCommDestroy(comm)); + return NULL; + } + + if (myRank == 0) { + printf("GIN Ring Exchange: ctas=%d, puts_per_cta=%d, elems_per_put=%d\n", + CTA_COUNT, PUTS_PER_CTA, ELEMS_PER_PUT); + } + + int* hSendBuff = (int*)malloc(BUFFER_SIZE_BYTES); + int* hRecvBuff = (int*)malloc(BUFFER_SIZE_BYTES); + if (hSendBuff == NULL || hRecvBuff == NULL) { + printf("ERROR: rank %d failed to allocate host buffers\n", myRank); + free(hSendBuff); + free(hRecvBuff); + NCCLCHECK(ncclCommFinalize(comm)); + NCCLCHECK(ncclCommDestroy(comm)); + return NULL; + } + + void* dSendBuff = NULL; + void* dRecvBuff = NULL; + ncclWindow_t sendWin; + ncclWindow_t recvWin; + NCCLCHECK(ncclMemAlloc(&dSendBuff, BUFFER_SIZE_BYTES)); + NCCLCHECK(ncclMemAlloc(&dRecvBuff, BUFFER_SIZE_BYTES)); + + NCCLCHECK(ncclCommWindowRegister(comm, dSendBuff, BUFFER_SIZE_BYTES, &sendWin, NCCL_WIN_COLL_SYMMETRIC)); + NCCLCHECK(ncclCommWindowRegister(comm, dRecvBuff, BUFFER_SIZE_BYTES, &recvWin, NCCL_WIN_COLL_SYMMETRIC)); + + cudaStream_t stream; + CUDACHECK(cudaStreamCreate(&stream)); + + ncclDevComm devComm; + ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; + reqs.worldGinBarrierCount = CTA_COUNT; + reqs.ginSignalCount = CTA_COUNT; + reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; + reqs.ginStrongSignalsRequired = false; + NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); + + if (myRank == 0) { + printf("\n=== Comparing GIN ring-exchange implementations ===\n"); + printf("Timing columns: us/batch is whole-batch time; us/put is derived from us/batch / (ctas * puts_per_cta).\n"); + printf("Speedups are relative to configuration case 1.\n"); + printf("\nConfiguration cases:\n"); + printf(" %-4s %-14s %-22s\n", "Case", "Producer", "Completion"); + printf(" %-4s %-14s %-22s\n", "----", "--------------", "----------------------"); + printf(" 1 %-14s %-22s (baseline)\n", "cta_coop", "weak per-put signals"); + printf(" 2 %-14s %-22s\n", "cta_coop", "weak AggregateRequests"); + printf(" 3 %-14s %-22s\n", "thread_per_put", "weak per-put signals"); + printf(" 4 %-14s %-22s\n", "thread_per_put", "weak AggregateRequests"); + } + + fillSendBuffer(hSendBuff, myRank); + CUDACHECK(cudaMemcpy(dSendBuff, hSendBuff, BUFFER_SIZE_BYTES, cudaMemcpyHostToDevice)); + + float elapsedMs[4]; + bool passed[4]; + bool success = false; + + // Case 1: CTA-cooperative producer without aggregate-request hints. + elapsedMs[0] = runTimedKernel(myRank, 0, comm, sendWin, recvWin, + dRecvBuff, hRecvBuff, devComm, stream, &success); + passed[0] = success; + + // Case 2: CTA-cooperative producer with aggregate-request hints. + success = false; + elapsedMs[1] = runTimedKernel(myRank, 1, comm, sendWin, recvWin, + dRecvBuff, hRecvBuff, devComm, stream, &success); + passed[1] = success; + + // Case 3: One producer thread per put without aggregate-request hints. + success = false; + elapsedMs[2] = runTimedKernel(myRank, 2, comm, sendWin, recvWin, + dRecvBuff, hRecvBuff, devComm, stream, &success); + passed[2] = success; + + // Case 4: One producer thread per put with aggregate-request hints. + success = false; + elapsedMs[3] = runTimedKernel(myRank, 3, comm, sendWin, recvWin, + dRecvBuff, hRecvBuff, devComm, stream, &success); + passed[3] = success; + + const bool allCasesPassed = passed[0] && passed[1] && passed[2] && passed[3]; + if (!allCasesPassed) { + validationSuccess.store(false, std::memory_order_relaxed); + } + if (myRank == 0) { + printf("\n %-4s %12s %12s %13s %8s\n", + "Case", "us/batch", "us/put", "speedup", "result"); + printf(" %-4s %12s %12s %13s %8s\n", + "----", "------------", "------------", "-------------", "--------"); + for (int caseIndex = 0; caseIndex < 4; ++caseIndex) { + const double usPerBatch = elapsedMs[caseIndex] * 1000.0 / TIMED_ITERS; + const double usPerPut = usPerBatch / (CTA_COUNT * PUTS_PER_CTA); + const double speedup = (elapsedMs[0] > 0.0 && elapsedMs[caseIndex] > 0.0) ? + elapsedMs[0] / elapsedMs[caseIndex] : 0.0; + + printf(" %-4d %12.2f %12.2f %12.2fx %8s\n", + caseIndex + 1, usPerBatch, usPerPut, speedup, + passed[caseIndex] ? "PASSED" : "FAILED"); + } + printf("\nGIN Ring Exchange result: %s\n", allCasesPassed ? "PASSED" : "FAILED"); + } + + NCCLCHECK(ncclDevCommDestroy(comm, &devComm)); + NCCLCHECK(ncclCommWindowDeregister(comm, sendWin)); + NCCLCHECK(ncclCommWindowDeregister(comm, recvWin)); + NCCLCHECK(ncclMemFree(dSendBuff)); + NCCLCHECK(ncclMemFree(dRecvBuff)); + + CUDACHECK(cudaStreamDestroy(stream)); + free(hSendBuff); + free(hRecvBuff); + + NCCLCHECK(ncclCommFinalize(comm)); + NCCLCHECK(ncclCommDestroy(comm)); + + return NULL; +} + +int main(int argc, char* argv[]) { + const int result = run_example(argc, argv, ringExchangeExample); + if (result != 0) return result; + return validationSuccess.load(std::memory_order_relaxed) ? 0 : 1; +} diff --git a/projects/rccl/docs/examples/09_gin_optimizations/Makefile b/projects/rccl/docs/examples/09_gin_optimizations/Makefile new file mode 100644 index 000000000000..a1ac6ac758b2 --- /dev/null +++ b/projects/rccl/docs/examples/09_gin_optimizations/Makefile @@ -0,0 +1,56 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# See LICENSE.txt for more license information +# + +# NCCL GIN Optimizations +EXAMPLES = 01_ring_exchange + +# Default target +all: $(EXAMPLES) + +# Build individual examples +$(EXAMPLES): + $(MAKE) -C $@/c + +# Clean all build artifacts +clean: + for example in $(EXAMPLES); do \ + $(MAKE) -C $$example/c clean; \ + done + +# Test all examples +test: all + for example in $(EXAMPLES); do \ + echo "Testing $$example..."; \ + $(MAKE) -C $$example/c test || exit 1; \ + done + +# Install all examples +install: all + for example in $(EXAMPLES); do \ + echo "Installing $$example..."; \ + $(MAKE) -C $$example/c install || exit 1; \ + done + +# Help +help: + @echo "NCCL GIN Optimizations" + @echo "======================" + @echo "" + @echo "Targets:" + @echo " all - Build all examples" + @echo " clean - Clean all build artifacts" + @echo " test - Test all examples" + @echo " install - Install all examples" + @echo " help - Show this help" + @echo "" + @echo "Examples:" + @echo " 01_ring_exchange - Compare GIN put optimizations with ring-exchange traffic" + @echo "" + @echo "To build/run individual examples:" + @echo " make -C 01_ring_exchange/c" + +.PHONY: all clean test install help $(EXAMPLES) diff --git a/projects/rccl/docs/examples/09_gin_optimizations/README.md b/projects/rccl/docs/examples/09_gin_optimizations/README.md new file mode 100644 index 000000000000..f5e3de7c1c39 --- /dev/null +++ b/projects/rccl/docs/examples/09_gin_optimizations/README.md @@ -0,0 +1,75 @@ + + +# NCCL GIN Optimizations + +## Overview + +This directory contains focused examples for understanding and tuning +GPU-Initiated Networking (GIN) in NCCL Device API applications. The examples +demonstrate how to structure, measure, and validate reusable GIN optimization +techniques. + +Each subdirectory defines its own communication pattern, compared +implementations, workload, measurement methodology, and correctness checks. +Consult the individual example README for those details. + +## Examples + +### [01_ring_exchange](01_ring_exchange/) + +**GIN Ring Exchange** + +This example measures GIN producer models and aggregate-request hints under one +comparable ring-exchange workload. Each rank sends to one peer and receives from +one peer. The executable launches multiple CTAs, maps each CTA to +`ctaIndex % devComm.ginContextCount`, and uses one signal index per CTA. + +The executable compares: + +- **Producer model**: CTA-cooperative puts with `ncclCoopCta` versus one + `ncclCoopThread` put per producer thread inside each CTA. +- **Request aggregation**: weak per-put signals using default request flags + versus the same signals with aggregate-request hints on non-final puts. + +These two dimensions form the four implementations: + +| Case | Producer model | Request flags | +| ---: | --- | --- | +| 1 | CTA-cooperative | Default flags on every put | +| 2 | CTA-cooperative | `AggregateRequests` on non-final puts | +| 3 | One producer thread per put | Default flags on every put | +| 4 | One producer thread per put | `AggregateRequests` on non-final puts | + +`ncclGinOptFlagsAggregateRequests` hints to the backend that more requests are +coming. A backend may use this hint to reduce notification overhead; for +example, it can delay ringing its NIC doorbell until a later unhinted request +and publish several requests with one ring. + +All four implementations run the same fixed workload and use only weak payload +signals. Speedups are reported relative to the CTA-cooperative configuration +using default request flags. Rank 0 reports whole-batch time and an end-to-end +amortized cost per logical put, while validation checks the complete payload +layout across all ranks. + +#### Buffer Layout + +Each `ctaIndex` owns a contiguous buffer region, and each `putIndex` inside that +CTA identifies a contiguous put-sized region: + +```text +idx = ((ctaIndex * putsPerCta) + putIndex) * elemsPerPut + elemIndex +``` + +The send buffer encodes source rank, CTA index, put index, and element index in +the payload. Receive-side validation uses the same layout, so it detects an +incorrect source rank, `ctaIndex`, `putIndex`, or element placement. + +The example creates an `ncclGin` context and calls `context.put` and +`context.waitSignal`. It also uses `ncclGinOptFlagsAggregateRequests`, +`ncclCoopCta`, `ncclCoopThread`, `ncclGin_WeakSignalInc`, and +`ncclGinBarrierSession`. diff --git a/projects/rccl/docs/how-to/device-api-gin.rst b/projects/rccl/docs/how-to/device-api-gin.rst index 39d0c4959ceb..0f905f548196 100644 --- a/projects/rccl/docs/how-to/device-api-gin.rst +++ b/projects/rccl/docs/how-to/device-api-gin.rst @@ -14,6 +14,9 @@ RCCL 2.31 incorporates Device API and GIN enhancements from upstream NCCL 2.31.2. This page describes those APIs and the limits of the AMD host-proxy backend. +To supply a GIN backend rather than consume one, see +:ref:`using-rccl-gin-plugin`. + ``NCCL_GIN_TYPE`` values for AMD backends are **not compatible with 2.30.7**: rocSHMEM GDA moved from 5 to 6 and Anvil SDMA from 6 to 7 because NCCL 2.31 placed EFA GDA at 5. The IB proxy remains ``NCCL_GIN_TYPE=2``. @@ -91,10 +94,17 @@ request; use the value returned in ``devComm.ginContextCount`` when assigning work to contexts. ``ginTrafficClass`` overrides the host communicator traffic class for this -device communicator. ``NCCL_IB_SL`` independently overrides the InfiniBand -service level, and ``NCCL_IB_TC`` independently overrides the RoCE traffic -class. Set ``reqs.ginTrafficClass`` only when the fabric administrator provides -an appropriate value. +device communicator. The service level of the GIN connections comes from +``NCCL_IB_SL`` when it is set, otherwise from ``ginTrafficClass`` or the host +communicator. Their RoCE traffic class comes from ``NCCL_GIN_IB_TC`` when it is +set, then from ``NCCL_IB_TC``, and only then from ``ginTrafficClass`` or the +host communicator. Setting ``NCCL_GIN_IB_TC`` moves GIN traffic to its own RoCE +traffic class while the collective and point-to-point connections keep +``NCCL_IB_TC``. InfiniBand link layers don't carry the traffic class, so +``NCCL_GIN_IB_TC`` has no effect there; GIN traffic can be separated only +through the service level, by setting ``ginTrafficClass`` and leaving +``NCCL_IB_SL`` unset. Set these values only when the fabric administrator +provides appropriate values. Device code creates an ``ncclGin`` object for one returned context: @@ -114,7 +124,7 @@ For example, a CTA can issue a put and wait for local queue completion: gin.put(ncclTeamWorld(devComm), peer, destination, /*destinationOffset=*/0, source, /*sourceOffset=*/0, bytes, - ncclGin_SignalInc{/*signal=*/0}); + ncclGin_WeakSignalInc{/*signal=*/0}); } gin.flush(ncclCoopCta()); } @@ -128,6 +138,63 @@ the destination bytes. resource-sharing behavior on direct device backends. The AMD GIN host-proxy backend uses the same proxy queue behavior for both modes. +Choose strong or weak signal semantics +====================================== + +Explicit signal types state which earlier puts become visible when a peer +observes the signal: + +* A **weak signal** guarantees visibility of only the put carrying that signal. + Use it for independent per-operation completion, as in the example above. It + makes no guarantee about earlier puts, although a backend can order them. +* A **strong signal** also guarantees visibility of every preceding put issued + to the same peer through the same GIN context. Use it as a publication point + for a sequence of puts. + +For example, one strong signal can publish two buffers without attaching a +signal to the first put: + +.. code-block:: cpp + + if (threadIdx.x == 0) { + auto team = ncclTeamWorld(devComm); + gin.put(team, peer, destination, 0, source, 0, firstBytes, + ncclGin_None{}); + gin.put(team, peer, destination, secondOffset, + source, secondOffset, secondBytes, + ncclGin_StrongSignalInc{/*signal=*/0}); + } + gin.flush(ncclCoopCta()); + +After the peer's ``waitSignal`` returns, both puts are visible. This ordering is +limited to the same peer and GIN context. A strong signal does not publish puts +issued through another context or to another peer. + +The explicit indexed actions are ``ncclGin_StrongSignalInc``, +``ncclGin_StrongSignalAdd``, ``ncclGin_WeakSignalInc``, and +``ncclGin_WeakSignalAdd``. Equivalent ``StrongVASignal`` and ``WeakVASignal`` +actions target a window and offset; their window must be registered with +``NCCL_WIN_STRICT_ORDERING``. + +``ginStrongSignalsRequired`` defaults to ``true`` in +``NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER``. The runtime rejects device +communicator creation if the selected backend cannot provide strong signals. +A kernel that uses only weak signals can set the field to ``false`` to permit +such a backend: + +.. code-block:: cpp + + ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; + reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; + reqs.ginSignalCount = 1; + reqs.ginStrongSignalsRequired = false; // Kernel issues only weak signals. + +Do not issue a strong action from a device communicator created with +``ginStrongSignalsRequired=false``; that usage is undefined. The older +``ncclGin_SignalInc``, ``ncclGin_SignalAdd``, and VA equivalents are +deprecated. Their strength is selected globally by +``ginStrongSignalsRequired``; new code should use an explicit type. + Use world-team barriers and timeouts ==================================== @@ -176,11 +243,21 @@ Version and backend notes ``ncclDevComm`` is versioned. The upstream NCCL 2.30.3 and 2.30.4 release notes require applications using GIN APIs to be rebuilt with the matching release. -The upstream NCCL 2.30.7 release notes add ``ncclGinFenceLevel`` semantics for -GIN barriers (``None``, ``Put``, ``Get``, default ``Put | Get``). RCCL accepts -compatible layouts within the 2.30 family, but applications using pre-2.30 GIN -device code must be rebuilt with compatible RCCL headers. The runtime rejects -pre-2.30 requirements that request indexed GIN resources. +The upstream NCCL 2.30.7 release notes describe explicit Strong and Weak +signals and add ``ncclGinFenceLevel`` semantics for GIN barriers (``None``, +``Put``, ``Get``, default ``Put | Get``). The explicit signal API is marked +available since NCCL 2.30.5. RCCL accepts compatible layouts within the 2.30 +family, but applications using pre-2.30 GIN device code must be rebuilt with +compatible RCCL headers. The runtime rejects pre-2.30 requirements that request +indexed GIN resources. + +``NCCL_GIN_IB_TC``, added in upstream NCCL 2.30.7, applies to the connections +of the IB proxy backend (``NCCL_GIN_TYPE=2``). The one-sided host RMA +operations run on the same backend and use the same traffic class. The +rocSHMEM GDA backend (``NCCL_GIN_TYPE=6``) doesn't read ``NCCL_GIN_IB_TC``, +``NCCL_IB_TC``, or ``ginTrafficClass``; its queue pairs take their traffic +class from ``ROCSHMEM_GDA_TRAFFIC_CLASS``. Anvil SDMA (``NCCL_GIN_TYPE=7``) +moves data within a node without a NIC, so traffic class doesn't apply to it. The 128-byte, versioned GIN proxy descriptor and per-context proxy progress are internal implementation details and require no application configuration. diff --git a/projects/rccl/docs/how-to/using-rccl-gin-plugin-api.rst b/projects/rccl/docs/how-to/using-rccl-gin-plugin-api.rst new file mode 100644 index 000000000000..4491304ec433 --- /dev/null +++ b/projects/rccl/docs/how-to/using-rccl-gin-plugin-api.rst @@ -0,0 +1,117 @@ +.. meta:: + :description: How to provide a GIN backend for RCCL + :keywords: RCCL, ROCm, library, API, GIN, RMA, plugin + +.. _using-rccl-gin-plugin: + +********************************** +Using the RCCL GIN plugin API +********************************** + +GIN (GPU-Initiated Networking) lets a kernel issue network operations itself, +through the device-side API described in :doc:`device-api-gin`. This topic covers the +other side of that interface: how to supply the *backend* that carries those +operations over your network. + +RCCL supports two kinds of GIN backend, and which one you write determines which +plugin API you implement: + +* A **proxy** backend, where RCCL implements the device side for you and converts + each GIN request into a host-initiated network operation. You only implement a + host-side API, and that API is the :ref:`RMA plugin API `. +* A **device-initiated** backend, where your code runs on the GPU and talks to the + NIC directly. This requires both device-side and host-side work inside RCCL and + is not a plugin-only deliverable. + +.. _gin-proxy-backend: + +Proxy backends use the RMA plugin API +===================================== + +Proxy GIN is built in and enabled by default. RCCL registers an internal backend, +``ncclGinProxy``, which implements the GIN device API and drains each request through +a host progress thread. It does not move data itself: it forwards to whatever RMA +backend the communicator adopted, so the way to put a custom network behind proxy +GIN is to write an RMA plugin. See :ref:`using-rccl-rma-plugin` for the v15 +interface, compatibility behavior and loading instructions. + +When the RMA plugin is adopted, RCCL reports the first line below. If an external +proxy-type GIN plugin is also selected through ``NCCL_GIN_PLUGIN``, RCCL skips it +in favor of the built-in proxy and reports the second line: + +.. code-block:: shell + + RMA/Plugin: Assigned plugin Example to comm + GIN/Plugin: ... using NCCL GIN proxy over RMA backend Example + +.. note:: + + An external plugin loaded through ``NCCL_GIN_PLUGIN`` that reports + ``NCCL_NET_DEVICE_GIN_PROXY`` is deliberately *not* used: the built-in + ``ncclGinProxy`` already provides that backend type, and RCCL keeps it rather + than the external one. This is why a custom proxy backend belongs behind + ``NCCL_RMA_PLUGIN`` and not ``NCCL_GIN_PLUGIN``. + +Device-initiated backends +========================= + +A device-initiated backend cannot be supplied by an external plugin alone. It needs, +inside RCCL: + +* a value in ``ncclGinType_t`` (``src/include/nccl_device/core.h``) matching the + corresponding ``NCCL_NET_DEVICE_GIN_*`` device type, +* a ``case`` for that type, with its supported version list, in the backend + dispatch in ``src/gin/gin_host.cc``, which rejects unknown types, and +* a device-side implementation of the GIN API, compiled per GPU architecture and + linked into the device binary. + +The in-tree device-initiated backends are built this way: ``ROCSHMEM_GDA`` and +``ANVIL_SDMA``, both enabled by configuring with ``--rocshmem-gin``. + +What the GIN example plugin shows +================================= + +``plugins/gin/example`` is a reference for the *shape* of the host-side GIN vtable +and for exporting several ``ncclGinPlugin_vNN`` versions from one object. Because it +reports the proxy device type, RCCL supersedes it as described above, so only +``init``, ``devices``, ``getProperties`` and ``finalize`` ever run. Treat it as an +API reference rather than a working backend, and use ``plugins/rma/example`` as the +starting point for a custom proxy backend. + +Environment variables +===================== + +.. list-table:: + :header-rows: 1 + :widths: 30,50,20 + + * - Variable + - Description + - Default + + * - ``NCCL_GIN_PLUGIN`` + - Comma-separated list of GIN plugins to load, each a path or a short name + resolved against the ``librccl-gin`` prefix. + - Unset + + * - ``NCCL_GIN_ENABLE`` + - Set to ``0`` to register no GIN backend, so GIN reports as unsupported. + RCCL-specific. + - ``1`` + + * - ``NCCL_GIN_TYPE`` + - Require a specific backend type, skipping backends of any other type: + ``2`` proxy, ``3`` GDAKI, ``4`` GPI, ``5`` EFA GDA, ``6`` rocSHMEM GDA, + ``7`` Anvil SDMA. ``-1`` disables the generic type filter, but does not + auto-enable the in-tree device backends: rocSHMEM GDA requires ``6``, + while Anvil SDMA accepts an unset value or ``7``. + - Unset + + * - ``NCCL_RMA_PLUGIN`` + - Comma-separated list of RMA plugins to load, each a path or a short name + resolved against the ``librccl-rma`` prefix. This is the backend proxy GIN + forwards to. + - Unset + +For the consumer side of GIN, including how a kernel issues put, signal and flush +operations, see :doc:`device-api-gin`. diff --git a/projects/rccl/docs/how-to/using-rccl-rma-plugin-api.rst b/projects/rccl/docs/how-to/using-rccl-rma-plugin-api.rst new file mode 100644 index 000000000000..e564497b1c5b --- /dev/null +++ b/projects/rccl/docs/how-to/using-rccl-rma-plugin-api.rst @@ -0,0 +1,77 @@ +.. meta:: + :description: How to provide an external one-sided RMA backend for RCCL + :keywords: RCCL, ROCm, RMA, plugin, one-sided + +.. _using-rccl-rma-plugin: + +********************************* +Using the RCCL RMA plugin API +********************************* + +The RMA plugin API lets an external library provide the host-side network +operations used by RCCL's one-sided communication path. RCCL loads the newest +supported interface exported by the library, trying ``ncclRmaPlugin_v16``, +``ncclRmaPlugin_v15``, ``ncclRmaPlugin_v14`` and then ``ncclRmaPlugin_v13``. + +Implementing version 16 +======================= + +Implement ``ncclRma_v16_t`` from ``src/include/plugin/rma/rma_v16.h`` and +export the vtable as ``ncclRmaPlugin_v16``. The following callbacks are +required: + +* ``init``, ``devices``, ``getRmaProperties``, ``getProperties``, ``listen`` + and ``connect`` +* ``createContext``, ``regMrSym``, ``deregMrSym`` and ``destroyContext`` +* ``closeColl`` and ``closeListen`` +* ``iput``, ``iputSignal``, ``iget``, ``test`` and ``finalize`` + +``regMrSymDmaBuf`` is also required when ``getProperties`` advertises +``NCCL_PTR_DMABUF``. ``iflush``, ``rmaProgress`` and ``queryLastError`` are +optional. + +Version 16 adds ``getRmaProperties``, which RCCL calls once per collective +communicator when it sets up the GIN proxy. It fills ``ncclRmaProperties_v16_t``: + +* ``flushesAllPutsOnAnySignal``: set to ``true`` only when the backend makes + every put already received on the communicator visible whenever any signal + arrives, from any peer, on every NIC the communicator uses. RCCL's device + barrier then skips its per-peer flush. Return ``false`` when unsure; that keeps + the per-peer flush and is what RCCL reports for v13 to v15 plugins. + +Version 15 added an ``optFlags`` argument to ``iput``, ``iputSignal`` and +``iget``, which v16 keeps. Plugins must accept these values: + +* ``ncclRmaOptFlagsDefault`` (``0``): no optional behavior requested. +* ``ncclRmaOptFlagsAggregateRequests`` (``1 << 0``): the caller permits the + backend to aggregate compatible requests. This is a hint; it must not change + the operation's data or ordering semantics. + +Compatibility with older plugins +================================= + +RCCL continues to load v15, v14 and v13 plugins. Their compatibility wrappers +report ``flushesAllPutsOnAnySignal`` as ``false``. The v14 and v13 wrappers also +adapt the older callback signatures and discard ``optFlags``, because those +versions cannot consume the hint. New plugins should export v16. + +Loading a plugin +================ + +Set ``NCCL_RMA_PLUGIN`` to an absolute path, a shared-library file name or a +short name. RCCL passes the raw value to ``dlopen`` first. If that fails, it +retries a short name with the ``librccl-rma`` prefix: + +.. code-block:: shell + + export LD_LIBRARY_PATH=/path/to/plugin:$LD_LIBRARY_PATH + export NCCL_RMA_PLUGIN=mybackend + +The command above loads a file literally named ``mybackend`` when one is on the +library search path, and otherwise resolves ``librccl-rma-mybackend.so``. +During communicator initialization, ``NCCL_DEBUG=INFO`` reports the selected +interface version and the backend assigned to the communicator. + +``plugins/rma/example`` is a minimal v15 implementation that demonstrates the +lifecycle, registration and data-operation callback signatures. RCCL loads it +through the v15 compatibility wrapper. diff --git a/projects/rccl/docs/index.rst b/projects/rccl/docs/index.rst index e837ec092bf1..29c4150d126a 100644 --- a/projects/rccl/docs/index.rst +++ b/projects/rccl/docs/index.rst @@ -27,6 +27,8 @@ The RCCL public repository is located within the rocm-systems repo at `` + * :doc:`Using the RCCL RMA plugin API <./how-to/using-rccl-rma-plugin-api>` + * :doc:`Using the RCCL GIN plugin API <./how-to/using-rccl-gin-plugin-api>` * :doc:`Using the RCCL Tuner plugin <./how-to/using-rccl-tuner-plugin-api>` * :doc:`Using the RCCL environment plugin <./how-to/using-rccl-env-plugin-api>` * :doc:`Using the NCCL Net plugin <./how-to/using-nccl>` diff --git a/projects/rccl/docs/perf/README.md b/projects/rccl/docs/perf/README.md new file mode 100644 index 000000000000..38af482f8821 --- /dev/null +++ b/projects/rccl/docs/perf/README.md @@ -0,0 +1,25 @@ +# NCCL Performance Data Guidance + +NCCL publishes reference performance data to: + +1. Provide reference points that help users align performance expectations. +2. Help users validate their system setup. +3. Reduce repeated requests to the NCCL team for basic performance numbers. + +These results are references, and NOT product-level guarantees that the same +performance is achievable on every system. Performance depends on a complex +combination of software versions, system configuration, hardware, and operating +conditions, including factors outside NCCL's control. A difference within 5% is +generally considered acceptable variance due to differences in the underlying +systems. + +We publish peak bandwidth for a selection of commonly used platforms. We do not +currently publish latency because it is typically more sensitive to factors +outside NCCL's control. + +If this data is useful to the community and the NCCL team, future releases may +cover more systems, message sizes, and performance metrics. + +If your workload differs significantly from the published results, open an +issue in the [NCCL repository](https://github.com/NVIDIA/nccl/issues) or contact +NVIDIA Support. We will try our best to help. diff --git a/projects/rccl/docs/perf/v2.31.2-1.md b/projects/rccl/docs/perf/v2.31.2-1.md new file mode 100644 index 000000000000..2ea9e66f8a4c --- /dev/null +++ b/projects/rccl/docs/perf/v2.31.2-1.md @@ -0,0 +1,151 @@ +# NCCL 2.31.2 Performance - Peak BusBW + +
+NCCL Version : 2.31.2-1
+CUDA Version : 13.0
+Metric       : Bus BW in GB/s
+Aggregation  : 3-run average, out-of-place only
+
+ +See the [performance data guidance](README.md) for how to interpret these +reference results. + +## Test Commands + +Tests can be launched as follows: + +```bash +# Legacy (MessageSize=8GB) +${MPIRUN_COMMAND} all_reduce_perf -t1 -g1 -f2 -b8G -e8G -w5 -n20 -dfloat -R0 -G10 +${MPIRUN_COMMAND} all_gather_perf -t1 -g1 -f2 -b8G -e8G -w5 -n20 -dfloat -R0 -G10 +${MPIRUN_COMMAND} reduce_scatter_perf -t1 -g1 -f2 -b8G -e8G -w5 -n20 -dfloat -R0 -G10 +${MPIRUN_COMMAND} alltoall_perf -t1 -g1 -f2 -b8G -e8G -w5 -n20 -duint8 -R0 -G10 +${MPIRUN_COMMAND} sendrecv_perf -t1 -g1 -f2 -b8G -e8G -w5 -n20 -duint8 -R0 -G10 + +# Symmetric (MessageSize=2GB) +${MPIRUN_COMMAND} all_reduce_perf -t1 -g1 -f2 -b2G -e2G -w5 -n20 -dfloat -R2 -G10 +${MPIRUN_COMMAND} reduce_scatter_perf -t1 -g1 -f2 -b2G -e2G -w5 -n20 -dfloat -R2 -G10 +${MPIRUN_COMMAND} env NCCL_SYM_CE_THRESHOLD=17179869184 all_gather_perf -t1 -g1 -f2 -b2G -e2G -w5 -n20 -dfloat -R2 -G10 + +# CE (MessageSize=8GB) +${MPIRUN_COMMAND} all_gather_perf -t1 -g1 -f2 -b8G -e8G -w5 -n20 -dfloat -R2 -x2 +${MPIRUN_COMMAND} alltoall_perf -t1 -g1 -f2 -b8G -e8G -w5 -n20 -duint8 -R2 -x2 +``` + +Notes: + +- `-t1` and `-g1` are included to make the one-GPU-per-process mode explicit. +- Adjust `${MPIRUN_COMMAND}` for your system and MPI library. Reference runs used SLURM as:
+ `srun -N ${NODES} --ntasks-per-node=${TASKS_PER_NODE} --mpi=pmix`. +- Symmetric kernels have a known performance issue for message sizes greater than 2 GB. + Performance has improved on the `dev` branch, and additional work is in progress. +- Versions: + + | Platform | SLURM | MPI | + | ----------- | :-----: | :--------: | + | GB300, B300 | 25.11.2 | HPCX v2.24 | + | H100 | 24.05.5 | HPCX v2.23 | +
+ +## GB300 + +| Configuration | NODES | TASKS PER NODE | +| ------------- | :---: | :------------: | +| NVL4 | 1 | 4 | +| NVL8 | 2 | 4 | +| NVL16 | 4 | 4 | +| NVL32 | 8 | 4 | +| NVL64 | 16 | 4 | + +**Note:** Use `--segment NODES` to allocate nodes in one NVLink domain on SLURM clusters. + +### Legacy Kernels - Bus BW in GB/s (MessageSize=8GB) + +| Collective | NVL4 | NVL8 | NVL16 | NVL32 | NVL64 | +| ------------- | :---: | :---: | :---: | :---: | :---: | +| AllReduce | 694 | 838 | 909 | 903 | 905 | +| AllGather | 708 | 711 | 710 | 710 | 709 | +| ReduceScatter | 660 | 688 | 702 | 707 | 704 | +| Alltoall | 701 | 704 | 706 | 694 | 666 | +| Sendrecv | 678 | 677 | 677 | 677 | 677 | + +### Symmetric Kernels - Bus BW in GB/s (MessageSize=2GB) + +| Collective | NVL4 | NVL8 | NVL16 | NVL32 | NVL64 | +| ------------- | :---: | :---: | :---: | :---: | :---: | +| AllReduce | 687 | 843 | 923 | 959 | 991 | +| AllGather | 673 | 627 | 625 | 603 | 705 | +| ReduceScatter | 435 | 607 | 645 | 650 | 724 | + +### CE Kernels - Bus BW in GB/s (MessageSize=8GB) + +| Collective | NVL4 | NVL8 | NVL16 | NVL32 | NVL64 | +| ---------- | :---: | :---: | :---: | :---: | :---: | +| AllGather | 721 | 751 | 737 | 742 | 738 | +| Alltoall | 722 | 751 | 737 | 742 | 736 | + +
+ +## B300 + +| Configuration | NODES | TASKS PER NODE | +| ------------- | :---: | :------------: | +| NVL8 | 1 | 8 | + +### Legacy Kernels - Bus BW in GB/s (MessageSize=8GB) + +| Collective | NVL8 | +| ------------- | :---: | +| AllReduce | 843 | +| AllGather | 711 | +| ReduceScatter | 692 | +| Alltoall | 703 | +| Sendrecv | 675 | + +### Symmetric Kernels - Bus BW in GB/s (MessageSize=2GB) + +| Collective | NVL8 | +| ------------- | :---: | +| AllReduce | 840 | +| AllGather | 627 | +| ReduceScatter | 605 | + +### CE Kernels - Bus BW in GB/s (MessageSize=8GB) + +| Collective | NVL8 | +| ---------- | :---: | +| AllGather | 750 | +| Alltoall | 750 | + +
+ +## H100 + +| Configuration | NODES | TASKS PER NODE | +| ------------- | :---: | :------------: | +| NVL8 | 1 | 8 | + +### Legacy Kernels - Bus BW in GB/s (MessageSize=8GB) + +| Collective | NVL8 | +| ------------- | :---: | +| AllReduce | 483 | +| AllGather | 364 | +| ReduceScatter | 362 | +| Alltoall | 351 | +| Sendrecv | 367 | + +### Symmetric Kernels - Bus BW in GB/s (MessageSize=2GB) + +| Collective | NVL8 | +| ------------- | :---: | +| AllReduce | 483 | +| AllGather | 327 | +| ReduceScatter | 319 | + +### CE Kernels - Bus BW in GB/s (MessageSize=8GB) + +| Collective | NVL8 | +| ---------- | :---: | +| AllGather | 379 | +| Alltoall | 379 | diff --git a/projects/rccl/docs/sphinx/_toc.yml.in b/projects/rccl/docs/sphinx/_toc.yml.in index 5914b9a4c5cb..0dff501f0313 100644 --- a/projects/rccl/docs/sphinx/_toc.yml.in +++ b/projects/rccl/docs/sphinx/_toc.yml.in @@ -18,6 +18,10 @@ subtrees: entries: - file: how-to/device-api-gin title: Use the device API and GIN + - file: how-to/using-rccl-rma-plugin-api + title: Using the RCCL RMA plugin API + - file: how-to/using-rccl-gin-plugin-api + title: Using the RCCL GIN plugin API - file: how-to/using-rccl-env-plugin-api title: Using the RCCL environment plugin - file: how-to/using-rccl-tuner-plugin-api diff --git a/projects/rccl/docs/userguide/Makefile b/projects/rccl/docs/userguide/Makefile index 2cb4f34e9288..bb371732f1dc 100644 --- a/projects/rccl/docs/userguide/Makefile +++ b/projects/rccl/docs/userguide/Makefile @@ -38,6 +38,8 @@ $(DOCPREPDIR)/% : $(DOCSRC)/%.in ../../makefiles/version.mk SPHINXOPTS = -W --keep-going SPHINXBUILD = sphinx-build PAPER = +SKIP_LLMSTXT ?= 0 +export SKIP_LLMSTXT # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) diff --git a/projects/rccl/docs/userguide/requirements.txt b/projects/rccl/docs/userguide/requirements.txt index d8badc9cf733..3ebdad05c8d0 100644 --- a/projects/rccl/docs/userguide/requirements.txt +++ b/projects/rccl/docs/userguide/requirements.txt @@ -4,4 +4,5 @@ jinja2==3.1.6 markupsafe==2.1.5 alabaster==0.7.13 docutils==0.20.1 +sphinx-llm==1.0 diff --git a/projects/rccl/docs/userguide/source/api/colls.rst b/projects/rccl/docs/userguide/source/api/colls.rst index 8570356b1966..de6ba4314d2a 100644 --- a/projects/rccl/docs/userguide/source/api/colls.rst +++ b/projects/rccl/docs/userguide/source/api/colls.rst @@ -117,3 +117,45 @@ ncclScatter In-place operation will happen if ``recvbuff == sendbuff + root * count``. Related links: :ref:`scatter`. + +Per-Collective Configuration Variants +-------------------------------------- + +Every collective above also has an ``*Config`` variant that takes a trailing +:c:type:`ncclCollConfig_t` argument for per-call customization (algorithm +selection, CTA counts, and similar overrides described under +:ref:`ncclcollconfig`). Initialize the config with +:c:macro:`NCCL_COLLCONFIG_INITIALIZER` and set it identically on every rank. +Passing ``config == NULL`` is equivalent to calling the plain API. + +.. c:function:: ncclResult_t ncclAllReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) + + Configurable variant of :c:func:`ncclAllReduce`. + +.. c:function:: ncclResult_t ncclBroadcastConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) + + Configurable variant of :c:func:`ncclBroadcast`. + +.. c:function:: ncclResult_t ncclReduceConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) + + Configurable variant of :c:func:`ncclReduce`. + +.. c:function:: ncclResult_t ncclAllGatherConfig(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) + + Configurable variant of :c:func:`ncclAllGather`. + +.. c:function:: ncclResult_t ncclReduceScatterConfig(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) + + Configurable variant of :c:func:`ncclReduceScatter`. + +.. c:function:: ncclResult_t ncclAlltoAllConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) + + Configurable variant of :c:func:`ncclAlltoAll`. + +.. c:function:: ncclResult_t ncclGatherConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) + + Configurable variant of :c:func:`ncclGather`. + +.. c:function:: ncclResult_t ncclScatterConfig(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream, const ncclCollConfig_t* config) + + Configurable variant of :c:func:`ncclScatter`. diff --git a/projects/rccl/docs/userguide/source/api/device.rst b/projects/rccl/docs/userguide/source/api/device.rst index 9073e0d38cd1..4ca9acd335a5 100644 --- a/projects/rccl/docs/userguide/source/api/device.rst +++ b/projects/rccl/docs/userguide/source/api/device.rst @@ -9,6 +9,8 @@ following areas: pointer functions, and related types. * **Memory and LSA** — Load/store accessible (LSA) memory, barriers, pointer accessors, and multimem. * **GIN (GPU-Initiated Networking)** — One-sided transfers, signals, counters, and network barriers. +* **CFT (Compute Fabric Transport)** — CUDA fabric logical endpoint queries, fabric transfers, reductions, and + CFT-backed barriers. * **Reduce, Broadcast, and Fused Building Blocks** — Building blocks for computation-fused kernels: reduce, copy (broadcast), and reduce-then-copy; used to implement algorithms such as AllReduce, AllGather, and ReduceScatter. @@ -20,4 +22,5 @@ For an introduction and usage examples, see :doc:`Device-Initiated Communication device_setup device_memory device_gin + device_cft device_reducecopy diff --git a/projects/rccl/docs/userguide/source/api/device_cft.rst b/projects/rccl/docs/userguide/source/api/device_cft.rst new file mode 100644 index 000000000000..cd8323081622 --- /dev/null +++ b/projects/rccl/docs/userguide/source/api/device_cft.rst @@ -0,0 +1,335 @@ +.. _device_api_cft: + +Device API - CFT +**************** + +Compute Fabric Transport +======================== + +Compute Fabric Transport (CFT) support is available starting with NCCL 2.31. +CFT exposes CUDA fabric logical endpoints to device kernels so they can move +data across a CFT-capable fabric using fabric instructions. NCCL provides the +team, logical endpoint, transfer, reduction, and barrier helpers needed to use +those endpoints from the Device API. + +CFT is intended for systems where all participating ranks can create compatible +CUDA fabric logical endpoints. Kernels that use the CFT helpers require CUDA +Toolkit support for fabric PTX instructions and must be compiled for +architectures that support those instructions. + +CFT Communicator Requirements +============================= + +Request CFT resources when creating the device communicator with :c:func:`ncclDevCommCreate`. + +``cftCaps`` +----------- + + Bitmask of :c:macro:`NCCL_CFT` and :c:macro:`NCCL_CFT_MULTIMEM` values. + If CFT resources are requested on a communicator where not all ranks support + CFT, :c:func:`ncclDevCommCreate` fails. + +``cftBarrierCount`` +------------------- + + Number of CFT barriers to allocate for :cpp:class:`ncclCftBarrierSession`. + This should match the number of independently addressed barrier slots used + by the kernel, commonly one per CTA with ``blockIdx.x`` as the barrier index. + +.. code-block:: C + + ncclDevComm devComm; + ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; + + reqs.cftCaps = NCCL_CFT | NCCL_CFT_MULTIMEM; + reqs.cftBarrierCount = nCTAs; + + NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); + +Use :c:macro:`NCCL_CFT` to request unicast CFT logical endpoints. Use :c:macro:`NCCL_CFT_MULTIMEM` +when the kernel also uses multicast CFT operations or multimem CFT barriers. + +.. c:macro:: NCCL_CFT_NONE + + No CFT capability is requested. + +.. c:macro:: NCCL_CFT + + Request unicast CFT logical endpoint support. + +.. c:macro:: NCCL_CFT_MULTIMEM + + Request multicast CFT logical endpoint support. + + +CFT Teams +========= + +CFT operations may use :c:type:`ncclTeam_t` and :cpp:type:`ncclTeam` values to +address peers. + +.. c:function:: ncclTeam_t ncclTeamCft(ncclComm_t comm, ncclCftTeamMode_t mode) + + Returns the host-side unicast CFT team containing ranks that are part of the CFT unicast group. + *comm* is the host-side communicator and *mode* is a selector used to filter the subset of CFT + unicast group ranks that are part of the team (useful to build hierarchical communication patterns). + +.. c:function:: ncclTeam_t ncclTeamCftMultimem(ncclComm_t comm) + + Returns the host-side multicast CFT team containing all the ranks that are part of the CFT multicast group. + +.. cpp:function:: ncclTeam ncclTeamCft(ncclDevComm const& comm, ncclCftTeamMode_t mode) + + Returns the device-side unicast CFT team containing ranks that are part of the CFT unicast group. + *comm* is the device-side communicator and *mode* is a selector used to filter the subset of CFT + unicast group ranks that are part of the team (useful to build hierarchical communication patterns). + +.. cpp:function:: ncclTeam ncclTeamCftMultimem(ncclDevComm const& comm) + + Returns the device-side multicast CFT team containing all the ranks that are part of the CFT multicast group. + +.. c:type:: ncclCftTeamMode_t + + Selects the CFT team layout returned by :c:func:`ncclTeamCft`. + + .. c:macro:: NCCL_CFT_TEAM_FLAT + + Flat CFT team includes all the ranks that are part of the CFT unicast group. + + .. c:macro:: NCCL_CFT_TEAM_HIER_MULTIMEM + + Hierarchical CFT team includes ranks that are part of the CFT unicast group and share the same index across different multicast CFT groups. + + .. c:macro:: NCCL_CFT_TEAM_HIER_LSA + + Hierarchical CFT team includes ranks that are part of the CFT unicast group and share the same index across different LSA groups. + +Logical Endpoint Queries +======================== + +CFT data movement routines operate on a logical endpoint ID and an offset within +that endpoint. NCCL provides host-side and device-side helpers for translating +registered windows and peer ranks into those values. + +.. c:function:: ncclResult_t ncclGetCftDeviceLeInfo(ncclWindow_t window, size_t offset, int peerCft, ncclTeam_t cftTeam, ncclCftLeId* leId, size_t* leOffset) + + Host-side query for the logical endpoint *leId* and *leOffset* corresponding to *window*, byte *offset*, and *peerCft* within *cftTeam*. + +.. c:function:: ncclResult_t ncclGetPeerDeviceLeInfo(ncclWindow_t window, size_t offset, int peerWorld, ncclCftLeId* leId, size_t* leOffset) + + Host-side query for the logical endpoint *leId* and *leOffset* corresponding to *window*, byte *offset* and *peerWorld* within the world team. + +.. c:function:: ncclResult_t ncclGetMultimemDeviceLeInfo(ncclWindow_t window, size_t offset, ncclCftLeId* leId, size_t* leOffset) + + Host-side query for the multicast logical endpoint *leId* and *leOffset* corresponding to *window* and byte *offset*. + +.. cpp:function:: void ncclGetCftLeInfo(ncclWindow_t w, size_t offset, int peerCft, ncclTeam cftTeam, ncclDevComm const& comm, ncclCftLeId* leId, size_t* leOffset) + + Device-side query for the logical endpoint *leId* and *leOffset* corresponding to *window*, byte *offset*, *peerCft* within *cftTeam* and device *comm*. + +.. cpp:function:: void ncclGetPeerLeInfo(ncclWindow_t w, size_t offset, int peerWorld, ncclDevComm const& comm, ncclCftLeId* leId, size_t* leOffset) + + Device-side query for the logical endpoint *leId* and *leOffset* corresponding to *window*, byte *offset*, *peerWorld* within the world team and device *comm*. + +.. cpp:function:: void ncclGetMultimemLeInfo(ncclWindow_t w, size_t offset, ncclDevComm const& comm, ncclCftLeId* leId, size_t* leOffset) + + Device-side query for the multicast logical endpoint *leId* and *leOffset* corresponding to *window*, byte *offset* and device *comm*. + +Host-side CFT queries require the communicator to create CFT logical endpoints +before the query is issued. Set ``hostCftMode`` in :c:type:`ncclConfig_t` when +creating the communicator if the application needs to query CFT endpoint +information from host code before creating a CFT-enabled device communicator. + +CFT Operations +============== + +.. cpp:struct:: ncclCftSmem + + A user-managed shared memory state object used by :cpp:class:`ncclCft` to track progress of outstanding CFT operations. + The object can track a specific amount of in-flight data, limiting the number of outstanding CFT operations. Specifically, + the object can track up to 16MB of non-fetching CFT operations (i.e., ``put``, ``red`` and their multicast variants) and up + to 1MB of fetching CFT operations (i.e., ``get``, ``pullred``). + +.. cpp:class:: template ncclCft + + A class encompassing major elements of CFT support. + + .. cpp:function:: ncclCft(Coop coop, ncclCftSmem& cftSmem) + + Initializes a new ``ncclCft`` object. *cftSmem* is a user-managed shared memory state used + by threads in *coop* to make progress on fabric operations and to check for their completion. + + .. cpp:function:: void put(OpCoop coop, ncclCftLeId leId, size_t leOffset, void* smemSource, uint32_t bytes) + + Initiates an asynchronous transfer of *bytes* bytes from the shared memory buffer referenced by *smemSource* + to the unicast logical endpoint referenced by *leId* at offset *leOffset*. + + *coop* can include a subset of the threads in the cooperative group used to create the CFT object. + Cooperative groups initiating fabric operations independently under the same CFT object share its state. + + .. cpp:function:: void putCpMask(OpCoop coop, ncclCftLeId leId, size_t leOffset, void* smemSource, uint32_t bytes, uint16_t cpMask) + + Variant of :cpp:func:`ncclCft::put` with an explicit copy mask. The copy mask is a 16-bit integer where bits + correspond to the bytes of every 16-byte word being copied. A bit is set to 1 if the corresponding byte ought + to be copied to the destination logical endpoint, or it is set to 0 otherwise. + + .. cpp:function:: void putMultimem(OpCoop coop, ncclCftLeId leId, size_t leOffset, void* smemSource, uint32_t bytes) + + Multicast variant of :cpp:func:`ncclCft::put` where *leId* references a logical endpoint for which the + the :c:macro:`NCCL_CFT_MULTIMEM` bit was set in the :c:macro:`cftCaps` bitmask at device communicator + creation time. + + .. cpp:function:: void putMultimemCpMask(OpCoop coop, ncclCftLeId leId, size_t leOffset, void* smemSource, uint32_t bytes, uint16_t cpMask) + + Multicast variant of :cpp:func:`ncclCft::put` with an explicit copy mask. + + .. cpp:function:: void get(OpCoop coop, ncclCftLeId leId, size_t leOffset, void* smemDestination, uint32_t bytes) + + Initiates an asynchronous transfer of *bytes* bytes from the unicast logical endpoint referenced by *leId* at offset + *leOffset* to the shared memory buffer referenced by *smemDestination*. + + *coop* can include a subset of the threads in the cooperative group used to create the CFT object. + Cooperative groups initiating fabric operations independently under the same CFT object share its state. + + .. cpp:function:: void red(OpCoop coop, ncclCftLeId leId, size_t leOffset, RedOp const& red, void* smemSource, uint32_t bytes) + + Initiates an asynchronous reduction operation (see :ref:`cft_reduction_operators` for reduction tags and data types) + for *bytes* bytes from the shared memory buffer referenced by *smemSource* to the unicast logical endpoint referenced by + *leId* at offset *leOffset*. + + *coop* can include a subset of the threads in the cooperative group used to create the CFT object. + Cooperative groups initiating fabric operations independently under the same CFT object share its state. + + .. cpp:function:: void redMultimem(OpCoop coop, ncclCftLeId leId, size_t leOffset, RedOp const& red, void* smemSource, uint32_t bytes) + + Multicast reduction variant of :cpp:func:`ncclCft::red` where *leId* references a logical endpoint for which + the :c:macro:`NCCL_CFT_MULTIMEM` bit was set in the :c:macro:`cftCaps` bitmask at device communicator creation + time. + + .. cpp:function:: void pullRed(OpCoop coop, ncclCftLeId leId, size_t leOffset, RedOp const& red, void* smemDestination, uint32_t bytes) + + Initiates an asynchronous reduction operation (see :ref:`cft_reduction_operators` for reduction tags and data types) + for *bytes* bytes from the multicast logical endpoint referenced by *leId* at offset *leOffset* to the shared memory buffer + referenced by *smemDestination*. *leId* references a logical endpoint for which the :c:macro:`NCCL_CFT_MULTIMEM` bit + was set in the :c:macro:`cftCaps` bitmask at device communicator creation time. + + *coop* must be ncclCoopWarp. + + .. cpp:function:: void submit(OpCoop coop) + + Submits previously initiated CFT operations for all the threads in *coop*, allowing them to make forward + progress. Every cooperative group (``OpCoop``) that initiated fabric operations must call submit before + :cpp:func:`ncclCft::flushSmem` and :cpp:func:`ncclCft::flush`. + + .. cpp:function:: void flushSmem(OpCoop coop) + + Makes all the threads in *coop* wait for CFT operations that consume shared memory data. All the + shared memory buffers that source data to outstanding fabric operations can be reused after the + function returns. Users can call this function to wait for shared memory buffers to be consumed + by previously initiated CFT operations before initiating new ones, and call :cpp:func:`ncclCft::flush` + only when they need to wait for the completion of all the outstanding CFT operations, e.g., because + they reached the number of in-flight bytes that the CFT shared memory state object can track. + + .. cpp:function:: void flush(OpCoop coop, bool* hasReport = nullptr, uint32_t* report = nullptr) + + Flushes outstanding CFT operations for all the threads in *coop* and waits for their completion. + When flush returns shared memory buffers are safe to reuse. *coop* must include the threads of + all cooperative groups that initiated fabric operations under the same CFT object. No other + cooperative group can initiate fabric operations until flush has completed. If the operation + succeeds *hasReport* is set to false and *report* is not modified. If the operation fails + *hasReport* is set to ``true`` and *report* is set to an error code that can be decoded via + the ``cudaFabricOpErrorStatusGet/Count``. + +``smemSource`` and ``smemDestination`` pointers must be 16-byte aligned, and ``bytes`` must be a multiple of 16. +CFT operates on shared memory; a typical kernel stages data through shared memory, issues one or more operations, +then calls :cpp:func:`ncclCft::submit` and :cpp:func:`ncclCft::flush` or :cpp:func:`ncclCft::flushSmem` +as required by the reuse and ordering rules of the kernel. + +.. _cft_reduction_operators: + +Reduction Operations +==================== + +Reduction fabric operations use operation tag types: + +.. cpp:struct:: ncclCftOpSum +.. cpp:struct:: ncclCftOpAnd +.. cpp:struct:: ncclCftOpXor +.. cpp:struct:: ncclCftOpOr +.. cpp:struct:: ncclCftOpMin +.. cpp:struct:: ncclCftOpMax + +Refer to CFT PTX documentation for supported reductions and data types. + +CFT Barriers +============ + +Applications request CFT barrier resources with ``cftBarrierCount`` in +:c:type:`ncclDevCommRequirements`, then use :cpp:class:`ncclCftBarrierSession` +from device code. The barrier count should cover every barrier index used by +the kernel. + +.. cpp:class:: template ncclCftBarrierSession + + A CFT-backed barrier session. + + .. cpp:function:: ncclCftBarrierSession(Coop coop, ncclDevComm const& comm, uint32_t index, bool multimem = false) + + Initializes a new CFT barrier session. *coop* represents a cooperative group (see :ref:`devapi_coops`) of threads + that synchronize throught the barrier. *comm* is the device communicator for which barrier resources were requested. + *index* selects the barrier slot from the device communicator. *multimem* can be set to ``true`` to select multicast + CFT barrier resources (requires CFT barriers to be created with :c:macro:`NCCL_CFT_MULTIMEM` capability). + + .. cpp:function:: void arrive(Coop coop, cuda::memory_order order, ncclMemProxyType producer, ncclMemProxyType consumer) + + Signals arrival at the barrier. *order* dictates the semantics used to order memory accesses from different proxies + around the barrier. *producer* indicates the type of proxy that stored the data into global memory. *consumer* + indicates the type of proxy that will be loading the *producer* data from global memory. + + .. cpp:function:: void wait(Coop coop, cuda::memory_order order, ncclMemProxyType producer, ncclMemProxyType consumer) + + Waits for the barrier to complete. It can be used to establish a ``happens-before`` relationship between *producer* + and *consumer* when *order* complements the memory ordering semantics of the matching + :cpp:func:`ncclCftBarrierSession::arrive`. + + .. cpp:function:: void sync(Coop coop, cuda::memory_order order, ncclMemProxyType producer, ncclMemProxyType consumer) + + Combines :cpp:func:`ncclCftBarrierSession::arrive` and :cpp:func:`ncclCftBarrierSession::wait`. + + .. cpp:function:: ncclResult_t wait(Coop coop, cuda::memory_order order, ncclMemProxyType producer, ncclMemProxyType consumer, uint64_t timeoutCycles) + .. cpp:function:: ncclResult_t sync(Coop coop, cuda::memory_order order, ncclMemProxyType producer, ncclMemProxyType consumer, uint64_t timeoutCycles) + + Timeout-returning variants of :cpp:func:`ncclCftBarrierSession::wait` and + :cpp:func:`ncclCftBarrierSession::sync`. + +.. cpp:function:: void ncclMemFence(Coop coop, cuda::memory_order order, ncclMemProxyType producer, ncclMemProxyType consumer, ncclMemFenceScope scope) + + Issues a memory fence between producer and consumer proxy domains for the participating threads. *coop* represents a cooperative + group of threads which memory accesses are ordered around the fence. *order* dictates the semantics used to order memory accesses + from different proxies around the fence. *producer* indicates the type of proxy that stored the data into memory. *consumer* + indicates the type of proxy that will be loading the *producer* data from memory. *scope* indicates the point of consistency for + the memory accesses around the fence. + +.. cpp:enum-class:: ncclMemProxyType + + Describes which memory proxy produced or consumes memory around a CFT barrier or a fence. + + .. cpp:enumerator:: Generic + + Indicates that the method of memory access is the SM load/store unit. + + .. cpp:enumerator:: Fabric + + Indicates that the method of memory access is the fabric. + +.. cpp:enum-class:: ncclMemFenceScope + + Describes the scope of a CFT memory fence. + + .. cpp:enumerator:: Cta + + Indicates that the point of consistency for memory accesses is the SM shared memory. + + .. cpp:enumerator:: Sys + + Indicates that the point of consistency for memory accesses is the L2 cache. diff --git a/projects/rccl/docs/userguide/source/api/device_gin.rst b/projects/rccl/docs/userguide/source/api/device_gin.rst index 3e3e8791706a..a94c6191c517 100644 --- a/projects/rccl/docs/userguide/source/api/device_gin.rst +++ b/projects/rccl/docs/userguide/source/api/device_gin.rst @@ -37,7 +37,7 @@ ncclGin *peer* is a rank within *team* (see :ref:`devapi_teams`); it may refer to the local rank (a loopback). The destination and source buffers are each specified using the window (*dstWnd*, *srcWnd*) and a byte-based offset (*dstOffset*, *srcOffset*). *bytes* specifies the data transfer count in bytes. If GIN is initialized with connection - type :c:macro:`NCCL_GIN_CONNECTION_RAIL`, *peer* must be within the same rail team as the local rank. + type :c:enumerator:`NCCL_GIN_CONNECTION_RAIL `, *peer* must be within the same rail team as the local rank. Arguments beyond the first seven are optional. *remoteAction* and *localAction* specify actions to undertake on the destination peer and on the local rank when the payload has been settled and the input has been @@ -66,7 +66,7 @@ ncclGin *peer* is a rank within *team* (see :ref:`devapi_teams`); it may refer to the local rank (a loopback). The remote and local buffers are each specified using the window (*remoteWnd*, *localWnd*) and a byte-based offset (*remoteOffset*, *localOffset*). *bytes* specifies the data transfer count in bytes. If GIN is initialized with connection - type :c:macro:`NCCL_GIN_CONNECTION_RAIL`, *peer* must be within the same rail team as the local rank. + type :c:enumerator:`NCCL_GIN_CONNECTION_RAIL `, *peer* must be within the same rail team as the local rank. *bufType* specifies the physical memory composition of the source and destination buffers (see :ref:`devapi_segment_types`); it defaults to ``ncclGin_SegmentDevice``. @@ -297,3 +297,55 @@ ncclGinBarrierSession Same fence contract as the overload without a timeout. Returns ``ncclTimeout`` if not every team member arrives within *timeoutCycles*. The fence still applies when the call returns success. + +ncclBarrierSession +------------------ + +.. cpp:class:: template ncclBarrierSession + + A class representing a hybrid barrier session. It combines a memory barrier over the :ref:`LSA ` + team (:cpp:class:`ncclLsaBarrierSession`) with a network barrier (:cpp:class:`ncclGinBarrierSession`). + + .. cpp:function:: ncclBarrierSession(Coop coop, ncclTeamTagWorld tag, ncclGin gin, uint32_t index, bool multimem = false) + + Initializes a new hybrid barrier session covering the world team (see :ref:`devapi_teams`). *coop* represents a + cooperative group (see :ref:`devapi_coops`). *gin* is a previously initialized :cpp:class:`ncclGin` object, + used for the network part of the barrier. *index* identifies the underlying barrier to use (it should be + unique for each *coop*; typically set to ``blockIdx.x`` to ensure uniqueness between CTAs). *multimem* + requests memory multicast for the LSA part of the barrier. + + .. cpp:function:: ncclBarrierSession(Coop coop, ncclTeamTagRail tag, ncclGin gin, uint32_t index) + + Initializes a new hybrid barrier session covering the rail team, i.e., all peers on the same rail as the local + rank (see :ref:`devapi_teams`). + + .. cpp:function:: ncclBarrierSession(Coop coop, ncclTeamTagLsa tag, ncclDevComm const& comm, uint32_t index, bool multimem = false) + + Initializes a new hybrid barrier session covering the LSA team. Since no network traffic is involved, this + variant takes the device communicator *comm* (created using :c:func:`ncclDevCommCreate`) instead of an + :cpp:class:`ncclGin` object, and only the LSA barrier takes part in the session. + + .. cpp:function:: ncclBarrierSession(Coop coop, ncclTeam innerTeam, ncclTeam outerTeam, ncclGin gin, \ + ncclLsaBarrierHandle innerBarHandle, ncclGinBarrierHandle outerBarHandle, uint32_t index, \ + bool multimem = false, ncclMultimemHandle innerMmHandle = {}) + + Initializes a new hybrid barrier session. This is the general-purpose variant, to be used when the teams and + the barrier locations are not the ones baked into the constructors above. *innerTeam* and *innerBarHandle* + describe the memory barrier, while *outerTeam* and *outerBarHandle* describe the network barrier. *innerMmHandle* provides + the multicast handle backing the *multimem* implementation of the memory barrier. + + .. cpp:function:: void sync(Coop coop, cuda::memory_order order, \ + ncclGinFenceLevel fence = ncclGinFenceLevel::Put | ncclGinFenceLevel::Get) + + Synchronizes all threads of all team members that participate in the barrier session. *fence* selects which + prior network operations must be complete after the barrier returns; it has the same meaning and the same + default as in :cpp:func:`ncclGinBarrierSession::sync`. + + The selected barrier implementation is decided at run-time from the session's team, the GIN connectivity of the device communicator + (see :c:member:`ginConnectionType`), and *fence*. + + .. cpp:function:: ncclLsaBarrierSession& lsaBarrier() + .. cpp:function:: ncclGinBarrierSession& ginBarrier() + + Returns the reference to the memory barrier or the rail network barrier underlying this hybrid session, so that it can be + driven individually. diff --git a/projects/rccl/docs/userguide/source/api/device_setup.rst b/projects/rccl/docs/userguide/source/api/device_setup.rst index d5792e4c534f..810855afa429 100644 --- a/projects/rccl/docs/userguide/source/api/device_setup.rst +++ b/projects/rccl/docs/userguide/source/api/device_setup.rst @@ -102,22 +102,22 @@ ncclDevCommRequirements .. c:member:: bool ginForceEnable **Deprecated.** Forces GIN (GPU-Initiated Networking) support to be enabled by automatically setting - ``ginConnectionType`` to :c:macro:`NCCL_GIN_CONNECTION_FULL`. This field is deprecated in favor of explicitly + ``ginConnectionType`` to :c:enumerator:`NCCL_GIN_CONNECTION_FULL `. This field is deprecated in favor of explicitly setting :c:member:`ginConnectionType` to the desired value. When set to ``true``, it overrides the ``ginConnectionType`` field. New code should use :c:member:`ginConnectionType` directly instead of this field. Available since NCCL 2.28.7, deprecated since NCCL 2.29.7. .. c:member:: ncclGinConnectionType_t ginConnectionType - Specifies the type of GIN (GPU-Initiated Networking) connection to establish for the device communicator. - This field controls whether GIN is enabled and how it is configured. When set to :c:macro:`NCCL_GIN_CONNECTION_FULL`, - GIN is initialized and all ranks connect to all other ranks in the communicator. When set to :c:macro:`NCCL_GIN_CONNECTION_RAIL`, - GIN is initialized and each rank connects to other ranks in the same rail team. If GIN resources are requested via ``ginSignalCount``, - ``ginCounterCount``, ``barrierCount``, or ``railGinBarrierCount`` while this field is set to - :c:macro:`NCCL_GIN_CONNECTION_NONE`, device communicator creation will fail with :c:macro:`ncclInvalidArgument`. - Available since NCCL 2.29.7. + Specifies the type of GIN connection to establish (see :c:enum:`ncclGinConnectionType_t`). + If GIN resources are requested (e.g., ``ginSignalCount``, ``ginCounterCount``, ``barrierCount``, or + ``railGinBarrierCount``) while this field is :c:enumerator:`NCCL_GIN_CONNECTION_NONE `, device communicator + creation fails with :c:macro:`ncclInvalidArgument`. Available since NCCL 2.29.7. + + .. c:member:: int ginCustomStride - See :c:type:`ncclGinConnectionType_t` for possible values. + Specifies the rank stride when :c:member:`ginConnectionType` is :c:enumerator:`NCCL_GIN_CONNECTION_CUSTOM_STRIDE `. + Available since NCCL 2.31. .. c:member:: ncclDevResourceRequirements_t* resourceRequirementsList @@ -141,7 +141,7 @@ ncclDevCommRequirements Set to false if kernels using this communicator will not use strong signal operations (such as :cpp:struct:`ncclGin_StrongSignalInc` and :cpp:struct:`ncclGin_StrongVASignalAdd`). Default is true. Available since NCCL 2.30.5. - + .. c:member:: bool ginVaSignalsRequired @@ -149,6 +149,18 @@ ncclDevCommRequirements do not use GIN VA signals (such as :cpp:struct:`ncclGin_WeakVASignalInc` and :cpp:struct:`ncclGin_StrongVASignalAdd`). Default is true. Available since NCCL 2.30.5. + .. c:member:: int cftCaps + + Bitmask of CFT capabilities requested for the device communicator (see :ref:`device_api_cft`). + Set :c:macro:`NCCL_CFT` for unicast CFT logical endpoint support, :c:macro:`NCCL_CFT_MULTIMEM` + for multicast CFT support, :c:macro:`NCCL_CFT_NONE` for no CFT support (default). + Available since NCCL 2.31. + + .. c:member:: int cftBarrierCount + + Specifies the number of CFT barriers to allocate (see :cpp:class:`ncclCftBarrierSession`). + Available since NCCL 2.31. + ncclCommQueryProperties ----------------------- @@ -193,8 +205,8 @@ ncclCommProperties_t .. c:member:: ncclGinType_t ginType - The GIN type supported by the communicator. If equal to :c:macro:`NCCL_GIN_TYPE_NONE`, a - :c:type:`ncclDevComm` cannot be created with GIN connection type :c:macro:`NCCL_GIN_CONNECTION_FULL`. + The GIN type supported by the communicator. If equal to :c:enumerator:`NCCL_GIN_TYPE_NONE `, a + :c:type:`ncclDevComm` cannot be created with GIN connection type :c:enumerator:`NCCL_GIN_CONNECTION_FULL `. .. c:member:: int nLsaTeams @@ -202,39 +214,54 @@ ncclCommProperties_t .. c:member:: ncclGinType_t railedGinType - The railed GIN type supported by the communicator. If equal to :c:macro:`NCCL_GIN_TYPE_NONE`, a - :c:type:`ncclDevComm` cannot be created with GIN connection type :c:macro:`NCCL_GIN_CONNECTION_RAIL`. + The railed GIN type supported by the communicator. If equal to :c:enumerator:`NCCL_GIN_TYPE_NONE `, a + :c:type:`ncclDevComm` cannot be created with GIN connection type :c:enumerator:`NCCL_GIN_CONNECTION_RAIL `. Available since NCCL 2.29.7. .. c:member:: uint64_t commHash Communicator hash identifier shared across all ranks in the communicator. Available since NCCL 2.31. + .. c:member:: int ginMinStride + + The minimum value allowed for :c:member:`ginCustomStride`. This value is based + on the network connectivity. For example, when :ref:`env_NCCL_CROSS_NIC` is ``0``, ``ginMinStride`` is increased so + that strides spanning rail boundaries are not possible. ``ginMinStride`` is ``INT_MAX`` if GIN connectivity + does not follow a uniform stride pattern, in which case :c:enumerator:`NCCL_GIN_CONNECTION_CUSTOM_STRIDE ` cannot + be used. Available since NCCL 2.31. + ncclGinType_t ------------- -.. c:type:: ncclGinType_t +.. c:enum:: ncclGinType_t GIN type. Communication between different GIN types is not supported. Possible values include: - .. c:macro:: NCCL_GIN_TYPE_NONE + .. c:enumerator:: NCCL_GIN_TYPE_NONE GIN is not supported. - .. c:macro:: NCCL_GIN_TYPE_PROXY + .. c:enumerator:: NCCL_GIN_TYPE_PROXY Host Proxy GIN type. - .. c:macro:: NCCL_GIN_TYPE_GDAKI + .. c:enumerator:: NCCL_GIN_TYPE_GDAKI GPUDirect Async Kernel-Initiated (GDAKI) GIN type. - .. c:macro:: NCCL_GIN_TYPE_GPI + .. c:enumerator:: NCCL_GIN_TYPE_GPI GPU-Push Interface (GPI) GIN type. Requires SpectrumX - see - SpectrumX documentation for details. Added as an experimental - feature in NCCL 2.30.6. + `SpectrumX documentation `_ + for details. Added as an experimental feature in NCCL 2.30.6. + + .. c:enumerator:: NCCL_GIN_TYPE_EFA_GDA + + AWS EFA GPUDirect Async (GDA) GIN type. Requires the AWS OFI plugin for NCCL - + see AWS EFA `documentation `_ and + `getting started guide `_ + for details. Available since NCCL 2.31. .. c:macro:: NCCL_GIN_TYPE_EFA_GDA @@ -252,24 +279,28 @@ ncclGinType_t ncclGinConnectionType_t ----------------------- -.. c:type:: ncclGinConnectionType_t +.. c:enum:: ncclGinConnectionType_t Specifies the type of GIN connection for device communicators. This enum controls whether GIN (GPU-Initiated Networking) resources should be allocated and what connection type to use. Used in :c:type:`ncclDevCommRequirements` when creating device communicators. Available since NCCL 2.29.7. - .. c:macro:: NCCL_GIN_CONNECTION_NONE + .. c:enumerator:: NCCL_GIN_CONNECTION_NONE No GIN connectivity. - .. c:macro:: NCCL_GIN_CONNECTION_FULL + .. c:enumerator:: NCCL_GIN_CONNECTION_FULL Full GIN connectivity. Each rank is connected to all other ranks. - .. c:macro:: NCCL_GIN_CONNECTION_RAIL + .. c:enumerator:: NCCL_GIN_CONNECTION_RAIL Railed GIN connectivity. Each rank is connected to other ranks in the same rail team. + .. c:enumerator:: NCCL_GIN_CONNECTION_CUSTOM_STRIDE + + Strided GIN connectivity. Each rank is connected to other ranks separated by a fixed stride. Available since NCCL 2.31. + .. _device_api_host_functions: Host-Accessible Device Pointer Functions diff --git a/projects/rccl/docs/userguide/source/api/param.rst b/projects/rccl/docs/userguide/source/api/param.rst index 8734c855a50e..ee01329c088d 100644 --- a/projects/rccl/docs/userguide/source/api/param.rst +++ b/projects/rccl/docs/userguide/source/api/param.rst @@ -20,7 +20,7 @@ Types .. c:type:: ncclParamHandle_t - Opaque handle that represents a binding between a caller and a + Opaque handle that represents a binding between a caller and a parameter. It is created by :c:func:`ncclParamBind`. The handle remains valid for duration of the NCCL module's lifetime; callers must not free it or expect it to be accessible during static destruction or process tear down. diff --git a/projects/rccl/docs/userguide/source/api/types.rst b/projects/rccl/docs/userguide/source/api/types.rst index ab8a62dff322..0a00070b68bd 100644 --- a/projects/rccl/docs/userguide/source/api/types.rst +++ b/projects/rccl/docs/userguide/source/api/types.rst @@ -432,6 +432,225 @@ ncclConfig_t host-side CFT in case of error. :c:macro:`ncclHostCftDefault` selects the library-defined default behavior. + .. c:macro:: nvlsHostMode + + (since 2.31) + + Controls host-side use of NVLink SHARP (NVLS) for a single communicator. + The value is a bitmask that can independently disable the NVLS collective + transport and multimem use by NCCL's internal symmetric kernels and Copy + Engine (CE) collectives. Explicit Device API multimem requests remain + available. Selectively disabling host NVLS components on small or infrequently + used communicators can preserve limited hardware multicast resources for other + communicators that can utilize them more effectively. + + Supported values (``ncclNvlsHostMode_t``): + + * ``ncclNvlsHostModeDefault`` — Library-defined default. Currently equivalent + to ``ncclNvlsHostModeEnable``. + * ``ncclNvlsHostModeEnable`` — Enable all host NVLS components. + * ``ncclNvlsHostModeDisableTransport`` — Disable the NVLS collective transport + and its registered-buffer optimization. The NVLS, NVLS_TREE, and multi-RPN + PAT algorithms will be disabled. + * ``ncclNvlsHostModeDisableSymmetricMultimem`` — Disable multimem use by NCCL's + internal symmetric kernels and CE collectives. UC symmetric kernels and CE + fallbacks remain available. + * ``ncclNvlsHostModeDisable`` — Disable all present and future host NVLS + components. This value sets all non-reserved bits for forward compatibility. + + The component flags can be combined with the bitwise OR operator. These flags + do not affect the communicator's reported Device API multimem capability and + do not prevent explicit Device API calls from allocating multimem resources. + +.. _ncclcollconfig: + +ncclCollConfig_t +---------------- + +.. c:type:: ncclCollConfig_t + + Per-collective configuration passed through an ``nccl*Config`` collective API. + Every :c:type:`ncclCollConfig_t` must be initialized with + :c:macro:`NCCL_COLLCONFIG_INITIALIZER` before any member is set. Passing + ``NULL`` to a config-taking collective is equivalent to calling the + corresponding collective without a configuration. + + The application owns the configuration and any storage it references, and must + keep them valid for the duration of each call that uses them. Set the same + configuration on every rank unless a member is documented otherwise. + + For options that also have environment-variable and communicator-level forms, + NCCL applies values in the following order: environment variable, per-collective + configuration, communicator configuration, and library default. Leave resource and CTA-policy + members at ``NCCL_CONFIG_UNDEF_INT`` to fall through to the next source in this list. + + .. c:macro:: NCCL_COLLCONFIG_INITIALIZER + + (since 2.31) + + Initializes a new :c:type:`ncclCollConfig_t` with default values. + + .. c:member:: ncclConfigExt_t* ext + + (since 2.31) + + Head of a linked list of vendor-specific :c:type:`ncclConfigExt_t` options, + or ``NULL``. The official NCCL library does not read this member; it exists + for vendor libraries layered on top of NCCL to forward their own + vendor-defined options through the same call. + + .. c:member:: int minCTAs + + (since 2.31) + + Per-collective override of ``minCTAs`` in :ref:`ncclconfig`. Accepts the + same value range as :ref:`NCCL_MIN_CTAS`; an out-of-range value is ignored + and NCCL falls through to the next priority source. If the resolved value + exceeds the resolved ``maxCTAs``, NCCL resets ``minCTAs`` to ``1``. + + .. c:member:: int maxCTAs + + (since 2.31) + + Per-collective override of ``maxCTAs`` in :ref:`ncclconfig`. Accepts the + same value range as :ref:`NCCL_MAX_CTAS`; an out-of-range value is ignored + and NCCL falls through to the next priority source. The resolved value is + further clamped to the communicator's ``maxCTAs`` and applies only to this + collective, including when calls are grouped. + + .. c:member:: int nvlsCTAs + + (since 2.31) + + Per-collective override of ``nvlsCTAs`` in :ref:`ncclconfig`. This setting applies only to NVLS algorithms. + + .. c:member:: int cgaClusterSize + + (since 2.31) + + Per-collective override of ``cgaClusterSize`` in :ref:`ncclconfig`. Accepts + the same value range as :ref:`NCCL_CGA_CLUSTER_SIZE`; an out-of-range value + is ignored and NCCL falls through to the next priority source. All + collectives in a group must use the same value; inconsistent values result + in undefined behavior. + + .. c:member:: const char* algSelection + + (since 2.31) + + A case-insensitive expression that filters the algorithms and kernels NCCL may + use for this call. ``NULL`` or an empty string requests automatic selection. + + Each name is matched as a prefix from the start of a known name. For example, + ``ring`` selects ``RING_LL``, ``RING_LL128``, and ``RING_SIMPLE``, but does not + select ``SYMK_RailRing_LsaSTMC``; similarly, ``nvls`` selects both ``NVLS_SIMPLE`` + and ``NVLSTREE_SIMPLE``. The supported operators are ``+`` (AND), ``,`` + (OR), and ``^`` (NOT); parentheses are not supported. For example, + ``tree,ring`` permits either family, ``ring + ^RING_LL128`` permits Ring except + LL128, and ``^symk`` excludes symmetric kernels. + + The selectable built-in names and the collectives for which they are valid are: + + * AllReduce: ``TREE_LL``, ``TREE_LL128``, ``TREE_SIMPLE``, ``RING_LL``, + ``RING_LL128``, ``RING_SIMPLE``, ``COLLNET_DIRECT_SIMPLE``, + ``COLLNET_CHAIN_SIMPLE``, ``NVLS_SIMPLE``, ``NVLSTREE_SIMPLE``, + ``SYMK_AGxLL_R``, ``SYMK_AGxLLMC_R``, ``SYMK_RSxTmaLD_AGxTmaST``, + ``SYMK_RSxLD_AGxST``, and ``SYMK_RSxLDMC_AGxSTMC``. + * AllGather: ``RING_LL``, ``RING_LL128``, ``RING_SIMPLE``, + ``COLLNET_DIRECT_SIMPLE``, ``NVLS_SIMPLE``, ``PAT_SIMPLE``, ``SYMK_LL``, + ``SYMK_LLMC``, ``SYMK_TmaST``, ``SYMK_ST``, ``SYMK_TmaSTMC``, + ``SYMK_STMC``, and ``SYMK_RailRing_LsaSTMC``. + * ReduceScatter: ``RING_LL``, ``RING_LL128``, ``RING_SIMPLE``, + ``COLLNET_DIRECT_SIMPLE``, ``NVLS_SIMPLE``, ``PAT_SIMPLE``, ``SYMK_LL``, + ``SYMK_TmaLD``, ``SYMK_LD``, ``SYMK_LDMC``, ``SYMK_RailA2A_LsaLD``, and + ``SYMK_RailA2A_LsaLDMC``. + * Broadcast and Reduce: ``RING_LL``, ``RING_LL128``, and ``RING_SIMPLE``. + + The ``SYMK_*`` names refer to symmetric-memory kernels built on the NCCL + Device API. ``LL`` is a low-latency protocol that tags data with an inline + flag instead of a separate synchronization step; ``LD``/``ST`` are ordinary + load/store data movement and ``TmaLD``/``TmaST`` use the Tensor Memory + Accelerator for bulk asynchronous copies; the ``MC`` suffix (as in + ``LLMC``, ``LDMC``, ``STMC``) means the load, store, or reduction is done + through NVLink SHARP multimem (multicast) instead of per-peer accesses; and + ``Rail``/``Lsa`` names are multi-node kernels that combine a local + NVLink-domain (``Lsa``) step with a network exchange between same-rail ranks + across nodes. + + Algorithm names are implementation details and may evolve. AllToAll, + Gather, and Scatter lower to point-to-point sends and receives and expose + no selectable names; Copy Engine paths are likewise not selectable here. + Request a Copy Engine path with ``CTAPolicy`` instead. + + .. c:member:: int forceAlgSelection + + (since 2.31) + + Controls what happens when ``algSelection`` is invalid or cannot be honored. + With the default value, ``1``, NCCL returns an error. Set it to ``0`` to + fall back to automatic selection instead. + + .. c:member:: int CTAPolicy + + (since 2.31) + + Per-collective override of the communicator's CTA policy. A set value replaces + the communicator value; see :ref:`cta_policy_flags`. If both + ``NCCL_CTA_POLICY_ZERO`` and ``NCCL_CTA_POLICY_EFFICIENCY`` are set, Zero-CTA + takes precedence. Copy Engine paths are selected through this policy, not + ``algSelection``. + + .. c:member:: uint64_t userProfilerTag + + (since 2.31) + + An opaque value delivered with the call's events to profiler plugins that + support profiler interface v7. Zero denotes an untagged call; older profiler + interfaces ignore the tag. + + The tag does not affect execution. NCCL reserves values with the + most-significant bit set for possible future use; applications should use + values with that bit clear. + + .. c:member:: cudaEvent_t launchCompletionEvent + + (since 2.32) + + A caller-owned CUDA event that NCCL records at the collective kernel-launch completion + boundary. Create the event with `cudaEventCreateWithFlags and cudaEventDisableTiming + `_. It must + not be an interprocess or interop event. + + A NULL event requests no update. Event presence must match across ranks: every rank in the + corresponding collective must provide a rank-local non-NULL event, or every rank must provide + NULL. A group supports at most one non-NULL event per communicator. + + Keep the event alive through all queued waits and graph executions that use it. For a grouped + collective, enqueue a dependent ``cudaStreamWaitEvent`` only after ``ncclGroupEnd`` returns + ``ncclSuccess``. With a nonblocking communicator, first poll ``ncclCommGetAsyncError`` until it + returns ``ncclSuccess``; do not enqueue the wait after an error. + + NCCL records the event only for kernel launches; Copy Engine and RMA work do not record it. + + CUDA 12.3 or newer provides launch-completion semantics. On older CUDA platforms, NCCL + emits a warning and records the event before the kernel launch; this fallback does not + provide a launch-completion guarantee. CUDA launch completion is best effort and does not + guarantee that dependent work overlaps the collective. Graph capture is supported with a + capture-compatible event; sharing a recorded event across graphs is not supported. + +.. c:type:: ncclConfigExt_t + + A vendor-specific key-value option that can be linked through its ``next`` + member and attached to :c:member:`ncclCollConfig_t.ext`. Each option has a key + consisting of ``key.vendorId`` and ``key.optionId`` and one value in the + ``val.i``, ``val.s``, or ``val.raw`` union member. The list is unordered and + must not contain duplicate keys. + + Vendors should choose a non-zero ``vendorId`` under ``2^24`` that is unlikely + to collide with other vendor libraries, for example by picking a random + number; ``optionId`` then distinguishes options within that vendor's own + namespace. + .. _ncclsiminfo: ncclSimInfo_t diff --git a/projects/rccl/docs/userguide/source/conf.py.in b/projects/rccl/docs/userguide/source/conf.py.in index 68903ef5daed..b6de74fc1e03 100644 --- a/projects/rccl/docs/userguide/source/conf.py.in +++ b/projects/rccl/docs/userguide/source/conf.py.in @@ -13,6 +13,8 @@ # documentation root, resolve it to an absolute path as shown here. import sphinx_rtd_theme import sphinx +import importlib +import os import re import subprocess import sys @@ -50,8 +52,21 @@ extensions = [ 'sphinx.ext.ifconfig', 'sphinx.ext.extlinks', 'sphinx_rtd_theme', + 'sphinx_llm.txt', ] +llms_txt_extension = 'sphinx_llm.txt' +try: + importlib.import_module(llms_txt_extension) +except ImportError as exc: + if os.environ.get('SKIP_LLMSTXT') == '1': + extensions.pop(extensions.index(llms_txt_extension)) + else: + raise RuntimeError( + 'sphinx-llm is required to generate llms.txt. Install it using docs/userguide/requirements.txt' + ' or set SKIP_LLMSTXT=1 to build without generating llms.txt.' + ) from exc + # Intersphinx mapping. cuda.core symbols (Buffer, Device, Stream, # cuda.core.typing.IsStreamType) appear in nccl4py type annotations but # cuda.core is mocked in autodoc_mock_imports, so they have no entry in diff --git a/projects/rccl/docs/userguide/source/env.rst b/projects/rccl/docs/userguide/source/env.rst index b6c9006a5b5b..73ba8878ad52 100644 --- a/projects/rccl/docs/userguide/source/env.rst +++ b/projects/rccl/docs/userguide/source/env.rst @@ -173,7 +173,8 @@ if it would result in a better performance. NCCL_IB_HCA ----------- -The ``NCCL_IB_HCA`` variable specifies which Host Channel Adapter (RDMA) interfaces to use for communication. +The ``NCCL_IB_HCA`` variable specifies which Host Channel Adapter (RDMA) interfaces to use for communication. On Windows, +NetworkDirect reuses it to select adapters by Windows interface alias. Values accepted ^^^^^^^^^^^^^^^ @@ -203,8 +204,34 @@ Examples: Note: using ``mlx5_1`` without a preceding ``=`` will select ``mlx5_1`` as well as ``mlx5_10`` to ``mlx5_19``, if they exist. It is therefore always recommended to add the ``=`` prefix to ensure an exact match. +For NetworkDirect, specify comma-separated Windows interface aliases without IB port, rail, or plane fields. For example, +``=Ethernet 4,Ethernet 10`` selects exactly those two interfaces. + Note: There is a fixed upper limit of 32 Host Channel Adapter (HCA) devices supported in NCCL. +NCCL_IB_RAIL_POLICY +------------------- +(since 2.30.5) + +Controls automatic rail and plane assignment for supported ConnectX SuperNIC devices. This can be used with +``NCCL_NET_MERGE_POLICY=RAIL`` to fuse only ports that belong to the same auto-detected rail. +The automatic rail and plane assignment will not overwrite the rails or planes given by the user (see ``NCCL_IB_HCA``). + +Values accepted +^^^^^^^^^^^^^^^ + +``NONE`` : Disable automatic rail and plane assignment. + +``CX9`` (default) : Currently only supported on aarch64 systems with CX9 HCAs, NCCL will assign the rails and planes assuming the reference architecture configuration. + +The per-socket rail layout can be selected explicitly with a suffix: + +``CX9:FLIP`` : Flipped layout ``[0 1 | 1 0]``. Same as ``CX9``. + +``CX9:ALT`` : Alternate layout ``[0 1 0 1]``. + +``CX9:BLOCK`` : Block layout ``[0 0 1 1]``. + NCCL_IB_TIMEOUT --------------- The ``NCCL_IB_TIMEOUT`` variable controls the InfiniBand Verbs Timeout. @@ -406,6 +433,25 @@ Values accepted ^^^^^^^^^^^^^^^ The default value is the traffic class set by NCCL_IB_TC, which defaults to 0 if not set. +NCCL_GIN_IB_TC +-------------- +(since 2.30.7) + +Defines the InfiniBand traffic class for GPU-initiated networking (GIN) +connections, independently of NCCL_IB_TC. GIN traffic can then use a different +RoCE traffic class from the collective and point-to-point connections, which +keep using NCCL_IB_TC. The one-sided host RMA operations run on the same IB +proxy backend and therefore also use this traffic class. + +In RCCL, only the IB proxy GIN backend (``NCCL_GIN_TYPE=2``) reads this variable. +The RCCL device API and GIN how-to describes the other backends. + +Values accepted +^^^^^^^^^^^^^^^ +The default value is the traffic class set by NCCL_IB_TC. If neither is set, GIN +uses the device communicator ``ginTrafficClass``, then the communicator traffic +class, and 0 if none is set. + NCCL_IB_RETURN_ASYNC_EVENTS --------------------------- (since 2.23) @@ -417,6 +463,28 @@ Values accepted ^^^^^^^^^^^^^^^ The default value is 1, set to 0 to disable +NCCL_IB_EVENT_BASED_LB +---------------------- +(since 2.31) + +Enables event-based load balancing across the physical devices of a fused (virtual) IB device (see ``NCCL_IB_MERGE_NICS``). +By default, NCCL splits each message equally across the devices of a fused device. When this variable is enabled, NCCL +instead splits messages proportionally to the current link speed of each device. The current link speed of a device is +queried with the ``ibv_query_port_speed`` API whenever that device reports an ``IBV_EVENT_DEVICE_SPEED_CHANGE`` +asynchronous event, and the distribution is recomputed at runtime from the new speeds. + +This feature is intended for NICs that expose a single interface with multiple planes underneath, where the interface +can continue to operate at a reduced speed when a subset of its planes goes down. A full interface or NIC going down +is considered a failure and is handled by the failover-recovery feature. + +Detecting speed changes requires ``IBV_EVENT_DEVICE_SPEED_CHANGE`` and ``ibv_query_port_speed`` support in the +InfiniBand verbs library (``IBVERBS_1.16`` or newer). This variable has no effect on non-fused devices, since all data +is sent on their single physical device regardless of its speed. + +Values accepted +^^^^^^^^^^^^^^^ +The default value is 0 (data is split equally across devices), set to 1 to enable. + NCCL_OOB_NET_ENABLE ------------------- (since 2.23) @@ -485,7 +553,7 @@ Forces NCCL to use a specific network, for example to make sure NCCL uses an ext Values accepted ^^^^^^^^^^^^^^^ -The value of NCCL_NET has to match exactly the name of the NCCL network used (case-insensitive). Internal network names are "IB" (generic IB verbs) and "Socket" (TCP/IP sockets). External network plugins define their own names. Default value is undefined. +The value of NCCL_NET has to match exactly the name of the NCCL network used (case-insensitive). On Linux, internal network names are "IB" (generic IB verbs) and "Socket" (TCP/IP sockets). On Windows, internal network names are "NetworkDirect" and "Socket"; when NCCL_NET is unset, NCCL tries NetworkDirect before falling back to Socket. External network plugins define their own names. Default value is undefined. NCCL_NET_PLUGIN --------------- @@ -504,6 +572,51 @@ Values accepted Plugin suffix, plugin file name, or "none". +.. _NCCL_GIN_PLUGIN: + +NCCL_GIN_PLUGIN +--------------- +(since 2.29.3) + +Set it to either a suffix string or to a library name to choose among multiple NCCL GIN (GPU-Initiated Networking) plugins; GIN is used by the device API. It also accepts a comma-separated list which are all considered. This setting will cause NCCL to look for the GIN plugin library using the following strategy, applied to each entry of the list: + - If NCCL_GIN_PLUGIN is set, attempt loading the library with name specified by NCCL_GIN_PLUGIN; + - If NCCL_GIN_PLUGIN is set and previous failed, attempt loading libnccl-gin-.so; + - If NCCL_GIN_PLUGIN is not set, attempt loading libnccl-gin.so; + - Additionally, look for the GIN symbols in the NET plugin; + - Additionally, use NCCL's internal GIN implementations. + +For example, setting ``NCCL_GIN_PLUGIN=foo`` will cause NCCL to try to load ``foo`` and, if ``foo`` cannot be found, ``libnccl-gin-foo.so`` (provided that it exists on the system). + +NCCL does not stop at the first plugin that initializes successfully: every GIN implementation it manages to initialize stays available, and the one to use is picked when a device communicator is created. Only one implementation is chosen per backend using the rank-order from above. + +Values accepted +^^^^^^^^^^^^^^^ + +Comma-separated list of plugin suffixes and/or plugin file names, or "none". + + +.. _NCCL_RMA_PLUGIN: + +NCCL_RMA_PLUGIN +--------------- +(since 2.30.7) + +Set it to either a suffix string or to a library name to choose among multiple NCCL RMA plugins. The RMA plugin implements host-driven, one-sided network operations; it is used by the host RMA API and by the GIN proxy implementation. It also accepts a comma-separated list, which are tried in order. This setting will cause NCCL to look for the RMA plugin library using the following strategy: + - If NCCL_RMA_PLUGIN is set, attempt loading the library with name specified by NCCL_RMA_PLUGIN; + - If NCCL_RMA_PLUGIN is set and previous failed, attempt loading libnccl-rma-.so; + - If NCCL_RMA_PLUGIN is not set, attempt loading libnccl-rma.so; + - If no plugin was found look for the RMA symbols in the GIN plugin, then in the NET plugin; + - If no plugin was found (neither user defined nor default), use the internal RMA plugin over InfiniBand. + +For example, setting ``NCCL_RMA_PLUGIN=foo`` will cause NCCL to try to load ``foo`` and, if ``foo`` cannot be found, ``libnccl-rma-foo.so`` (provided that it exists on the system). + +The first plugin that initializes successfully and reports at least one usable device is the one that is used; the remaining external plugins are then disabled for the lifetime of the process. + +Values accepted +^^^^^^^^^^^^^^^ + +Comma-separated list of plugin suffixes and/or plugin file names, or "none". + NCCL_TUNER_PLUGIN ----------------- @@ -604,12 +717,42 @@ Values accepted ^^^^^^^^^^^^^^^ VERSION - Prints the NCCL version at the start of the program. -WARN - Prints an explicit error message whenever any NCCL call errors out. +WARN - Prints error messages when NCCL calls return an error code. + +ATTN (Attention) - Prints WARN messages plus informational notices (cleanup failures, configuration overrides, error backtraces, async event notifications). (since 2.32) INFO - Prints debug information. TRACE - Prints replayable trace information on every call. +.. _NCCL_DEBUG_LEVELS: + +NCCL_DEBUG_LEVELS +----------------- +(since 2.32) + +The ``NCCL_DEBUG_LEVELS`` variable adds individual levels to the inclusive +selection made by ``NCCL_DEBUG``. The effective selection is the logical union +of the levels selected by both variables. The value is a comma-separated list +of ``VERSION``, ``WARN``, ``ATTN``, ``INFO``, ``ABORT``, ``TRACE``, or ``ALL``. + +For example, the following maintains ``WARN`` as the baseline while adding +``ATTN`` on NCCL versions that support this variable. ``NCCL_DEBUG=WARN`` +selects ``VERSION`` and ``WARN``; adding ``ATTN`` produces an effective +selection of ``VERSION,WARN,ATTN``: + +.. code:: shell + + NCCL_DEBUG=WARN NCCL_DEBUG_LEVELS=ATTN ./my_app + +Older NCCL versions ignore ``NCCL_DEBUG_LEVELS`` and retain the ``WARN`` +baseline. Unknown level names are ignored so a site-wide setting can include +levels introduced by a newer NCCL version without disabling recognized levels. + +Do not set ``NCCL_DEBUG=ATTN`` as a site-wide setting in a mixed-version +deployment. NCCL versions before 2.32 do not recognize ``ATTN`` and fall back +to no debug logging. Use ``NCCL_DEBUG=WARN NCCL_DEBUG_LEVELS=ATTN`` instead. + .. _NCCL_DEBUG_FILE: NCCL_DEBUG_FILE @@ -705,13 +848,13 @@ which log lines get a timestamp depending upon the level of the log. Value accepted ^^^^^^^^^^^^^^ The value should be a comma separated list of the levels which should -have the timestamp. Valid levels are: ``VERSION``, ``WARN``, ``INFO``, -``ABORT``, and ``TRACE``. In addition, ``ALL`` can be used to turn it -on for all levels. Setting it to an empty value disables it for all -levels. If the value is prefixed with a caret (``^``) then the listed -levels will NOT log a timestamp, and the rest will. -The default is to enable timestamps for ``WARN``, but disable it for -the rest. +have the timestamp. Valid levels are: ``VERSION``, ``WARN``, ``ATTN``, +``INFO``, ``ABORT``, and ``TRACE``. In addition, ``ALL`` can be used to +turn it on for all levels. Setting it to an empty value disables it for +all levels. If the value is prefixed with a caret (``^``) then the +listed levels will NOT log a timestamp, and the rest will. +The default is to enable timestamps for ``WARN`` and ``ATTN``, but +disable it for the rest. For example, ``NCCL_DEBUG_TIMESTAMP_LEVELS=WARN,INFO,TRACE`` will turn it on for warnings, info logs, and traces. Or, @@ -755,6 +898,39 @@ Set multiple non-legacy policies with the ``|`` operator. For more explanation about NCCL policies, please see :ref:`cta_policy_flags`. +NCCL_CE_CHUNK_SIZE +------------------ +(since 2.32) + +Chunk size, in bytes, used by copy-engine (CE) collectives when a batched copy requests round-robin chunking. +Chunking is opt-in per batch, so this only affects batches that ask for it. + +Values accepted +^^^^^^^^^^^^^^^ +Default is 8388608 (8 MiB). + +NCCL_CE_INTRA_GPU_MEMCPY_ENABLE +------------------------------- +(since 2.32) + +When ``NCCL_CTA_POLICY`` is ``ZERO``, copy-engine collectives request that intra-GPU copies overlap with compute, which +offloads those self-copies to the copy engine. Set to 0 to leave intra-GPU copies on the compute units instead. + +Values accepted +^^^^^^^^^^^^^^^ +Default is 1 (enabled); set to 0 to disable. RCCL always requested the overlap before this variable existed, so the +default keeps that behavior. + +NCCL_HIER_CE_COLL_AG_RAIL_RING_ENABLE +------------------------------------- +(since 2.32) + +Selects the inter-node rail algorithm of the hierarchical copy-engine AllGather. + +Values accepted +^^^^^^^^^^^^^^^ +A positive value selects the ring algorithm. 0 or a negative value keeps the direct algorithm. Default is -1 (direct). + NCCL_NETDEVS_POLICY ------------------- (since 2.28) @@ -765,11 +941,10 @@ For each GPU, NCCL detects automatically available network devices, taking into Value accepted ^^^^^^^^^^^^^^ -If set to ``AUTO`` (default), NCCL also takes into account the other GPUs in the same communicator in order to assign network devices. +By default, NCCL assigns each GPU to network devices by considering the set of available devices, as well as other GPUs in the same communicator. In specific scenarios, this policy might lead to different GPUs from different communicators sharing the same network devices, and therefore impacts performance. If set to ``MAX:N``, NCCL uses up to N of the network devices available to each GPU. -This is intended to be used when device sharing happens with ``AUTO`` and impacts the performance. If set to ``ALL``, NCCL will use all the available network devices for each GPU, disregarding other GPUs. @@ -815,7 +990,14 @@ Value accepted ^^^^^^^^^^^^^^ A path to an accessible file describing part or all of the topology. -Note: For multi-node NVLink systems, despite NCCL_TOPO_DUMP_FILE producing a file with the full NVLink domain topology, NCCL_TOPO_FILE should only include the single node topology. +The topology file must describe the topology of a single host (even for multi-node NVLink +systems). + +``NCCL_TOPO_DUMP_FILE`` produces a file containing the topology of the NVLink domain. +For multi-node NVLink systems, such a file should not be used directly as input to ``NCCL_TOPO_FILE`` without first +trimming it to a single host's topology. +To obtain a suitable topology file, either use ``NCCL_TOPO_DUMP_FILE`` from a single-host run or +manually extract a single host's portion from a multi-host dump. NCCL_TOPO_DUMP_FILE ------------------- @@ -827,7 +1009,11 @@ Value accepted ^^^^^^^^^^^^^^ A path to a file which will be created or overwritten. -Note: For multi-node NVLink systems, the dumped file will contain the full NVLink domain topology. +The dumped file reflects the topology as seen by NCCL. For multi-node NVLink systems, the file +contains the topology from all the hosts that are part of the same NVLink domain. This file is +useful for debugging and inspection, but should be reduced to a sigle host before use with +``NCCL_TOPO_FILE`` (see the notes above). + NCCL_SET_THREAD_NAME -------------------- @@ -1033,12 +1219,13 @@ This is deprecated in 2.9 and may be removed in future versions. NCCL_IB_DISABLE --------------- -The ``NCCL_IB_DISABLE`` variable prevents the IB/RoCE transport from being used by NCCL. Instead, NCCL will fall back to -using IP sockets. +The ``NCCL_IB_DISABLE`` variable prevents the IB/RoCE transport from being used by NCCL. On Windows, the same variable +also disables the NetworkDirect transport. NCCL will instead fall back to another available transport such as IP sockets. Values accepted ^^^^^^^^^^^^^^^ -Define and set to 1 to disable the use of InfiniBand Verbs for communication (and force another method, e.g. IP sockets). +Define and set to 1 to disable the use of InfiniBand Verbs and NetworkDirect for communication (and force another method, +e.g. IP sockets). NCCL_IB_AR_THRESHOLD -------------------- @@ -1070,6 +1257,7 @@ NCCL_IB_QPS_PER_CONNECTION Number of IB queue pairs to use for each connection between two ranks. This can be useful on multi-level fabrics which need multiple queue pairs to have good routing entropy. See ``NCCL_IB_SPLIT_DATA_ON_QPS`` for different ways to split data on multiple QPs, as it can affect performance. +NetworkDirect reuses this variable as the QP count per physical adapter in a connection. Values accepted ^^^^^^^^^^^^^^^ @@ -1671,6 +1859,17 @@ Value accepted ^^^^^^^^^^^^^^ 0 or 1. Default is 0 (init at first window registration). +NCCL_RMA_IB_WR_BATCHSIZE +------------------------ +(since 2.32) + +Maximum number of InfiniBand work requests the RMA proxy posts to a queue pair in one batch. + +Values accepted +^^^^^^^^^^^^^^^ +Default is 64, which is also the maximum. Values below 2 are raised to 2, because one RMA operation can need two work +requests (put and signal). + NCCL_DMABUF_ENABLE ------------------ (since 2.13) @@ -1709,12 +1908,22 @@ NCCL_P2P_LL128_THRESHOLD ------------------------ (since 2.28) -The ``NCCL_P2P_LL128_THRESHOLD`` is the maximum per-channel message size (in bytes) at or below which RCCL uses the LL128 low-latency protocol for P2P (send/recv) operations. Above this size, RCCL uses the SIMPLE protocol. This threshold is used instead of ``NCCL_P2P_LL_THRESHOLD`` whenever the LL128 P2P send/recv path is active (gfx942/gfx950 with ``NCCL_ALLOC_P2P_NET_LL_BUFFERS=1`` and LL128 enabled for the communicator). LL128's much lower per-line flag overhead keeps it faster than SIMPLE to larger per-channel sizes than legacy LL, so it can be set higher than ``NCCL_P2P_LL_THRESHOLD``. +The ``NCCL_P2P_LL128_THRESHOLD`` is the maximum per-channel message size (in bytes) at or below which RCCL uses the LL128 low-latency protocol for P2P (send/recv) operations. Above this size, RCCL uses the SIMPLE protocol. This threshold is used instead of ``NCCL_P2P_LL_THRESHOLD`` whenever the LL128 P2P send/recv path is active (gfx942/gfx950 with ``NCCL_ALLOC_P2P_NET_LL_BUFFERS=1``, or gfx1250 with ``NCCL_P2P_LL128_ENABLE=1`` on rank counts that have no SendRecv window). On gfx1250 with ``ENABLE=1`` and 4, 8, or 16 ranks, ``ncclSend``/``ncclRecv`` use the message-size caps documented under ``NCCL_P2P_LL128_ENABLE`` instead of this per-channel threshold. LL128's much lower per-line flag overhead (~1/8 on gfx9, ~1/16 on gfx1250) keeps it faster than SIMPLE to larger per-channel sizes than legacy LL, so it can be set higher than ``NCCL_P2P_LL_THRESHOLD``. A value of 0 means no upper bound. Values accepted ^^^^^^^^^^^^^^^ Decimal number. Default is 16384. +NCCL_P2P_LL128_ENABLE +--------------------- +(since 2.28) + +``NCCL_P2P_LL128_ENABLE`` controls gfx1250 LL128 send/recv. Default ``-1`` keeps the usual P2P path: legacy LL up to ``NCCL_P2P_LL_THRESHOLD`` (4 KiB per channel) and SIMPLE above it. Set to ``1`` to use LL128 for ``ncclSend``/``ncclRecv`` only (not AlltoAll) from 0 through these inclusive caps on 4 GPU/node: 1-node 4 GPU 1 MiB, 2-node 8 GPU 512 KiB, 4-node 16 GPU 256 KiB. Above the cap uses SIMPLE. Protocol is a function of message size, so mixed send/recv sizes under the cap are both LL128 and share one kernel. On other rank counts, ``ENABLE=1`` still uses ``NCCL_P2P_LL128_THRESHOLD``. Set to ``0`` to disable. gfx942/gfx950 do not use this flag; they select LL128 via ``NCCL_ALLOC_P2P_NET_LL_BUFFERS``. Internodal gfx1250 LL128 still needs the net staging buffer; RCCL allocates it when ``ENABLE=1`` even if ``NCCL_ALLOC_P2P_NET_LL_BUFFERS`` is unset. + +Values accepted +^^^^^^^^^^^^^^^ +-1 (default: legacy LL then SIMPLE, no LL128), 0 (disabled), or 1 (force on). Default value is -1. + NCCL_ALLOC_P2P_NET_LL_BUFFERS ----------------------------- (since 2.14) @@ -1736,6 +1945,8 @@ Values accepted ^^^^^^^^^^^^^^^ 0 or 1. 1 indicates blocking communicators, and 0 indicates nonblocking communicators. The default value is undefined. +.. _NCCL_CGA_CLUSTER_SIZE: + NCCL_CGA_CLUSTER_SIZE --------------------- (since 2.16) @@ -1751,6 +1962,8 @@ Values accepted ^^^^^^^^^^^^^^^ 0 to 8. Default value is undefined. +.. _NCCL_MAX_CTAS: + NCCL_MAX_CTAS ------------- (since 2.17) @@ -1779,6 +1992,8 @@ Values accepted Set to a positive integer value up to 64 (32 prior to 2.25). Default value is undefined. +.. _NCCL_MIN_CTAS: + NCCL_MIN_CTAS ------------- (since 2.17) @@ -1836,6 +2051,17 @@ Values accepted ^^^^^^^^^^^^^^^ Default is 1 (enabled), define and set to 0 to disable NIC merging +NCCL_IB_SORT_MERGE_NICS +----------------------- +(since 2.32) + +When NICs are merged into one virtual device, sort the merged sub-devices by plane, so that sub-device indices line +up by plane across all merged devices. + +Values accepted +^^^^^^^^^^^^^^^ +Default is 0 (disabled) in RCCL; set to 1 to enable. Upstream NCCL enables it by default. + NCCL_NET_MERGE_POLICY --------------------- (since 2.30.5) @@ -1936,6 +2162,59 @@ Values accepted ^^^^^^^^^^^^^^^ Default is 1. A positive floating-point number. Values less than or equal to 0 are ignored and replaced with 1. +NCCL_PROGRESS_COUNTERS +---------------------- +(since 2.32) + +Enable GPU-resident progress counters, which RAS uses to report how far each rank's collectives have progressed. +Counters are only allocated when RAS is also enabled (``NCCL_RAS_ENABLE=1``). + +Values accepted +^^^^^^^^^^^^^^^ +Default is 0 (disabled); set to 1 to enable. + +NCCL_PROGRESS_COUNTER_MONITOR_POLL_MS +------------------------------------- +(since 2.32) + +Interval, in milliseconds, at which a per-GPU monitor thread copies the progress counters to host memory. Only used +when ``NCCL_PROGRESS_COUNTERS`` is enabled. + +Values accepted +^^^^^^^^^^^^^^^ +Default is 1000. Values below 50 are raised to 50. + +NCCL_PROGRESS_COUNTER_MONITOR_STALE_MS +-------------------------------------- +(since 2.32) + +Time, in milliseconds, after which an unfinished progress-counter copy is reported as stale. + +Values accepted +^^^^^^^^^^^^^^^ +Default is 5000. Values below 1000 are raised to 1000. + +NCCL_PROGRESS_COUNTER_MONITOR_STALE_WARN_SEC +-------------------------------------------- +(since 2.32) + +Minimum time, in seconds, between two stale-copy messages for the same GPU. + +Values accepted +^^^^^^^^^^^^^^^ +Default is 600. 0 or a negative value disables the messages. + +NCCL_DIAGNOSTICS_IB_BW_TIMEOUT +------------------------------ +(since 2.32) + +Time limit, in seconds, for the network bandwidth check that ``NCCL_RUN_DIAGNOSTICS=1`` runs at communicator +initialization. The check only runs when the communicator spans at least two nodes over the network transport. + +Values accepted +^^^^^^^^^^^^^^^ +Default is 5. Values below 1 are replaced with 5. + .. _NCCL_LAUNCH_ORDER_IMPLICIT: NCCL_LAUNCH_ORDER_IMPLICIT diff --git a/projects/rccl/docs/userguide/source/nccl4py/communicator.rst b/projects/rccl/docs/userguide/source/nccl4py/communicator.rst index 0ee35e746913..6148ecc24e28 100644 --- a/projects/rccl/docs/userguide/source/nccl4py/communicator.rst +++ b/projects/rccl/docs/userguide/source/nccl4py/communicator.rst @@ -7,8 +7,8 @@ Communicator The :py:class:`Communicator` class and its methods, organized by lifecycle stage and operation kind: -- :doc:`communicator/class` — the class itself, its constructor, and - per-instance properties for identity and device-API capability. +- :doc:`communicator/class` — the class itself, its constructor, its + per-instance properties, and the predefined teams. - :doc:`communicator/lifecycle` — creating, splitting, growing, and tearing down communicators. - :doc:`communicator/collectives` — collective communication methods diff --git a/projects/rccl/docs/userguide/source/nccl4py/communicator/class.rst b/projects/rccl/docs/userguide/source/nccl4py/communicator/class.rst index fea3b33e07cb..bf15300e3d26 100644 --- a/projects/rccl/docs/userguide/source/nccl4py/communicator/class.rst +++ b/projects/rccl/docs/userguide/source/nccl4py/communicator/class.rst @@ -11,6 +11,13 @@ Communicator Class Properties ========== +:py:attr:`Communicator.properties` returns an :py:class:`NCCLCommProperties` +holding the properties NCCL reports for the communicator, including fields that +have no dedicated accessor. The groups below provide per-field accessors for +the values needed most often. + +.. autoattribute:: Communicator.properties + Identity -------- @@ -35,17 +42,32 @@ structure. .. autoattribute:: Communicator.host_rma_support .. autoattribute:: Communicator.railed_gin_type +NCCLCommProperties +------------------ + +Covers the accessors in both groups above, plus the fields that have none. + +.. autoclass:: NCCLCommProperties + :members: + Teams ===== A :py:class:`NCCLTeam` names a strided subset of the communicator's ranks. -These properties return the predefined teams; pass one to -:py:class:`TeamRequirement` or to the rank converters below. See +The members below return the predefined teams; pass one to +:py:class:`TeamRequirement` or to the rank converters. See :ref:`devapi_teams` for team semantics. .. autoattribute:: Communicator.team_world .. autoattribute:: Communicator.team_lsa .. autoattribute:: Communicator.team_rail +.. autoattribute:: Communicator.team_cft_multimem +.. automethod:: Communicator.team_cft .. automethod:: Communicator.team_rank_to_world .. automethod:: Communicator.team_rank_to_lsa + +NcclCftTeamMode +--------------- +.. autoclass:: NcclCftTeamMode + :members: diff --git a/projects/rccl/docs/userguide/source/nccl4py/communicator/collectives.rst b/projects/rccl/docs/userguide/source/nccl4py/communicator/collectives.rst index 6a3f2b5cd704..2aaf13fa5366 100644 --- a/projects/rccl/docs/userguide/source/nccl4py/communicator/collectives.rst +++ b/projects/rccl/docs/userguide/source/nccl4py/communicator/collectives.rst @@ -7,6 +7,9 @@ Collective Communication Methods Methods on :py:class:`Communicator` for collective communication. See :doc:`../../api/colls` for the corresponding C API. +Each collective below takes an optional :py:class:`NCCLCollConfig` as +``config`` to customize that one call; see :doc:`../configuration`. + allreduce --------- .. automethod:: Communicator.allreduce diff --git a/projects/rccl/docs/userguide/source/nccl4py/configuration.rst b/projects/rccl/docs/userguide/source/nccl4py/configuration.rst index 1e0aae4fa732..5b170bbd2d69 100644 --- a/projects/rccl/docs/userguide/source/nccl4py/configuration.rst +++ b/projects/rccl/docs/userguide/source/nccl4py/configuration.rst @@ -4,8 +4,8 @@ Configuration ************* -Configuration objects passed to communicator creation methods, plus the -flag enums they consume. +Configuration objects passed to communicator creation methods and to +individual collectives, plus the flag enums they consume. NCCLConfig ========== @@ -19,6 +19,37 @@ consumed. .. autoclass:: NCCLConfig :members: +NcclHostCftMode +--------------- + +Value of :py:attr:`NCCLConfig.host_cft_mode`. + +.. autoclass:: NcclHostCftMode + :members: + +NcclNvlsHostMode +---------------- + +Value of :py:attr:`NCCLConfig.nvls_host_mode`. + +.. autoclass:: NcclNvlsHostMode + :members: + +NCCLCollConfig +============== + +Accepted as the ``config`` argument of every collective on +:py:class:`Communicator`. See the individual field documentation for unset +behavior and usage requirements. + +.. autoclass:: NCCLCollConfig + :members: + +VendorOption +------------ +.. autoclass:: VendorOption + :members: + CTAPolicy ========= @@ -35,6 +66,14 @@ Used by :py:meth:`Communicator.create_dev_comm`. Fields left unset .. autoclass:: NCCLDevCommRequirements :members: +NcclCftCap +---------- + +Bitmask value of :py:attr:`NCCLDevCommRequirements.cft_caps`. + +.. autoclass:: NcclCftCap + :members: + Requirement entries =================== diff --git a/projects/rccl/docs/userguide/source/nccl4py/resources.rst b/projects/rccl/docs/userguide/source/nccl4py/resources.rst index 166327b94403..ec6c5b9bc960 100644 --- a/projects/rccl/docs/userguide/source/nccl4py/resources.rst +++ b/projects/rccl/docs/userguide/source/nccl4py/resources.rst @@ -31,6 +31,15 @@ RegisteredWindowHandle :members: :inherited-members: +CftLeInfo +========= + +Returned by the :py:class:`RegisteredWindowHandle` logical-endpoint queries. +CFT operations address memory by this pair rather than by pointer. + +.. autoclass:: CftLeInfo + :members: + CustomRedOp =========== .. autoclass:: CustomRedOp diff --git a/projects/rccl/docs/userguide/source/nccl4py/types.rst b/projects/rccl/docs/userguide/source/nccl4py/types.rst index 3295b72607d9..8c3e017f520f 100644 --- a/projects/rccl/docs/userguide/source/nccl4py/types.rst +++ b/projects/rccl/docs/userguide/source/nccl4py/types.rst @@ -105,8 +105,8 @@ NCCLTeam Type aliases ============ -Aliases naming the argument types accepted by public methods. Each expands -to a union of concrete types, so a value of any member type is accepted. +Aliases naming value types accepted by the public API. Each expands to a +union of concrete types, so a value of any member type is accepted. .. autodata:: NcclBufferSpec :annotation: @@ -120,6 +120,9 @@ to a union of concrete types, so a value of any member type is accepted. .. autodata:: NcclStreamSpec :annotation: +.. autodata:: NcclEventSpec + :annotation: + Exceptions ========== diff --git a/projects/rccl/docs/userguide/source/setup.rst b/projects/rccl/docs/userguide/source/setup.rst index 72b92167fe68..c29189115255 100644 --- a/projects/rccl/docs/userguide/source/setup.rst +++ b/projects/rccl/docs/userguide/source/setup.rst @@ -6,6 +6,30 @@ NCCL is a communication library providing optimized GPU-to-GPU communication for It is not, like MPI, providing a parallel environment including a process launcher and manager. NCCL relies therefore on the application's process management system and CPU-side communication system for its own bootstrap. -Similarly to MPI and other libraries which are optimized for performance, NCCL does not provide secure network -communication between GPUs. It is therefore the responsibility of the user to ensure NCCL operates over a secure network, -both for bootstrap (controlled by :ref:`NCCL_SOCKET_IFNAME`) and for high-speed communication. +By default, NCCL does not encrypt network communication. It is therefore the responsibility of the user to ensure NCCL +operates over a secure network, both for bootstrap (controlled by :ref:`NCCL_SOCKET_IFNAME`) and for high-speed +communication. An OpenSSL-enabled build can encrypt NCCL-owned TCP traffic carried by ``ncclSocket`` when an encryption +key is configured. Non-Socket transports remain outside that protection boundary. + +Building with TLS support +========================= + +TLS support requires OpenSSL 3 headers and shared libraries. Install the OpenSSL development package through the system +package manager so the headers are available during the build and NCCL can load the system libraries at run time. + +For Make, select OpenSSL when building NCCL: + +.. code-block:: shell + + make TLS_BACKEND=OPENSSL3 src.build + +For CMake, select the same backend during configuration: + +.. code-block:: shell + + cmake -S . -B build-cmake -DTLS_BACKEND=OPENSSL3 + cmake --build build-cmake -j + +An unset or empty ``TLS_BACKEND`` builds NCCL without TLS support or an OpenSSL dependency. Other nonempty values are +rejected. Both build systems require OpenSSL 3 when the backend is selected and fail if its headers or libraries cannot +be found. diff --git a/projects/rccl/docs/userguide/source/troubleshooting/diagnostics.rst b/projects/rccl/docs/userguide/source/troubleshooting/diagnostics.rst index e23ebd3ec51f..b32fd6921412 100644 --- a/projects/rccl/docs/userguide/source/troubleshooting/diagnostics.rst +++ b/projects/rccl/docs/userguide/source/troubleshooting/diagnostics.rst @@ -49,7 +49,7 @@ output of the process hosting rank 0 of the communicator; each report line is pr .. code:: NCCL DIAG === NCCL Diagnostics === - NCCL DIAG [OK] p2p: all 56 directed GPU P2P edges verified + NCCL DIAG [OK] p2p: verified P2P access in both directions between every GPU pair (56 peer accesses) NCCL DIAG NCCL diagnostics completed in 245.3 ms across 8 ranks Result lines use the following tags: @@ -58,6 +58,13 @@ Result lines use the following tags: * ``[INFO]`` identifies a condition that may require review, such as a failed verification or a check that could not be completed. +For a partial P2P result, the summary reports the number of GPU-to-GPU peer accesses that passed +verification out of the total number tested: + +.. code:: + + NCCL DIAG [INFO] p2p: only 52/56 GPU-to-GPU peer accesses passed verification + Diagnostics are informational only: a reported issue does not abort communicator initialization. Available Checks @@ -66,11 +73,67 @@ Available Checks P2P Check """"""""" -The P2P check verifies direct Peer-to-Peer (P2P) communication between GPUs. For every pair of -GPUs in the communicator that NCCL expects to communicate directly (e.g., over NVLink or PCIe), -it transfers data between the two GPUs and validates that the data arrives intact. +The P2P check verifies direct Peer-to-Peer (P2P) access between GPUs. For every pair of GPUs in the +communicator that NCCL expects to communicate directly (e.g., over NVLink or PCIe), each GPU reads +from and writes to its peer, and the diagnostic validates the transferred data. The summary counts +each direction as one peer access. GPU pairs that communicate through an intermediate GPU rather +than direct peer access are excluded from the summary; their ``reason=indirect`` skip records are +available with ``NCCL_DEBUG=INFO`` and ``NCCL_DEBUG_SUBSYS=INIT``. When a transfer fails, the report identifies the affected GPU pair and the connection path between them, narrowing the investigation down to a specific link or device. Conversely, a passing P2P check makes intra-node or NVLink connectivity an unlikely culprit, directing the investigation toward other components, such as the network between nodes or the application itself. + +Network Bandwidth Check +""""""""""""""""""""""" + +The network bandwidth check runs ``ib_write_bw`` from the ``perftest`` package over the physical +InfiniBand devices selected by NCCL. It runs only when the communicator uses a network transport +across at least two hosts and requires ``ib_write_bw`` to be installed on every participating +node. The client endpoint of +each measurement connects to the server endpoint's host name, so host names must be resolvable +between the nodes of the communicator. See :doc:`networking_troubleshooting` for instructions on +running the tool manually. + +Nodes (ordered by their first rank) form a ring and are checked pairwise in exactly two phases: +pairs starting at even positions run in phase 0 and pairs starting at odd positions in phase 1 +(on an odd ring the wrapping pair is skipped; its nodes are covered by their other neighbors). +Within a pair, every device of the server node that the client node also has anchors one check +against that same-named client device; when the selected NCCL topology enables cross-NIC +communication (``NCCL_CROSS_NIC``), phase 0 instead checks against the client's next device after +the anchor, wrapping around the client's device list (a single cross-device rotation). Ranks +sharing a device are paired by occurrence -- the k-th such rank with the k-th on the paired +node -- and measure the shared device concurrently, splitting its bandwidth; a single ``[INFO]`` +line from rank 0 notes when shared devices are present. +Each rank measures only the device of its first channel; when ranks use more devices than the one +tested (e.g. a merged device), a single ``[INFO]`` line from rank 0 notes it. +Every rank still runs at most one tool instance per phase, and within a phase all checks proceed +in parallel. Nodes may differ in rank and device counts; a device without a same-named +counterpart on the paired node anchors no check. + +Both phases measure from GPU memory, including DMA-BUF, when both endpoints and the installed +tool support it, and from host memory otherwise. Successful runs are not reported individually; +failed runs are reported per device pair. Each endpoint separately averages the successful same-NIC +and cross-NIC measurements in which it participated; the per-rank averages are then shared across all +ranks, and rank 0 prints the minimum, median, and maximum for each mode (tagged ``[OK]``, or +``[INFO]`` when the mode has outliers or not every scheduled measurement produced a result) and +emits an ``[INFO]`` +line for each rank deviating by more than 30 percent from that mode's median, in either direction +(at most 8 such lines per mode; a final line reports the count of any further outliers). +Ranks without a successful measurement +in a mode are excluded from that comparison. Example output is shown below: + +.. code:: + + node0:24563 NCCL DIAG [OK] net bw: 376.1/391.5/398.2 Gbit/s min/median/max same-nic bw (across 8 ranks) in comm 0x7a3f9b2c41d05e88 + node0:24563 NCCL DIAG [INFO] net bw: 265.1/391.5/398.0 Gbit/s min/median/max cross-nic bw (across 8 ranks) in comm 0x7a3f9b2c41d05e88 + node0:24563 NCCL DIAG [INFO] net bw: cross-nic rank 3 (node0): 265.1 Gbit/s, >30% off median 391.5 Gbit/s in comm 0x7a3f9b2c41d05e88 + +Each measurement streams 1000 messages of 64 KiB. A healthy link completes a measurement in well +under a second; a severely degraded link runs into the per-tool timeout and is reported as +``tool run timed out``. +The check runs under a fixed overall time allocation, separate from the per-tool timeout that +guards each individual ``ib_write_bw`` run. When that time runs out, all nodes skip the remaining +measurements together and a ``test unable to complete in allocated time`` line reports how many +test phases completed. diff --git a/projects/rccl/docs/userguide/source/troubleshooting/logging.rst b/projects/rccl/docs/userguide/source/troubleshooting/logging.rst index a89f4b0107e1..2e17beb05326 100644 --- a/projects/rccl/docs/userguide/source/troubleshooting/logging.rst +++ b/projects/rccl/docs/userguide/source/troubleshooting/logging.rst @@ -12,7 +12,10 @@ The following environment variables control NCCL logging behavior: +------------------------------------+-----------------------------------------------+ | Variable | Description | +====================================+===============================================+ -| :ref:`NCCL_DEBUG` | Sets verbosity: VERSION, WARN, INFO, TRACE | +| :ref:`NCCL_DEBUG` | Sets verbosity: VERSION, WARN, ATTN, INFO, | +| | TRACE | ++------------------------------------+-----------------------------------------------+ +| :ref:`NCCL_DEBUG_LEVELS` | Adds individual logging levels | +------------------------------------+-----------------------------------------------+ | :ref:`NCCL_DEBUG_SUBSYS` | Filters log output by subsystem | +------------------------------------+-----------------------------------------------+ @@ -35,13 +38,19 @@ NCCL supports several logging levels, from least to most verbose: +=========+============================================+=============================+ | VERSION | Prints NCCL version at startup | Verify installation | +---------+--------------------------------------------+-----------------------------+ -| WARN | Warnings and errors | Production minimum | +| WARN | Errors returned by NCCL | Production baseline | ++---------+--------------------------------------------+-----------------------------+ +| ATTN | Noteworthy messages that are not errors | Production diagnostics | +---------+--------------------------------------------+-----------------------------+ | INFO | Detailed operational information | Diagnose runtime issues | +---------+--------------------------------------------+-----------------------------+ | TRACE | Replayable traces, plus CALL APIs | Deep debugging / NCCL dev | +---------+--------------------------------------------+-----------------------------+ +Use ``NCCL_DEBUG=ATTN`` when diagnosing production jobs that need noteworthy +operational notices without the volume of ``INFO`` output. It includes all +``WARN`` messages. + Setting the Logging Level ------------------------- @@ -51,6 +60,27 @@ Set the ``NCCL_DEBUG`` environment variable: NCCL_DEBUG=INFO ./my_app +Adding Logging Levels +--------------------- + +``NCCL_DEBUG_LEVELS`` adds selected levels to the inclusive ``NCCL_DEBUG`` +setting. The effective selection is the logical union of the levels selected by +both variables. This is useful for a mixed NCCL deployment where older +releases need to retain a ``WARN`` baseline: + +.. code:: shell + + NCCL_DEBUG=WARN NCCL_DEBUG_LEVELS=ATTN ./my_app + +Here, ``NCCL_DEBUG=WARN`` selects VERSION and WARN, while +``NCCL_DEBUG_LEVELS=ATTN`` selects ATTN. Newer releases therefore emit +VERSION, WARN, and ATTN messages. Older releases ignore ``NCCL_DEBUG_LEVELS`` +and emit their normal WARN output. + +Do not set ``NCCL_DEBUG=ATTN`` as a site-wide setting in a mixed-version +deployment. NCCL versions before 2.32 do not recognize ``ATTN`` and fall back +to no debug logging. + Example Output -------------- @@ -68,6 +98,12 @@ The snippets below are excerpts from NCCL logs; only the relevant lines are show [2026-05-05 06:16:47] node-01:189884:189884 [3] plugin/net.cc:334 NCCL WARN Failed to initialize any NET plugin +**NCCL_DEBUG=ATTN** - Errors and noteworthy non-fatal messages are printed: + +.. code:: shell + + [2026-05-05 06:16:47] node-01:189884:189884 [3] plugin/net.cc:208 (ncclNetPluginInit) NCCL ATTN Failed to initialize NET plugin IB + **NCCL_DEBUG=INFO** - Detailed information about NCCL operations: .. code:: shell diff --git a/projects/rccl/docs/userguide/source/troubleshooting/ras.rst b/projects/rccl/docs/userguide/source/troubleshooting/ras.rst index 5a559a386f9b..ec7071775c21 100644 --- a/projects/rccl/docs/userguide/source/troubleshooting/ras.rst +++ b/projects/rccl/docs/userguide/source/troubleshooting/ras.rst @@ -429,27 +429,80 @@ the graph is recaptured. RAS Diagnostics --------------- -RAS diagnostics provide a readiness probe for NCCL jobs by checking the selected GPU, CUDA driver, and NCCL -configuration information across ranks. +RAS diagnostics provide a readiness probe for NCCL jobs by comparing GPU inventory, PCI configuration, CUDA and +NVIDIA graphics driver versions, and NCCL configuration across ranks. + +RAS diagnostics run asynchronously, so collection and reporting may be delayed. For short-lived communicators, this may +result in incomplete information. + +Available Checks +^^^^^^^^^^^^^^^^ The report includes results for the following checks: * **GPU inventory:** Checks that communicator ranks report the same number of GPUs and use the same GPU model. * **CUDA driver version:** Checks that all ranks report the same CUDA version supported by the driver. +* **NVIDIA graphics driver version:** Checks that the graphics driver version reported through NVML is consistent across + all ranks. * **Volatile ECC errors:** Checks volatile SRAM and DRAM ECC error counters for the GPU used by each rank. -* **NVLink state:** Checks that ranks report the same number of NVLinks and that all reported links are active. +* **Xid and SXid events:** Scans the host kernel log for GPU Xids and NVSwitch SXids, maps each event to the host + where it was detected, and classifies it into predefined guidance groups using the + `Xid Error Catalog `_ and + `Fabric Manager guide `_. +* **NVLink state:** Checks that ranks report the same number of NVLinks, that every link is active, and that all active + links run at the same speed. * **NCCL environment:** Checks that ``NCCL_*`` environment-variable names and values are consistent across ranks. +* **PCI configuration:** Runs ``rdma_topo check`` to validate the PCI topology and PCI Access Control Services (ACS) + configuration used for GPUDirect RDMA. +* **IOMMU mode and groups:** If ``rdma_topo check`` cannot provide a conclusive result, checks the IOMMU mode of + topology-selected GPUs and NICs and compares the IOMMU groups of associated GPU/NIC pairs. +* **ATS state:** If ``rdma_topo check`` cannot provide a conclusive result, checks the Address Translation Services + (ATS) state of topology-selected NICs. .. note:: RAS diagnostics do not exercise NCCL data paths and do not provide a comprehensive assessment of cluster health. +PCI-Related Checks +"""""""""""""""""" + +RAS first runs ``rdma_topo check``. If the check passes or identifies a configuration problem, the result is conclusive +and reported directly. Otherwise, RAS performs heuristic fallback checks on the GPU/NIC pairs selected by NCCL: + +* Determines the IOMMU mode of each relevant GPU and NIC (disabled, passthrough, or DMA remapping). +* Compares the IOMMU groups of associated GPU/NIC pairs when groups are present. +* Inspects ``ATSCtl`` in ``lspci -D -vvv`` output to determine the ATS state of each relevant NIC. + +The fallback checks can identify conditions that merit review, but do not provide the same platform-aware conclusion +as ``rdma_topo check``. + +Xid and SXid Events +""""""""""""""""""" + +The Xid/SXid check first tries ``/dev/kmsg`` and falls back to ``dmesg``, then +``/var/log/kern.log``, ``/var/log/messages``, and ``/var/log/syslog`` when necessary. When a regular log file is used, +only the active file is scanned; rotated log files are not inspected. Its first run reports events recorded since the +local RAS thread started. Each later run reports events recorded since the preceding Xid/SXid scan. The scan interval +is shared by initialization-triggered and on-demand RAS diagnostics in the process. + +.. note:: + + Xid guidance assumes an NVIDIA Ampere architecture or newer GPU. For earlier GPU architectures, use the + `archived NVIDIA Xid documentation `_ to interpret + the reported code. + Prerequisites ^^^^^^^^^^^^^ RAS diagnostics require the RAS subsystem to be enabled, which is the default; see :ref:`env_NCCL_RAS_ENABLE`. +.. note:: + + Some checks can provide more complete results when optional system tools are available or the process has elevated + privileges. The PCI-related checks use ``rdma_topo`` for the platform-aware audit and ``lspci`` for ATS inspection. + Access to the required PCI information may require elevated privileges. + Running During Communicator Initialization ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -506,8 +559,11 @@ For example, an on-demand report with no reported issues might look like: node042:12345 NCCL DIAG [OK] GPU inventory: 8x NVIDIA H100 per node consistent across 8 ranks in comm 0x1234 node042:12345 NCCL DIAG [OK] CUDA driver version: 13000 consistent across 8 ranks in comm 0x1234 node042:12345 NCCL DIAG [OK] ECC: no uncorrected volatile errors across 8 ranks in comm 0x1234 - node042:12345 NCCL DIAG [OK] NVLink: found 18 link(s) per device, all active across 8 ranks in comm 0x1234 + node042:12345 NCCL DIAG [OK] Xid/SXid: no events requiring immediate attention across 2 hosts + node042:12345 NCCL DIAG [OK] NVLink: 18 links per GPU, all active at consistent speed across 8 ranks in comm 0x1234 node042:12345 NCCL DIAG [OK] NCCL environment: NCCL_* env vars consistent across 8 ranks in comm 0x1234 + node042:12345 NCCL DIAG [OK] rdma_topo check: passed on 8/8 ranks in comm 0x1234 + node042:12345 NCCL DIAG [OK] NVIDIA graphics driver version: 610.47.03 consistent across 8 ranks in comm 0x1234 node042:12345 NCCL DIAG RAS diagnostics completed in 12.3 ms across 8 RAS peers .. note:: diff --git a/projects/rccl/docs/userguide/source/usage.rst b/projects/rccl/docs/userguide/source/usage.rst index 76654f869240..91b1bb54f2fd 100644 --- a/projects/rccl/docs/userguide/source/usage.rst +++ b/projects/rccl/docs/userguide/source/usage.rst @@ -32,3 +32,4 @@ The creation of a communicator is the first step needed before launching any com usage/cudagraph usage/bufferreg usage/deviceapi + usage/cft diff --git a/projects/rccl/docs/userguide/source/usage/cft.rst b/projects/rccl/docs/userguide/source/usage/cft.rst new file mode 100644 index 000000000000..505e5b88429c --- /dev/null +++ b/projects/rccl/docs/userguide/source/usage/cft.rst @@ -0,0 +1,199 @@ +.. _usage_cft: + +Compute Fabric Transport +************************ + +Starting with NCCL 2.31, the Device API includes Compute Fabric Transport (CFT) +helpers for kernels that communicate through CUDA fabric logical endpoints. +CFT is useful when an application launches custom device-side communication +kernels on CFT-capable systems and needs direct fabric put, get, reduction, or +barrier operations. + +Requirements +============ + +CFT kernels require hardware, driver, and CUDA Toolkit support for fabric +logical endpoints and fabric PTX instructions. In this release, the example CFT +barrier program documents the practical requirements as CUDA Toolkit 13.3 and +SM_100 architectures. + +Before using CFT in an application: + +* Allocate and register symmetric memory windows with NCCL. +* Create a device communicator with CFT capabilities in + :c:type:`ncclDevCommRequirements`. +* Use CFT team helpers to address peers by CFT team rank. +* Query logical endpoint IDs and offsets before issuing CFT operations. +* Match the number of requested CFT barriers to the number of barrier slots used + by the kernel. + +Device Communicator Setup +========================= + +The host code requests CFT capability when creating the device communicator. + +.. code-block:: C + + ncclDevComm devComm; + ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; + + reqs.cftCaps = NCCL_CFT; + reqs.cftBarrierCount = nCTAs; + + NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); + +Request :c:macro:`NCCL_CFT_MULTIMEM` in addition to :c:macro:`NCCL_CFT` when a +kernel uses multicast CFT operations or multimem CFT barriers: + +.. code-block:: C + + reqs.cftCaps = NCCL_CFT | NCCL_CFT_MULTIMEM; + +If a communicator cannot provide the requested CFT capability on all ranks, +:c:func:`ncclDevCommCreate` fails. + +Teams and Endpoints +=================== + +CFT operations may address peers through CFT teams rather than directly through +world ranks. Use :c:func:`ncclTeamCft` or :c:func:`ncclTeamCftMultimem` on the +host side, and the corresponding device overloads in kernels, to obtain the +team layout. + +Use logical endpoint query helpers to translate a registered window, byte +offset, and peer into the values consumed by :cpp:class:`ncclCft` operations. +For device-side kernels, :cpp:func:`ncclGetCftLeInfo` accepts a CFT-team peer +rank and :cpp:func:`ncclGetPeerLeInfo` accepts a world-rank peer. + +.. code-block:: C + + ncclTeam cftTeam = ncclTeamCft(devComm); + ncclCftLeId leId; + size_t leOffset; + + ncclGetCftLeInfo(win, byteOffset, peerCft, cftTeam, devComm, + &leId, &leOffset); + +Host code can use :c:func:`ncclGetCftDeviceLeInfo`, +:c:func:`ncclGetPeerDeviceLeInfo`, and +:c:func:`ncclGetMultimemDeviceLeInfo`. If host-side endpoint queries are needed +before creating a CFT-enabled device communicator, configure ``hostCftMode`` in +:c:type:`ncclConfig_t` during communicator initialization. + +CFT Operations +============== + +A CFT kernel stages data through shared memory, creates a :cpp:class:`ncclCft` +object, issues operations, and then submits and flushes the work. + +.. code-block:: C + + __global__ void cftPutKernel(ncclDevComm devComm, ncclWindow_t win) { + __shared__ ncclCftSmem cftSmem; + __shared__ alignas(16) char smem[128]; + + ncclCoopCta coop; + ncclCft cft{coop, cftSmem}; + + ncclTeam cftTeam = ncclTeamCft(devComm); + int peer = (cftTeam.rank + 1) % cftTeam.nRanks; + + ncclCftLeId leId; + size_t leOffset; + ncclGetCftLeInfo(win, 0, peer, cftTeam, devComm, &leId, &leOffset); + + cft.put(coop, leId, leOffset, smem, sizeof(smem)); + cft.submit(coop); + cft.flush(coop); + } + +CFT source and destination shared memory pointers must be 16-byte aligned, and +the byte count must be a multiple of 16. See :ref:`device_api_cft` for the full +list of put, get, multicast, reduction, and pull-reduce helpers. + +CFT Barriers +============ + +CFT barriers synchronize device threads across the CFT team. The common setup is +to request one barrier per CTA and use ``blockIdx.x`` as the barrier index. + +.. code-block:: C + + __global__ void cftBarrierKernel(ncclDevComm devComm) { + ncclCoopCta coop; + ncclCftBarrierSession bar{coop, devComm, blockIdx.x}; + + bar.sync(coop, cuda::memory_order_acq_rel, + ncclMemProxyType::Generic, ncclMemProxyType::Fabric); + } + +For multicast CFT barriers, create the device communicator with +:c:macro:`NCCL_CFT_MULTIMEM` and pass ``multimem=true`` to the barrier session +constructor. + +Examples +======== + +See the CFT barrier example under +``docs/examples/06_device_api/04_cft_barrier`` for a complete runnable CFT +setup and kernel. + +Cross-proxy Fences +================== + +Cross-proxy fences enforce memory ordering between operations issued by different +producer and consumer proxies. The common use case for fence is ordering of data +and flag updates to establish a ``happens-before`` relationship between producer +stores and consumer loads to the same memory region. In the following example, the +producer is the Fabric proxy, while the consumer (ommitted) is the Generic proxy. +The consumer polls on the flag using relaxed loads and, after observing the flag +update, issues a fence with memory_order_acquire, Fabric producer, Generic consumer +and memory scope Sys. + +.. code-block:: C + + __global__ void cftPutKernel(ncclDevComm devComm, ncclWindow_t win) { + __shared__ ncclCftSmem cftSmem; + __shared__ alignas(16) char smem[128]; + __shared__ alignas(16) int flag[4]; + + ncclCoopCta coop; + ncclCft cft{coop, cftSmem}; + + ncclTeam cftTeam = ncclTeamCft(devComm); + int peer = (cftTeam.rank + 1) % cftTeam.nRanks; + + ncclCftLeId leId; + size_t leOffset; + ncclGetCftLeInfo(win, 0, peer, cftTeam, devComm, &leId, &leOffset); + + cft.put(coop, leId, leOffset, smem, sizeof(smem)); + cft.submit(coop); + cft.flush(coop); + + ncclMemFence(coop, cuda::memory_order_release, ncclMemProxyType::Fabric, ncclMemProxyType::Generic, ncclMemFenceScope::Sys); + + cft.put(coop, leId, leOffset + flagOffset, flag, 16); + cft.submit(coop); + cft.flush(coop); + } + +Another use case for fence is ordering accesses to shared memory between the Generic +proxy and the Fabric proxy. In the following example, the Generic proxy updates shared +memory and makes the updates visible to the Fabric proxy using a fence. + +.. code-block:: C + + __shared__ncclCftSmem cftSmem; + ncclCoopCta coop; + ncclCft cft{coop, cftSmem}; + + __shared__ alignas(16) payload[4]; + + payload[0] = payload[1] = payload[2] = payload[3] = 1; + + ncclMemFence(coop, cuda::memory_order_release, ncclMemProxyType::Generic, ncclMemProxyType::Fabric, ncclMemFenceScope::Cta); + + cft.put(coop, leId, leOffset, payload, 16); + cft.submit(coop); + cft.flush(coop); diff --git a/projects/rccl/docs/userguide/source/usage/communicators.rst b/projects/rccl/docs/userguide/source/usage/communicators.rst index 2e3ae2d41ff0..296f925d8c4b 100644 --- a/projects/rccl/docs/userguide/source/usage/communicators.rst +++ b/projects/rccl/docs/userguide/source/usage/communicators.rst @@ -45,6 +45,37 @@ Related links: * :c:func:`ncclGetUniqueId` * :c:func:`ncclCommInitRank` +.. _using_nccl_with_mig: + +Using MIG instances +------------------- +(since 2.31) (experimental) + +Starting with NCCL 2.31, a communicator can use multiple Multi-Instance GPU +(MIG) instances. Each rank must select a distinct MIG instance as its CUDA +device; multiple ranks using the same MIG instance are not supported. Because +each MIG instance is a distinct CUDA device, this configuration does not +require ``NCCL_MULTI_RANK_GPU_ENABLE``. + +Since MIG does not support NVLink P2P from CUDA driver, Multi-Node NVLink +(MNNVL) needs to be disabled when using NCCL with MIG: ``NCCL_MNNVL_ENABLE=0``. + +NCCL will not use NVLink SHARP because NVLS is not supported with MIG. + +Requires CUDA driver version 13.0 or greater. + +.. warning:: + + Note that MIG support in NCCL is experimental and not thoroughly tested with + large scale deployment. In the 2.31 release, we have validated up to 8 GPUs with + 7 MIG instances on each GPU and running the typical ``nccl-tests`` collectives. + Given the massive validation combinations possible with this feature, we will + keep expanding test coverage in future releases. + +For information about configuring MIG instances and selecting them as CUDA +devices, see the `NVIDIA Multi-Instance GPU User +Guide `_. + .. _init-rank-config: Creating a communicator with options diff --git a/projects/rccl/docs/userguide/source/usage/deviceapi.rst b/projects/rccl/docs/userguide/source/usage/deviceapi.rst index 9c4438c55d90..6a1d62dcfc60 100644 --- a/projects/rccl/docs/userguide/source/usage/deviceapi.rst +++ b/projects/rccl/docs/userguide/source/usage/deviceapi.rst @@ -18,6 +18,8 @@ Device API consists of the following modules: * **Multimem** -- for communication between devices using the hardware multicast feature provided by NVLink SHARP (available on some datacenter GPUs since the Hopper generation). * **GIN (GPU-Initiated Networking)** -- for communication over the network (since NCCL 2.28.7). + * **CFT (Compute Fabric Transport)** -- for communication through CUDA fabric logical endpoints + (since NCCL 2.31). See :ref:`usage_cft`. * **Reduce, Broadcast, and Fused Building Blocks** — Building Blocks for Computation-Fused Kernels: reduce, copy (broadcast), and reduce-then-copy (see :ref:`device_api_reducecopy` in the API reference). @@ -26,18 +28,30 @@ Requirements The device API relies on symmetric memory (see :ref:`window_reg`), which in turn depends on GPU virtual memory management (see :ref:`env_NCCL_CUMEM_ENABLE`) and optionally -- for multimem support -- on NVLink SHARP (see -:ref:`env_NCCL_NVLS_ENABLE`). +:ref:`env_NCCL_NVLS_ENABLE`). GIN supports muiltiple networking backends, each with their own set of requirements. GIN has the following requirements: * CUDA 12.2 or later when compiling the GPU code * NVIDIA GPUs: Volta or newer. NVIDIA GPU drivers >= 510.40.3 -* NVIDIA NICs: CX4 or newer. rdma-core >= 44.0 -* GPU Direct RDMA: GIN host proxy requires DMA-BUF or nvidia-peermem support. GIN GDAKI requires DMA-BUF with kernel version >= 6.1 or nvidia-peermem support -* Network topology: Requires full NIC connectivity. Does not support topologies where NICs cannot communicate across rails. Also does not support ``NCCL_CROSS_NIC=0``. -* Fused NICs are not supported. To use GIN on dual-port NICs, set ``NCCL_IB_MERGE_NICS=0`` * Using GIN for buffers that are backed by multiple cuMem segments requires DMA-BUF +The GIN **CPU Proxy** backend has the following requirements: + +* NVIDIA NICs: CX4 or newer +* GPUDirect RDMA with the internal plugin: + * If using DMA-BUF, rdma-core >= 34.0. + * If using nvidia-peermem, Mellanox OFED >= 5.0 or DOCA. + +The GIN **GDAKI** backend has the following requirements: + +* NVIDIA NICs: CX4 or newer. rdma-core >= 44.0 +* GPU Direct RDMA: + * If using DMA-BUF, linux kernel >= 6.1. + * If using nvidia-peermem on baremetal, linux kernel >= 5.12 and either Mellanox OFED or DOCA driver. + * If using nvidia-peermem on virtualized environments, linux kernel >= 6.1 and either Mellanox OFED or DOCA driver. + * Mellanox OFED and DOCA driver packages may replace some parts of the upstream drivers. If you plan to install any of those packages, you need Mellanox OFED >= 5.8 or DOCA >= 3.1.0. + When using host-backed buffers, the following additional limitations apply: * Host segments must be allocated with ``CU_MEM_LOCATION_TYPE_HOST_NUMA``. @@ -49,6 +63,13 @@ Using the host RMA API requires CUDA 12.5 or greater. Building with EMIT_LLVM_IR=1 (to generate readable LLVM intermediate representation code) requires CUDA 12. +CFT has the following requirements: + +* CUDA 13.3 or later when compiling the GPU code. +* NVIDIA GPUs: Blackwell or newer. NVIDIA GPU drivers >= 610.43.02 + +See :ref:`usage_cft` and :ref:`device_api_cft` for CFT setup and API details. + Cross-Version Compatibility ---------------------------- NCCL assumes the compile-time version of the device code is the same as the compile-time version of the @@ -214,12 +235,14 @@ Teams ----- To address remote ranks or perform barriers, NCCL refers to subsets of ranks within a communicator as "teams". -NCCL provides three predefined ones: +NCCL provides five predefined ones: * ``ncclTeamWorld()`` -- the "world" team, encompassing all the ranks of a given communicator. * ``ncclTeamLsa()`` -- all the peers accessible from the local rank using load/store operations. * ``ncclTeamRail()`` -- the set of peers that have the same rank number within their LSA team (a rail team is orthogonal to an LSA team). + * ``ncclTeamCft()`` -- the CFT team for CUDA fabric logical endpoint unicast operations. + * ``ncclTeamCftMultimem()`` -- the CFT team for CUDA fabric logical endpoint multicast operations. The ``ncclTeam`` structure contains fairly self-explanatory elements ``nRanks``, ``rank``, and ``stride``. The device API contains functions to verify team membership, convert rank numbers between teams, etc. The world and LSA teams are @@ -361,7 +384,7 @@ In :c:type:`ncclDevCommRequirements`, :c:macro:`worldGinBarrierCount` reserves s in the launch grid (here ``NCCL_DEVICE_CTA_COUNT``), matching ``gridDim.x``, so each thread block uses ``blockIdx.x`` as its barrier index and signal index. GIN relies on these barriers and signals for cross-rank synchronization and for tracking asynchronous work. Set ``ginConnectionType`` to -:c:macro:`NCCL_GIN_CONNECTION_FULL` to connect each rank to all peers (see :c:type:`ncclGinConnectionType_t`). +:c:enumerator:`NCCL_GIN_CONNECTION_FULL ` to connect each rank to all peers (see :c:enum:`ncclGinConnectionType_t`). :c:func:`ncclDevCommCreate` fails if GIN cannot be provided. On the device, GIN barriers synchronize across ranks over the network. Each thread block uses ``blockIdx.x`` to select its @@ -415,7 +438,7 @@ the following as needed. :cpp:class:`ncclGinBarrierSession` with ``ncclTeamTagWorld()``. * **Before NCCL 2.29.7 — no** :c:member:`ginConnectionType` — Set :c:member:`ginForceEnable` to ``true`` to enable full GIN - connectivity (equivalent to :c:macro:`NCCL_GIN_CONNECTION_FULL` once :c:member:`ginConnectionType` exists). The + connectivity (equivalent to :c:enumerator:`NCCL_GIN_CONNECTION_FULL ` once :c:member:`ginConnectionType` exists). The :c:member:`ginConnectionType` field is available starting with NCCL 2.29.7 (see :c:type:`ncclDevCommRequirements` in :ref:`device_api_setup`). diff --git a/projects/rccl/examples/01_communicators/03_one_device_per_process_mpi/python/one_device_per_process_mpi.py b/projects/rccl/examples/01_communicators/03_one_device_per_process_mpi/python/one_device_per_process_mpi.py index 2c30aba176d5..44a82e02e43f 100644 --- a/projects/rccl/examples/01_communicators/03_one_device_per_process_mpi/python/one_device_per_process_mpi.py +++ b/projects/rccl/examples/01_communicators/03_one_device_per_process_mpi/python/one_device_per_process_mpi.py @@ -112,7 +112,7 @@ def main() -> int: # ========================================================================= # STEP 5: Clean shutdown and resource cleanup # ========================================================================= - + print("Cleaning up resources") if mpi_rank == 0: diff --git a/projects/rccl/examples/06_device_api/01_allreduce_lsa/c/README.md b/projects/rccl/examples/06_device_api/01_allreduce_lsa/c/README.md index f3fd2b9939a7..39600a2d8a30 100644 --- a/projects/rccl/examples/06_device_api/01_allreduce_lsa/c/README.md +++ b/projects/rccl/examples/06_device_api/01_allreduce_lsa/c/README.md @@ -80,11 +80,11 @@ The NCCL Device API enables GPU kernels to perform inter-GPU communication direc ```c ncclDevComm devComm; -ncclDevCommRequirements requirements = {0}; -requirements.lsaBarrierCount = numBlocks; // One barrier per thread block +ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; +reqs.lsaBarrierCount = numBlocks; // One barrier per thread block // Create device communicator -ncclDevCommCreate(&devComm, comm, &requirements); +ncclDevCommCreate(comm, &reqs, &devComm); ``` ### Symmetric Memory Windows (Host-side) @@ -114,7 +114,7 @@ GPUs. // synchronize independently with their counterparts on other GPUs. ncclLsaBarrierSession bar { ncclCoopCta(), // Barrier scope: entire CTA (thread block) - devComm, ncclTeamLsa(devComm), devComm.lsaBarrier, + devComm, ncclTeamTagLsa(), blockIdx.x // Barrier index: matches our CTA index (0 to lsaBarrierCount-1) }; diff --git a/projects/rccl/examples/06_device_api/01_allreduce_lsa/c/main.cu b/projects/rccl/examples/06_device_api/01_allreduce_lsa/c/main.cu index e2ff19344cdc..202b20d4e71d 100644 --- a/projects/rccl/examples/06_device_api/01_allreduce_lsa/c/main.cu +++ b/projects/rccl/examples/06_device_api/01_allreduce_lsa/c/main.cu @@ -65,8 +65,7 @@ __global__ void simpleAllReduceKernel(ncclWindow_t sendwin, size_t sendoffset, // LSA barriers enable coordination between GPU threads across different ranks // Barrier scope: CTA (all threads in this block participate) // Barrier index: blockIdx.x selects this CTA's dedicated barrier (one barrier per CTA) - ncclLsaBarrierSession bar { ncclCoopCta(), devComm, ncclTeamLsa(devComm), - devComm.lsaBarrier, blockIdx.x }; + ncclLsaBarrierSession bar { ncclCoopCta(), devComm, ncclTeamTagLsa(), blockIdx.x }; bar.sync(ncclCoopCta(), cuda::memory_order_acquire); const int rank = devComm.rank, nRanks = devComm.nRanks; diff --git a/projects/rccl/examples/06_device_api/02_alltoall_gin/c/README.md b/projects/rccl/examples/06_device_api/02_alltoall_gin/c/README.md index 104b77fe6b71..065860e7975b 100644 --- a/projects/rccl/examples/06_device_api/02_alltoall_gin/c/README.md +++ b/projects/rccl/examples/06_device_api/02_alltoall_gin/c/README.md @@ -66,9 +66,9 @@ goes through the network. ncclDevComm devComm; ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; // GIN barriers enable cross-node synchronization over the network -reqs.railGinBarrierCount = NCCL_DEVICE_CTA_COUNT; +reqs.worldGinBarrierCount = NCCL_DEVICE_CTA_COUNT; // GIN signals provide completion notifications for asynchronous operations -reqs.ginSignalCount = 1; +reqs.ginSignalCount = NCCL_DEVICE_CTA_COUNT; // Enable full GIN connectivity, i.e., connect each rank to all other ranks reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; @@ -100,17 +100,16 @@ to progress independently while coordinating with corresponding blocks on other ncclGinBarrierSession bar { ncclCoopCta(), // Barrier scope: entire CTA (thread block) gin, // GIN context for network operations - ncclTeamWorld(devComm), // Team spanning all ranks - devComm.railGinBarrier, // GIN barrier handle + ncclTeamTagWorld(), // Team spanning all ranks blockIdx.x // Barrier index: matches our CTA index }; -bar.sync(ncclCoopCta(), cuda::memory_order_acquire, ncclGinFenceLevel::Relaxed); +bar.sync(ncclCoopCta(), cuda::memory_order_acquire, ncclGinFenceLevel::None); ``` ### GIN Put Operations (Device-side) GIN provides one-sided put operations for direct remote memory writes over the -network. The `ncclGin_SignalInc` parameter increments a signal counter, enabling +network. The `ncclGin_WeakSignalInc` parameter increments a signal counter, enabling asynchronous completion detection. ```cpp @@ -120,7 +119,7 @@ for (int r = tid; r < devComm.nRanks; r += nthreads) { gin.put(ncclTeamWorld(devComm), r, recvwin, recvoffset + devComm.rank * size, // Destination: peer r's buffer sendwin, sendoffset + r * size, // Source: data for peer r - size, ncclGin_SignalInc{signalIndex}); // Signal increment for completion + size, ncclGin_WeakSignalInc{signalIndex}); // Signal increment for completion } ``` diff --git a/projects/rccl/examples/06_device_api/03_alltoall_hybrid/c/README.md b/projects/rccl/examples/06_device_api/03_alltoall_hybrid/c/README.md index 8e43a6c070f7..916c0a7c54a0 100644 --- a/projects/rccl/examples/06_device_api/03_alltoall_hybrid/c/README.md +++ b/projects/rccl/examples/06_device_api/03_alltoall_hybrid/c/README.md @@ -76,12 +76,10 @@ resources. This dual setup enables optimal communication for each peer type. ```cpp ncclDevComm devComm; ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; -// LSA barriers enable direct memory access coordination for local peers -reqs.lsaBarrierCount = NCCL_DEVICE_CTA_COUNT; -// GIN barriers enable cross-node synchronization over the network -reqs.railGinBarrierCount = NCCL_DEVICE_CTA_COUNT; +// Hybrid barriers coordinate both LSA and GIN peers across the world team +reqs.barrierCount = NCCL_DEVICE_CTA_COUNT; // GIN signals provide completion notifications for asynchronous network operations -reqs.ginSignalCount = 1; +reqs.ginSignalCount = NCCL_DEVICE_CTA_COUNT; // Enable full GIN connectivity, i.e., connect each rank to all other ranks reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; @@ -115,7 +113,7 @@ ncclBarrierSession bar { gin, // GIN context for network coordination blockIdx.x // Barrier index: matches our CTA index }; -bar.sync(ncclCoopCta(), cuda::memory_order_acquire, ncclGinFenceLevel::Relaxed); +bar.sync(ncclCoopCta(), cuda::memory_order_acquire, ncclGinFenceLevel::None); ``` ### Peer Classification (Device-side) @@ -143,7 +141,7 @@ T* recvPtr = (T*)ncclGetLsaPointer(recvwin, recvoffset, lp); // Remote peers: network operations (GIN) gin.put(world, r, recvwin, recvoffset + world.rank * size, - sendwin, sendoffset + r * size, size, ncclGin_SignalInc{signalIndex}); + sendwin, sendoffset + r * size, size, ncclGin_WeakSignalInc{signalIndex}); ``` ### Receiving CTA (Device-side) diff --git a/projects/rccl/install.sh b/projects/rccl/install.sh index 2557c124258c..7db207a84f79 100755 --- a/projects/rccl/install.sh +++ b/projects/rccl/install.sh @@ -63,7 +63,7 @@ quiet_warnings=false # Does NOT enable alltoall_wg offload. build_rocshmem_support=false build_rocshmem_gin=false -rocshmem_mono_hash="33d980d7ca1f0bf90cfe4ff9106310abcf47b550" +rocshmem_mono_hash="f636cc5be66716349dd9ed42eebb7b4d651fbb20" custom_cmake_options="" # ################################################# @@ -328,11 +328,13 @@ fi # rocSHMEM worktree setup (must run before cd-ing into the build directory) # ################################################# rocshmem_source_dir="" +rocshmem_is_monorepo=false if [[ "${build_rocshmem_support}" == true || "${build_rocshmem_gin}" == true ]] && [[ -z "${ROCSHMEM_INSTALL_DIR}" ]]; then # Prefer mono-repo layout (projects/rocshmem alongside projects/rccl) mono_root=$(git rev-parse --show-toplevel 2>/dev/null) if [[ -n "$mono_root" ]] && [[ -f "$mono_root/projects/rocshmem/CMakeLists.txt" ]]; then rocshmem_source_dir="$mono_root/projects/rocshmem" + rocshmem_is_monorepo=true echo "=== Using rocSHMEM from mono-repo: ${rocshmem_source_dir} ===" else setup_rocshmem_worktree @@ -511,14 +513,14 @@ if [[ "${build_rocshmem_support}" == true ]]; then cmake_common_options="${cmake_common_options} -DENABLE_ROCSHMEM=ON" if [[ -n "${ROCSHMEM_INSTALL_DIR}" ]]; then cmake_common_options="${cmake_common_options} -DROCSHMEM_INSTALL_DIR=${ROCSHMEM_INSTALL_DIR}" - elif [[ -n "${rocshmem_source_dir}" ]]; then + elif [[ -n "${rocshmem_source_dir}" ]] && [[ "${rocshmem_is_monorepo}" == false ]]; then cmake_common_options="${cmake_common_options} -DROCSHMEM_SOURCE_DIR=${rocshmem_source_dir}" fi elif [[ "${build_rocshmem_gin}" == true ]]; then cmake_common_options="${cmake_common_options} -DENABLE_ROCSHMEM=OFF -DENABLE_ROCSHMEM_GIN=ON" if [[ -n "${ROCSHMEM_INSTALL_DIR}" ]]; then cmake_common_options="${cmake_common_options} -DROCSHMEM_INSTALL_DIR=${ROCSHMEM_INSTALL_DIR}" - elif [[ -n "${rocshmem_source_dir}" ]]; then + elif [[ -n "${rocshmem_source_dir}" ]] && [[ "${rocshmem_is_monorepo}" == false ]]; then cmake_common_options="${cmake_common_options} -DROCSHMEM_SOURCE_DIR=${rocshmem_source_dir}" fi else diff --git a/projects/rccl/makefiles/common.mk b/projects/rccl/makefiles/common.mk index 642c36ffc1db..a744b1c17983 100644 --- a/projects/rccl/makefiles/common.mk +++ b/projects/rccl/makefiles/common.mk @@ -7,6 +7,7 @@ CUDA_HOME ?= /usr/local/cuda PREFIX ?= /usr/local +TLS_BACKEND ?= VERBOSE ?= 0 KEEP ?= 0 DEBUG ?= 0 @@ -47,6 +48,12 @@ NVCC ?= $(CUDA_HOME)/bin/nvcc CUDA_LIB ?= $(CUDA_HOME)/lib64 CUDA_INC ?= $(CUDA_HOME)/include +ifeq ($(TLS_BACKEND),OPENSSL3) +CXXFLAGS += -DNCCL_TLS_BACKEND_OPENSSL3=1 +else ifneq ($(strip $(TLS_BACKEND)),) +$(error Unsupported TLS_BACKEND '$(TLS_BACKEND)'; expected OPENSSL3 or empty) +endif +NCCL_STATIC_DL_LIBS ?= -ldl CUDA_VERSION = $(strip $(shell which $(NVCC) >/dev/null && $(NVCC) --version | grep release | sed 's/.*release //' | sed 's/\,.*//')) #CUDA_VERSION ?= $(shell ls $(CUDA_LIB)/libcudart.so.* | head -1 | rev | cut -d "." -f -2 | rev) CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) @@ -64,6 +71,7 @@ CUDA12_GENCODE = -gencode=arch=compute_90,code=sm_90 CUDA12_8_GENCODE = -gencode=arch=compute_100,code=sm_100 \ -gencode=arch=compute_120,code=sm_120 CUDA13_GENCODE = -gencode=arch=compute_110,code=sm_110 +CUDA13_4_GENCODE = -gencode=arch=compute_107,code=sm_107 CUDA8_PTX = -gencode=arch=compute_61,code=compute_61 CUDA9_PTX = -gencode=arch=compute_70,code=compute_70 @@ -71,7 +79,10 @@ CUDA11_PTX = -gencode=arch=compute_80,code=compute_80 CUDA12_PTX = -gencode=arch=compute_90,code=compute_90 CUDA13_PTX = -gencode=arch=compute_120,code=compute_120 -ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 13; echo $$?),0) +ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 13 -a "0$(CUDA_MINOR)" -ge 4 -o "0$(CUDA_MAJOR)" -gt 13; echo $$?),0) +# Include Rubin support from CUDA 13.4 onwards + NVCC_GENCODE ?= $(CUDA10_GENCODE) $(CUDA11_GENCODE) $(CUDA12_GENCODE) $(CUDA12_8_GENCODE) $(CUDA13_GENCODE) $(CUDA13_4_GENCODE) $(CUDA13_PTX) +else ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 13; echo $$?),0) # Prior to SM75 is deprecated from CUDA13.0 onwards NVCC_GENCODE ?= $(CUDA10_GENCODE) $(CUDA11_GENCODE) $(CUDA12_GENCODE) $(CUDA12_8_GENCODE) $(CUDA13_GENCODE) $(CUDA13_PTX) else ifeq ($(shell test "0$(CUDA_MAJOR)" -eq 12 -a "0$(CUDA_MINOR)" -ge 8; echo $$?),0) @@ -98,7 +109,8 @@ else endif CXXFLAGS := -DCUDA_MAJOR=$(CUDA_MAJOR) -DCUDA_MINOR=$(CUDA_MINOR) -fPIC -fvisibility=hidden \ - -Wall -Wno-unused-function -Wno-sign-compare $(CXXSTD) -Wvla \ + -Wall -Wextra -Wno-missing-field-initializers -Wno-unused-parameter \ + -Wno-unused-function -Wno-sign-compare $(CXXSTD) -Wvla \ -I $(CUDA_INC) -I $(CUDA_INC)/cccl \ $(CXXFLAGS) # Maxrregcount needs to be set accordingly to NCCL_MAX_NTHREADS (otherwise it will cause kernel launch errors) @@ -175,6 +187,7 @@ endif ifneq ($(WERROR), 0) CXXFLAGS += -Werror +NVCUFLAGS += -Werror=all-warnings -Xcompiler -Werror endif ifneq ($(KEEP), 0) diff --git a/projects/rccl/makefiles/version.mk b/projects/rccl/makefiles/version.mk index 942a49641bed..db924347f062 100644 --- a/projects/rccl/makefiles/version.mk +++ b/projects/rccl/makefiles/version.mk @@ -7,7 +7,7 @@ ##### version NCCL_MAJOR := 2 -NCCL_MINOR := 31 -NCCL_PATCH := 2 +NCCL_MINOR := 32 +NCCL_PATCH := 3 NCCL_SUFFIX := PKG_REVISION := 1 diff --git a/projects/rccl/plugins/gin/README.md b/projects/rccl/plugins/gin/README.md index 369ac9494127..74b7971b43b1 100644 --- a/projects/rccl/plugins/gin/README.md +++ b/projects/rccl/plugins/gin/README.md @@ -33,12 +33,12 @@ are just examples, custom implementations may make other design decisions. ### Plugin name and supporting multiple GIN plugins -When NCCL is initialized, it will look for a `libnccl-gin.so` library and +When RCCL is initialized, it will look for a `librccl-gin.so` library and dynamically load it, then look for symbols inside the library. The `NCCL_GIN_PLUGIN` environment variable allows users to select one or more -GIN plugins. If set to a bare plugin name such as `mygin`, NCCL can load a -library named `libnccl-gin-mygin.so`. `NCCL_GIN_PLUGIN` can also be set to a +GIN plugins. If set to a bare plugin name such as `mygin`, RCCL can load a +library named `librccl-gin-mygin.so`. `NCCL_GIN_PLUGIN` can also be set to a shared library file name or an absolute path to the plugin file. Multiple plugins can be specified as a comma-separated list. @@ -46,8 +46,8 @@ For example, any of the following can be used to load a plugin: ```shell export NCCL_GIN_PLUGIN=mygin -export NCCL_GIN_PLUGIN=libnccl-gin-mygin.so -export NCCL_GIN_PLUGIN=/path/to/your/plugin/libnccl-gin-mygin.so +export NCCL_GIN_PLUGIN=librccl-gin-mygin.so +export NCCL_GIN_PLUGIN=/path/to/your/plugin/librccl-gin-mygin.so ``` Set the `LD_LIBRARY_PATH` to include the plugin directory when using a bare name @@ -111,18 +111,18 @@ and the plugins. NCCL decides how user contexts map to plugin connections and co ### Operations -**Put/PutValue** +**Put/PutValue** Put moves data from a local source buffer to a (likely remote) target buffer. PutValue places a value in a target buffer. Puts are not required to be completed in the order they are requested. A single call to `Put` or `PutValue` may include a signal or counter operation. See below for more details. -**Get** +**Get** Get moves data from a (likely remote) source buffer to a local target buffer. Gets are not required to be completed in the order they are requested. -**Signal** +**Signal** Signal increments a target memory address by a fixed value. There's 2 types of visibility guarantees: strong and weak. The visibility of a *strong* signal guarantees the completion/visibility of all previous puts and signals, including the bundled put in the case of put+signal. The visibility of a *weak* @@ -135,12 +135,12 @@ current signal value, while `waitSignal` blocks until the signal meets or exceed value. `resetSignal` reinitializes a signal before reuse and must not race with concurrent signal updates. -**Flush** +**Flush** Flush ensures all previous operations are locally complete. In the case of gets, flush indicates the data is visible and ready to use. In the case of puts, flush indicates source buffers are ready for reuse. -**Counter** +**Counter** Counter increments a local memory address by a fixed value. The completion/visibility of a counter guarantees the local completion of a bundled put. It does not guarantee completion at the target, nor local completion of any previous puts. @@ -254,7 +254,7 @@ A `ncclGinCtx` contains metadata associated with a GIN context. The most importa This is the pointer returned as `ginHandle` in `regMrSym`. -**Coop** +**Coop** In many places, the backend API supplies a `coop` argument. `coop` specifies the threads calling/participating in the operation. `coop` can be used for both correctness and performance: @@ -276,14 +276,14 @@ Some backends choose to "reset" indexed signals by caching an offset (the value Accordingly, `GetSignalPtr` returns an offset. Backends can return an offset of 0 if `ResetSignal` actually sets the memory location of the indexed signal to 0. -**Abort flag** +**Abort flag** Many blocking functions supply an `abortFlag` argument. When true, the function should return immediately. The function (and corresponding state) can have undefined behavior if it returns early due to an abort flag. `abortFlag` may be null, in which case the function should return only when the requested operation is complete. -**Counter** +**Counter** A counter is a 64-bit *local* memory location allocated by the custom implementation. All writes are done via the backend API (e.g. either `ResetCounter` or `Put` (with counter)). @@ -291,14 +291,14 @@ All reads are done via `GetCounterPtr`, which should return a device-side addres be read using CUDA atomic operations (e.g. `cuda::atomic_ref`). Similar to signals, `GetCounterPtr` returns an offset in case backends wish to "reset" counters by caching an offset. -**DescriptorSmem** +**DescriptorSmem** `DescriptorSmem` is a 64-byte, user-allocated, shared-memory scratch pad. As an optional performance improvement, implementations may use this scratch pad instead of reallocating internal structs on each API call. Users can reuse one `DescriptorSmem` across many non-overlapping GIN calls, but they do not have to. -**Optimization flags** +**Optimization flags** GIN exposes several optimization flags in the form of `ncclGinOptFlags`. The flags include: @@ -351,7 +351,7 @@ make -C plugins/gin/example Set the `LD_LIBRARY_PATH` to include the example plugin directory: ```shell -export LD_LIBRARY_PATH=/path/to/nccl/plugins/gin/example:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/path/to/rccl/plugins/gin/example:$LD_LIBRARY_PATH ``` Set `NCCL_GIN_PLUGIN` to either the plugin name, the shared library file name, or @@ -359,8 +359,8 @@ the absolute path to the plugin file. Any of the following can work: ```shell export NCCL_GIN_PLUGIN=example -export NCCL_GIN_PLUGIN=libnccl-gin-example.so -export NCCL_GIN_PLUGIN=/path/to/nccl/plugins/gin/example/libnccl-gin-example.so +export NCCL_GIN_PLUGIN=librccl-gin-example.so +export NCCL_GIN_PLUGIN=/path/to/rccl/plugins/gin/example/librccl-gin-example.so ``` -NCCL will automatically discover and load the plugin based on the exported symbol names. +RCCL will automatically discover and load the plugin based on the exported symbol names. diff --git a/projects/rccl/plugins/profiler/example/event.h b/projects/rccl/plugins/profiler/example/event.h index ac461ea78d19..ee13e2df18ea 100644 --- a/projects/rccl/plugins/profiler/example/event.h +++ b/projects/rccl/plugins/profiler/example/event.h @@ -25,6 +25,8 @@ typedef enum { CE_TIMING_GPU = 1 } CeTimingMode_t; +// Must match MAXCHANNELS in RCCL's src/include/device.h (not exposed by the public or +// plugin API). Core reports raw channel ids in [0, MAXCHANNELS) and does not clamp them. #define MAX_CHANNELS 256 // RCCL MAXCHANNELS #define MAX_STEPS 1024 #define MAX_OPS 16 // Up to 64K ranks for PAT diff --git a/projects/rccl/plugins/profiler/example/nccl/profiler.h b/projects/rccl/plugins/profiler/example/nccl/profiler.h index 5da2a39aa54a..ef33eac8bddd 100644 --- a/projects/rccl/plugins/profiler/example/nccl/profiler.h +++ b/projects/rccl/plugins/profiler/example/nccl/profiler.h @@ -32,7 +32,8 @@ enum { ncclProfileCeColl = (1 << 12), // CE collective operation ncclProfileCeSync = (1 << 13), // CE synchronization operation ncclProfileCeBatch = (1 << 14), // CE batch operation - // Kernel phase events (v7) + // Symmetric kernels only; implies ncclProfileKernelCh. Regular and P2P kernels + // emit no phase sub-events. ncclProfileKernelPhase = (1 << 15), // kernel barrier phase sub-event }; diff --git a/projects/rccl/plugins/profiler/example/plugin.cc b/projects/rccl/plugins/profiler/example/plugin.cc index d4cd97ab391a..8142b7503b84 100644 --- a/projects/rccl/plugins/profiler/example/plugin.cc +++ b/projects/rccl/plugins/profiler/example/plugin.cc @@ -874,6 +874,7 @@ __hidden ncclResult_t exampleProfilerStartEvent(void* context, void** eHandle, n } else if (eDescr->type == ncclProfileKernelCh) { struct taskEventBase* eventBase = (struct taskEventBase *)eDescr->parentObj; if (eventBase == NULL) return ncclSuccess; + if (eDescr->kernelCh.channelId >= MAX_CHANNELS) return ncclSuccess; if (eventBase->type == ncclProfileColl) { struct collective* parent = (struct collective *)eDescr->parentObj; int channelId = eDescr->kernelCh.channelId; @@ -1271,4 +1272,79 @@ ncclProfiler_v6_t ncclProfiler_v6 = { exampleProfilerFinalize, }; -// ===================================================================== \ No newline at end of file +// ============================================================================ +// v7 implementation: kernel phase sub-events + symmetric-kernel coll metadata +// ============================================================================ + +#include "nccl/profiler_v7.h" + +__hidden ncclResult_t exampleProfilerStartEvent_v7(void* context, void** eHandle, ncclProfilerEventDescr_v7_t* eDescr) { + struct context* ctx = (struct context*)context; + if (ctx == NULL) { *eHandle = NULL; return ncclSuccess; } + + // v7 adds the kernel phase sub-event: a child of a kernelCh, stored in the parent. + if (eDescr->type == ncclProfileKernelPhase) { + *eHandle = NULL; + if (__atomic_load_n(&ctx->finalizing, __ATOMIC_RELAXED)) return ncclSuccess; + struct kernelCh* parent = (struct kernelCh*)eDescr->parentObj; + if (parent == NULL || parent->type != ncclProfileKernelCh) return ncclSuccess; + int idx = eDescr->kernelPhase.phaseId; + if (idx < 0 || idx >= MAX_KERNEL_PHASES) return ncclSuccess; + struct kernelPhase* phase = &parent->phases[idx]; + phase->type = ncclProfileKernelPhase; + phase->channelId = eDescr->kernelPhase.channelId; + phase->phaseId = eDescr->kernelPhase.phaseId; + phase->phaseName = eDescr->kernelPhase.phaseName; + phase->startGpuClk = eDescr->kernelPhase.pTimer; + phase->stopGpuClk = 0; + // Place the 100 MHz GPU timestamp on the parent channel's host timeline. + uint64_t elapsedTicks = phase->startGpuClk >= parent->startGpuClk + ? phase->startGpuClk - parent->startGpuClk + : 0; + phase->startTs = parent->startTs + (double)elapsedTicks / 100.0; + phase->parent = parent; + *eHandle = phase; + return ncclSuccess; + } + + // Other events: the v6 coll layout is a prefix of v7, so the v6 handler reads it + // safely. Capture the v7-only coll metadata on the created event afterwards. + ncclResult_t ret = exampleProfilerStartEvent_v6(context, eHandle, (ncclProfilerEventDescr_v6_t*)eDescr); + if (eDescr->type == ncclProfileColl && *eHandle) { + struct collective* c = (struct collective*)*eHandle; + c->kernelVariant = eDescr->coll.kernelVariant; + c->isSymColl = eDescr->coll.isSymColl; + } + return ret; +} + +__hidden ncclResult_t exampleProfilerStopEvent_v7(void* eHandle) { + if (eHandle == NULL) return ncclSuccess; + if (*(uint64_t*)eHandle == ncclProfileKernelPhase) { + struct kernelPhase* phase = (struct kernelPhase*)eHandle; + phase->stopTs = gettime() - startTime; + return ncclSuccess; + } + return exampleProfilerStopEvent_v6(eHandle); +} + +__hidden ncclResult_t exampleProfilerRecordEventState_v7(void* eHandle, ncclProfilerEventState_v7_t eState, ncclProfilerEventStateArgs_v7_t* eStateArgs) { + if (eHandle == NULL) return ncclSuccess; + if (*(uint64_t*)eHandle == ncclProfileKernelPhase) { + if (eState == ncclProfilerKernelPhaseStop) { + struct kernelPhase* phase = (struct kernelPhase*)eHandle; + phase->stopGpuClk = eStateArgs->kernelCh.pTimer; + } + return ncclSuccess; + } + return exampleProfilerRecordEventState_v6(eHandle, (ncclProfilerEventState_v6_t)eState, (ncclProfilerEventStateArgs_v6_t*)eStateArgs); +} + +ncclProfiler_v7_t ncclProfiler_v7 = { + "Example-profiler-v7", + exampleProfilerInit, + exampleProfilerStartEvent_v7, + exampleProfilerStopEvent_v7, + exampleProfilerRecordEventState_v7, + exampleProfilerFinalize, +}; diff --git a/projects/rccl/plugins/profiler/example/print_event.cc b/projects/rccl/plugins/profiler/example/print_event.cc index 42e33cd89af1..15f8c951410a 100644 --- a/projects/rccl/plugins/profiler/example/print_event.cc +++ b/projects/rccl/plugins/profiler/example/print_event.cc @@ -168,13 +168,13 @@ __hidden void printKernelChEventTrailer(FILE* fh, struct kernelCh* event) { "KernelCh", kernelId, getpid(), 1, event->stopTs); } -// v7: emit each kernel phase as a complete event; duration is from the GPU globaltimer (ns->us) +// v7: emit each kernel phase as a complete event; RCCL's GPU globaltimer is 100 MHz. __hidden void printKernelPhaseEvents(FILE* fh, struct kernelCh* event) { if (event->type != ncclProfileKernelCh) return; for (int i = 0; i < MAX_KERNEL_PHASES; i++) { struct kernelPhase* p = &event->phases[i]; if (p->type != ncclProfileKernelPhase) continue; - double durUs = (p->stopGpuClk > p->startGpuClk) ? (double)(p->stopGpuClk - p->startGpuClk) / 1000.0 : 0.0; + double durUs = (p->stopGpuClk > p->startGpuClk) ? (double)(p->stopGpuClk - p->startGpuClk) / 100.0 : 0.0; fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GPU\", \"ph\": \"X\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"dur\": %f, \"args\": {\"Channel\": %d, \"PhaseId\": %d, \"StartGpuClk\": %lu, \"StopGpuClk\": %lu}},\n", p->phaseName ? p->phaseName : "phase", kernelId, getpid(), 1, p->startTs, durUs, p->channelId, p->phaseId, p->startGpuClk, p->stopGpuClk); } diff --git a/projects/rccl/plugins/profiler/inspector/CMakeLists.txt b/projects/rccl/plugins/profiler/inspector/CMakeLists.txt index 270a34349214..87b75e453908 100644 --- a/projects/rccl/plugins/profiler/inspector/CMakeLists.txt +++ b/projects/rccl/plugins/profiler/inspector/CMakeLists.txt @@ -57,6 +57,7 @@ add_library(rccl-profiler-inspector SHARED ${CMAKE_CURRENT_SOURCE_DIR}/inspector.cc ${CMAKE_CURRENT_SOURCE_DIR}/inspector_json.cc ${CMAKE_CURRENT_SOURCE_DIR}/inspector_prom.cc + ${CMAKE_CURRENT_SOURCE_DIR}/inspector_otel.cc ${CMAKE_CURRENT_SOURCE_DIR}/inspector_plugin.cc ${CMAKE_CURRENT_SOURCE_DIR}/inspector_cudawrap.cc ${CMAKE_CURRENT_SOURCE_DIR}/inspector_ring.cc diff --git a/projects/rccl/plugins/profiler/inspector/README.md b/projects/rccl/plugins/profiler/inspector/README.md index d6dd06f4ba5f..a8c776bff9fc 100644 --- a/projects/rccl/plugins/profiler/inspector/README.md +++ b/projects/rccl/plugins/profiler/inspector/README.md @@ -96,6 +96,8 @@ export NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS=500 Enables verbose output including event trace information. - `NCCL_INSPECTOR_PROM_DUMP=<0|1>` (default: `0`) Enables Prometheus format for textfile node exporter output instead of custom JSON. +- `NCCL_INSPECTOR_PROM_DUMP_STATS=<0|1>` (default: `0`) + In Prometheus mode, additionally emit the per-device stats metrics (`nccl_collectives_total`, `nccl_collectives_dropped_total`, `nccl_p2p_total`, `nccl_p2p_dropped_total`). Off by default to keep the default Prometheus output minimal; enable it to plumb drop/rate signals to a dashboard. Has no effect on JSON mode (whose per-dump `dump_stats` record is always emitted) or OTLP mode (whose equivalent stats ride under `NCCL_INSPECTOR_OTEL_VERBOSE`). - `NCCL_INSPECTOR_OTEL_EXPORT=<0|1>` (default: `0`) Enables OTLP HTTP metrics export. Accepted values are `0` (disabled) and `1` (enabled). Default OTLP export emits aggregated bucket metrics. - `NCCL_INSPECTOR_OTEL_VERBOSE=<0|1>` (default: `0`) @@ -120,6 +122,13 @@ export NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS=500 Per-communicator completed-collective ring buffer capacity. - `NCCL_INSPECTOR_DUMP_P2P_RING_SIZE=` (default: `1024`) Per-communicator completed-P2P ring buffer capacity. + + If operations complete faster than the dump thread drains the ring, the oldest + entries are overwritten before they can be dumped. When this happens the + Inspector logs a one-time warning and reports drop counts in the output + (`dropped_total` / `dropped_since_last_dump` in the JSON `dump_stats` record; + `nccl_collectives_dropped_total` / `nccl_p2p_dropped_total` in Prometheus/OTLP + stats). Increasing the ring size retains more entries under bursts. - `NCCL_INSPECTOR_COLL_POOL_SIZE=` (default: `256`) Collective pool initial size/stride. - `NCCL_INSPECTOR_P2P_POOL_SIZE=` (default: `256`) @@ -209,9 +218,18 @@ Note: Prometheus mode enforces a minimum dump interval of 30 seconds (30,000,000 When P2P tracking is enabled (`NCCL_INSPECTOR_ENABLE_P2P=1`, the default), Prometheus output includes P2P metrics with a `p2p_operation` label (e.g., `Send`, `Recv`). +**Opt-in per-device stats metrics** (emitted only when `NCCL_INSPECTOR_PROM_DUMP_STATS=1`): +- `nccl_collectives_total` - Cumulative collectives enqueued into the ring buffer, per device (counter) +- `nccl_collectives_dropped_total` - Cumulative collective records overwritten before being dumped, per device (counter) +- `nccl_p2p_total` - Cumulative P2P operations enqueued into the ring buffer, per device (counter) +- `nccl_p2p_dropped_total` - Cumulative P2P records overwritten before being dumped, per device (counter) + +These are cumulative counters (not per-window gauges), so `rate(nccl_collectives_dropped_total[$__rate_interval]) / rate(nccl_collectives_total[$__rate_interval])` gives the fraction of collectives being lost, robust to scrape/dump interval alignment. They are emitted for every known device each dump so the counters stay continuous. + **Labels:** - Collectives: `version`, `cluster`, `slurm_job_id`, `node`, `gpu`, `comm_name`, `n_nodes`, `nranks`, `collective`, `message_size`, `algo_proto` - P2P: `version`, `cluster`, `slurm_job_id`, `node`, `gpu`, `comm_name`, `n_nodes`, `nranks`, `p2p_operation`, `message_size` +- Per-device stats metrics: `version`, `slurm_job_id`, `node`, `gpu` `cluster` is `NCCL_INSPECTOR_CLUSTER`, else `SLURM_CLUSTER_NAME`, else `unknown`. `slurm_job_id` is taken from `SLURM_JOB_ID`, then `SLURM_JOBID`, `PBS_JOBID`, `LSB_JOBID`, else `unknown`. `message_size` is a bucketed range string (for example `8-9MB`). @@ -324,7 +342,7 @@ Each output file contains JSON objects with the following structure: "nnodes": 1 }, "metadata": { - "inspector_output_format_version": "v4.0", + "inspector_output_format_version": "v4.1", "git_rev": "", "rec_mechanism": "nccl_profiler_interface", "dump_timestamp_us": 1748030377748202, @@ -342,6 +360,39 @@ Each output file contains JSON objects with the following structure: } ``` +### Per-Dump Stats Record + +Once per dump cycle, each communicator also emits a single stats record (identified by the `dump_stats` key) as a marker at the start of its dump. It reports how many records were written and how many were dropped — overwritten in the ring buffer before they could be dumped (see `NCCL_INSPECTOR_DUMP_COLL_RING_SIZE`). `dropped_total` is cumulative for the communicator; `dropped_since_last_dump` covers only the current dump. Non-zero drops mean the output is an incomplete sample and the ring should be enlarged. + +Because this record has no `coll_perf`/`p2p_perf` body, consumers that iterate per-collective should filter on the presence of `coll_perf`/`p2p_perf` and read `dump_stats` separately rather than assuming every record is an operation. + +```json +{ + "header": { + "id": "0x7f8c496ae9f661", + "rank": 2, + "n_ranks": 8, + "nnodes": 1 + }, + "metadata": { + "inspector_output_format_version": "v4.1", + "git_rev": "", + "rec_mechanism": "nccl_profiler_interface", + "dump_timestamp_us": 1748030377748202, + "hostname": "example-hostname", + "pid": 1639453 + }, + "dump_stats": { + "coll_records": 1024, + "coll_dropped_total": 4096, + "coll_dropped_since_last_dump": 512, + "p2p_records": 0, + "p2p_dropped_total": 0, + "p2p_dropped_since_last_dump": 0 + } +} +``` + ## Output Example Verbose To enable verbose output with event trace information, set the `NCCL_INSPECTOR_DUMP_VERBOSE=1` environment variable: @@ -361,7 +412,7 @@ This will include additional event trace information in the JSON output, showing "nnodes": 1 }, "metadata": { - "inspector_output_format_version": "v4.0", + "inspector_output_format_version": "v4.1", "git_rev": "9019a1912-dirty", "rec_mechanism": "nccl_profiler_interface", "dump_timestamp_us": 1752867229276385, diff --git a/projects/rccl/plugins/profiler/inspector/inspector.cc b/projects/rccl/plugins/profiler/inspector/inspector.cc index 9e950333d9af..6fd677953646 100644 --- a/projects/rccl/plugins/profiler/inspector/inspector.cc +++ b/projects/rccl/plugins/profiler/inspector/inspector.cc @@ -49,6 +49,10 @@ static bool enableNcclInspectorDumpVerbose = false; // Global flag to control prometheus format dumping static bool enableNcclInspectorPromDump = false; static bool warnedOtelPromDumpConflict = false; +// Opt-in flag for the extra per-device stats metrics (collectives/p2p totals +// and drop counters) in Prometheus mode; kept off so the default Prometheus +// output stays minimal. Non-static: read from inspector_prom.cc. +bool enableNcclInspectorPromStats = false; // Per-communicator completed-collective ring buffer capacity static uint32_t ncclInspectorDumpCollRingSize = 1024; // Per-communicator completed-P2P ring buffer capacity @@ -956,6 +960,7 @@ static void showInspectorEnvVars() { {"NCCL_INSPECTOR_DUMP_DIR", getenv("NCCL_INSPECTOR_DUMP_DIR"), "(auto-generated)", "Output directory for inspector logs"}, {"NCCL_INSPECTOR_DUMP_VERBOSE", getenv("NCCL_INSPECTOR_DUMP_VERBOSE"), "0", "Enable/disable verbose dumping (event_trace)"}, {"NCCL_INSPECTOR_PROM_DUMP", getenv("NCCL_INSPECTOR_PROM_DUMP"), "0", "Enable/disable Prometheus format output dump"}, + {"NCCL_INSPECTOR_PROM_DUMP_STATS", getenv("NCCL_INSPECTOR_PROM_DUMP_STATS"), "0", "Enable/disable extra per-device Prometheus stats metrics (collectives/p2p totals and drop counters)"}, {"NCCL_INSPECTOR_OTEL_EXPORT", getenv("NCCL_INSPECTOR_OTEL_EXPORT"), "0", "Enable/disable OTLP HTTP metrics export"}, {"NCCL_INSPECTOR_OTEL_VERBOSE", getenv("NCCL_INSPECTOR_OTEL_VERBOSE"), "0", "Enable/disable high-cardinality per-operation OTLP metrics"}, {"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"), "http://localhost:4318/v1/metrics", "OTLP HTTP metrics endpoint"}, @@ -1122,6 +1127,10 @@ static inspectorResult_t initDumpThreadFromEnv() { enableNcclInspectorPromDump = enable == 0 ? false : true; warnedOtelPromDumpConflict = false; + str = getenv("NCCL_INSPECTOR_PROM_DUMP_STATS"); + enable = str ? atoi(str) : 0; + enableNcclInspectorPromStats = enable == 0 ? false : true; + inspectorOtelInitFromEnv(); str = getenv("NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS"); diff --git a/projects/rccl/plugins/profiler/inspector/inspector.h b/projects/rccl/plugins/profiler/inspector/inspector.h index b3724cff1915..b8e994b667ee 100644 --- a/projects/rccl/plugins/profiler/inspector/inspector.h +++ b/projects/rccl/plugins/profiler/inspector/inspector.h @@ -337,6 +337,8 @@ inline int ncclTypeSize(ncclDataType_t type) { // Global flag to control P2P tracking extern bool enableNcclInspectorP2p; extern bool requireKernelTiming; +// Opt-in flag for extra per-device Prometheus stats metrics (totals + drops) +extern bool enableNcclInspectorPromStats; // Minimum message size (bytes) to be `tracked by inspector extern size_t ncclInspectorDumpMinSizeBytes; diff --git a/projects/rccl/plugins/profiler/inspector/inspector_json.cc b/projects/rccl/plugins/profiler/inspector/inspector_json.cc index 77db0403232b..31b3e7ec2eb6 100644 --- a/projects/rccl/plugins/profiler/inspector/inspector_json.cc +++ b/projects/rccl/plugins/profiler/inspector/inspector_json.cc @@ -58,7 +58,7 @@ static inspectorResult_t inspectorCommInfoHeader(jsonFileOutput* jfo, static inspectorResult_t inspectorCommInfoMetaHeader(jsonFileOutput* jfo) { JSON_CHK(jsonStartObject(jfo)); { - JSON_CHK(jsonKey(jfo, "inspector_output_format_version")); JSON_CHK(jsonStr(jfo, "v4.0")); + JSON_CHK(jsonKey(jfo, "inspector_output_format_version")); JSON_CHK(jsonStr(jfo, "v4.1")); JSON_CHK(jsonKey(jfo, "git_rev")); JSON_CHK(jsonStr(jfo, get_git_version_info())); JSON_CHK(jsonKey(jfo, "rec_mechanism")); JSON_CHK(jsonStr(jfo, "nccl_profiler_interface")); JSON_CHK(jsonKey(jfo, "dump_timestamp_us")); JSON_CHK(jsonUint64(jfo, inspectorGetTime())); @@ -278,94 +278,117 @@ static inline inspectorResult_t inspectorCompletedP2p(jsonFileOutput* jfo, * inspectorResult_t - success or error code. * */ -static inspectorResult_t inspectorCommInfoDumpColl(jsonFileOutput* jfo, - inspectorCommInfo* commInfo, - bool* needs_writing) { - if (commInfo == nullptr) { - return inspectorSuccess; - } +// Per-dump statistics for one communicator: records written this dump and +// cumulative / since-last-dump ring drop counts for each op type. +struct inspectorCommDumpStats { + uint64_t collRecords = 0; + uint64_t collDroppedTotal = 0; + uint64_t collDroppedSinceLastDump = 0; + uint64_t p2pRecords = 0; + uint64_t p2pDroppedTotal = 0; + uint64_t p2pDroppedSinceLastDump = 0; +}; - thread_local std::vector drainedColl; - drainedColl.clear(); +/* + * Description: + * Snapshots a ring's drop counters and advances its reported watermark so the + * next dump reports only newly dropped entries. + * + * Thread Safety: + * Not thread-safe (caller must hold commInfo->guard). + */ +static inline void inspectorCommInfoUpdateDropStats(struct inspectorCompletedRing* ring, + uint64_t* total, + uint64_t* sinceLastDump) { + *total = ring->dropped; + *sinceLastDump = ring->dropped - ring->droppedReported; + ring->droppedReported = ring->dropped; +} +// Drains the collective ring under the comm guard (one lock hold for this ring) +// and snapshots its drop stats. Records are appended to drained. +static inspectorResult_t inspectorCommInfoDrainColl(inspectorCommInfo* commInfo, + std::vector& drained, + inspectorCommDumpStats* stats) { inspectorLockWr(&commInfo->guard); if (commInfo->dump_coll) { // Make sure we won't allocate while draining (steady-state: no-op). if (commInfo->completedCollRing.size > 0 - && drainedColl.capacity() < commInfo->completedCollRing.size) { - drainedColl.reserve(commInfo->completedCollRing.size); + && drained.capacity() < commInfo->completedCollRing.size) { + drained.reserve(commInfo->completedCollRing.size); } INS_CHK(inspectorRingDrain(&commInfo->completedCollRing, - drainedColl)); + drained)); commInfo->dump_coll = inspectorRingNonEmpty(&commInfo->completedCollRing); } + inspectorCommInfoUpdateDropStats(&commInfo->completedCollRing, + &stats->collDroppedTotal, + &stats->collDroppedSinceLastDump); inspectorUnlockRWLock(&commInfo->guard); - - if (!drainedColl.empty()) { - *needs_writing = true; - JSON_CHK(jsonLockOutput(jfo)); - for (size_t i = 0; i < drainedColl.size(); i++) { - JSON_CHK(jsonStartObject(jfo)); - { - JSON_CHK(jsonKey(jfo, "header")); - inspectorCommInfoHeader(jfo, commInfo); - - JSON_CHK(jsonKey(jfo, "metadata")); - inspectorCommInfoMetaHeader(jfo); - - JSON_CHK(jsonKey(jfo, "coll_perf")); - INS_CHK(inspectorCompletedColl(jfo, &drainedColl[i])); - } - JSON_CHK(jsonFinishObject(jfo)); - JSON_CHK(jsonNewline(jfo)); - } - JSON_CHK(jsonUnlockOutput(jfo)); - } + stats->collRecords = drained.size(); return inspectorSuccess; } -static inspectorResult_t inspectorCommInfoDumpP2p(jsonFileOutput* jfo, - inspectorCommInfo* commInfo, - bool* needs_writing) { - if (commInfo == nullptr) { - return inspectorSuccess; - } - - thread_local std::vector drainedP2p; - drainedP2p.clear(); - +// Drains the P2P ring under the comm guard (one lock hold for this ring) +// and snapshots its drop stats. +static inspectorResult_t inspectorCommInfoDrainP2p(inspectorCommInfo* commInfo, + std::vector& drained, + inspectorCommDumpStats* stats) { inspectorLockWr(&commInfo->guard); if (commInfo->dump_p2p) { if (commInfo->completedP2pRing.size > 0 - && drainedP2p.capacity() < commInfo->completedP2pRing.size) { - drainedP2p.reserve(commInfo->completedP2pRing.size); + && drained.capacity() < commInfo->completedP2pRing.size) { + drained.reserve(commInfo->completedP2pRing.size); } INS_CHK(inspectorRingDrain(&commInfo->completedP2pRing, - drainedP2p)); + drained)); commInfo->dump_p2p = inspectorRingNonEmpty(&commInfo->completedP2pRing); } + inspectorCommInfoUpdateDropStats(&commInfo->completedP2pRing, + &stats->p2pDroppedTotal, + &stats->p2pDroppedSinceLastDump); inspectorUnlockRWLock(&commInfo->guard); + stats->p2pRecords = drained.size(); + return inspectorSuccess; +} - if (!drainedP2p.empty()) { - *needs_writing = true; - JSON_CHK(jsonLockOutput(jfo)); - for (size_t i = 0; i < drainedP2p.size(); i++) { - JSON_CHK(jsonStartObject(jfo)); - { - JSON_CHK(jsonKey(jfo, "header")); - inspectorCommInfoHeader(jfo, commInfo); - - JSON_CHK(jsonKey(jfo, "metadata")); - inspectorCommInfoMetaHeader(jfo); - - JSON_CHK(jsonKey(jfo, "p2p_perf")); - INS_CHK(inspectorCompletedP2p(jfo, &drainedP2p[i])); - } - JSON_CHK(jsonFinishObject(jfo)); - JSON_CHK(jsonNewline(jfo)); +/* + * Description: + * Writes a single per-dump stats record for a communicator: a marker at the + * start of the comm's dump cycle carrying records-written and drop counts. + * Emitted once per dump per comm, so consumers must not assume every record + * carries coll_perf/p2p_perf — the "dump_stats" key identifies this record. + * + * Thread Safety: + * Not thread-safe (caller must hold the JSON output lock). + */ +static inspectorResult_t inspectorCommInfoDumpStats(jsonFileOutput* jfo, + inspectorCommInfo* commInfo, + const inspectorCommDumpStats* stats) { + JSON_CHK(jsonStartObject(jfo)); + { + JSON_CHK(jsonKey(jfo, "header")); + inspectorCommInfoHeader(jfo, commInfo); + + JSON_CHK(jsonKey(jfo, "metadata")); + inspectorCommInfoMetaHeader(jfo); + + JSON_CHK(jsonKey(jfo, "dump_stats")); + JSON_CHK(jsonStartObject(jfo)); + { + JSON_CHK(jsonKey(jfo, "coll_records")); JSON_CHK(jsonUint64(jfo, stats->collRecords)); + JSON_CHK(jsonKey(jfo, "coll_dropped_total")); JSON_CHK(jsonUint64(jfo, stats->collDroppedTotal)); + JSON_CHK(jsonKey(jfo, "coll_dropped_since_last_dump")); + JSON_CHK(jsonUint64(jfo, stats->collDroppedSinceLastDump)); + JSON_CHK(jsonKey(jfo, "p2p_records")); JSON_CHK(jsonUint64(jfo, stats->p2pRecords)); + JSON_CHK(jsonKey(jfo, "p2p_dropped_total")); JSON_CHK(jsonUint64(jfo, stats->p2pDroppedTotal)); + JSON_CHK(jsonKey(jfo, "p2p_dropped_since_last_dump")); + JSON_CHK(jsonUint64(jfo, stats->p2pDroppedSinceLastDump)); } - JSON_CHK(jsonUnlockOutput(jfo)); + JSON_CHK(jsonFinishObject(jfo)); } + JSON_CHK(jsonFinishObject(jfo)); + JSON_CHK(jsonNewline(jfo)); return inspectorSuccess; } @@ -373,9 +396,61 @@ static inspectorResult_t inspectorCommInfoDump(jsonFileOutput* jfo, inspectorCommInfo* commInfo, bool* needs_writing) { *needs_writing = false; + if (commInfo == nullptr) { + return inspectorSuccess; + } + + thread_local std::vector drainedColl; + thread_local std::vector drainedP2p; + drainedColl.clear(); + drainedP2p.clear(); + + inspectorCommDumpStats stats; + // One guard hold per ring, matching the original per-ring locking. + INS_CHK(inspectorCommInfoDrainColl(commInfo, drainedColl, &stats)); + INS_CHK(inspectorCommInfoDrainP2p(commInfo, drainedP2p, &stats)); + + // Emit only when this comm produced records or has newly dropped entries to + // report, so idle communicators don't generate empty stats records. + bool haveDrops = stats.collDroppedSinceLastDump != 0 || stats.p2pDroppedSinceLastDump != 0; + if (drainedColl.empty() && drainedP2p.empty() && !haveDrops) { + return inspectorSuccess; + } + + *needs_writing = true; + JSON_CHK(jsonLockOutput(jfo)); + INS_CHK(inspectorCommInfoDumpStats(jfo, commInfo, &stats)); + for (size_t i = 0; i < drainedColl.size(); i++) { + JSON_CHK(jsonStartObject(jfo)); + { + JSON_CHK(jsonKey(jfo, "header")); + inspectorCommInfoHeader(jfo, commInfo); - INS_CHK(inspectorCommInfoDumpColl(jfo, commInfo, needs_writing)); - INS_CHK(inspectorCommInfoDumpP2p(jfo, commInfo, needs_writing)); + JSON_CHK(jsonKey(jfo, "metadata")); + inspectorCommInfoMetaHeader(jfo); + + JSON_CHK(jsonKey(jfo, "coll_perf")); + INS_CHK(inspectorCompletedColl(jfo, &drainedColl[i])); + } + JSON_CHK(jsonFinishObject(jfo)); + JSON_CHK(jsonNewline(jfo)); + } + for (size_t i = 0; i < drainedP2p.size(); i++) { + JSON_CHK(jsonStartObject(jfo)); + { + JSON_CHK(jsonKey(jfo, "header")); + inspectorCommInfoHeader(jfo, commInfo); + + JSON_CHK(jsonKey(jfo, "metadata")); + inspectorCommInfoMetaHeader(jfo); + + JSON_CHK(jsonKey(jfo, "p2p_perf")); + INS_CHK(inspectorCompletedP2p(jfo, &drainedP2p[i])); + } + JSON_CHK(jsonFinishObject(jfo)); + JSON_CHK(jsonNewline(jfo)); + } + JSON_CHK(jsonUnlockOutput(jfo)); return inspectorSuccess; } diff --git a/projects/rccl/plugins/profiler/inspector/inspector_otel.cc b/projects/rccl/plugins/profiler/inspector/inspector_otel.cc index e5ead89bfb75..7bd3e454713b 100644 --- a/projects/rccl/plugins/profiler/inspector/inspector_otel.cc +++ b/projects/rccl/plugins/profiler/inspector/inspector_otel.cc @@ -111,6 +111,12 @@ struct inspectorOtelDevice { inspectorOtelCollBucketMap collBuckets; inspectorOtelP2pBucketMap p2pBuckets; bool hasData = false; + // Cumulative counts summed across this device's communicators: total ops + // ever enqueued and total dropped (overwritten before being dumped). + uint64_t collTotal = 0; + uint64_t collDroppedTotal = 0; + uint64_t p2pTotal = 0; + uint64_t p2pDroppedTotal = 0; }; static const char* inspectorOtelFirstEnv(const char* name0, @@ -534,6 +540,51 @@ static inspectorResult_t inspectorOtelBuildAggregatedPoints( return inspectorSuccess; } +// Emits the per-device ring-buffer stats (cumulative op totals and drop +// counters) for one device. Low-cardinality (node/gpu), and monotonic so +// rate(dropped)/rate(total) yields the fraction of operations lost. +static inspectorResult_t inspectorOtelAddDeviceTotals( + std::vector& points, + const inspectorOtelDevice& device, + uint64_t timestampUsec) { + std::vector attrs; + inspectorOtelAddAttr(attrs, "version", + inspectorOtelFormatVersion(INSPECTOR_OTEL_DEFAULT_FORMAT_MINOR)); + inspectorOtelAddAttr(attrs, "node", gOtelNodeName); + inspectorOtelAddAttr(attrs, "gpu", + device.gpuName.empty() ? "unknown" : device.gpuName); + + inspectorOtelAddMetricPoint(points, "nccl_collectives_total", + "NCCL collectives enqueued into the inspector ring buffer", + "1", attrs, + static_cast(device.collTotal), timestampUsec); + inspectorOtelAddMetricPoint(points, "nccl_collectives_dropped_total", + "NCCL collectives overwritten in the ring buffer before being dumped", + "1", attrs, + static_cast(device.collDroppedTotal), timestampUsec); + inspectorOtelAddMetricPoint(points, "nccl_p2p_total", + "NCCL P2P operations enqueued into the inspector ring buffer", + "1", attrs, + static_cast(device.p2pTotal), timestampUsec); + inspectorOtelAddMetricPoint(points, "nccl_p2p_dropped_total", + "NCCL P2P operations overwritten in the ring buffer before being dumped", + "1", attrs, + static_cast(device.p2pDroppedTotal), timestampUsec); + return inspectorSuccess; +} + +// Emits per-device stats for every known device so the counters stay +// continuous (usable with rate()) even across dumps with no new records. +static inspectorResult_t inspectorOtelBuildDeviceStatsPoints( + const std::map& devices, + std::vector& points, + uint64_t timestampUsec) { + for (const auto& entry : devices) { + INS_CHK(inspectorOtelAddDeviceTotals(points, entry.second, timestampUsec)); + } + return inspectorSuccess; +} + static bool inspectorOtelUrlHasPath(const std::string& endpoint) { size_t scheme = endpoint.find("://"); size_t hostStart = scheme == std::string::npos ? 0 : scheme + 3; @@ -810,13 +861,28 @@ static bool inspectorOtelSendAll(int fd, const std::string& request) { static bool inspectorOtelParseStatus(int fd, int* statusCode) { char response[512]; - ssize_t got = recv(fd, response, sizeof(response) - 1, 0); - if (got <= 0) return false; - response[got] = '\0'; + size_t used = 0; + char* lineEnd = nullptr; + while (used < sizeof(response) - 1 && lineEnd == nullptr) { + ssize_t got = recv(fd, response + used, sizeof(response) - 1 - used, 0); + if (got < 0) { + if (errno == EINTR) continue; + return false; + } + if (got == 0) return false; + used += (size_t)got; + response[used] = '\0'; + lineEnd = strstr(response, "\r\n"); + if (lineEnd == nullptr) lineEnd = strchr(response, '\n'); + } + if (lineEnd == nullptr || strncmp(response, "HTTP/", 5) != 0) return false; char* space = strchr(response, ' '); - if (space == nullptr) return false; - *statusCode = atoi(space + 1); + if (space == nullptr || space >= lineEnd) return false; + char* codeEnd = nullptr; + long code = strtol(space + 1, &codeEnd, 10); + if (codeEnd == space + 1 || codeEnd > lineEnd || code < 100 || code > 999) return false; + *statusCode = (int)code; return *statusCode >= 200 && *statusCode < 300; } @@ -1040,6 +1106,8 @@ static inspectorResult_t inspectorOtelCommInfoDumpColl( drainedColl)); commInfo->dump_coll = inspectorRingNonEmpty(&commInfo->completedCollRing); } + device.collTotal += commInfo->completedCollRing.enqueued; + device.collDroppedTotal += commInfo->completedCollRing.dropped; inspectorUnlockRWLock(&commInfo->guard); if (!drainedColl.empty()) { @@ -1100,6 +1168,8 @@ static inspectorResult_t inspectorOtelCommInfoDumpP2p( drainedP2p)); commInfo->dump_p2p = inspectorRingNonEmpty(&commInfo->completedP2pRing); } + device.p2pTotal += commInfo->completedP2pRing.enqueued; + device.p2pDroppedTotal += commInfo->completedP2pRing.dropped; inspectorUnlockRWLock(&commInfo->guard); if (!drainedP2p.empty()) { @@ -1223,6 +1293,11 @@ inspectorResult_t inspectorOtelCommInfoListDump(struct inspectorCommInfoList* co if (!verbose) { INS_CHK_GOTO(inspectorOtelBuildAggregatedPoints(devices, points), res, unlock); + } else { + // Per-device ring-buffer stats (totals + drops) ride under verbose mode + // so the default aggregated output stays minimal. + INS_CHK_GOTO(inspectorOtelBuildDeviceStatsPoints(devices, points, currentTime), + res, unlock); } shouldExport = true; } diff --git a/projects/rccl/plugins/profiler/inspector/inspector_plugin.cc b/projects/rccl/plugins/profiler/inspector/inspector_plugin.cc index c6397aa30549..680576c40f66 100644 --- a/projects/rccl/plugins/profiler/inspector/inspector_plugin.cc +++ b/projects/rccl/plugins/profiler/inspector/inspector_plugin.cc @@ -190,19 +190,33 @@ static void inspectorPluginCollInfoCleanup(struct inspectorCollInfo *collInfo) { inspectorEventPoolReleaseColl(collInfo); } +// Set once (per process) when the first ring overwrite is detected, so the +// overflow condition is reported exactly once rather than per dropped entry. +static bool ringDropWarned = false; + static void inspectorUpdateCommOpInfo(struct inspectorCommInfo *commInfo, struct inspectorCompletedOpInfo *completedOp) { struct inspectorCompletedRing *ring = completedOp->isP2p ? &commInfo->completedP2pRing : &commInfo->completedCollRing; inspectorLockWr(&commInfo->guard); inspectorComputeOpBw(commInfo, completedOp); + uint64_t droppedBefore = ring->dropped; inspectorRingEnqueue(ring, completedOp); + bool overflowed = ring->dropped != droppedBefore; if (completedOp->isP2p) { commInfo->dump_p2p = inspectorRingNonEmpty(&commInfo->completedP2pRing); } else { commInfo->dump_coll = inspectorRingNonEmpty(&commInfo->completedCollRing); } inspectorUnlockRWLock(&commInfo->guard); + if (overflowed && !__atomic_exchange_n(&ringDropWarned, true, __ATOMIC_RELAXED)) { + WARN_INSPECTOR( + "NCCL Inspector: completed-op ring buffer overflowed on comm %s (%s ring size %u); " + "oldest entries are being dropped before they can be dumped. Increase " + "NCCL_INSPECTOR_DUMP_COLL_RING_SIZE/NCCL_INSPECTOR_DUMP_P2P_RING_SIZE to retain more. " + "Dropped counts are reported in the output.", + commInfo->commHashStr, completedOp->isP2p ? "p2p" : "coll", ring->size); + } } inspectorResult_t inspectorPluginP2pInfoRef(struct inspectorP2pInfo *p2pInfo) { diff --git a/projects/rccl/plugins/profiler/inspector/inspector_prom.cc b/projects/rccl/plugins/profiler/inspector/inspector_prom.cc index 63aa5e254fdf..fe7541cf41e2 100644 --- a/projects/rccl/plugins/profiler/inspector/inspector_prom.cc +++ b/projects/rccl/plugins/profiler/inspector/inspector_prom.cc @@ -29,6 +29,7 @@ extern inspectorResult_t inspectorCommInfoListFinalize(struct inspectorCommInfoL extern const char* inspectorTimingSourceToString(inspectorTimingSource_t timingSource); extern const char* ncclFuncToString(ncclFunc_t fn); +extern bool enableNcclInspectorPromStats; struct inspectorPromBucketAgg { uint64_t count = 0; @@ -88,6 +89,12 @@ struct inspectorPromDevice { inspectorPromCollBucketMap collBuckets; inspectorPromP2pBucketMap p2pBuckets; bool hasData = false; + // Cumulative counts summed across this device's communicators: total ops + // ever enqueued and total dropped (overwritten before being dumped). + uint64_t collTotal = 0; + uint64_t collDroppedTotal = 0; + uint64_t p2pTotal = 0; + uint64_t p2pDroppedTotal = 0; }; static const int kInspectorPromFormatMinor = 1; @@ -595,6 +602,8 @@ static inspectorResult_t inspectorPromCommInfoDumpColl(struct inspectorCommInfo* drainedColl)); commInfo->dump_coll = inspectorRingNonEmpty(&commInfo->completedCollRing); } + device.collTotal += commInfo->completedCollRing.enqueued; + device.collDroppedTotal += commInfo->completedCollRing.dropped; inspectorUnlockRWLock(&commInfo->guard); if (!drainedColl.empty()) { @@ -646,6 +655,8 @@ static inspectorResult_t inspectorPromCommInfoDumpP2p(struct inspectorCommInfo* drainedP2p)); commInfo->dump_p2p = inspectorRingNonEmpty(&commInfo->completedP2pRing); } + device.p2pTotal += commInfo->completedP2pRing.enqueued; + device.p2pDroppedTotal += commInfo->completedP2pRing.dropped; inspectorUnlockRWLock(&commInfo->guard); if (!drainedP2p.empty()) { @@ -747,6 +758,78 @@ static inspectorResult_t inspectorPromFillDeviceBuckets(struct inspectorCommInfo * Thread Safety: * Not thread-safe (caller must hold commList lock). */ +/* + * Description: + * Formats device-level labels (node/gpu/version only, no bucket-key fields) + * for the per-device Prometheus stats metrics. + * + * Thread Safety: + * Not thread-safe. Onus of thread safety is on the caller. + */ +static inspectorResult_t inspectorPromGetLabelsDevice(char* labels, + size_t labelSize, + const char* nodeName, + const char* gpuName, + const char* version) { + const char* jobId = getenv("SLURM_JOB_ID"); + int ret = snprintf(labels, labelSize, + "version=\"%s\",slurm_job_id=\"%s\",node=\"%s\",gpu=\"%s\"", + (version && version[0]) ? version : "unknown", + jobId ? jobId : "unknown", + (nodeName && nodeName[0]) ? nodeName : "unknown", + (gpuName && gpuName[0]) ? gpuName : "unknown"); + if (ret < 0 || (size_t)ret >= labelSize) { + return inspectorMemoryError; + } + return inspectorSuccess; +} + +/* + * Description: + * Writes the opt-in per-device stats metrics: cumulative op totals and + * ring-buffer drop counters. Counters (not per-window gauges), so rate() + * and rate(dropped)/rate(total) work. + * + * Thread Safety: + * Not thread-safe. Onus of thread safety is on the caller/owner of the file. + */ +static inspectorResult_t inspectorPromWriteDeviceTotals(FILE* file, + const inspectorPromDevice& device) { + if (!file) { + return inspectorFileOpenError; + } + + char version[16]; + inspectorPromGetVersion(version, sizeof(version)); + + char labels[1024]; + memset(labels, 0, sizeof(labels)); + INS_CHK(inspectorPromGetLabelsDevice(labels, + sizeof(labels), + device.nodeName.c_str(), + device.gpuName.c_str(), + version)); + + char buffer[2048]; + int written = snprintf(buffer, sizeof(buffer), + "nccl_collectives_total{%s} %llu\n" + "nccl_collectives_dropped_total{%s} %llu\n" + "nccl_p2p_total{%s} %llu\n" + "nccl_p2p_dropped_total{%s} %llu\n", + labels, (unsigned long long)device.collTotal, + labels, (unsigned long long)device.collDroppedTotal, + labels, (unsigned long long)device.p2pTotal, + labels, (unsigned long long)device.p2pDroppedTotal); + if (written < 0 || (size_t)written >= sizeof(buffer)) { + return inspectorMemoryError; + } + if (fwrite(buffer, 1, written, file) != (size_t)written) { + return inspectorFileOpenError; + } + fflush(file); + return inspectorSuccess; +} + static inspectorResult_t inspectorPromWriteDeviceBuckets(std::map& devices, const char* output_root, @@ -754,7 +837,11 @@ static inspectorResult_t inspectorPromWriteDeviceBuckets(std::mapsize = size; ring->head = 0; ring->tail = 0; + ring->enqueued = 0; + ring->dropped = 0; + ring->droppedReported = 0; return inspectorSuccess; } @@ -64,6 +67,9 @@ void inspectorRingFinalize(struct inspectorCompletedRing* ring) { free(ring->entries); ring->entries = nullptr; ring->size = ring->head = ring->tail = 0; + ring->enqueued = 0; + ring->dropped = 0; + ring->droppedReported = 0; } /* @@ -93,9 +99,11 @@ inspectorResult_t inspectorRingEnqueue(struct inspectorCompletedRing* ring, if ((ring->tail + 1) % bufSize == ring->head) { // Ring is full: advance head to overwrite the oldest entry ring->head = (ring->head + 1) % bufSize; + ring->dropped++; } memcpy(ringSlot(ring, ring->tail), entry, ring->entrySize); ring->tail = (ring->tail + 1) % bufSize; + ring->enqueued++; return inspectorSuccess; } diff --git a/projects/rccl/plugins/profiler/inspector/inspector_ring.h b/projects/rccl/plugins/profiler/inspector/inspector_ring.h index e9421797e6d1..389a139e2172 100644 --- a/projects/rccl/plugins/profiler/inspector/inspector_ring.h +++ b/projects/rccl/plugins/profiler/inspector/inspector_ring.h @@ -26,6 +26,9 @@ struct inspectorCompletedRing { uint32_t size{0}; // user-visible capacity (size+1 slots are allocated) uint32_t head{0}; // index of oldest element uint32_t tail{0}; // index where next element will be written + uint64_t enqueued{0}; // cumulative entries ever enqueued (drained + dropped) + uint64_t dropped{0}; // cumulative entries overwritten before being drained + uint64_t droppedReported{0}; // value of 'dropped' at the last drain (for deltas) }; /* diff --git a/projects/rccl/plugins/rma/README.md b/projects/rccl/plugins/rma/README.md index bbad8e3596fc..af70cc9890fd 100644 --- a/projects/rccl/plugins/rma/README.md +++ b/projects/rccl/plugins/rma/README.md @@ -8,6 +8,51 @@ the network layer for GIN CPU Proxy and for one-sided NCCL operations (e.g. `ncc This document uses `ncclImpl` to refer to NCCL's internal (non-customized) implementations of these APIs. References to `ncclImpl` are just examples, custom implementations may make other design decisions. +## Plugin architecture + +### Plugin name and supporting multiple RMA plugins + +When NCCL is initialized, it will look for a `libnccl-rma.so` library and +dynamically load it, then look for symbols inside the library. + +The `NCCL_RMA_PLUGIN` environment variable allows users to select one or more +RMA plugins. If set to a bare plugin name such as `myrma`, NCCL can load a +library named `libnccl-rma-myrma.so`. `NCCL_RMA_PLUGIN` can also be set to a +shared library file name or an absolute path to the plugin file. Multiple +plugins can be specified as a comma-separated list. + +For example, any of the following can be used to load a plugin: + +```shell +export NCCL_RMA_PLUGIN=myrma +export NCCL_RMA_PLUGIN=libnccl-rma-myrma.so +export NCCL_RMA_PLUGIN=/path/to/your/plugin/libnccl-rma-myrma.so +``` + +Set the `LD_LIBRARY_PATH` to include the plugin directory when using a bare name +or shared library file name: + +```shell +export LD_LIBRARY_PATH=/path/to/your/plugin:$LD_LIBRARY_PATH +``` + +If none of the listed plugins can be loaded, NCCL looks for the RMA symbols in +the GIN plugin and in the NET plugin, and finally falls back to its internal RMA +implementation over InfiniBand. + +Only one RMA implementation is used: NCCL picks the first one that both succeeds +`init` and reports at least one device from `devices`, and disables the remaining +external plugins. + +### Struct versioning + +Once a library is found, NCCL will look for a symbol named `ncclRmaPlugin_vX`, +with `X` increasing over time. The versioning ensures that the plugin and NCCL +core are compatible. + +Plugins are encouraged to provide multiple of those symbols, implementing +multiple versions of the NCCL RMA API, so that the same plugin can be compiled +and support a wide range of NCCL versions. ## RMA Concepts @@ -37,24 +82,24 @@ nothing prevents applications from creating more or less. ### Operations -**Put** +**Put** Put moves data from a local source buffer to a (likely remote) target buffer. A single call to `Put` may include a signal operation. See below for more details. -**Get** +**Get** Get moves data from a (likely remote) source buffer to a local target buffer. Gets are not required to be completed in the order they are requested. -**Signal** +**Signal** Signal increments a target memory address by a fixed value. There's 2 types of visibility guarantees: strong and weak. The visibility of a *strong* signal guarantees the completion/visibility of all previous puts and signals, including the bundled put in the case of put+signal. The visibility of a *weak* signal guarantees the completion/visibility of the bundled put (in the case of put+signal), but makes no guarantees on previous puts or previous signals. -**Flush** +**Flush** Flush flushes the PCIe path from a NIC to a GPU. The completion of a flush must guarantee that the data -from all previous *completed* gets is visible on the GPU. Gets that are not complete at the time flush +from all previous *completed* gets is visible on the GPU. Gets that are not complete at the time flush is issued need not have data visible. ## `ncclRma` API @@ -120,3 +165,22 @@ not a functional network transport. Build it with: ```shell make -C plugins/rma/example ``` + +### Loading the example plugin + +Set the `LD_LIBRARY_PATH` to include the example plugin directory: + +```shell +export LD_LIBRARY_PATH=/path/to/nccl/plugins/rma/example:$LD_LIBRARY_PATH +``` + +Set `NCCL_RMA_PLUGIN` to either the plugin name, the shared library file name, or +the absolute path to the plugin file. Any of the following can work: + +```shell +export NCCL_RMA_PLUGIN=example +export NCCL_RMA_PLUGIN=libnccl-rma-example.so +export NCCL_RMA_PLUGIN=/path/to/nccl/plugins/rma/example/libnccl-rma-example.so +``` + +NCCL will automatically discover and load the plugin based on the exported symbol names. diff --git a/projects/rccl/plugins/rma/example/CMakeLists.txt b/projects/rccl/plugins/rma/example/CMakeLists.txt index 93ecbbe4505c..cdb70877d41f 100644 --- a/projects/rccl/plugins/rma/example/CMakeLists.txt +++ b/projects/rccl/plugins/rma/example/CMakeLists.txt @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # +# Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# # See LICENSE.txt for more license information # @@ -10,16 +12,16 @@ set(SRC_FILES ) # Create shared library -add_library(nccl-rma-example SHARED ${SRC_FILES}) +add_library(rccl-rma-example SHARED ${SRC_FILES}) # Set include directories -target_include_directories(nccl-rma-example PRIVATE +target_include_directories(rccl-rma-example PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nccl ) # Set output name to match Makefile -set_target_properties(nccl-rma-example PROPERTIES - OUTPUT_NAME "nccl-rma-example" +set_target_properties(rccl-rma-example PROPERTIES + OUTPUT_NAME "rccl-rma-example" PREFIX "lib" POSITION_INDEPENDENT_CODE ON LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test/unit/plugins diff --git a/projects/rccl/plugins/rma/example/Makefile b/projects/rccl/plugins/rma/example/Makefile index 69a3e9c56708..32a1b2a58e82 100644 --- a/projects/rccl/plugins/rma/example/Makefile +++ b/projects/rccl/plugins/rma/example/Makefile @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # +# Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# # See LICENSE.txt for more license information # .DEFAULT_GOAL: build @@ -11,12 +13,12 @@ NCCLDIR := $(BUILDDIR) SRC_FILES := $(wildcard *.c) -build: ${BUILDDIR}/libnccl-rma-example.so +build: ${BUILDDIR}/librccl-rma-example.so -${BUILDDIR}/libnccl-rma-example.so: ${SRC_FILES} +${BUILDDIR}/librccl-rma-example.so: ${SRC_FILES} $(wildcard nccl/*.h) @printf "Compiling %-35s > %s\n" $< $@ @mkdir -p ${BUILDDIR} - $(CC) -Inccl -fPIC -shared -o $@ $^ + $(CC) -Inccl -fPIC -shared -o $@ ${SRC_FILES} clean: - rm -f ${BUILDDIR}/libnccl-rma-example.so + rm -f ${BUILDDIR}/librccl-rma-example.so diff --git a/projects/rccl/plugins/rma/example/nccl/rma.h b/projects/rccl/plugins/rma/example/nccl/rma.h index a4794406b9c7..aa77e6a17d23 100644 --- a/projects/rccl/plugins/rma/example/nccl/rma.h +++ b/projects/rccl/plugins/rma/example/nccl/rma.h @@ -58,9 +58,9 @@ typedef struct { } ncclNetProperties_v12_t; /* RMA plugin versioned interfaces */ -#include "rma_v14.h" +#include "rma_v15.h" -typedef ncclRma_v14_t ncclRma_t; -typedef ncclRmaConfig_v14_t ncclRmaConfig_t; +typedef ncclRma_v15_t ncclRma_t; +typedef ncclRmaConfig_v15_t ncclRmaConfig_t; #endif // end include guard diff --git a/projects/rccl/plugins/rma/example/nccl/rma_v15.h b/projects/rccl/plugins/rma/example/nccl/rma_v15.h new file mode 100644 index 000000000000..7eefb738d82c --- /dev/null +++ b/projects/rccl/plugins/rma/example/nccl/rma_v15.h @@ -0,0 +1,52 @@ +/************************************************************************* + * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for more license information + *************************************************************************/ + +#ifndef RMA_V15_H_ +#define RMA_V15_H_ + +#include "rma_v14.h" + +typedef ncclRmaConfig_v14_t ncclRmaConfig_v15_t; + +enum ncclRmaOptFlags { + ncclRmaOptFlagsDefault = 0, + ncclRmaOptFlagsAggregateRequests = (1 << 0), +}; + +typedef struct { + const char* name; + ncclResult_t (*init)(void** ctx, uint64_t commId, ncclDebugLogger_t logFunction); + ncclResult_t (*devices)(int* ndev); + ncclResult_t (*getProperties)(int dev, ncclNetProperties_v12_t* props); + ncclResult_t (*listen)(void* ctx, int dev, void* handle, void** listenComm); + ncclResult_t (*connect)(void* ctx, void* handles[], int nranks, int rank, void* listenComm, void** collComm); + ncclResult_t (*createContext)(void* collComm, ncclRmaConfig_v15_t* config, void** rmaCtx); + ncclResult_t (*regMrSym)(void* collComm, void* data, size_t size, int type, uint64_t mrFlags, void** mhandle); + ncclResult_t (*regMrSymDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, + uint64_t mrFlags, void** mhandle); + ncclResult_t (*deregMrSym)(void* collComm, void* mhandle); + ncclResult_t (*destroyContext)(void* rmaCtx); + ncclResult_t (*closeColl)(void* collComm); + ncclResult_t (*closeListen)(void* listenComm); + ncclResult_t (*iput)(void* rmaCtx, int context, uint64_t srcOff, void* srcMhandle, size_t size, uint64_t dstOff, + void* dstMhandle, uint32_t rank, uint32_t optFlags, void** request); + ncclResult_t (*iputSignal)(void* rmaCtx, int context, uint64_t srcOff, void* srcMhandle, size_t size, uint64_t dstOff, + void* dstMhandle, uint32_t rank, uint64_t signalOff, void* signalMhandle, + uint64_t signalValue, uint32_t signalOp, bool isStrongSignal, uint32_t optFlags, + void** request); + ncclResult_t (*iget)(void* rmaCtx, int context, uint64_t remoteOff, void* remoteMhandle, size_t size, + uint64_t localOff, void* localMhandle, uint32_t rank, uint32_t optFlags, void** request); + ncclResult_t (*iflush)(void* rmaCtx, int context, void* mhandle, uint32_t rank, void** request); + ncclResult_t (*test)(void* collComm, void* request, int* done); + ncclResult_t (*rmaProgress)(void* rmaCtx); + ncclResult_t (*queryLastError)(void* rmaCtx, bool* hasError); + ncclResult_t (*finalize)(void* ctx); +} ncclRma_v15_t; + +#endif // end include guard diff --git a/projects/rccl/plugins/rma/example/plugin.c b/projects/rccl/plugins/rma/example/plugin.c index f5092bcb0eaa..01b057a3630f 100644 --- a/projects/rccl/plugins/rma/example/plugin.c +++ b/projects/rccl/plugins/rma/example/plugin.c @@ -41,6 +41,7 @@ struct rmaMemHandle { struct rmaRequest { int done; + uint32_t optFlags; }; /* Shared functions */ @@ -100,7 +101,7 @@ __hidden ncclResult_t rmaConnect(void* ctx, void* handles[], int nranks, int ran return ncclSuccess; } -__hidden ncclResult_t rmaCreateContext(void* collComm, ncclRmaConfig_v14_t* config, void** rmaCtxOut) { +__hidden ncclResult_t rmaCreateContext(void* collComm, ncclRmaConfig_v15_t* config, void** rmaCtxOut) { struct rmaCtx* ctx = (struct rmaCtx*)calloc(1, sizeof(*ctx)); if (ctx == NULL) return ncclSystemError; ctx->nContexts = config->nContexts; @@ -153,15 +154,17 @@ __hidden ncclResult_t rmaFinalize(void* ctx) { return ncclSuccess; } -__hidden struct rmaRequest* rmaAllocRequest(void) { - return (struct rmaRequest*)calloc(1, sizeof(struct rmaRequest)); +__hidden struct rmaRequest* rmaAllocRequest(uint32_t optFlags) { + struct rmaRequest* request = (struct rmaRequest*)calloc(1, sizeof(struct rmaRequest)); + if (request != NULL) request->optFlags = optFlags; + return request; } /* Data operations */ __hidden ncclResult_t rmaIput(void* rmaCtx, int context, uint64_t srcOff, void* srcMhandle, size_t size, - uint64_t dstOff, void* dstMhandle, uint32_t rank, void** request) { - struct rmaRequest* r = rmaAllocRequest(); + uint64_t dstOff, void* dstMhandle, uint32_t rank, uint32_t optFlags, void** request) { + struct rmaRequest* r = rmaAllocRequest(optFlags); if (r == NULL) return ncclSystemError; *request = r; return ncclSuccess; @@ -170,23 +173,23 @@ __hidden ncclResult_t rmaIput(void* rmaCtx, int context, uint64_t srcOff, void* __hidden ncclResult_t rmaIputSignal(void* rmaCtx, int context, uint64_t srcOff, void* srcMhandle, size_t size, uint64_t dstOff, void* dstMhandle, uint32_t rank, uint64_t signalOff, void* signalMhandle, - uint64_t signalValue, uint32_t signalOp, bool isStrongSignal, void** request) { - struct rmaRequest* r = rmaAllocRequest(); + uint64_t signalValue, uint32_t signalOp, bool isStrongSignal, uint32_t optFlags, void** request) { + struct rmaRequest* r = rmaAllocRequest(optFlags); if (r == NULL) return ncclSystemError; *request = r; return ncclSuccess; } __hidden ncclResult_t rmaIget(void* rmaCtx, int context, uint64_t remoteOff, void* remoteMhandle, size_t size, - uint64_t localOff, void* localMhandle, uint32_t rank, void** request) { - struct rmaRequest* r = rmaAllocRequest(); + uint64_t localOff, void* localMhandle, uint32_t rank, uint32_t optFlags, void** request) { + struct rmaRequest* r = rmaAllocRequest(optFlags); if (r == NULL) return ncclSystemError; *request = r; return ncclSuccess; } __hidden ncclResult_t rmaIflush(void* rmaCtx, int context, void* mhandle, uint32_t rank, void** request) { - struct rmaRequest* r = rmaAllocRequest(); + struct rmaRequest* r = rmaAllocRequest(ncclRmaOptFlagsDefault); if (r == NULL) return ncclSystemError; *request = r; return ncclSuccess; @@ -204,7 +207,7 @@ __hidden ncclResult_t rmaProgress(void* rmaCtx) { /* Exported plugin struct */ -const ncclRma_v14_t ncclRmaPlugin_v14 = { +const ncclRma_v15_t ncclRmaPlugin_v15 = { .name = "Example", .init = rmaInit, .devices = rmaDevices, diff --git a/projects/rccl/plugins/tuner/example/plugin.c b/projects/rccl/plugins/tuner/example/plugin.c index dd7267745dd4..dec60670b99c 100644 --- a/projects/rccl/plugins/tuner/example/plugin.c +++ b/projects/rccl/plugins/tuner/example/plugin.c @@ -211,11 +211,9 @@ static ncclResult_t loadConfig(TunerContext* ctx, const char* filename) { strncpy(lineCopy, line, sizeof(lineCopy)); lineCopy[sizeof(lineCopy) - 1] = '\0'; - // Tokenize by comma. Use strtok_r (reentrant): strtok keeps parser state in - // a shared static, so concurrent multi-comm init would corrupt each other's - // parse and load a partial config set, causing per-rank algo split-brain. - char* saveptr = NULL; - token = strtok_r(lineCopy, ",", &saveptr); + // Tokenize by comma + char* savePtr = NULL; + token = strtok_r(lineCopy, ",", &savePtr); while (token != NULL && tokenCount < CONFIG_FIELDS_MAX) { // Trim whitespace while (*token == ' ' || *token == '\t') token++; @@ -225,7 +223,7 @@ static ncclResult_t loadConfig(TunerContext* ctx, const char* filename) { end--; } tokens[tokenCount++] = token; - token = strtok_r(NULL, ",", &saveptr); + token = strtok_r(NULL, ",", &savePtr); } // Validate field count: support required fields (8), with pipeOps (9), or with regBuff (10) diff --git a/projects/rccl/src/CMakeLists.txt b/projects/rccl/src/CMakeLists.txt index f14139425974..cb250b9972e2 100644 --- a/projects/rccl/src/CMakeLists.txt +++ b/projects/rccl/src/CMakeLists.txt @@ -95,6 +95,8 @@ set(SRC_FILES diagnostics/device/p2p.cu diagnostics/p2p.cc diagnostics/p2p.h + diagnostics/ib_write_bw.cc + diagnostics/ib_write_bw.h enqueue/enqueue.cc enqueue/mgmt_task_enq.cc enqueue/task_prep/task_classify.cc @@ -117,6 +119,8 @@ set(SRC_FILES gin/proxy_gpucontext/gpucontext.h gin/proxy_gpucontext/gpucontext_v1.cc gin/proxy_gpucontext/gpucontext_v1.h + gin/proxy_gpucontext/gpucontext_v2.cc + gin/proxy_gpucontext/gpucontext_v2.h graph/connect.cc graph/rccl_graph_gen.cc graph/paths.cc @@ -184,6 +188,7 @@ set(SRC_FILES include/config/collconfig.h include/core.h include/cpuset.h + include/crypt.h include/cudawrap.h algorithms/dda/all_gather/dda_all_gather.h algorithms/dda/all_reduce/dda_all_reduce.h @@ -209,11 +214,14 @@ set(SRC_FILES include/gdr_peermem.h include/gdrwrap.h include/gin.h + include/gin/gin_anvil_conn_check.h include/gin/gin_host.h include/gin/gin_host_anvil_sdma.h include/gin/gin_host_proxy.h include/gin/gin_host_rocshmem_gda.h include/gin/gin_rocshmem_gda_factory.h + include/gin/gin_host_anvil_sdma.h + include/gin/gin_anvil_sdma_factory.h include/git_version.h include/graph.h include/group.h @@ -236,11 +244,13 @@ set(SRC_FILES include/latency_profiler/CollTraceFunc.h include/latency_profiler/CollTraceUtils.h include/latency_profiler/EventQueue.h + include/mc_arena.h include/mem_manager.h include/mlx5/mlx5dvcore.h include/mlx5/mlx5dvsymbols.h include/mlx5/mlx5dvwrap.h include/mnnvl.h + include/multicast.h include/nccl_common.h include/nccl_device.h include/nccl_device/barrier.h @@ -266,6 +276,7 @@ set(SRC_FILES include/nccl_device/gin/rocshmem_gda/gin_rocshmem_device_host_common_gda.h include/nccl_device/gin/rocshmem_gda/gin_rocshmem_gda.h include/nccl_device/gin_barrier.h + include/nccl_device/host.h include/nccl_device/impl/barrier__funcs.h include/nccl_device/impl/barrier__types.h include/nccl_device/impl/cft__funcs.h @@ -339,6 +350,7 @@ set(SRC_FILES include/nvtx3/nvtxDetail/nvtxTypes.h include/nvtx_payload_schemas.h include/nvtx_stub.h + include/nvls_ub.h include/os.h include/os/linux.h include/os/windows.h @@ -387,12 +399,14 @@ set(SRC_FILES include/plugin/rma/rma_v13.h include/plugin/rma/rma_v14.h include/plugin/rma/rma_v15.h + include/plugin/rma/rma_v16.h include/plugin/tuner/tuner_v2.h include/plugin/tuner/tuner_v3.h include/plugin/tuner/tuner_v4.h include/plugin/tuner/tuner_v5.h include/plugin/tuner/tuner_v6.h include/profiler.h + include/progress_monitor.h include/proxy.h include/proxy_diag_counters.h include/ras.h @@ -435,6 +449,7 @@ set(SRC_FILES misc/api_trace.cc misc/archinfo.cc misc/argcheck.cc + misc/crypt.cc misc/gdr_peermem.cc misc/gdr_probe.cc misc/ibvsymbols.cc @@ -499,6 +514,7 @@ set(SRC_FILES plugin/rma/rma_v13.cc plugin/rma/rma_v14.cc plugin/rma/rma_v15.cc + plugin/rma/rma_v16.cc plugin/tuner.cc plugin/tuner/csv_tuner.cc plugin/tuner/tuner_v2.cc @@ -517,7 +533,11 @@ set(SRC_FILES ras/diagnostics_checks_common.h ras/diagnostics_env.cc ras/diagnostics_gpu.cc + ras/diagnostics_paths.cc + ras/diagnostics_pci.cc + ras/diagnostics_xid.cc ras/peers.cc + ras/progress_monitor.cc ras/ras.cc ras/ras_internal.h ras/ras_param.cc @@ -539,6 +559,8 @@ set(SRC_FILES transport.cc transport/coll_net.cc transport/generic.cc + transport/mc_arena.cc + transport/multicast.cc transport/net.cc transport/net_telemetry.cc transport/net_ib/common.cc @@ -550,6 +572,7 @@ set(SRC_FILES transport/net_ib/gin.cc transport/net_ib/gin.h transport/net_ib/init.cc + transport/net_ib/net_ib_gid_inspect.cc transport/net_ib/p2p.cc transport/net_ib/p2p.h transport/net_ib/p2p_resiliency.cc @@ -561,10 +584,19 @@ set(SRC_FILES transport/net_ib/wqe_lat_mon.h transport/net_ib_cast/net_ib_cast_inspect.h transport/net_ib_cast/net_ib_fault_inject.h + transport/net_ib_gid_inspect.h transport/net_ib_limits.h transport/net_socket.cc + transport/net_socket/common.cc + transport/net_socket/common.h transport/nvls.cc + transport/nvls_ub.cc transport/p2p.cc + transport/rma_socket.cc + transport/rma_socket.h + transport/rma_socket_mem.cc + transport/rma_socket_progress_recver.cc + transport/rma_socket_setup.cc transport/shm.cc tuning/ce_model.cc tuning/collnet.cc @@ -574,6 +606,12 @@ set(SRC_FILES tuning/pat.cc tuning/ring.cc tuning/sym_model.cc + tuning/sym_model/gin.cc + tuning/sym_model/lsa.cc + tuning/sym_model/lsa_a2a.cc + tuning/sym_model/lsa_a2a.h + tuning/sym_model/lsa_base.cc + tuning/sym_model/model.h tuning/tree.cc tuning/tuning.cc tuning/tuning_general.cc @@ -590,10 +628,13 @@ if(ENABLE_ROCSHMEM_GIN) src/gin/gin_plugin_rocshmem_gda.cc src/gin/gin_rocshmem_gda_factory.cc src/gin/gin_plugin_anvil_sdma.cc + src/gin/gin_anvil_sdma_factory.cc src/gin/gin_anvil_ipc_table_host.cc + src/gin/gin_anvil_conn_check_device.cc src/algorithms/gin/sdma/gin_alltoall_sdma.cu - src/algorithms/gin/sdma/gin_all_reduce_sdma.cu + src/algorithms/gin/sdma/gin_all_reduce_sdma.cu ) + # GIN shim headers: in SRC_FILES so hipify copies them to the build tree. list(APPEND SRC_FILES src/include/nccl_device/gin/rocshmem_gda/log.hpp src/include/nccl_device/gin/rocshmem_gda/util.hpp @@ -601,6 +642,8 @@ if(ENABLE_ROCSHMEM_GIN) src/include/nccl_device/gin/rocshmem_gda/gda/ibv_core.hpp src/include/nccl_device/gin/rocshmem_gda/rocshmem/rocshmem_common.hpp ) + # anvil.cpp is compiled from rocshmem source via target_sources after + # add_library (bypasses SRC_FILES/hipify). See block after add_library. endif() if(USE_AMDSMI) @@ -807,8 +850,10 @@ add_dependencies(hipify_all nccl_device_header_symlinks) # param/common.h vs device/common.h, net_ib/p2p.h vs p2p.h). add_file_unique() gives # the later-listed one the _tmp name, but consumers such as alloc.h still # #include "utils.h", so symlink the un-suffixed names back, same as above. +# NCCL 2.32 added transport/net_socket/common.h (included by net_socket/common.cc and rma_socket.cc). set(COLLIDING_HEADER_SYMLINKS "") -foreach(_hdr include/utils include/p2p include/param/common include/param/param ras/diagnostics) +foreach(_hdr include/utils include/p2p include/param/common include/param/param ras/diagnostics + transport/net_socket/common) get_filename_component(_dir "${HIPIFY_DIR}/src/${_hdr}" DIRECTORY) get_filename_component(_name "${_hdr}" NAME) add_custom_command( @@ -921,6 +966,7 @@ if(ENABLE_DEVICE_LINKER) # needs full HIP compilation rather than --offload-host-only. list(FILTER HIP_SOURCES EXCLUDE REGEX "gin_alltoall_sdma\\.cu\\.cpp$") list(FILTER HIP_SOURCES EXCLUDE REGEX "gin_all_reduce_sdma\\.cu\\.cpp$") + list(FILTER HIP_SOURCES EXCLUDE REGEX "gin_anvil_conn_check_device\\.cc$") list(FILTER HIP_SOURCES EXCLUDE REGEX "dda_all_reduce_fabric\\.cu\\.cpp$") list(FILTER HIP_SOURCES EXCLUDE REGEX "dda_all_reduce_fabric_ll\\.cu\\.cpp$") list(FILTER HIP_SOURCES EXCLUDE REGEX "dda_all_reduce_fabric_ll128\\.cu\\.cpp$") @@ -1054,6 +1100,10 @@ if (ENABLE_ROCSHMEM OR ENABLE_ROCSHMEM_GIN) target_include_directories(rccl PRIVATE ${ROCSHMEM_INCLUDE_DIR}) endif() + # GDA factory (gin_rocshmem_gda_factory.cc) directly includes rocshmem + # host headers (ibv_wrapper, topology, envvar, debug_gda) that are not + # yet shimmed or copied. These paths will be removed when GDA host + # dependencies are severed (like SDMA was). if (ROCSHMEM_SOURCE_DIR) target_include_directories(rccl PRIVATE ${ROCSHMEM_SOURCE_DIR}/src) target_include_directories(rccl PRIVATE ${ROCSHMEM_SOURCE_DIR}/include) @@ -1141,6 +1191,57 @@ if (ENABLE_ROCSHMEM_GIN) ) add_custom_target(copy_rocshmem_source_headers DEPENDS ${GIN_GDA_ROCSHMEM_SOURCE_HEADERS}) add_dependencies(copy_rocshmem_headers copy_rocshmem_source_headers) + + # SDMA anvil host code: copied into build tree (no sed — host code needs + # unqualified log.hpp to find the host shim via include path ordering). + set(_GIN_SDMA_DIR "${PROJECT_BINARY_DIR}/gin/sdma") + foreach(_f anvil.cpp anvil.hpp) + add_custom_command( + OUTPUT "${_GIN_SDMA_DIR}/${_f}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${_GIN_SDMA_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${ROCSHMEM_SOURCE_DIR}/src/sdma/${_f}" "${_GIN_SDMA_DIR}/${_f}" + # Qualify device header includes so they resolve from the installed + # layout (nccl_device/gin/anvil_sdma/sdma/) without extra include dirs. + COMMAND sed -i + "s|#include \"anvil_device\\.hpp\"|#include \"nccl_device/gin/anvil_sdma/sdma/anvil_device.hpp\"|g" + "${_GIN_SDMA_DIR}/${_f}" + COMMAND sed -i + "s|#include \"sdma_pkt_struct\\.h\"|#include \"nccl_device/gin/anvil_sdma/sdma/sdma_pkt_struct.h\"|g" + "${_GIN_SDMA_DIR}/${_f}" + COMMAND sed -i + "s|#include \"sdma_pkt_struct_mi4\\.h\"|#include \"nccl_device/gin/anvil_sdma/sdma/sdma_pkt_struct_mi4.h\"|g" + "${_GIN_SDMA_DIR}/${_f}" + COMMAND sed -i + "s|#include \"sdma_opcodes\\.h\"|#include \"nccl_device/gin/anvil_sdma/sdma/sdma_opcodes.h\"|g" + "${_GIN_SDMA_DIR}/${_f}" + DEPENDS "${ROCSHMEM_SOURCE_DIR}/src/sdma/${_f}" + COMMENT "Copying ${_f} -> ${_GIN_SDMA_DIR}/${_f}" + VERBATIM) + list(APPEND _GIN_SDMA_OUTPUTS "${_GIN_SDMA_DIR}/${_f}") + endforeach() + add_custom_target(copy_rocshmem_sdma_sources DEPENDS ${_GIN_SDMA_OUTPUTS}) + add_dependencies(copy_rocshmem_headers copy_rocshmem_sdma_sources) + target_sources(rccl PRIVATE "${_GIN_SDMA_DIR}/anvil.cpp") + # src/gin/ so host log.hpp wins over the device shim for anvil.cpp. + target_include_directories(rccl BEFORE PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/gin") + + # Copy SDMA device headers next to gin_anvil_sdma.h so "sdma/anvil_device.hpp" + # resolves during build (same layout as the install tree). + set(_SDMA_DEV_DST "${PROJECT_BINARY_DIR}/include/nccl_device/gin/anvil_sdma/sdma") + foreach(_f anvil_device.hpp sdma_opcodes.h sdma_pkt_struct.h sdma_pkt_struct_mi4.h) + add_custom_command( + OUTPUT "${_SDMA_DEV_DST}/${_f}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${_SDMA_DEV_DST}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${ROCSHMEM_SOURCE_DIR}/src/sdma/${_f}" "${_SDMA_DEV_DST}/${_f}" + DEPENDS "${ROCSHMEM_SOURCE_DIR}/src/sdma/${_f}" + COMMENT "Copying sdma/${_f} -> ${_SDMA_DEV_DST}/${_f}" + VERBATIM) + list(APPEND _SDMA_DEV_OUTPUTS "${_SDMA_DEV_DST}/${_f}") + endforeach() + add_custom_target(copy_sdma_device_headers DEPENDS ${_SDMA_DEV_OUTPUTS}) + add_dependencies(copy_rocshmem_headers copy_sdma_device_headers) endif() # Add rule to install ibvcore.h so that it can be found by ibv_core.hpp @@ -1299,28 +1400,63 @@ if(ENABLE_CODE_COVERAGE) message(STATUS "Code coverage is enabled with build type '${CMAKE_BUILD_TYPE}'.") + # Host-only and host+device coverage instrument with the same two flags and + # differ only in whether the instrumentation is restricted to host code. + # Device coverage (ROCm 7.15+) drops -Xarch_host so the flags also reach the + # device compilation; host-only keeps -Xarch_host so device code is untouched. + if(ENABLE_FULL_COVERAGE) + message(STATUS "Device-side coverage is enabled (host + device instrumented).") + set(_coverage_instr_flags + -fprofile-instr-generate + -fcoverage-mapping) + else() + set(_coverage_instr_flags + "SHELL:-Xarch_host -fprofile-instr-generate" + "SHELL:-Xarch_host -fcoverage-mapping") + endif() target_compile_options(rccl PRIVATE -fvisibility=default - "SHELL:-Xarch_host -fprofile-instr-generate" - "SHELL:-Xarch_host -fcoverage-mapping") + ${_coverage_instr_flags}) set(COVERAGE_SHARED_LINKER_FLAGS -fprofile-generate -Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN ) - set(COVERAGE_EXE_LINKER_FLAGS - -fprofile-generate - -Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN/../lib - ) - target_link_options(rccl PRIVATE ${COVERAGE_SHARED_LINKER_FLAGS}) - target_link_options(rccl PRIVATE ${COVERAGE_EXE_LINKER_FLAGS}) # Expose explicit profile-flush wrappers so test harnesses that bypass # atexit() handlers (e.g. ProcessIsolatedTestRunner using _exit()) can # still flush librccl's private LLVM profile runtime instance. target_sources(rccl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/misc/coverage_flush.cc) + + # Force-link the HIP device-data drain symbol so the host profile RT's + # InstrProfilingPlatformROCm.cpp.o gets pulled in (it's only included on + # demand). That .o registers the atexit hook which walks loaded HSA code + # objects, reads each device.elf's __llvm_prf_cnts via HSA introspection, + # and merges device counters into the same .profraw the host writes. + # Without this, .profraw contains host counters only and llvm-cov reports + # 0% on every device kernel. + # + # We add it on librccl's link rather than relying on the clang driver + # heuristic (the driver only force-links it for explicit `-x hip` link + # invocations; .o-only links from rccl-tests and friends don't qualify). + # One reference anywhere in the process is sufficient — once librccl is + # loaded, the atexit handler is registered for the whole process. + if(ENABLE_FULL_COVERAGE) + # Newer compiler-rt packages keep the ROCm collector and its interception + # dependencies in a self-contained companion archive. Explicitly link it + # for this object-only host link; older packages leave this path empty and + # continue to use the generic profile runtime. + if(RCCL_HOST_ROCM_PROFILE_RT) + message(STATUS "Host ROCm profile runtime: ${RCCL_HOST_ROCM_PROFILE_RT}") + target_link_libraries(rccl PRIVATE "${RCCL_HOST_ROCM_PROFILE_RT}") + else() + message(STATUS "Host ROCm profile runtime: none found, using the generic profile runtime") + endif() + target_link_options(rccl PRIVATE + "LINKER:-u,__llvm_profile_hip_collect_device_data") + endif() elseif(BUILD_TESTS) # Enable default/hidden visibility based on build type and ROCM_VERSION # MPI tests include internal headers (comm.h, ce_coll, mem_manager, transports) # and link those symbols. Debug already exports them; Release must too when @@ -1403,6 +1539,26 @@ if(RCCL_ROCPROFILER_REGISTER) target_link_libraries( rccl PRIVATE rocprofiler-register::rocprofiler-register) endif() + +# Poison the __hip_atomic_* builtins +#================================================================================================== +# RCCL spells device atomics with the Clang scoped-atomic builtins so that memory +# scope is explicit at every call site. Force-include (rather than #include) the +# poison header so it also covers hipified and generated sources, and set it here, +# before DeviceLinker.cmake snapshots COMPILE_OPTIONS, so the per-kernel device +# compiles inherit it too. See src/include/poison_hip_atomics.h for the coverage +# this does and does not give. +if(RCCL_POISON_HIP_ATOMICS) + # Joined --include= rather than separate-argument -include : as a + # single token it needs no cooperation from anything that forwards flags one + # at a time, so it survives rccl-device-compile's catch-all for unrecognized + # `-*` tokens as-is. That script also parses the separate-argument spelling + # (see the -include/-imacros branch in parse_compiler_flags), so either form + # works today; the joined one is used because it does not depend on that. + target_compile_options(rccl PRIVATE --include=${RCCL_SOURCE_DIR}/src/include/poison_hip_atomics.h) + message(STATUS "RCCL_POISON_HIP_ATOMICS enabled - __hip_atomic_* builtins rejected in RCCL sources") +endif() + # Device Linker: include pipeline and wire up dependencies # (placed after all target_compile_definitions so DeviceLinker.cmake # can read the rccl target's definitions directly) @@ -1495,7 +1651,6 @@ if(ENABLE_ROCSHMEM) # of the final librccl.so link. target_link_libraries(rccl PRIVATE ${ROCSHMEM_LIBRARY}) target_link_libraries(rccl PRIVATE ${HSAKMT_LIBRARY}) - target_link_libraries(rccl PRIVATE ${IBVERBS}) set(_rocshmem_offload_arch_flags "") foreach(_gpu_raw ${GPU_TARGETS}) string(REGEX REPLACE ":.*" "" _gpu "${_gpu_raw}") @@ -1508,15 +1663,10 @@ elseif(ENABLE_ROCSHMEM_GIN) # With the device linker (default), per-arch .bc files are injected directly # into the per-arch device link step via DeviceLinker.cmake. # - # Pull the SDMA host objects out of librocshmem.a so apps running only SDMA do - # not have to link it. Seeding by symbol rather than by file survives renames. - # GDA stays unresolved, as it needs constmem registration this mode cannot do. + # Seed rocshmem host symbols needed by GDA into librccl.so. SDMA factory + # symbols are compiled in directly (gin_anvil_sdma_factory.cc) and no longer + # need pulling from librocshmem.a. set(_gin_rocshmem_syms - gin_anvil_sdma_probe - gin_anvil_sdma_create - gin_anvil_sdma_destroy - gin_anvil_sdma_get_num_channels - gin_anvil_sdma_get_channel_stride _ZN8rocshmem3ibvE) set(_gin_rocshmem_undefine "") foreach(_sym ${_gin_rocshmem_syms}) diff --git a/projects/rccl/src/Makefile b/projects/rccl/src/Makefile index 0ca38d679091..25b0aaf4a9dd 100644 --- a/projects/rccl/src/Makefile +++ b/projects/rccl/src/Makefile @@ -22,6 +22,7 @@ LIBSRCFILES := \ $(wildcard enqueue/task_sched/*.cc) \ $(wildcard misc/*.cc) \ $(wildcard transport/*.cc) \ + $(wildcard transport/net_socket/*.cc) \ $(wildcard transport/net_ib/*.cc) \ $(wildcard transport/net_ib/gdaki/*.cc) \ $(wildcard transport/net_ib/gdaki/gpucontext/*.cc) \ @@ -40,6 +41,7 @@ LIBSRCFILES := \ $(wildcard rma/*.cc) \ $(wildcard devcomm/*.cc) \ $(wildcard tuning/*.cc) \ + $(wildcard tuning/sym_model/*.cc) \ $(filter-out param/ncclparam.cc,$(wildcard param/*.cc)) \ $(filter-out ras/client.cc,$(wildcard ras/*.cc)) BINSRCFILES := ras/client.cc @@ -290,8 +292,8 @@ $(OBJDIR)/misc/git_version.o: $(GIT_VERSION_FILE) $(DOCA_OBJDIR)/%.o : $(DOCA_HOME)/src/%.cpp @printf "Compiling %-35s > %s\n" $< $@ mkdir -p `dirname $@` - $(CXX) -I$(DOCA_HOME)/src -I$(DOCA_HOME)/include $(CXXFLAGS) -c $< -o $@ - @$(CXX) -I$(DOCA_HOME)/src -I$(DOCA_HOME)/include $(CXXFLAGS) -MM $< > $(@:%.o=%.d.tmp) + $(CXX) -Iinclude -I$(DOCA_HOME)/src -I$(DOCA_HOME)/include $(CXXFLAGS) -c $< -o $@ + @$(CXX) -Iinclude -I$(DOCA_HOME)/src -I$(DOCA_HOME)/include $(CXXFLAGS) -MM $< > $(@:%.o=%.d.tmp) @sed "0,/^.*:/s//$(subst /,\/,$@):/" $(@:%.o=%.d.tmp) > $(@:%.o=%.d) @sed -e 's/.*://' -e 's/\\$$//' < $(@:%.o=%.d.tmp) | fmt -1 | \ sed -e 's/^ *//' -e 's/$$/:/' >> $(@:%.o=%.d) diff --git a/projects/rccl/src/algorithms/dda/all_gather/all_gather_dda_fabric_ll128.h b/projects/rccl/src/algorithms/dda/all_gather/all_gather_dda_fabric_ll128.h index b6d9f20450d8..1b8eea61c8ab 100644 --- a/projects/rccl/src/algorithms/dda/all_gather/all_gather_dda_fabric_ll128.h +++ b/projects/rccl/src/algorithms/dda/all_gather/all_gather_dda_fabric_ll128.h @@ -117,7 +117,7 @@ __launch_bounds__(1024) ddaLL128PollWire(gatherSlot + s * (size_t)kDdaLL128WireWordsPerSlice + 2 * lane, vr, flag, lane); ddaLL128StoreRegs(dstBytes + dataByte, vr, eltInSlice, lane, flagLane); } -#if defined(__gfx1250__) +#if (defined(__gfx1250__) || defined(__gfx1250_strict__)) asm volatile("s_wait_storecnt 0x0" ::: "memory"); #endif __syncthreads(); diff --git a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric.cu b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric.cu index 63f59b3865ab..704b4e65ae91 100644 --- a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric.cu +++ b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric.cu @@ -15,6 +15,7 @@ #include "algorithms/dda/fabric/fabric_gpu_barrier.h" #include +#include #include #include @@ -60,24 +61,25 @@ static ncclResult_t ncclAllGatherDdaFabricTyped(const void* sendbuff, void* recv INFO(NCCL_COLL, "DDA fabric AllGather: launching kernel: nRanks=%d sendcount=%zu grid=%u block=%u%s", nRanks, sendcount, grid.x, block.x, (nRanks == 4 || nRanks == 8) ? " (unrolled)" : " (runtime)"); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // Specialize the kernel for common clique sizes (compile-time NRANKS -> the // unrolled CollCommon allGather), and fall back to the runtime kernel // (NRANKS_CT == 0) for any other size. switch (nRanks) { case 4: - dda::common::ddaAllGatherFabric<<>>(d_ipcbuffs, static_cast(recvbuff), sendcount, - static_cast(sendbuff), comm->rank, - nRanks, barrierHost); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabric), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), sendcount, + static_cast(sendbuff), comm->rank, nRanks, barrierHost); break; case 8: - dda::common::ddaAllGatherFabric<<>>(d_ipcbuffs, static_cast(recvbuff), sendcount, - static_cast(sendbuff), comm->rank, - nRanks, barrierHost); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabric), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), sendcount, + static_cast(sendbuff), comm->rank, nRanks, barrierHost); break; default: - dda::common::ddaAllGatherFabric<<>>(d_ipcbuffs, static_cast(recvbuff), sendcount, - static_cast(sendbuff), comm->rank, - nRanks, barrierHost); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabric), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), sendcount, + static_cast(sendbuff), comm->rank, nRanks, barrierHost); break; } diff --git a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu index fde723910fed..32f0036154fc 100644 --- a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu +++ b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu @@ -16,6 +16,7 @@ #include "algorithms/dda/fabric/fabric_gpu_barrier.h" // dda::common::kDdaMaxNranks #include +#include #include #include @@ -78,22 +79,23 @@ static ncclResult_t ncclAllGatherDdaFabricLLTyped( INFO(NCCL_COLL, "DDA fabric AllGather LL: nRanks=%d perRankBytes=%zu grid=%ux%u block=%u (block-per-peer, bpp=%u)", nRanks, perRankBytes, grid.x, grid.y, block.x, blocksPerPeer); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled; 0: runtime fallback. switch (nRanks) { case 4: - dda::common::ddaAllGatherFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), perRankBytes, - comm->rank, nRanks, epochDev, epochLen, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perRankBytes, comm->rank, nRanks, epochDev, epochLen, bankSize); break; case 8: - dda::common::ddaAllGatherFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), perRankBytes, - comm->rank, nRanks, epochDev, epochLen, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perRankBytes, comm->rank, nRanks, epochDev, epochLen, bankSize); break; default: - dda::common::ddaAllGatherFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), perRankBytes, - comm->rank, nRanks, epochDev, epochLen, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perRankBytes, comm->rank, nRanks, epochDev, epochLen, bankSize); break; } diff --git a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu index a4518f654dad..25a7f24c0171 100644 --- a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu +++ b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu @@ -15,6 +15,7 @@ #include "rccl_common.h" #include +#include #include #include @@ -102,22 +103,23 @@ static ncclResult_t ncclAllGatherDdaFabricLL128Typed( nRanks, perRankBytes, slices, grid.x, grid.y, block.x, blocksPerPeer, ddaLL128AgSlotWords(bankSize, nRanks)); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled; 0: runtime fallback. switch (nRanks) { case 4: - dda::common::ddaAllGatherFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), perRankBytes, - comm->rank, nRanks, epochDev, epochLen, slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabricLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perRankBytes, comm->rank, nRanks, epochDev, epochLen, slices, bankSize); break; case 8: - dda::common::ddaAllGatherFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), perRankBytes, - comm->rank, nRanks, epochDev, epochLen, slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabricLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perRankBytes, comm->rank, nRanks, epochDev, epochLen, slices, bankSize); break; default: - dda::common::ddaAllGatherFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), perRankBytes, - comm->rank, nRanks, epochDev, epochLen, slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherFabricLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perRankBytes, comm->rank, nRanks, epochDev, epochLen, slices, bankSize); break; } diff --git a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_ipc.cu b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_ipc.cu index b0efe4fd70e0..0a7a3449c3ed 100644 --- a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_ipc.cu +++ b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_ipc.cu @@ -15,6 +15,7 @@ #include "algorithms/dda/dda_init_detail.h" #include +#include #include #include @@ -59,8 +60,10 @@ static ncclResult_t ncclAllGatherDdaIpcTyped(const void* sendbuff, void* recvbuf void* peerPtrsDev = comm->ddaPeerPtrsDev; T** d_ipcbuffs = reinterpret_cast(peerPtrsDev); - dda::common::ddaAllGatherIpc<<>>( - d_ipcbuffs, static_cast(recvbuff), sendcount, static_cast(sendbuff), comm->rank, barrierHost); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); + hipExtLaunchKernelGGL((dda::common::ddaAllGatherIpc), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + sendcount, static_cast(sendbuff), comm->rank, barrierHost); CUDACHECK(cudaGetLastError()); diff --git a/projects/rccl/src/algorithms/dda/all_reduce/all_reduce_dda_ll128.h b/projects/rccl/src/algorithms/dda/all_reduce/all_reduce_dda_ll128.h index 67a3af54b5a6..758f0fe9417a 100644 --- a/projects/rccl/src/algorithms/dda/all_reduce/all_reduce_dda_ll128.h +++ b/projects/rccl/src/algorithms/dda/all_reduce/all_reduce_dda_ll128.h @@ -135,7 +135,7 @@ __launch_bounds__(512) ddaLL128StoreRegs(dstBytes + dataByte, acc, eltInSlice, lane, flagLane); } -#if defined(__gfx1250__) +#if defined(__gfx1250__) || defined(__gfx1250_strict__) asm volatile("s_wait_storecnt 0x0" ::: "memory"); #endif diff --git a/projects/rccl/src/algorithms/dda/all_reduce/all_reduce_dda_ll128_twoshot.h b/projects/rccl/src/algorithms/dda/all_reduce/all_reduce_dda_ll128_twoshot.h index fef4d3f75635..c0b38010c49a 100644 --- a/projects/rccl/src/algorithms/dda/all_reduce/all_reduce_dda_ll128_twoshot.h +++ b/projects/rccl/src/algorithms/dda/all_reduce/all_reduce_dda_ll128_twoshot.h @@ -207,7 +207,7 @@ __launch_bounds__(512) } } -#if defined(__gfx1250__) +#if defined(__gfx1250__) || defined(__gfx1250_strict__) asm volatile("s_wait_storecnt 0x0" ::: "memory"); #endif diff --git a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric.cu b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric.cu index 9581791d29b4..25f4fc7102aa 100644 --- a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric.cu +++ b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric.cu @@ -15,6 +15,7 @@ #include "algorithms/dda/fabric/fabric_gpu_barrier.h" #include +#include #include #include @@ -74,48 +75,49 @@ static ncclResult_t ncclAllReduceDdaFabricTyped(const void* sendbuff, void* recv treeOk ? "tree (two-shot)" : "flat (one-shot)", nRanks, count, sizeBytes, grid.x, block.x, (nRanks == 4 || nRanks == 8) ? " (unrolled)" : " (runtime)"); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); if (treeOk) { CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, count * sizeof(T), cudaMemcpyDeviceToDevice, stream)); // NRANKS_CT 4/8: unrolled CollCommon reduce; 0: runtime fallback. switch (nRanks) { case 4: INFO(NCCL_COLL, "DDA fabric AllReduce: tree path, NRANKS_CT=4 (unrolled)"); - dda::common::ddaAllReduceTreeFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTreeFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + count, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; case 8: INFO(NCCL_COLL, "DDA fabric AllReduce: tree path, NRANKS_CT=8 (unrolled)"); - dda::common::ddaAllReduceTreeFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTreeFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + count, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; default: INFO(NCCL_COLL, "DDA fabric AllReduce: tree path, NRANKS_CT=0 (runtime, nRanks=%d)", nRanks); - dda::common::ddaAllReduceTreeFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTreeFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + count, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; } } else { switch (nRanks) { case 4: INFO(NCCL_COLL, "DDA fabric AllReduce: flat path, NRANKS_CT=4 (unrolled)"); - dda::common::ddaAllReduceFlatFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + count, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; case 8: INFO(NCCL_COLL, "DDA fabric AllReduce: flat path, NRANKS_CT=8 (unrolled)"); - dda::common::ddaAllReduceFlatFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + count, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; default: INFO(NCCL_COLL, "DDA fabric AllReduce: flat path, NRANKS_CT=0 (runtime, nRanks=%d)", nRanks); - dda::common::ddaAllReduceFlatFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + count, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; } } diff --git a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll.cu b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll.cu index 4623dd8b67c6..f6870fa12937 100644 --- a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll.cu +++ b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll.cu @@ -28,6 +28,7 @@ #include "param.h" #include +#include #include #include @@ -97,23 +98,24 @@ static ncclResult_t ncclAllReduceDdaFabricLLTyped(const void* sendbuff, void* re INFO(NCCL_COLL, "DDA fabric AllReduce LL: nRanks=%d bytes=%zu nPk=%zu grid=%u block=%u", nRanks, bytes, nPk, grid.x, block.x); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled reduce loop; 0: runtime fallback (up to // kDdaMaxNranks). switch (nRanks) { case 4: - dda::common::ddaAllReduceFlatLL<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, nRanks, epochDev, epochLen, - bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, bankSize); break; case 8: - dda::common::ddaAllReduceFlatLL<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, nRanks, epochDev, epochLen, - bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, bankSize); break; default: - dda::common::ddaAllReduceFlatLL<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, nRanks, epochDev, epochLen, - bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, bankSize); break; } @@ -158,22 +160,23 @@ static ncclResult_t ncclAllReduceDdaFabricLL128OneShotTyped(const void* sendbuff "(warp-per-slice, bankSize=%zu)", nRanks, bytes, slices, grid.x, block.x, bankSize); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled reduce loop; 0: runtime fallback. switch (nRanks) { case 4: - dda::common::ddaAllReduceFlatLL128<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), bytes, comm->rank, nRanks, epochDev, epochLen, - slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + bytes, comm->rank, nRanks, epochDev, epochLen, slices, bankSize); break; case 8: - dda::common::ddaAllReduceFlatLL128<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), bytes, comm->rank, nRanks, epochDev, epochLen, - slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + bytes, comm->rank, nRanks, epochDev, epochLen, slices, bankSize); break; default: - dda::common::ddaAllReduceFlatLL128<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), bytes, comm->rank, nRanks, epochDev, epochLen, - slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + bytes, comm->rank, nRanks, epochDev, epochLen, slices, bankSize); break; } @@ -216,22 +219,26 @@ static ncclResult_t ncclAllReduceDdaFabricLL128TwoShotTyped(const void* sendbuff "(warp-per-slice, bankSize=%zu)", nRanks, bytes, shardBytes, slices, grid.x, block.x, bankSize); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled peer loops; 0: runtime fallback. switch (nRanks) { case 4: - dda::common::ddaAllReduceTwoShotLL128<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), shardBytes, comm->rank, nRanks, epochDev, - epochLen, slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTwoShotLL128), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, peers, static_cast(recvbuff), + static_cast(sendbuff), shardBytes, comm->rank, nRanks, epochDev, epochLen, slices, + bankSize); break; case 8: - dda::common::ddaAllReduceTwoShotLL128<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), shardBytes, comm->rank, nRanks, epochDev, - epochLen, slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTwoShotLL128), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, peers, static_cast(recvbuff), + static_cast(sendbuff), shardBytes, comm->rank, nRanks, epochDev, epochLen, slices, + bankSize); break; default: - dda::common::ddaAllReduceTwoShotLL128<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), shardBytes, comm->rank, nRanks, epochDev, - epochLen, slices, bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTwoShotLL128), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, peers, static_cast(recvbuff), + static_cast(sendbuff), shardBytes, comm->rank, nRanks, epochDev, epochLen, slices, + bankSize); break; } @@ -273,23 +280,24 @@ static ncclResult_t ncclAllReduceDdaFabricLLTwoShotTyped(const void* sendbuff, v INFO(NCCL_COLL, "DDA fabric AllReduce LL two-shot: nRanks=%d bytes=%zu nPk=%zu grid=%u block=%u", nRanks, bytes, nPk, grid.x, block.x); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled reduce loop; 0: runtime fallback (up to // kDdaMaxNranks). switch (nRanks) { case 4: - dda::common::ddaAllReduceTwoShotLL<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, nRanks, epochDev, epochLen, - bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTwoShotLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, bankSize); break; case 8: - dda::common::ddaAllReduceTwoShotLL<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, nRanks, epochDev, epochLen, - bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTwoShotLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, bankSize); break; default: - dda::common::ddaAllReduceTwoShotLL<<>>( - peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, nRanks, epochDev, epochLen, - bankSize); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTwoShotLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, bankSize); break; } diff --git a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu index 6f31d12bd4d7..ae0745ea5d29 100644 --- a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu +++ b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu @@ -19,6 +19,7 @@ #include "rccl_common.h" #include +#include #include #include @@ -101,22 +102,23 @@ static ncclResult_t ncclAllReduceDdaFabricLL128Typed(const void* sendbuff, void* INFO(NCCL_COLL, "DDA fabric AllReduce LL128: nRanks=%d bytes=%zu numLines=%zu grid=%u block=%u", nRanks, bytes, numLines, grid.x, block.x); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled reduce loop; 0: runtime fallback. switch (nRanks) { case 4: - dda::common::ddaAllReduceFlatLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, - nRanks, epochDev, epochLen, slotStrideLines); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, slotStrideLines); break; case 8: - dda::common::ddaAllReduceFlatLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, - nRanks, epochDev, epochLen, slotStrideLines); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, slotStrideLines); break; default: - dda::common::ddaAllReduceFlatLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), count, comm->rank, - nRanks, epochDev, epochLen, slotStrideLines); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + count, comm->rank, nRanks, epochDev, epochLen, slotStrideLines); break; } diff --git a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_ipc.cu b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_ipc.cu index ed229320fbaa..fd6772369c44 100644 --- a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_ipc.cu +++ b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_ipc.cu @@ -15,6 +15,7 @@ #include "algorithms/dda/dda_init_detail.h" #include +#include #include #include @@ -70,13 +71,16 @@ static ncclResult_t ncclAllReduceDdaIpcTyped(const void* sendbuff, void* recvbuf void* peerPtrsDev = comm->ddaPeerPtrsDev; T** d_ipcbuffs = reinterpret_cast(peerPtrsDev); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); if (treeOk) { CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, count * sizeof(T), cudaMemcpyDeviceToDevice, stream)); - dda::common::ddaAllReduceTreeIpc<<>>( - d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), comm->rank, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceTreeIpc), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), count, + static_cast(sendbuff), comm->rank, barrierHost, nullptr); } else { - dda::common::ddaAllReduceFlatIpc<<>>( - d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), comm->rank, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaAllReduceFlatIpc), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), count, + static_cast(sendbuff), comm->rank, barrierHost, nullptr); } CUDACHECK(cudaGetLastError()); diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric.cu b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric.cu index 8c5e284a1819..0c7799ea81dd 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric.cu +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric.cu @@ -15,6 +15,7 @@ #include "algorithms/dda/fabric/fabric_gpu_barrier.h" #include +#include #include #include @@ -66,18 +67,22 @@ static ncclResult_t ncclAllToAllDdaFabricTyped(const void* sendbuff, void* recvb // the fabric path. CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, totalBytes, cudaMemcpyDeviceToDevice, stream)); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); switch (nRanks) { case 4: - dda::common::ddaAllToAllFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, comm->rank, nRanks, barrierHost); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabric), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), count, comm->rank, nRanks, + barrierHost); break; case 8: - dda::common::ddaAllToAllFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, comm->rank, nRanks, barrierHost); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabric), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), count, comm->rank, nRanks, + barrierHost); break; default: - dda::common::ddaAllToAllFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), count, comm->rank, nRanks, barrierHost); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabric), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), count, comm->rank, nRanks, + barrierHost); break; } diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu index 1c4fb1e6d597..74851447d7c3 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu @@ -17,6 +17,7 @@ #include "algorithms/dda/fabric/fabric_gpu_barrier.h" // dda::common::kDdaMaxNranks #include +#include #include #include @@ -79,21 +80,22 @@ static ncclResult_t ncclAllToAllDdaFabricLLTyped( INFO(NCCL_COLL, "DDA fabric AllToAll LL: nRanks=%d perChunkBytes=%zu grid=%ux%u block=%u (block-per-peer, bpp=%d)", nRanks, perChunkBytes, grid.x, grid.y, block.x, blocksPerPeer); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); switch (nRanks) { case 4: - dda::common::ddaAllToAllFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), perChunkBytes, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perChunkBytes, comm->rank, nRanks, epochDev, epochLen); break; case 8: - dda::common::ddaAllToAllFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), perChunkBytes, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perChunkBytes, comm->rank, nRanks, epochDev, epochLen); break; default: - dda::common::ddaAllToAllFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), perChunkBytes, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perChunkBytes, comm->rank, nRanks, epochDev, epochLen); break; } diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu index dced800b4d38..e4090ffa9ae2 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu @@ -18,6 +18,7 @@ #include "param.h" #include +#include #include #include @@ -119,21 +120,22 @@ static ncclResult_t ncclAllToAllDdaFabricLL128Typed( INFO(NCCL_COLL, "DDA fabric AllToAll LL128: nRanks=%d perChunkBytes=%zu grid=%ux%u block=%u (block-per-peer, bpp=%d)", nRanks, perChunkBytes, grid.x, grid.y, block.x, blocksPerPeer); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); switch (nRanks) { case 4: - dda::common::ddaAllToAllFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), perChunkBytes, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabricLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perChunkBytes, comm->rank, nRanks, epochDev, epochLen); break; case 8: - dda::common::ddaAllToAllFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), perChunkBytes, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabricLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perChunkBytes, comm->rank, nRanks, epochDev, epochLen); break; default: - dda::common::ddaAllToAllFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), perChunkBytes, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllFabricLL128), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + perChunkBytes, comm->rank, nRanks, epochDev, epochLen); break; } diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_ipc.cu b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_ipc.cu index 51eb29871a05..de3ddec3b026 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_ipc.cu +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_ipc.cu @@ -15,6 +15,7 @@ #include "algorithms/dda/dda_init_detail.h" #include +#include #include #include @@ -60,13 +61,16 @@ static ncclResult_t ncclAllToAllDdaIpcTyped(const void* sendbuff, void* recvbuff void* peerPtrsDev = comm->ddaPeerPtrsDev; T** d_ipcbuffs = reinterpret_cast(peerPtrsDev); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); if (dda::common::ddaAlltoAllSingleBlockGrid(count, sizeof(T))) { - dda::common::ddaAllToAllIpc<<>>( - d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), comm->rank, barrierHost); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllIpc), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), count, + static_cast(sendbuff), comm->rank, barrierHost); } else { CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, totalBytes, cudaMemcpyDeviceToDevice, stream)); - dda::common::ddaAllToAllIpc<<>>( - d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), comm->rank, barrierHost); + hipExtLaunchKernelGGL((dda::common::ddaAllToAllIpc), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), count, + static_cast(sendbuff), comm->rank, barrierHost); } CUDACHECK(cudaGetLastError()); diff --git a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric.cu b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric.cu index bd0c5c0553e4..4724c2de6fe9 100644 --- a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric.cu +++ b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric.cu @@ -15,6 +15,7 @@ #include "algorithms/dda/fabric/fabric_gpu_barrier.h" #include +#include #include #include @@ -58,21 +59,22 @@ static ncclResult_t ncclReduceScatterDdaFabricTyped(const void* sendbuff, void* INFO(NCCL_COLL, "DDA fabric ReduceScatter: launching kernel: nRanks=%d recvcount=%zu grid=%u block=%u%s", nRanks, recvcount, grid.x, block.x, (nRanks == 4 || nRanks == 8) ? " (unrolled)" : " (runtime)"); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); switch (nRanks) { case 4: - dda::common::ddaReduceScatterFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), recvcount, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + recvcount, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; case 8: - dda::common::ddaReduceScatterFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), recvcount, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + recvcount, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; default: - dda::common::ddaReduceScatterFabric - <<>>(d_ipcbuffs, static_cast(recvbuff), recvcount, static_cast(sendbuff), - comm->rank, nRanks, barrierHost, nullptr); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabric), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + recvcount, static_cast(sendbuff), comm->rank, nRanks, barrierHost, nullptr); break; } diff --git a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu index 7573a48aef22..f158db54fa2d 100644 --- a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu +++ b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu @@ -16,6 +16,7 @@ #include "algorithms/dda/fabric/fabric_gpu_barrier.h" // dda::common::kDdaMaxNranks #include +#include #include #include @@ -63,22 +64,23 @@ static ncclResult_t ncclReduceScatterDdaFabricLLTyped(const void* sendbuff, void INFO(NCCL_COLL, "DDA fabric ReduceScatter LL: nRanks=%d shardBytes=%zu nPk=%zu grid=%u block=%u", nRanks, bytes, nPk, grid.x, block.x); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled reduce loop; 0: runtime fallback. switch (nRanks) { case 4: - dda::common::ddaReduceScatterFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), recvcount, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + recvcount, comm->rank, nRanks, epochDev, epochLen); break; case 8: - dda::common::ddaReduceScatterFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), recvcount, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + recvcount, comm->rank, nRanks, epochDev, epochLen); break; default: - dda::common::ddaReduceScatterFabricLL<<>>(peers, static_cast(recvbuff), - static_cast(sendbuff), recvcount, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabricLL), grid, block, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, peers, static_cast(recvbuff), static_cast(sendbuff), + recvcount, comm->rank, nRanks, epochDev, epochLen); break; } diff --git a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll128.cu b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll128.cu index be0c7d59a261..1a31db7edc8c 100644 --- a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll128.cu +++ b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll128.cu @@ -17,6 +17,7 @@ #include "param.h" #include +#include #include #include @@ -84,22 +85,23 @@ static ncclResult_t ncclReduceScatterDdaFabricLL128Typed(const void* sendbuff, v INFO(NCCL_COLL, "DDA fabric ReduceScatter LL128: nRanks=%d shardBytes=%zu numLines=%zu grid=%u block=%u", nRanks, bytes, numLines, grid.x, block.x); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); // NRANKS_CT 4/8: unrolled reduce loop; 0: runtime fallback. switch (nRanks) { case 4: - dda::common::ddaReduceScatterFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), recvcount, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabricLL128), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, peers, static_cast(recvbuff), + static_cast(sendbuff), recvcount, comm->rank, nRanks, epochDev, epochLen); break; case 8: - dda::common::ddaReduceScatterFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), recvcount, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabricLL128), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, peers, static_cast(recvbuff), + static_cast(sendbuff), recvcount, comm->rank, nRanks, epochDev, epochLen); break; default: - dda::common::ddaReduceScatterFabricLL128 - <<>>(peers, static_cast(recvbuff), static_cast(sendbuff), recvcount, - comm->rank, nRanks, epochDev, epochLen); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterFabricLL128), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, peers, static_cast(recvbuff), + static_cast(sendbuff), recvcount, comm->rank, nRanks, epochDev, epochLen); break; } diff --git a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_ipc.cu b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_ipc.cu index a4015ed46ccc..5709536fbc9f 100644 --- a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_ipc.cu +++ b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_ipc.cu @@ -15,6 +15,7 @@ #include "algorithms/dda/dda_init_detail.h" #include +#include #include #include @@ -55,8 +56,10 @@ static ncclResult_t ncclReduceScatterDdaIpcTyped(const void* sendbuff, void* rec T** d_ipcbuffs = reinterpret_cast(peerPtrsDev); CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, totalCount * sizeof(T), cudaMemcpyDeviceToDevice, stream)); - dda::common::ddaReduceScatterIpc<<>>( - d_ipcbuffs, static_cast(recvbuff), recvcount, static_cast(sendbuff), comm->rank, barrierHost); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); + hipExtLaunchKernelGGL((dda::common::ddaReduceScatterIpc), grid, block, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, d_ipcbuffs, static_cast(recvbuff), + recvcount, static_cast(sendbuff), comm->rank, barrierHost); CUDACHECK(cudaGetLastError()); return ncclSuccess; diff --git a/projects/rccl/src/algorithms/gin/sdma/gin_all_reduce_sdma.cu b/projects/rccl/src/algorithms/gin/sdma/gin_all_reduce_sdma.cu index f62f7aa978b6..ef95002ef237 100755 --- a/projects/rccl/src/algorithms/gin/sdma/gin_all_reduce_sdma.cu +++ b/projects/rccl/src/algorithms/gin/sdma/gin_all_reduce_sdma.cu @@ -27,6 +27,7 @@ #include "param.h" #include +#include NCCL_PARAM(GinAllReduceEnable, "GIN_ALLREDUCE_ENABLE", 1); // When 0 (default), GIN AllReduce is eligible only for messages >= 256 MiB. @@ -104,9 +105,8 @@ static ncclResult_t ncclGinAllReduceInitOnce(ncclComm* comm) { } template -static ncclResult_t ncclAllReduceGinSdmaOneShotTyped(const void* sendbuff, void* recvbuff, size_t count, - ncclComm* comm, cudaStream_t stream, - struct ncclDevrWindow* sendWin, +static ncclResult_t ncclAllReduceGinSdmaOneShotTyped(const void* sendbuff, void* recvbuff, size_t count, ncclComm* comm, + cudaStream_t stream, struct ncclDevrWindow* sendWin, struct ncclDevrWindow* recvWin) { NCCLCHECK(ncclGinAllReduceInitOnce(comm)); @@ -115,8 +115,10 @@ static ncclResult_t ncclAllReduceGinSdmaOneShotTyped(const void* sendbuff, void* const size_t recvOff = static_cast(static_cast(recvbuff) - static_cast(recvWin->userPtr)); - gin::sdma::allReduceLsaOneShotKernel<<>>( - sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, count, comm->ginAllReduceState.devComm); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); + hipExtLaunchKernelGGL((gin::sdma::allReduceLsaOneShotKernel), kGinAllReduceLsaCtas, + kGinAllReduceLsaThreadsPerCta, 0, stream, /*startEvent=*/nullptr, stopEvent, /*flags=*/0, + sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, count, comm->ginAllReduceState.devComm); CUDACHECK(cudaGetLastError()); return ncclSuccess; } @@ -138,8 +140,10 @@ template static void ginAllReduceLaunchLsaTwoShot(ncclComm* comm, cudaStream_t stream, struct ncclDevrWindow* sendWin, size_t sendOff, struct ncclDevrWindow* recvWin, size_t recvOff, size_t countPerRank, int gridCtas) { - gin::sdma::lsaAllReduceTwoShotKernel<<>>( - comm->ginAllReduceState.devComm, sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, countPerRank, comm->nRanks); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); + hipExtLaunchKernelGGL((gin::sdma::lsaAllReduceTwoShotKernel), gridCtas, kGinAllReduceLsaThreadsPerCta, + 0, stream, /*startEvent=*/nullptr, stopEvent, /*flags=*/0, comm->ginAllReduceState.devComm, + sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, countPerRank, comm->nRanks); } template @@ -187,9 +191,11 @@ static ncclResult_t ncclAllReduceGinSdmaGinTwoShotTyped(const void* sendbuff, vo static_cast(static_cast(recvbuff) - static_cast(recvWin->userPtr)); const size_t countPerRank = count / static_cast(comm->nRanks); - gin::sdma::ginAllReduceTwoShotKernel<<>>( - comm->ginAllReduceState.devComm, sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, countPerRank, comm->nRanks, - comm->ginAllReduceState.intraGpuCtaBar); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); + hipExtLaunchKernelGGL((gin::sdma::ginAllReduceTwoShotKernel), kGinAllReduceLsaCtas, + kGinAllReduceLsaThreadsPerCta, 0, stream, /*startEvent=*/nullptr, stopEvent, /*flags=*/0, + comm->ginAllReduceState.devComm, sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, + countPerRank, comm->nRanks, comm->ginAllReduceState.intraGpuCtaBar); CUDACHECK(cudaGetLastError()); return ncclSuccess; } diff --git a/projects/rccl/src/algorithms/gin/sdma/gin_alltoall_sdma.cu b/projects/rccl/src/algorithms/gin/sdma/gin_alltoall_sdma.cu index 2a160baccd01..25ddaca03c15 100644 --- a/projects/rccl/src/algorithms/gin/sdma/gin_alltoall_sdma.cu +++ b/projects/rccl/src/algorithms/gin/sdma/gin_alltoall_sdma.cu @@ -19,6 +19,7 @@ #include "param.h" #include +#include #include @@ -228,11 +229,13 @@ ncclResult_t ncclAllToAllGinSdma(const void* sendbuff, void* recvbuff, size_t co bool useSdma = bytesPerPeer >= (size_t)ncclParamGinA2ASdmaMinBytes(); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); if (useSdma) { int sdmaThreads = ginA2ASdmaThreads(comm->nRanks); INFO(NCCL_COLL, "AllToAll GIN: transport=sdma bytesPerPeer=%zu threads=%d", bytesPerPeer, sdmaThreads); - ncclGinA2AKernel<<>>( - sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, bytesPerPeer, 0, comm->ginA2AState.devComm); + hipExtLaunchKernelGGL((ncclGinA2AKernel), kGinA2ASdmaCtas, sdmaThreads, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, bytesPerPeer, + /*chunkBytes=*/0, comm->ginA2AState.devComm); } else { // Under LSA the CTAs do the copying, so the grid scales with the message. int lsaChunks, lsaThreads; @@ -241,8 +244,9 @@ ncclResult_t ncclAllToAllGinSdma(const void* sendbuff, void* recvbuff, size_t co INFO(NCCL_COLL, "AllToAll GIN: transport=lsa bytesPerPeer=%zu chunks=%d ctas=%d threads=%d", bytesPerPeer, lsaChunks, comm->nRanks * lsaChunks, lsaThreads); - ncclGinA2AKernel<<nRanks, lsaChunks), lsaThreads, 0, stream>>>( - sendWin->vidmem, sendOff, recvWin->vidmem, recvOff, bytesPerPeer, lsaChunkBytes, comm->ginA2AState.devComm); + hipExtLaunchKernelGGL((ncclGinA2AKernel), dim3(comm->nRanks, lsaChunks), lsaThreads, 0, stream, + /*startEvent=*/nullptr, stopEvent, /*flags=*/0, sendWin->vidmem, sendOff, recvWin->vidmem, + recvOff, bytesPerPeer, lsaChunkBytes, comm->ginA2AState.devComm); } CUDACHECK(cudaGetLastError()); diff --git a/projects/rccl/src/allocator.cc b/projects/rccl/src/allocator.cc index 49e8ca612500..f1f5f4f0f284 100644 --- a/projects/rccl/src/allocator.cc +++ b/projects/rccl/src/allocator.cc @@ -215,7 +215,7 @@ static void insertSegment(struct ncclSpace* a, int index, int64_t lo, int64_t hi a->count = w; } -ncclResult_t ncclSpaceAlloc(struct ncclSpace* a, int64_t limit, int64_t size, int align, int64_t* outOffset) { +ncclResult_t ncclSpaceTryAlloc(struct ncclSpace* a, int64_t limit, int64_t size, int align, int64_t* outOffset) { // When allocating we try to locate the first empty segment which can hold // the allocation and move its lower cut upward. int i = a->count % 2; // First empty segment ends at cuts[i] @@ -235,11 +235,17 @@ ncclResult_t ncclSpaceAlloc(struct ncclSpace* a, int64_t limit, int64_t size, in } i += 2; // Next empty segment } - WARN("Allocation failed. No suitable space found to accommodate size=0x%lx within limit=0x%lx", (long)size, - (long)limit); return ncclInternalError; } +ncclResult_t ncclSpaceAlloc(struct ncclSpace* a, int64_t limit, int64_t size, int align, int64_t* outOffset) { + ncclResult_t res = ncclSpaceTryAlloc(a, limit, size, align, outOffset); + if (res != ncclSuccess) + WARN("Allocation failed. No suitable space found to accommodate size=0x%lx within limit=0x%lx", (long)size, + (long)limit); + return res; +} + ncclResult_t ncclSpaceFree(struct ncclSpace* a, int64_t offset, int64_t size) { if (a->count == 0 || a->cuts[a->count - 1] <= offset) { WARN("No allocation found at offset=0x%lx", (long)offset); @@ -289,6 +295,7 @@ void ncclShadowPoolConstruct(struct ncclShadowPool* pool) { pool->hbits = 0; pool->count = 0; pool->table = nullptr; + pool->memPool = nullptr; pool->pages = nullptr; } @@ -307,7 +314,8 @@ ncclResult_t ncclShadowPoolDestruct(struct ncclShadowPool* pool, cudaStream_t st pool->pages = page; } } else { - CUDACHECKIGNORE(cudaFreeAsync(obj->devObj, stream)); + if (pool->memPool) CUDACHECKIGNORE(cudaFreeAsync(obj->devObj, stream)); + else CUDACHECKIGNORE(cudaFree(obj->devObj)); } struct ncclShadowObject* next = obj->next; free(obj); @@ -318,14 +326,15 @@ ncclResult_t ncclShadowPoolDestruct(struct ncclShadowPool* pool, cudaStream_t st free(pool->table); while (pool->pages != nullptr) { - CUDACHECKIGNORE(cudaFreeAsync(pool->pages->devObjs, stream)); + if (pool->memPool) CUDACHECKIGNORE(cudaFreeAsync(pool->pages->devObjs, stream)); + else CUDACHECKIGNORE(cudaFree(pool->pages->devObjs)); struct ncclShadowPage* next = pool->pages->next; free(pool->pages); pool->pages = next; } CUDACHECKIGNORE(cudaStreamSynchronize(stream)); - CUDACHECKIGNORE(cudaMemPoolDestroy(pool->memPool)); + if (pool->memPool) CUDACHECKIGNORE(cudaMemPoolDestroy(pool->memPool)); } return ncclSuccess; } @@ -346,13 +355,19 @@ ncclResult_t ncclShadowPoolAlloc(struct ncclShadowPool* pool, size_t size, void* int hbits = pool->hbits; if (hbits == 0) { - cudaMemPoolProps props = {}; - props.allocType = cudaMemAllocationTypePinned; - props.handleTypes = cudaMemHandleTypeNone; - props.location.type = cudaMemLocationTypeDevice; - CUDACHECKIGNORE(cudaGetDevice(&props.location.id)); - props.maxSize = (size_t)ncclParamShadowMempoolMaxSize(); - CUDACHECK(cudaMemPoolCreate(&pool->memPool, &props)); + int cudaDev = 0; + int memoryPoolsSupported = 0; + CUDACHECK(cudaGetDevice(&cudaDev)); + CUDACHECK(cudaDeviceGetAttribute(&memoryPoolsSupported, cudaDevAttrMemoryPoolsSupported, cudaDev)); + if (memoryPoolsSupported) { + cudaMemPoolProps props = {}; + props.allocType = cudaMemAllocationTypePinned; + props.handleTypes = cudaMemHandleTypeNone; + props.location.type = cudaMemLocationTypeDevice; + props.location.id = cudaDev; + props.maxSize = (size_t)ncclParamShadowMempoolMaxSize(); + CUDACHECK(cudaMemPoolCreate(&pool->memPool, &props)); + } pool->hbits = hbits = 4; pool->table = (struct ncclShadowObject**)malloc(sizeof(struct ncclShadowObject*) << hbits); @@ -394,7 +409,8 @@ ncclResult_t ncclShadowPoolAlloc(struct ncclShadowPool* pool, size_t size, void* page->freeMask = uint64_t(-1) >> (64 - pageSize / pageObjSize); page->next = pool->pages; pool->pages = page; - CUDACHECK(cudaMallocFromPoolAsync(&page->devObjs, pageSize, pool->memPool, stream)); + if (pool->memPool) CUDACHECK(cudaMallocFromPoolAsync(&page->devObjs, pageSize, pool->memPool, stream)); + else CUDACHECK(cudaMalloc(&page->devObjs, pageSize)); CUDACHECK(cudaMemsetAsync(page->devObjs, 0, pageSize, stream)); // fall through... } @@ -408,7 +424,8 @@ ncclResult_t ncclShadowPoolAlloc(struct ncclShadowPool* pool, size_t size, void* } } else { page = nullptr; - CUDACHECK(cudaMallocFromPoolAsync(&devObj, size, pool->memPool, stream)); + if (pool->memPool) CUDACHECK(cudaMallocFromPoolAsync(&devObj, size, pool->memPool, stream)); + else CUDACHECK(cudaMalloc(&devObj, size)); CUDACHECK(cudaMemsetAsync(devObj, 0, size, stream)); } @@ -448,7 +465,8 @@ ncclResult_t ncclShadowPoolFree(struct ncclShadowPool* pool, void* devObj, cudaS int slot = ((char*)obj->devObj - (char*)obj->page->devObjs) / obj->page->objSize; obj->page->freeMask |= uint64_t(1) << slot; } else { - CUDACHECK(cudaFreeAsync(devObj, stream)); + if (pool->memPool) CUDACHECK(cudaFreeAsync(devObj, stream)); + else CUDACHECK(cudaFree(devObj)); } free(obj); pool->count -= 1; diff --git a/projects/rccl/src/bootstrap.cc b/projects/rccl/src/bootstrap.cc index b1b074fdbb5d..9d475940ee85 100644 --- a/projects/rccl/src/bootstrap.cc +++ b/projects/rccl/src/bootstrap.cc @@ -17,10 +17,13 @@ #include "signals.h" // [RCCL] #include "param.h" #include "ras.h" +#include "crypt.h" +#include #include #include "os.h" #include #include +#include #define BOOTSTRAP_N_CHECK_ABORT 10000 #define BOOTSTRAP_TAG_CONNECT (0x1 << 31) @@ -732,6 +735,16 @@ struct bootstrapListen_t { }; }; +struct bootstrapState; +struct bootstrapAsyncSend { + struct bootstrapState* state; + int peer; + int tag; + int size; + char* data; + struct bootstrapAsyncSend* next; +}; + struct bootstrapState { struct bootstrapRing_t ring; struct bootstrapListen_t listen; @@ -745,6 +758,12 @@ struct bootstrapState { int nranks; uint64_t magic; volatile uint32_t* abortFlag; + std::mutex asyncSendLock; + std::condition_variable asyncSendCond; + struct ncclIntruQueue asyncSendQueue; // in caller order + ncclResult_t asyncSendError; + bool asyncSendClosing; // rejects sends once drain/teardown begins + bool asyncSendSetAbort; // async failure set abortFlag; report it once before abort cleanup }; #define STATE_RING(s, f) (s->ring.f) #define STATE_LISTEN(s, f) (s->listen.f) @@ -857,15 +876,20 @@ static ncclResult_t netRingConnectFinish(void* ctx, ncclNet_t* net, struct boots // TCP connect/accept handshakes with local proxy/P2P socket setup, without adding // threads. socketRingConnect() below keeps the old blocking semantics for callers that // do not have useful work to overlap (notably bootstrapSplit). +// With TLS on the bootstrap sockets, the handshake is also progressed by the polling in +// socketRingConnectFinish, so connecting to next and accepting from prev make progress +// together on this single thread (upstream's bootstrapConcurrent spawns a helper thread for +// the same reason). static ncclResult_t socketRingConnectStart(ncclSocketAddress* addr, struct ncclSocket* sendSocket, struct ncclSocket* listenSock, struct ncclSocket* recvSocket, uint64_t magic, volatile uint32_t* abortFlag) { ncclResult_t ret = ncclSuccess; + // Init recvSocket first so callers can safely close both sockets on failure. + NCCLCHECK(ncclSocketInit(recvSocket)); NCCLCHECK(ncclSocketInit(sendSocket, addr, magic, ncclSocketTypeBootstrap, abortFlag, /*asyncFlag*/ 1)); NCCLCHECK(ncclSocketConnect(sendSocket)); int oldAsync = listenSock->asyncFlag; listenSock->asyncFlag = 1; - NCCLCHECKGOTO(ncclSocketInit(recvSocket), ret, exit); NCCLCHECKGOTO(ncclSocketAccept(recvSocket, listenSock), ret, exit); exit: listenSock->asyncFlag = oldAsync; @@ -885,12 +909,60 @@ static ncclResult_t socketRingConnectFinish(struct ncclSocket* sendSocket, struc return ncclSuccess; } +// With TLS on the bootstrap sockets, a connector cannot finish its handshake (and +// therefore its send) until the acceptor's application accepts the connection and +// runs the server half. The ring connect (connect to next, then accept from prev) +// would deadlock once every rank sits in its connect, so the connect side runs on +// a helper thread while this rank keeps serving its accept side. Plaintext mode +// keeps the stock single-threaded order. One-shot sends get the same property from +// the asynchronous bootstrapSend below. +// RCCL: the bootstrap ring itself uses the non-blocking socketRingConnectStart/Finish +// above instead; this helper serves the intra-node peer connect/accept. +struct bootstrapThreadOp { + ncclResult_t (*fn)(void*); + void* args; + ncclResult_t result; +}; + +static void bootstrapThreadRun(void* opaque) { + struct bootstrapThreadOp* op = (struct bootstrapThreadOp*)opaque; + op->result = op->fn(op->args); +} + +static ncclResult_t bootstrapConcurrent(ncclResult_t (*sendFn)(void*), void* sendArgs, ncclResult_t (*recvFn)(void*), + void* recvArgs) { + bool encrypted; + NCCLCHECK(ncclGetCryptConnectionMode(&encrypted)); + if (!encrypted) { + NCCLCHECK(sendFn(sendArgs)); + NCCLCHECK(recvFn(recvArgs)); + return ncclSuccess; + } + // We want to be able to do the initial TLS handshake connecting to the next rank around the ring + // and accepting from the previous rank around the ring. To keep this simple, we spawn a thread to + // handle the outgoing side. Unencrypted doesn't need this because no data flows on initial + // connect. + struct bootstrapThreadOp op = {sendFn, sendArgs, ncclInternalError}; + std::thread thread; + STDTHREADCREATE(thread, bootstrapThreadRun, &op); + ncclResult_t recvRes = recvFn(recvArgs); + NCCLCHECK(ncclThreadJoin(thread)); + NCCLCHECK(op.result); + NCCLCHECK(recvRes); + return ncclSuccess; +} + static ncclResult_t socketRingConnect(ncclSocketAddress* addr, struct ncclSocket* sendSocket, struct ncclSocket* listenSock, struct ncclSocket* recvSocket, uint64_t magic, volatile uint32_t* abortFlag) { - NCCLCHECK(socketRingConnectStart(addr, sendSocket, listenSock, recvSocket, magic, abortFlag)); - NCCLCHECK(socketRingConnectFinish(sendSocket, recvSocket, abortFlag)); + ncclResult_t ret = ncclSuccess; + NCCLCHECKGOTO(socketRingConnectStart(addr, sendSocket, listenSock, recvSocket, magic, abortFlag), ret, fail); + NCCLCHECKGOTO(socketRingConnectFinish(sendSocket, recvSocket, abortFlag), ret, fail); return ncclSuccess; +fail: + (void)ncclSocketClose(sendSocket); + (void)ncclSocketClose(recvSocket); + return ret; } // Set up the reverse direction of the bootstrap ring for the net OOB path: @@ -1077,8 +1149,7 @@ static ncclResult_t sendToRoot(struct ncclBootstrapHandle* handle, struct ncclCo NCCL_PARAM(StaggerRate, "UID_STAGGER_RATE", 7000); NCCL_PARAM(StaggerThreshold, "UID_STAGGER_THRESHOLD", 256); - -NCCL_PARAM(RasEnable, "RAS_ENABLE", 1); +extern int64_t ncclParamRasEnable(); ncclResult_t bootstrapInit(int nHandles, void* handles, struct ncclComm* comm, struct ncclComm* parent) { ncclResult_t result = ncclSuccess; @@ -1103,7 +1174,8 @@ ncclResult_t bootstrapInit(int nHandles, void* handles, struct ncclComm* comm, s // Multi-root rendezvous can't propagate revHandle cross-root; bidir requires nHandles==1. bool wantNetBidir = (nHandles == 1) && bootstrapBidirEnabled(nranks, 1); - NCCLCHECK(ncclCalloc(&state, 1)); + NEW_NOTHROW(state, bootstrapState); + ncclIntruQueueConstruct(&state->asyncSendQueue); state->rank = rank; state->nranks = nranks; state->cudaDev = comm->cudaDev; @@ -1368,7 +1440,8 @@ ncclResult_t bootstrapSplit(uint64_t magic, struct ncclComm* comm, struct ncclCo struct ncclSocket* proxySocket = NULL; struct bootstrapState* state; - NCCLCHECKGOTO(ncclCalloc(&state, 1), ret, fail); + NEW_NOTHROW_GOTO(state, bootstrapState, ret, fail); + ncclIntruQueueConstruct(&state->asyncSendQueue); state->rank = rank; state->nranks = nranks; state->cudaDev = comm->cudaDev; @@ -1462,6 +1535,7 @@ struct socketAckInfo { int rank; int tag; }; + static ncclResult_t socketConnect(void* commState, int peer, int tag, struct ncclSocket* sock) { ncclResult_t ret = ncclSuccess; struct bootstrapState* state = (struct bootstrapState*)commState; @@ -1477,7 +1551,7 @@ static ncclResult_t socketConnect(void* commState, int peer, int tag, struct ncc (void)ncclSocketClose(sock); return ret; } -ncclResult_t bootstrapSend(void* commState, int peer, int tag, void* data, int size) { +static ncclResult_t bootstrapSendSync(void* commState, int peer, int tag, void* data, int size) { ncclResult_t ret = ncclSuccess; struct ncclSocket sock; TRACE(NCCL_BOOTSTRAP, "Sending to peer=%d tag=%d size=%d", peer, tag, size); @@ -1490,6 +1564,130 @@ ncclResult_t bootstrapSend(void* commState, int peer, int tag, void* data, int s (void)ncclSocketClose(&sock); return ret; } + +// With TLS, sockets that are send-only at the data level still need to receive for TLS handshaking +// so we put them on their own threads with a copy of the payload and run them async. +// bootstrapClose waits for them to drain, and a send failure is stashed on the state and returned +// by the next bootstrap operation or close. It also sets the abort flag so an operation already +// blocked on this state can make progress and fail. +// +// The receiver matches connections by (peer, tag) in accept order, so two sends to the same (peer, +// tag) must reach it in call order (NVLS setup broadcasts to the same peers with the same tag +// several times during init). The in-flight queue keeps caller order; a send's thread waits until it +// is the oldest in-flight send for its (peer, tag) before connecting. Sends to distinct (peer, tag) +// stay concurrent. + +// Called with asyncSendLock held, which serializes competing async-send writers. +// asyncSendError is still atomic because bootstrap operations read it without +// taking the lock. Publish asyncSendSetAbort before asyncSendError so acquire +// readers that see asyncSendError also see the marker. +static void bootstrapAsyncSendSetError(struct bootstrapState* state, ncclResult_t res) { + if (res == ncclSuccess || COMPILER_ATOMIC_LOAD(&state->asyncSendError, std::memory_order_relaxed) != ncclSuccess) + return; + COMPILER_ATOMIC_STORE(&state->asyncSendSetAbort, true, std::memory_order_relaxed); + COMPILER_ATOMIC_STORE(&state->asyncSendError, res, std::memory_order_release); + COMPILER_ATOMIC_STORE(state->abortFlag, 1u, std::memory_order_release); +} + +static bool bootstrapAsyncSendMatches(struct bootstrapAsyncSend* a, struct bootstrapAsyncSend* b) { + return a == b; +} + +static void bootstrapAsyncSendMain(void* opaque) { + struct bootstrapAsyncSend* op = (struct bootstrapAsyncSend*)opaque; + struct bootstrapState* state = op->state; + std::unique_lock lock(state->asyncSendLock); + for (;;) { + struct bootstrapAsyncSend* earlier = ncclIntruQueueHead(&state->asyncSendQueue); + while (earlier != op && !(earlier->peer == op->peer && earlier->tag == op->tag)) earlier = earlier->next; + if (earlier == op) break; + // someone is queued ahead of us with the same (peer, tag), wait for it + state->asyncSendCond.wait(lock); + } + // Once any send on this state has failed, stop sending: a dropped message + // followed by a delivered one would be matched to the wrong bootstrapRecv on + // the receiver. The comm is coming down anyway; keep the first error. + bool skip = (COMPILER_ATOMIC_LOAD(&state->asyncSendError, std::memory_order_relaxed) != ncclSuccess); + ncclResult_t res = ncclSuccess; + if (!skip) { + lock.unlock(); + res = bootstrapSendSync(state, op->peer, op->tag, op->data, op->size); + lock.lock(); + } + bootstrapAsyncSendSetError(state, res); + ncclIntruQueueDelete(&state->asyncSendQueue, op, bootstrapAsyncSendMatches); + // we're done, wake up anyone that might be waiting for us to finish + state->asyncSendCond.notify_all(); + lock.unlock(); + free(op->data); + free(op); +} + +static ncclResult_t bootstrapAsyncSendDrain(struct bootstrapState* state) { + std::unique_lock lock(state->asyncSendLock); + state->asyncSendClosing = true; + while (!ncclIntruQueueEmpty(&state->asyncSendQueue)) state->asyncSendCond.wait(lock); + return COMPILER_ATOMIC_LOAD(&state->asyncSendError, std::memory_order_acquire); +} + +ncclResult_t bootstrapSend(void* commState, int peer, int tag, void* data, int size) { + struct bootstrapState* state = (struct bootstrapState*)commState; + ncclResult_t ret = ncclSuccess; + bool encrypted; + NCCLCHECK(ncclGetCryptConnectionMode(&encrypted)); + if (!encrypted) { + NCCLCHECK(bootstrapSendSync(commState, peer, tag, data, size)); + return ncclSuccess; + } + + char* copy = nullptr; + struct bootstrapAsyncSend* op = nullptr; + ncclResult_t pending; + bool closing; + std::thread thread; + if (size > 0) { + NCCLCHECKGOTO(ncclCalloc(©, size), ret, fail); + memcpy(copy, data, size); + } + NCCLCHECKGOTO(ncclCalloc(&op, 1), ret, fail); + op->state = state; + op->peer = peer; + op->tag = tag; + op->size = size; + op->data = copy; + + { + std::lock_guard lock(state->asyncSendLock); + pending = COMPILER_ATOMIC_LOAD(&state->asyncSendError, std::memory_order_acquire); + if (pending != ncclSuccess) COMPILER_ATOMIC_STORE(&state->asyncSendSetAbort, false, std::memory_order_relaxed); + closing = state->asyncSendClosing; + if (pending == ncclSuccess && !closing) { + STDTHREADCREATE_GOTO(thread, bootstrapAsyncSendMain, ret, fail, op); + // Append in caller order; bootstrapAsyncSendMain relies on it for the + // per-(peer, tag) ordering. + ncclIntruQueueEnqueue(&state->asyncSendQueue, op); + } + } + if (pending != ncclSuccess) { + WARN("bootstrapSend: an earlier asynchronous send to peer %d failed", peer); + ret = pending; + goto fail; + } + if (closing) { + WARN("bootstrapSend: bootstrap state is closing"); + ret = ncclInternalError; + goto fail; + } + + thread.detach(); + TRACE(NCCL_BOOTSTRAP, "Async send to peer=%d tag=%d size=%d", peer, tag, size); + return ncclSuccess; +fail: + free(copy); + free(op); + return ret; +} + // Bootstrap send/receive functions static ncclResult_t unexpectedEnqueue(struct bootstrapState* state, int peer, int tag, struct ncclSocket* sock) { // New unex @@ -1497,7 +1695,7 @@ static ncclResult_t unexpectedEnqueue(struct bootstrapState* state, int peer, in NCCLCHECK(ncclCalloc(&unex, 1)); unex->peer = peer; unex->tag = tag; - memcpy(&unex->sock, sock, sizeof(struct ncclSocket)); + ncclSocketMove(&unex->sock, sock); // Enqueue struct unexConn* list = state->unexpectedConnections; @@ -1522,7 +1720,7 @@ static ncclResult_t unexpectedDequeue(struct bootstrapState* state, int peer, in } else { prev->next = elem->next; } - memcpy(sock, &elem->sock, sizeof(struct ncclSocket)); + ncclSocketMove(sock, &elem->sock); free(elem); *found = 1; return ncclSuccess; @@ -1540,6 +1738,7 @@ static void unexpectedFree(struct bootstrapState* state) { while (elem) { prev = elem; elem = elem->next; + (void)ncclSocketClose(&prev->sock); free(prev); } return; @@ -1570,10 +1769,21 @@ static ncclResult_t socketAccept(void* commState, int peer, int tag, struct nccl (void)ncclSocketClose(sock); return ret; } + // We can't know who we'll receive from, so we need to receive everything at once ncclResult_t bootstrapRecv(void* commState, int peer, int tag, void* data, int size) { ncclResult_t ret; struct ncclSocket sock; + struct bootstrapState* state = (struct bootstrapState*)commState; + // Asynchronous send failures are stashed rather than returned to their caller; + // surface them at the next bootstrap operation so init fails on this rank + // instead of only hanging the peer. + ret = COMPILER_ATOMIC_LOAD(&state->asyncSendError, std::memory_order_acquire); + if (ret != ncclSuccess) COMPILER_ATOMIC_STORE(&state->asyncSendSetAbort, false, std::memory_order_relaxed); + if (ret != ncclSuccess) { + WARN("bootstrapRecv: an earlier asynchronous bootstrap send failed"); + return ret; + } NCCLCHECK(socketAccept(commState, peer, tag, &sock)); TRACE(NCCL_BOOTSTRAP, "Receiving tag=%d peer=%d size=%d", tag, peer, size); NCCLCHECKGOTO(socketRecv(&sock, ((char*)data), size), ret, fail); @@ -1859,16 +2069,39 @@ ncclResult_t bootstrapBarrier(void* commState, int rank, int nranks, int tag) { return ncclSuccess; } +// Like the ring connect: connecting to next and accepting from prev must make +// progress together once the connect includes a TLS handshake. +struct bootstrapPeerSocketOp { + void* commState; + int peer; + int tag; + struct ncclSocket* sock; +}; +static ncclResult_t bootstrapPeerConnectOp(void* opaque) { + struct bootstrapPeerSocketOp* op = (struct bootstrapPeerSocketOp*)opaque; + NCCLCHECK(socketConnect(op->commState, op->peer, op->tag, op->sock)); + return ncclSuccess; +} +static ncclResult_t bootstrapPeerAcceptOp(void* opaque) { + struct bootstrapPeerSocketOp* op = (struct bootstrapPeerSocketOp*)opaque; + NCCLCHECK(socketAccept(op->commState, op->peer, op->tag, op->sock)); + return ncclSuccess; +} + ncclResult_t bootstrapIntraNodeAllGather(void* commState, int* ranks, int rank, int nranks, void* allData, int size) { if (nranks == 1) return ncclSuccess; + ncclResult_t ret = ncclSuccess; TRACE(NCCL_INIT, "rank %d nranks %d size %d - ENTER", rank, nranks, size); int prevRank = ranks[(rank - 1 + nranks) % nranks]; int nextRank = ranks[(rank + 1) % nranks]; // intraNode bootstrap is done defacto using the socket-based implementation struct ncclSocket recvSocket, sendSocket; - NCCLCHECK(socketConnect(commState, nextRank, BOOTSTRAP_TAG_INTRANODE_ALLGATHER, &sendSocket)); - NCCLCHECK(socketAccept(commState, prevRank, BOOTSTRAP_TAG_INTRANODE_ALLGATHER, &recvSocket)); + struct bootstrapPeerSocketOp connectOp = {commState, nextRank, BOOTSTRAP_TAG_INTRANODE_ALLGATHER, &sendSocket}; + struct bootstrapPeerSocketOp acceptOp = {commState, prevRank, BOOTSTRAP_TAG_INTRANODE_ALLGATHER, &recvSocket}; + NCCLCHECK(ncclSocketInit(&recvSocket)); + NCCLCHECKGOTO(ncclSocketInit(&sendSocket), ret, fail); + NCCLCHECKGOTO(bootstrapConcurrent(bootstrapPeerConnectOp, &connectOp, bootstrapPeerAcceptOp, &acceptOp), ret, fail); // Intra-node AllGather is socket-only and so cannot reuse bootstrapBidirEnabled // (its `!netOn` clause is geared to inter-node OOB selection). Gate purely on the @@ -1876,16 +2109,20 @@ ncclResult_t bootstrapIntraNodeAllGather(void* commState, int* ranks, int rank, // (nranks==2 → single bidirectional socketSendRecv; nranks==1 is short-circuited // earlier in this function). if (ncclParamBootstrapBidirAllGather() != 0) { - NCCLCHECK(socketRingAllGather(&sendSocket, &recvSocket, rank, nranks, (char*)allData, size)); + NCCLCHECKGOTO(socketRingAllGather(&sendSocket, &recvSocket, rank, nranks, (char*)allData, size), ret, fail); } else { - NCCLCHECK(socketRingAllGatherUnidir(&sendSocket, &recvSocket, rank, nranks, (char*)allData, size)); + NCCLCHECKGOTO(socketRingAllGatherUnidir(&sendSocket, &recvSocket, rank, nranks, (char*)allData, size), ret, fail); } - NCCLCHECK(ncclSocketClose(&sendSocket)); - NCCLCHECK(ncclSocketClose(&recvSocket)); + NCCLCHECKGOTO(ncclSocketClose(&sendSocket), ret, fail); + NCCLCHECKGOTO(ncclSocketClose(&recvSocket), ret, fail); TRACE(NCCL_INIT, "rank %d nranks %d size %d - DONE", rank, nranks, size); return ncclSuccess; +fail: + (void)ncclSocketClose(&sendSocket); + (void)ncclSocketClose(&recvSocket); + return ret; } // [IntraNode] in-place Broadcast @@ -1927,6 +2164,15 @@ ncclResult_t bootstrapClose(void* commState) { if (commState == NULL) return ncclSuccess; struct bootstrapState* state = (struct bootstrapState*)commState; int nranks = state->nranks; + // In-flight asynchronous sends reference this state; let them finish before + // tearing it down. On abort they exit quickly via the abort flag. + ncclResult_t asyncSendError = bootstrapAsyncSendDrain(state); + if (asyncSendError != ncclSuccess && (COMPILER_ATOMIC_LOAD(state->abortFlag, std::memory_order_acquire) == 0 || + COMPILER_ATOMIC_LOAD(&state->asyncSendSetAbort, std::memory_order_relaxed))) { + WARN("bootstrapClose: an asynchronous bootstrap send failed"); + COMPILER_ATOMIC_STORE(&state->asyncSendSetAbort, false, std::memory_order_relaxed); + return asyncSendError; + } // close unexpected and return an error if we are not aborting and still operations in the pipe if (state->unexpectedConnections != NULL) { unexpectedFree(state); @@ -1953,16 +2199,6 @@ ncclResult_t bootstrapClose(void* commState) { // proxy things are free'd elsewhere free(state->peerP2pAddresses); - free(state); - return ncclSuccess; -} - -ncclResult_t bootstrapAbort(void* commState) { - if (commState == NULL) return ncclSuccess; - struct bootstrapState* state = (struct bootstrapState*)commState; - // when aborting we need to close the proxy here (maybe?) - free(state->peerProxyAddresses); - free(state->peerProxyAddressesUDS); - NCCLCHECK(bootstrapClose(commState)); + delete state; return ncclSuccess; } diff --git a/projects/rccl/src/ce_coll.cc b/projects/rccl/src/ce_coll.cc index 5619bff77d48..ecf513d80c9c 100644 --- a/projects/rccl/src/ce_coll.cc +++ b/projects/rccl/src/ce_coll.cc @@ -16,6 +16,7 @@ #include "group.h" #include "alloc.h" #include "ce_fault_inject.h" +#include "enqueue.h" #include "tuning.h" // User override: when set (>= 0) the cost model is bypassed and this byte @@ -72,6 +73,17 @@ static int ncclCeBatchAsyncEnable() { return 0; #endif } + +// Chunk size used when a cudaMemcpyBatchAsync CE data batch requests +// round-robin chunking. Chunking is opt-in per batch. +NCCL_PARAM(CeChunkSize, "CE_CHUNK_SIZE", 8 * 1024 * 1024); + +// On CUDA 13.1+, cudaMemcpyFlagPreferOverlapWithCompute is honored on non-Tegra platforms for cudaMemcpyBatchAsync. +// When set, intra-device copies offload to the CE. NCCL sets this flag on NCCL_CTA_POLICY=ZERO when NCCL_CE_INTRA_GPU_MEMCPY_ENABLE=1 (default). +// Set to 0 to omit the flag and leave intra-device self-copies on SMs. +// [RCCL] default 1 preserves pre-2.32 behaviour (RCCL always set hipMemcpyFlagPreferOverlapWithCompute); rule #28 exception. +NCCL_PARAM(CeMemcpyOverlapEnable, "CE_INTRA_GPU_MEMCPY_ENABLE", 1); + // Static constant for graph synchronization static const uint32_t GRAPH_SYNC_VALUE = 1; @@ -95,6 +107,20 @@ static constexpr size_t HIER_COLL_MAX_CHUNK_SIZE = 64 * 1024 * 1024; // builder and the chunk-width computation, which must agree on it. static constexpr size_t HIER_COLL_CHUNK_ALIGN = 8 * 1024; +// -------------------------------------------------------------------- +// Hierarchical CE AllGather: Ring selection and chunking +// -------------------------------------------------------------------- + +// The retained ring implementation only changes chunking once messages are +// clearly bandwidth-dominated. +static constexpr size_t HIER_COLL_AG_RING_HALF_AWARE_THRESHOLD = 128 * 1024 * 1024; +static constexpr size_t HIER_COLL_AG_RING_MIN_HALF_AWARE_CHUNK = 8 * 1024 * 1024; + +// Ring is opt-in in this MR: a positive value enables it, while zero or a +// negative value keeps the Direct path. The stacked tuning MR gives -1 its +// automatic-selection semantics. +NCCL_PARAM(HierCeCollAgRailRingEnable, "HIER_CE_COLL_AG_RAIL_RING_ENABLE", -1); + // Minimum per-peer transfer size (bytes) for the hierarchical CE collectives to // distribute their inter-node rail traffic across multiple internal RMA // contexts. Below the threshold the per-context launch/progress overhead @@ -337,6 +363,10 @@ bool ncclCeAvailable(struct ncclComm* comm, ncclFunc_t coll, int /*ncclDevRedOp_ TRACE(NCCL_TUNING, "Skipping CE collective: not implemented"); return false; } + if (!comm->symmetricSupport) { + TRACE(NCCL_TUNING, "Skipping CE collective: symmetric support is not enabled"); + return false; + } if (ncclDevrWindowHasSysmemSegment(sendWin) || ncclDevrWindowHasSysmemSegment(recvWin)) { TRACE(NCCL_TUNING, "Skipping CE collective: host-backed segments are not supported"); return false; @@ -358,10 +388,6 @@ bool ncclCeAvailable(struct ncclComm* comm, ncclFunc_t coll, int /*ncclDevRedOp_ TRACE(NCCL_TUNING, "Skipping CE collective: not all ranks have direct GPU-to-GPU connectivity"); return false; } - if (!comm->symmetricSupport) { - TRACE(NCCL_TUNING, "Skipping CE collective: symmetric support is not enabled"); - return false; - } if (winRegType != ncclSymSendRegRecvReg && winRegType != ncclSymSendNonregRecvReg) { TRACE(NCCL_TUNING, "Skipping CE collective: window registration type %d is not supported", winRegType); return false; @@ -687,6 +713,7 @@ ncclResult_t ncclCeInitBatchOpsParams(struct ncclCeBatchOpsParams* params, int n params->dsts = nullptr; params->sizes = nullptr; params->numOps = 0; + params->chunking = false; params->intraBatchSync = false; #ifdef CE_BATCH_ASYNC_SUPPORTED params->attrs = nullptr; @@ -725,6 +752,7 @@ void ncclCeFreeBatchOpsParams(struct ncclCeBatchOpsParams* params) { // batch, once at exit), and a stale numOps would make the second release look // like it still describes a populated batch. params->numOps = 0; + params->chunking = false; params->intraBatchSync = false; #ifdef CE_BATCH_ASYNC_SUPPORTED if (params->attrs) { @@ -739,10 +767,63 @@ void ncclCeFreeBatchOpsParams(struct ncclCeBatchOpsParams* params) { #endif } +#ifdef CE_BATCH_ASYNC_SUPPORTED +static ncclResult_t ncclCeLaunchChunkedMemcpyBatchAsync(struct ncclCeBatchOpsParams* params, size_t chunkSize, + cudaStream_t stream) { + ncclResult_t ret = ncclSuccess; + size_t maxSize = 0; + for (int i = 0; i < params->numOps; i++) { + maxSize = std::max(maxSize, params->sizes[i]); + } + size_t numRounds = maxSize == 0 ? 0 : 1 + (maxSize - 1) / chunkSize; + + ncclUniqueArrayPtr tmpDsts{nullptr}; + ncclUniqueArrayPtr tmpSrcs{nullptr}; + ncclUniqueArrayPtr tmpSizes{nullptr}; + NCCLCHECKGOTO(ncclCalloc(tmpDsts, params->numOps), ret, fail); + NCCLCHECKGOTO(ncclCalloc(tmpSrcs, params->numOps), ret, fail); + NCCLCHECKGOTO(ncclCalloc(tmpSizes, params->numOps), ret, fail); + + // Submit one batch per round. CUDA guarantees stream ordering between + // batches, but does not guarantee the execution order of copies within + // a batch, so flattening every chunk into one batch would not pace the + // destinations in round-robin waves. + for (size_t round = 0; round < numRounds; round++) { + size_t offset = round * chunkSize; + int nWaveOps = 0; + for (int i = 0; i < params->numOps; i++) { + if (offset >= params->sizes[i]) continue; + size_t bytes = std::min(params->sizes[i] - offset, chunkSize); + tmpDsts[nWaveOps] = (uint8_t*)params->dsts[i] + offset; + tmpSrcs[nWaveOps] = (uint8_t*)params->srcs[i] + offset; + tmpSizes[nWaveOps] = bytes; + nWaveOps++; + } + + if (nWaveOps == 0) continue; +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + CUDACHECKGOTO(hipMemcpyBatchAsync( +#else + CUDACHECKGOTO(cudaMemcpyBatchAsync( +#endif + tmpDsts.get(), tmpSrcs.get(), tmpSizes.get(), nWaveOps, params->attrs, params->attrIdxs, + params->numAttrs, nullptr, stream), + ret, fail); + } + +exit: + return ret; +fail: + goto exit; +} +#endif + ncclResult_t ncclCeLaunchBatchOps(struct ncclComm* comm, struct ncclCeBatchOpsParams* params, cudaStream_t stream, struct ncclCeCollArgs* args) { ncclResult_t ret = ncclSuccess; bool capturing; + int64_t chunkSizeParam = ncclParamCeChunkSize(); + size_t ceChunkSize = chunkSizeParam > 0 ? (size_t)chunkSizeParam : 0; void* ceBatchHandle = NULL; #ifdef ENABLE_FAULT_INJECTION @@ -791,15 +872,23 @@ ncclResult_t ncclCeLaunchBatchOps(struct ncclComm* comm, struct ncclCeBatchOpsPa params->attrs[0] = {}; #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) params->attrs[0].srcAccessOrder = hipMemcpySrcAccessOrderStream; - params->attrs[0].flags = hipMemcpyFlagPreferOverlapWithCompute; + if (ncclParamCeMemcpyOverlapEnable()) { + params->attrs[0].flags = hipMemcpyFlagPreferOverlapWithCompute; + } #else params->attrs[0].srcAccessOrder = cudaMemcpySrcAccessOrderStream; - params->attrs[0].flags = cudaMemcpyFlagPreferOverlapWithCompute; + if (ncclParamCeMemcpyOverlapEnable()) { + params->attrs[0].flags = cudaMemcpyFlagPreferOverlapWithCompute; + } #endif params->attrIdxs[0] = 0; params->numAttrs = 1; - if (params->intraBatchSync) { + if (params->chunking && ceChunkSize > 0) { + INFO(NCCL_COLL, "CE: rank %d -> Batch path with round-robin chunking (chunkSize=%zu), numOps=%zu", comm->rank, + ceChunkSize, params->numOps); + NCCLCHECKGOTO(ncclCeLaunchChunkedMemcpyBatchAsync(params, ceChunkSize, stream), ret, fail); + } else if (params->intraBatchSync) { // Break into multiple batches with sync between them int batchSize = comm->ceColl.intraBatchSyncFreq; for (int i = 0; i < params->numOps; i += batchSize) { @@ -906,9 +995,10 @@ ncclResult_t ncclCeAllGather(struct ncclComm* comm, struct ncclCeCollArgs* args, bool agUseMulticast; // [RCCL] The DDA path addresses peers through ddaPeerBases instead of a window, so it // cannot take the multicast route. - agUseMulticast = !args->useDda && ncclCeAllGatherUseMulticast(comm, chunkBytes, - ncclCudaGraphValid(comm->planner.capturingGraph), - mySendBuff == myRecvBuff); + agUseMulticast = + !args->useDda && + ncclCeAllGatherUseMulticast(comm, chunkBytes, ncclCudaGraphValid(comm->planner.capturingGraph), + ncclAllGatherIsInPlace(args->sendBuff, args->recvBuff, myLsaRank, chunkBytes)); if (agUseMulticast) { // Multicast path: a single write to the multicast pointer covers @@ -1252,18 +1342,16 @@ bool ncclHierCeAvailable(struct ncclComm* comm, ncclFunc_t coll, int /*ncclDevRe TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: only AllGather and AlltoAll are supported"); return false; } + if (!comm->symmetricSupport) { + TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: symmetric support is not enabled"); + return false; + } // Must be multi-node (single-node uses the regular CE path) if (comm->nNodes <= 1) { TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: not multi-node"); return false; } - // Sub-comms are only built at nNodes >= 8; rcclHierarchicalAlgoInfo dereferences - // them unconditionally, so bail out here if they are not initialized. - if (!comm->hierarchicalCommsInitialized) { - TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: hierarchical sub-comms not initialized"); - return false; - } // If LSA already spans the whole comm, use CE path instead if (ncclDevrIsOneLsaTeam(comm)) { TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: LSA spans the comm; use CE path instead"); @@ -1277,11 +1365,6 @@ bool ncclHierCeAvailable(struct ncclComm* comm, ncclFunc_t coll, int /*ncclDevRe TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: LSA team does not cover all local ranks"); return false; } - // Need symmetric support - if (!comm->symmetricSupport) { - TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: symmetric support is not enabled"); - return false; - } // Need RMA proxy for inter-node puts, and the internal RMA contexts that back // the rail step (provisioned under exactly this path's guard conditions). This // is independent of the user's config.numRmaCtx -- the rail uses the internal @@ -1306,10 +1389,10 @@ bool ncclHierCeDispatch(struct ncclComm* comm) { // span [chunkStart[p], chunkStart[p+1]); total chunks = chunkStart[nPeers]. struct ncclHierChunkPlan { int nPeers; - int* chunkStart; // [nPeers + 1] -- prefix sums - size_t* chunkBytes; // [chunkStart[nPeers]] -- per-chunk byte size - size_t* chunkOff; // [chunkStart[nPeers]] -- per-chunk offset within - // peer's perRankBytes slice + int* chunkStart; // [nPeers + 1] -- prefix sums + size_t* chunkBytes; // [chunkStart[nPeers]] -- per-chunk byte size + size_t* chunkOff; // [chunkStart[nPeers]] -- per-chunk offset within + // peer's perRankBytes slice }; // Build a uniform chunking plan @@ -1408,13 +1491,28 @@ static size_t ncclHierCollChunkWidth(size_t perPeerBytes, int numCtx) { size_t maxChunk = HIER_COLL_MAX_CHUNK_SIZE; if (numCtx > 1) { size_t perCtx = DIVUP(perPeerBytes, (size_t)numCtx); - size_t chunksPerCtx = DIVUP(perCtx, HIER_COLL_MAX_CHUNK_SIZE); // 1 unless the cap binds + size_t chunksPerCtx = DIVUP(perCtx, HIER_COLL_MAX_CHUNK_SIZE); // 1 unless the cap binds size_t target = alignUp(DIVUP(perPeerBytes, numCtx * chunksPerCtx), HIER_COLL_CHUNK_ALIGN); if (target < maxChunk) maxChunk = target; } return maxChunk; } +static size_t ncclHierCollRingChunkWidth(size_t perRankBytes, size_t cwBytes, size_t ccwBytes, int numCtx) { + size_t maxChunk = ncclHierCollChunkWidth(perRankBytes, numCtx); + if (numCtx < 4 || perRankBytes < HIER_COLL_AG_RING_HALF_AWARE_THRESHOLD) { + return maxChunk; + } + + size_t halfBytes = std::max(cwBytes, ccwBytes); + if (halfBytes == 0) return maxChunk; + + size_t halfAware = alignUp(DIVUP(halfBytes, (size_t)numCtx), HIER_COLL_CHUNK_ALIGN); + halfAware = std::min(halfAware, HIER_COLL_MAX_CHUNK_SIZE); + halfAware = std::max(halfAware, HIER_COLL_AG_RING_MIN_HALF_AWARE_CHUNK); + return std::min(maxChunk, halfAware); +} + // Cross-node rail-sync entry barrier for the hierarchical CE collectives. static ncclResult_t ncclRailSync(struct ncclComm* comm, struct ncclRmaProxyCtx* rmaProxyCtx, struct ncclKernelPlan* plan, int ctx, cudaStream_t stream) { @@ -1512,27 +1610,35 @@ static ncclResult_t ncclRailSync(struct ncclComm* comm, struct ncclRmaProxyCtx* goto exit; } -// Helper function to wait for a single peer's signals. -static ncclResult_t ncclProxyWaitOnePeer(struct ncclComm* comm, struct ncclRmaProxyCtx* rmaProxyCtx, - struct ncclKernelPlan* plan, int ctx, cudaStream_t stream, int peer, - int nsignals) { +// Helper function to wait for one or more distinct peers' signals. +static ncclResult_t ncclProxyWaitPeers(struct ncclComm* comm, struct ncclRmaProxyCtx* rmaProxyCtx, + struct ncclKernelPlan* plan, cudaStream_t stream, int npeers, const int* peersIn, + const int* nsignalsIn) { ncclResult_t ret = ncclSuccess; + int realPeers = 0; int* waitPeers = nullptr; int* waitSigCounts = nullptr; int* waitSignalIdxs = nullptr; struct ncclRmaProxyDesc* waitDesc = nullptr; CUstreamBatchMemOpParams* waitBatch = nullptr; - NCCLCHECKGOTO(ncclCalloc(&waitPeers, 1), ret, fail); - NCCLCHECKGOTO(ncclCalloc(&waitSigCounts, 1), ret, fail); - NCCLCHECKGOTO(ncclCalloc(&waitSignalIdxs, 1), ret, fail); - waitPeers[0] = peer; - waitSigCounts[0] = nsignals; + if (npeers <= 0) return ncclSuccess; + + NCCLCHECKGOTO(ncclCalloc(&waitPeers, npeers), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitSigCounts, npeers), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitSignalIdxs, npeers), ret, fail); + for (int i = 0; i < npeers; i++) { + if (nsignalsIn[i] <= 0) continue; + waitPeers[realPeers] = peersIn[i]; + waitSigCounts[realPeers] = nsignalsIn[i]; + realPeers++; + } + if (realPeers == 0) goto exit; NCCLCHECKGOTO(ncclCalloc(&waitDesc, 1), ret, fail); - NCCLCHECKGOTO(ncclRmaProxyWaitBuildDesc(comm, rmaProxyCtx, plan, 1, &waitPeers, &waitSigCounts, &waitSignalIdxs, - waitDesc), + NCCLCHECKGOTO(ncclRmaProxyWaitBuildDesc(comm, rmaProxyCtx, plan, realPeers, &waitPeers, &waitSigCounts, + &waitSignalIdxs, waitDesc), ret, fail); { @@ -1554,6 +1660,85 @@ static ncclResult_t ncclProxyWaitOnePeer(struct ncclComm* comm, struct ncclRmaPr goto exit; } +// Variant specialized for the 1-peer / 2-peer wait cases used by the retained +// hierarchical ring allgather implementation. It merges the clockwise and +// counterclockwise entries when they resolve to the same peer. +static ncclResult_t ncclProxyBuildWaitPeersRing(struct ncclComm* comm, struct ncclRmaProxyCtx* rmaProxyCtx, + struct ncclKernelPlan* plan, int npeers, const int* peersIn, + const int* nsignalsIn, struct ncclRmaProxyDesc** waitDescOut) { + ncclResult_t ret = ncclSuccess; + + int realPeers = 0; + int mergedPeers[2] = {0, 0}; + int mergedSignals[2] = {0, 0}; + int* waitPeers = nullptr; + int* waitSigCounts = nullptr; + int* waitSignalIdxs = nullptr; + struct ncclRmaProxyDesc* waitDesc = nullptr; + + *waitDescOut = nullptr; + if (npeers <= 0) return ncclSuccess; + if (npeers > 2) return ncclInternalError; + + if (npeers == 1) { + if (nsignalsIn[0] > 0) { + mergedPeers[0] = peersIn[0]; + mergedSignals[0] = nsignalsIn[0]; + realPeers = 1; + } + } else if (npeers == 2) { + int sig0 = nsignalsIn[0]; + int sig1 = nsignalsIn[1]; + if (sig0 > 0) { + mergedPeers[0] = peersIn[0]; + mergedSignals[0] = sig0; + realPeers = 1; + } + if (sig1 > 0) { + if (realPeers > 0 && mergedPeers[0] == peersIn[1]) { + mergedSignals[0] += sig1; + } else { + mergedPeers[realPeers] = peersIn[1]; + mergedSignals[realPeers] = sig1; + realPeers++; + } + } + } + if (realPeers == 0) goto exit; + + NCCLCHECKGOTO(ncclCalloc(&waitPeers, realPeers), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitSigCounts, realPeers), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitSignalIdxs, realPeers), ret, fail); + for (int i = 0; i < realPeers; i++) { + waitPeers[i] = mergedPeers[i]; + waitSigCounts[i] = mergedSignals[i]; + } + + NCCLCHECKGOTO(ncclCalloc(&waitDesc, 1), ret, fail); + NCCLCHECKGOTO(ncclRmaProxyWaitBuildDesc(comm, rmaProxyCtx, plan, realPeers, &waitPeers, &waitSigCounts, + &waitSignalIdxs, waitDesc), + ret, fail); + *waitDescOut = waitDesc; + waitDesc = nullptr; + +exit: + if (waitDesc != nullptr) (void)ncclRmaProxyDestroyDesc(comm, &waitDesc); + free(waitPeers); + free(waitSigCounts); + free(waitSignalIdxs); + return ret; +fail: + goto exit; +} + +// Helper function to wait for a single peer's signals. +static ncclResult_t ncclProxyWaitOnePeer(struct ncclComm* comm, struct ncclRmaProxyCtx* rmaProxyCtx, + struct ncclKernelPlan* plan, cudaStream_t stream, int peer, int nsignals) { + int peers[1] = {peer}; + int sigCounts[1] = {nsignals}; + return ncclProxyWaitPeers(comm, rmaProxyCtx, plan, stream, 1, peers, sigCounts); +} + // Hierarchical AllGather: railed all-to-all inter-node + intra-node CE scatter. // Each per-rank slice is split into chunks. A single PutGroup descriptor // bundles all nRemoteNodes * nChunks puts. @@ -1568,7 +1753,7 @@ static ncclResult_t ncclProxyWaitOnePeer(struct ncclComm* comm, struct ncclRmaPr // PutGroupDone // one memop blocks until all network puts complete // IntraNodeBarrier // gates user code reading recvbuf -ncclResult_t ncclHierCeAllGather(struct ncclComm* comm, struct ncclKernelPlan* plan, cudaStream_t stream) { +static ncclResult_t ncclHierCeAllGatherDirect(struct ncclComm* comm, struct ncclKernelPlan* plan, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; // Distribute the cross-node rail puts/waits across the NCCL-internal RMA proxy @@ -1606,15 +1791,15 @@ ncclResult_t ncclHierCeAllGather(struct ncclComm* comm, struct ncclKernelPlan* p // arrays are calloc'd (null-initialized) so the exit path frees them uniformly. int startOps = ncclRmaProxyPutGroupStartNumOps(persistent); int doneOps = ncclRmaProxyPutGroupDoneNumOps(persistent); - int* ctxOps = nullptr; // [numCtx] ops assigned per ctx - int* ctxFill = nullptr; // [numCtx] running fill index - struct ncclRmaProxyDesc** groupDesc = nullptr; // [numCtx] - struct ncclRmaPutSignalOp** groupOps = nullptr; // [numCtx] + int* ctxOps = nullptr; // [numCtx] ops assigned per ctx + int* ctxFill = nullptr; // [numCtx] running fill index + struct ncclRmaProxyDesc** groupDesc = nullptr; // [numCtx] + struct ncclRmaPutSignalOp** groupOps = nullptr; // [numCtx] // Start/done memop params for all active contexts, one contiguous slice per // context, so each phase fires a single stream batch instead of one per ctx. int nActiveCtx = 0; - CUstreamBatchMemOpParams* groupStartParams = nullptr; // [nActiveCtx * startOps] - CUstreamBatchMemOpParams* groupDoneParams = nullptr; // [nActiveCtx * doneOps] + CUstreamBatchMemOpParams* groupStartParams = nullptr; // [nActiveCtx * startOps] + CUstreamBatchMemOpParams* groupDoneParams = nullptr; // [nActiveCtx * doneOps] // Batch-ops scratch for intra-node broadcast. struct ncclCeBatchOpsParams ceBcastOps = {}; // Batch-ops scratch for per-chunk intra-node CE scatter. @@ -1663,7 +1848,7 @@ ncclResult_t ncclHierCeAllGather(struct ncclComm* comm, struct ncclKernelPlan* p // Pass 2: build each chunk's put op into its context's ops array. for (int s = 1; s < nNodes; s++) { - int p = s - 1; // peer index in plan + int p = s - 1; // peer index in plan int n = (comm->node + s) % nNodes; int railPeer = comm->nodeRanks[n].localRankToRank[localRank]; @@ -1707,9 +1892,8 @@ ncclResult_t ncclHierCeAllGather(struct ncclComm* comm, struct ncclKernelPlan* p // unicast loop (which is better at hiding tail latency). // declare-then-assign: NCCLCHECKGOTO's goto can't cross a scalar initialization. bool agUseMulticast; - agUseMulticast = - ncclCeAllGatherUseMulticast(comm, perRankBytes, ncclCudaGraphValid(comm->planner.capturingGraph), - (const uint8_t*)sendbuff == (const uint8_t*)recvbuff + myRank * perRankBytes); + agUseMulticast = ncclCeAllGatherUseMulticast(comm, perRankBytes, ncclCudaGraphValid(comm->planner.capturingGraph), + ncclAllGatherIsInPlace(sendbuff, recvbuff, myRank, perRankBytes)); // ==================================================================== // Phase 4: Self-broadcast (intra-node CE Broadcast of own chunk) @@ -1756,7 +1940,7 @@ ncclResult_t ncclHierCeAllGather(struct ncclComm* comm, struct ncclKernelPlan* p // ==================================================================== { for (int s = 1; s < nNodes; s++) { - int p = s - 1; // peer index in plan + int p = s - 1; // peer index in plan int n = (comm->node - s + nNodes) % nNodes; int railPeer = comm->nodeRanks[n].localRankToRank[localRank]; size_t peerSliceOffset = railPeer * perRankBytes; @@ -1771,7 +1955,7 @@ ncclResult_t ncclHierCeAllGather(struct ncclComm* comm, struct ncclKernelPlan* p // ----- Wait for this sub-chunk's signal from railPeer on the chunk's context ----- int k = (c - chunkPlan.chunkStart[p]) % numCtx; NCCLCHECKGOTO(ncclProxyWaitOnePeer(comm, (struct ncclRmaProxyCtx*)rmaProxyState->rmaProxyCtxs[baseCtx + k], - plan, baseCtx + k, stream, railPeer, /*nsignals=*/1), + plan, stream, railPeer, /*nsignals=*/1), ret, fail); // ----- CE scatter this sub-chunk to all other LSA peers ----- @@ -1834,6 +2018,551 @@ ncclResult_t ncclHierCeAllGather(struct ncclComm* comm, struct ncclKernelPlan* p goto exit; } +struct ncclHierCeAllGatherRingRound { + int originRankSendCw; + int originRankSendCcw; + int originRankRecvCw; + int originRankRecvCcw; + struct ncclDevrWindow* srcWinHostCw; + struct ncclDevrWindow* srcWinHostCcw; + size_t srcBaseOffsetCw; + size_t dstBaseOffsetCw; + size_t srcBaseOffsetCcw; + size_t dstBaseOffsetCcw; +}; + +// State consumed by the round-issue helper. The main ring routine owns the +// arrays and chunk plans; grouping the round-specific fields here keeps +// descriptor construction independent from setup, receive, and cleanup. +struct ncclHierCeAllGatherRingIssueState { + int numCtx; + int baseCtx; + int nActiveCtx; + int sendRailPeerCw; + int sendRailPeerCcw; + int* ctxOps; + int* ctxFill; + int* activeCtxs; + struct ncclHierChunkPlan* cwChunkPlan; + struct ncclHierChunkPlan* ccwChunkPlan; + struct ncclRmaProxyDesc** groupDesc; + struct ncclRmaPutSignalOp** groupOps; + CUstreamBatchMemOpParams** groupStartParams; + CUstreamBatchMemOpParams** groupDoneParams; + struct ncclHierCeAllGatherRingRound* rounds; +}; + +// Build and submit one bidirectional forwarding round. Two parameter slots are +// alternated by the caller so the next round can be submitted before the +// current round's local CE fanout and completion retirement. +static ncclResult_t ncclHierCeAllGatherRingIssueRound(struct ncclComm* comm, struct ncclKernelPlan* plan, + cudaStream_t stream, struct ncclHierCeAllGatherRingIssueState* st, + int step, int slot) { + ncclResult_t ret = ncclSuccess; + struct ncclRmaProxyState* rmaProxyState = &comm->rmaState.rmaProxyState; + struct ncclDevrWindow* recvWin = plan->ceCollArgs->recvWin; + struct ncclHierCeAllGatherRingRound* round = &st->rounds[step]; + int startOps = ncclRmaProxyPutGroupStartNumOps(plan->persistent); + int doneOps = ncclRmaProxyPutGroupDoneNumOps(plan->persistent); + + memset(st->ctxFill, 0, (size_t)st->numCtx * sizeof(int)); + for (int ai = 0; ai < st->nActiveCtx; ai++) { + int k = st->activeCtxs[ai]; + NCCLCHECKGOTO(ncclCalloc(&st->groupOps[k], st->ctxOps[k]), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&st->groupDesc[k], 1), ret, fail); + } + + if (st->cwChunkPlan->chunkStart != nullptr) { + for (int c = st->cwChunkPlan->chunkStart[0]; c < st->cwChunkPlan->chunkStart[1]; c++) { + int k = c % st->numCtx; + size_t subBytes = st->cwChunkPlan->chunkBytes[c]; + size_t off = st->cwChunkPlan->chunkOff[c]; + NCCLCHECKGOTO(ncclRmaProxyPutBuildOp(comm, (struct ncclRmaProxyCtx*)rmaProxyState->rmaProxyCtxs[st->baseCtx + k], + st->baseCtx + k, plan->persistent, round->srcWinHostCw, + round->srcBaseOffsetCw + off, recvWin, round->dstBaseOffsetCw + off, + subBytes, st->sendRailPeerCw, + /*signalIdx=*/0, NCCL_SIGNAL, &st->groupOps[k][st->ctxFill[k]++]), + ret, fail); + } + } + if (st->ccwChunkPlan->chunkStart != nullptr) { + for (int c = st->ccwChunkPlan->chunkStart[0]; c < st->ccwChunkPlan->chunkStart[1]; c++) { + int k = c % st->numCtx; + size_t subBytes = st->ccwChunkPlan->chunkBytes[c]; + size_t off = st->ccwChunkPlan->chunkOff[c]; + NCCLCHECKGOTO(ncclRmaProxyPutBuildOp(comm, (struct ncclRmaProxyCtx*)rmaProxyState->rmaProxyCtxs[st->baseCtx + k], + st->baseCtx + k, plan->persistent, round->srcWinHostCcw, + round->srcBaseOffsetCcw + off, recvWin, round->dstBaseOffsetCcw + off, + subBytes, st->sendRailPeerCcw, + /*signalIdx=*/0, NCCL_SIGNAL, &st->groupOps[k][st->ctxFill[k]++]), + ret, fail); + } + } + + for (int ai = 0; ai < st->nActiveCtx; ai++) { + int k = st->activeCtxs[ai]; + struct ncclRmaProxyCtx* proxyCtx = (struct ncclRmaProxyCtx*)rmaProxyState->rmaProxyCtxs[st->baseCtx + k]; + NCCLCHECKGOTO(ncclRmaProxyPutGroupBuildDesc(comm, proxyCtx, plan, st->ctxOps[k], &st->groupOps[k], st->baseCtx + k, + st->groupDesc[k]), + ret, fail); + NCCLCHECKGOTO(ncclRmaProxyPutGroupStartParams(st->groupDesc[k], st->groupStartParams[slot] + ai * startOps), ret, + fail); + NCCLCHECKGOTO(ncclRmaProxyPutGroupDoneParams(st->groupDesc[k], st->groupDoneParams[slot] + ai * doneOps), ret, + fail); + NCCLCHECKGOTO(ncclRmaProxyEnqueueDesc(proxyCtx, &st->groupDesc[k]), ret, fail); + } + if (st->nActiveCtx > 0) + NCCLCHECKGOTO(ncclCuStreamBatchMemOp(stream, st->nActiveCtx * startOps, st->groupStartParams[slot]), ret, fail); + +exit: + return ret; +fail: + goto exit; +} + +// Bidirectional ring variant of hierarchical CE allgather. +// +// DAG on the user stream: +// RailSync // cross-node entry barrier (net + wait) +// IntraNodeBarrier // gates LSA peers' recvbuf writes before local fanout starts +// SelfBcast // CE scatter of own slice to LSA peers +// Round1PutGroupSubmit // prime the pipeline with both clockwise and counterclockwise halves +// for each ring round: +// wait per active context // one wait covers that context's cw + ccw arrivals for the current round +// NextRoundPutGroupSubmit // submit the next round after current arrivals are known, before local fanout +// CE-scatter arrived halves // local fanout of the current round's newly received sub-chunks via LSA +// PutGroupDone // retire the current round's outbound puts +// IntraNodeBarrier // gates user code reading recvbuf +static ncclResult_t ncclHierCeAllGatherRing(struct ncclComm* comm, struct ncclKernelPlan* plan, cudaStream_t stream) { + ncclResult_t ret = ncclSuccess; + + struct ncclRmaProxyState* rmaProxyState = &comm->rmaState.rmaProxyState; + int baseCtx = comm->config.numRmaCtx; + int railCtx = baseCtx; + int myRank = comm->rank; + int localRank = comm->localRank; + int myNode = comm->node; + int nNodes = comm->nNodes; + int nRemoteNodes = nNodes - 1; + int myLsaRank = comm->devrState.lsaSelf; + int lsaSize = comm->devrState.lsaSize; + bool persistent = plan->persistent; + + struct ncclCeCollArgs* args = plan->ceCollArgs; + const void* sendbuff = args->sendBuff; + void* recvbuff = args->recvBuff; + struct ncclDevrWindow* sendWin = args->sendWin; + struct ncclDevrWindow* recvWin = args->recvWin; + size_t perRankBytes = args->nElts * args->eltSize; + size_t cwBytes = perRankBytes <= 1 ? perRankBytes : perRankBytes / 2; + size_t ccwBytes = perRankBytes - cwBytes; + bool agUseMulticast = + ncclCeAllGatherUseMulticast(comm, perRankBytes, ncclCudaGraphValid(comm->planner.capturingGraph), + (const uint8_t*)sendbuff == (const uint8_t*)recvbuff + myRank * perRankBytes); + int numCtx = ncclHierCollNumCtx(rmaProxyState, perRankBytes, persistent); + + struct ncclRmaProxyCtx* railProxyCtx = (struct ncclRmaProxyCtx*)rmaProxyState->rmaProxyCtxs[railCtx]; + struct ncclHierChunkPlan cwChunkPlan = {}; + struct ncclHierChunkPlan ccwChunkPlan = {}; + int startOps = ncclRmaProxyPutGroupStartNumOps(persistent); + int doneOps = ncclRmaProxyPutGroupDoneNumOps(persistent); + int* ctxOps = nullptr; + int* cwCtxOps = nullptr; + int* ccwCtxOps = nullptr; + int* ctxFill = nullptr; + int* cwChunkFill = nullptr; + int* ccwChunkFill = nullptr; + int* activeCtxs = nullptr; + int** cwChunksByCtx = nullptr; + int** ccwChunksByCtx = nullptr; + uint8_t** scatterPeerBases = nullptr; + int* waitNPeersByActiveCtx = nullptr; + int** waitPeersByActiveCtx = nullptr; + int** waitSignalsByActiveCtx = nullptr; + struct ncclRmaProxyDesc** waitDesc = nullptr; + struct ncclRmaProxyDesc** groupDesc = nullptr; + struct ncclRmaPutSignalOp** groupOps = nullptr; + int nActiveCtx = 0; + CUstreamBatchMemOpParams* groupStartParams[2] = {nullptr, nullptr}; + CUstreamBatchMemOpParams* groupDoneParams[2] = {nullptr, nullptr}; + CUstreamBatchMemOpParams* waitBatch = nullptr; + struct ncclCeBatchOpsParams ceBcastOps = {}; + struct ncclCeBatchOpsParams ceScatterOps = {}; + uint8_t* scatterMcBase = nullptr; + struct ncclHierCeAllGatherRingRound* roundMeta = nullptr; + struct ncclHierCeAllGatherRingIssueState issueState = {}; + + NCCLCHECKGOTO(ncclCalloc(&ctxOps, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&cwCtxOps, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&ccwCtxOps, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&ctxFill, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&cwChunkFill, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&ccwChunkFill, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&activeCtxs, numCtx), ret, fail); + if (lsaSize > 1) { + NCCLCHECKGOTO(ncclCalloc(&scatterPeerBases, lsaSize - 1), ret, fail); + } + NCCLCHECKGOTO(ncclCalloc(&cwChunksByCtx, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&ccwChunksByCtx, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitNPeersByActiveCtx, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitPeersByActiveCtx, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitSignalsByActiveCtx, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitDesc, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&groupDesc, numCtx), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&groupOps, numCtx), ret, fail); + + // ==================================================================== + // Phase 1: Rail sync (cross-node entry barrier) + // ==================================================================== + NCCLCHECKGOTO(ncclRailSync(comm, railProxyCtx, plan, railCtx, stream), ret, fail); + + // Build one-peer chunking that is reused by every ring round. + if (nRemoteNodes > 0) { + size_t maxChunk = ncclHierCollRingChunkWidth(perRankBytes, cwBytes, ccwBytes, numCtx); + if (cwBytes > 0) { + NCCLCHECKGOTO(ncclHierCollBuildChunk(cwBytes, 1, maxChunk, &cwChunkPlan), ret, fail); + for (int c = cwChunkPlan.chunkStart[0]; c < cwChunkPlan.chunkStart[1]; c++) { + int k = c % numCtx; + ctxOps[k]++; + cwCtxOps[k]++; + } + } + if (ccwBytes > 0) { + NCCLCHECKGOTO(ncclHierCollBuildChunk(ccwBytes, 1, maxChunk, &ccwChunkPlan), ret, fail); + for (int c = ccwChunkPlan.chunkStart[0]; c < ccwChunkPlan.chunkStart[1]; c++) { + int k = c % numCtx; + ctxOps[k]++; + ccwCtxOps[k]++; + } + } + for (int k = 0; k < numCtx; k++) { + if (ctxOps[k] == 0) continue; + activeCtxs[nActiveCtx] = k; + nActiveCtx++; + } + + for (int k = 0; k < numCtx; k++) { + if (cwCtxOps[k] > 0) NCCLCHECKGOTO(ncclCalloc(&cwChunksByCtx[k], cwCtxOps[k]), ret, fail); + if (ccwCtxOps[k] > 0) NCCLCHECKGOTO(ncclCalloc(&ccwChunksByCtx[k], ccwCtxOps[k]), ret, fail); + } + if (cwBytes > 0) { + for (int c = cwChunkPlan.chunkStart[0]; c < cwChunkPlan.chunkStart[1]; c++) { + int k = c % numCtx; + cwChunksByCtx[k][cwChunkFill[k]++] = c; + } + } + if (ccwBytes > 0) { + for (int c = ccwChunkPlan.chunkStart[0]; c < ccwChunkPlan.chunkStart[1]; c++) { + int k = c % numCtx; + ccwChunksByCtx[k][ccwChunkFill[k]++] = c; + } + } + + for (int i = 0; i < 2; i++) { + NCCLCHECKGOTO(ncclCalloc(&groupStartParams[i], (size_t)nActiveCtx * startOps), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&groupDoneParams[i], (size_t)nActiveCtx * doneOps), ret, fail); + } + } + + // ==================================================================== + // Phase 2: Initial intra-node barrier + // ==================================================================== + NCCLCHECKGOTO(ncclMemOpSync(comm, stream, args), ret, fail); + + if (agUseMulticast) { + void* mcBase = nullptr; + NCCLCHECKGOTO(ncclDevrGetLsaTeamPtrMC(comm, recvWin, 0, ncclTeamLsa(comm), &mcBase), ret, fail); + scatterMcBase = (uint8_t*)mcBase; + } else if (lsaSize > 1) { + for (int r = 1; r < lsaSize; r++) { + int targetLsaRank = (myLsaRank + r) % lsaSize; + void* peerBase = nullptr; + NCCLCHECKGOTO(ncclDevrGetLsaRankPtr(comm, recvWin, 0, targetLsaRank, &peerBase), ret, fail); + scatterPeerBases[r - 1] = (uint8_t*)peerBase; + } + } + + // ==================================================================== + // Phase 3: Self-broadcast (intra-node CE broadcast of own chunk) + // ==================================================================== + NCCLCHECKGOTO(ncclCeInitBatchOpsParams(&ceBcastOps, lsaSize), ret, fail); + { + uint8_t* myRecvSlot = (uint8_t*)recvbuff + myRank * perRankBytes; + size_t offset = myRecvSlot - (uint8_t*)recvWin->userPtr; + + if (agUseMulticast) { + ceBcastOps.srcs[ceBcastOps.numOps] = (void*)sendbuff; + ceBcastOps.dsts[ceBcastOps.numOps] = (void*)(scatterMcBase + offset); + ceBcastOps.sizes[ceBcastOps.numOps] = perRankBytes; + ceBcastOps.numOps++; + } else { + if (myRecvSlot != (const uint8_t*)sendbuff) { + ceBcastOps.srcs[ceBcastOps.numOps] = (void*)sendbuff; + ceBcastOps.dsts[ceBcastOps.numOps] = (void*)myRecvSlot; + ceBcastOps.sizes[ceBcastOps.numOps] = perRankBytes; + ceBcastOps.numOps++; + } + + for (int r = 1; r < lsaSize; r++) { + ceBcastOps.srcs[ceBcastOps.numOps] = (void*)sendbuff; + ceBcastOps.dsts[ceBcastOps.numOps] = (void*)(scatterPeerBases[r - 1] + offset); + ceBcastOps.sizes[ceBcastOps.numOps] = perRankBytes; + ceBcastOps.numOps++; + } + } + + NCCLCHECKGOTO(ncclCeLaunchBatchOps(comm, &ceBcastOps, stream, args), ret, fail); + } + + // ==================================================================== + // Phase 4: Ring rounds (next-node send, previous-node receive) + // ==================================================================== + if (nRemoteNodes > 0 && nActiveCtx > 0) { + int nRounds = nNodes - 1; + int nextNode = (myNode + 1) % nNodes; + int prevNode = (myNode - 1 + nNodes) % nNodes; + int sendRailPeerCw = comm->nodeRanks[nextNode].localRankToRank[localRank]; + int recvRailPeerCw = comm->nodeRanks[prevNode].localRankToRank[localRank]; + int sendRailPeerCcw = comm->nodeRanks[prevNode].localRankToRank[localRank]; + int recvRailPeerCcw = comm->nodeRanks[nextNode].localRankToRank[localRank]; + int nRoundChunks = (cwChunkPlan.chunkStart ? (cwChunkPlan.chunkStart[1] - cwChunkPlan.chunkStart[0]) : 0) + + (ccwChunkPlan.chunkStart ? (ccwChunkPlan.chunkStart[1] - ccwChunkPlan.chunkStart[0]) : 0); + int scatterCapacity = agUseMulticast ? nRoundChunks : nRoundChunks * (lsaSize - 1); + + for (int ai = 0; ai < nActiveCtx; ai++) { + int k = activeCtxs[ai]; + int waitNPeers = 0; + NCCLCHECKGOTO(ncclCalloc(&waitPeersByActiveCtx[k], 2), ret, fail); + NCCLCHECKGOTO(ncclCalloc(&waitSignalsByActiveCtx[k], 2), ret, fail); + if (cwCtxOps[k] > 0) { + waitPeersByActiveCtx[k][waitNPeers] = recvRailPeerCw; + waitSignalsByActiveCtx[k][waitNPeers] = cwCtxOps[k]; + waitNPeers++; + } + if (ccwCtxOps[k] > 0) { + waitPeersByActiveCtx[k][waitNPeers] = recvRailPeerCcw; + waitSignalsByActiveCtx[k][waitNPeers] = ccwCtxOps[k]; + waitNPeers++; + } + waitNPeersByActiveCtx[k] = waitNPeers; + } + + NCCLCHECKGOTO(ncclCalloc(&roundMeta, nRounds + 1), ret, fail); + for (int step = 1; step <= nRounds; step++) { + int originNodeSendCw = (myNode - (step - 1) + nNodes) % nNodes; + int originNodeSendCcw = (myNode + (step - 1)) % nNodes; + int originNodeRecvCw = (myNode - step + nNodes) % nNodes; + int originNodeRecvCcw = (myNode + step) % nNodes; + roundMeta[step].originRankSendCw = comm->nodeRanks[originNodeSendCw].localRankToRank[localRank]; + roundMeta[step].originRankSendCcw = comm->nodeRanks[originNodeSendCcw].localRankToRank[localRank]; + roundMeta[step].originRankRecvCw = comm->nodeRanks[originNodeRecvCw].localRankToRank[localRank]; + roundMeta[step].originRankRecvCcw = comm->nodeRanks[originNodeRecvCcw].localRankToRank[localRank]; + roundMeta[step].srcWinHostCw = step == 1 ? sendWin : recvWin; + roundMeta[step].srcWinHostCcw = step == 1 ? sendWin : recvWin; + roundMeta[step].srcBaseOffsetCw = step == 1 ? + (const uint8_t*)sendbuff - (const uint8_t*)sendWin->userPtr : + ((const uint8_t*)recvbuff + roundMeta[step].originRankSendCw * perRankBytes) - + (const uint8_t*)recvWin->userPtr; + roundMeta[step].dstBaseOffsetCw = + ((const uint8_t*)recvbuff + roundMeta[step].originRankSendCw * perRankBytes) - (const uint8_t*)recvWin->userPtr; + roundMeta[step].srcBaseOffsetCcw = + (step == 1 ? (const uint8_t*)sendbuff - (const uint8_t*)sendWin->userPtr : + ((const uint8_t*)recvbuff + roundMeta[step].originRankSendCcw * perRankBytes) - + (const uint8_t*)recvWin->userPtr) + + cwBytes; + roundMeta[step].dstBaseOffsetCcw = ((const uint8_t*)recvbuff + roundMeta[step].originRankSendCcw * perRankBytes) - + (const uint8_t*)recvWin->userPtr + cwBytes; + } + + issueState.numCtx = numCtx; + issueState.baseCtx = baseCtx; + issueState.nActiveCtx = nActiveCtx; + issueState.sendRailPeerCw = sendRailPeerCw; + issueState.sendRailPeerCcw = sendRailPeerCcw; + issueState.ctxOps = ctxOps; + issueState.ctxFill = ctxFill; + issueState.activeCtxs = activeCtxs; + issueState.cwChunkPlan = &cwChunkPlan; + issueState.ccwChunkPlan = &ccwChunkPlan; + issueState.groupDesc = groupDesc; + issueState.groupOps = groupOps; + issueState.groupStartParams = groupStartParams; + issueState.groupDoneParams = groupDoneParams; + issueState.rounds = roundMeta; + + if (scatterCapacity > 0) NCCLCHECKGOTO(ncclCeInitBatchOpsParams(&ceScatterOps, scatterCapacity), ret, fail); + + NCCLCHECKGOTO(ncclHierCeAllGatherRingIssueRound(comm, plan, stream, &issueState, /*step=*/1, /*slot=*/0), ret, + fail); + + for (int step = 1; step <= nRounds; step++) { + auto* meta = &roundMeta[step]; + int slot = (step - 1) & 1; + + // Wait for every active context before forwarding data read from recvBuff. + // Collect all contexts' waits into one stream memop batch. + int waitOpsTotal = 0; + for (int ai = 0; ai < nActiveCtx; ai++) { + int k = activeCtxs[ai]; + struct ncclRmaProxyCtx* pc = (struct ncclRmaProxyCtx*)rmaProxyState->rmaProxyCtxs[baseCtx + k]; + NCCLCHECKGOTO(ncclProxyBuildWaitPeersRing(comm, pc, plan, waitNPeersByActiveCtx[k], waitPeersByActiveCtx[k], + waitSignalsByActiveCtx[k], &waitDesc[k]), + ret, fail); + if (waitDesc[k] != nullptr) waitOpsTotal += ncclRmaProxyWaitNumStreamOps(waitDesc[k]); + } + + // The active contexts and their wait peer sets are invariant across rounds. + if (waitBatch == nullptr) NCCLCHECKGOTO(ncclCalloc(&waitBatch, waitOpsTotal), ret, fail); + int waitOff = 0; + for (int ai = 0; ai < nActiveCtx; ai++) { + int k = activeCtxs[ai]; + if (waitDesc[k] == nullptr) continue; + struct ncclRmaProxyCtx* pc = (struct ncclRmaProxyCtx*)rmaProxyState->rmaProxyCtxs[baseCtx + k]; + int waitOps = ncclRmaProxyWaitNumStreamOps(waitDesc[k]); + NCCLCHECKGOTO(ncclRmaProxyWaitParams(pc, waitDesc[k], waitBatch + waitOff), ret, fail); + NCCLCHECKGOTO(ncclRmaProxyEnqueueDesc(pc, &waitDesc[k]), ret, fail); + waitOff += waitOps; + } + NCCLCHECKGOTO(ncclCuStreamBatchMemOp(stream, waitOpsTotal, waitBatch), ret, fail); + + // Enqueue the next round as soon as its inputs are known to be ready, + // before spending host time constructing this round's CE scatter batch. + if (step < nRounds) + NCCLCHECKGOTO(ncclHierCeAllGatherRingIssueRound(comm, plan, stream, &issueState, step + 1, slot ^ 1), ret, + fail); + + ceScatterOps.numOps = 0; + for (int ai = 0; ai < nActiveCtx; ai++) { + int k = activeCtxs[ai]; + if (cwBytes > 0) { + for (int ci = 0; ci < cwCtxOps[k]; ci++) { + int c = cwChunksByCtx[k][ci]; + size_t subBytes = cwChunkPlan.chunkBytes[c]; + size_t off = cwChunkPlan.chunkOff[c]; + uint8_t* chunkSlot = (uint8_t*)recvbuff + meta->originRankRecvCw * perRankBytes + off; + size_t winOffset = chunkSlot - (uint8_t*)recvWin->userPtr; + + if (agUseMulticast) { + ceScatterOps.srcs[ceScatterOps.numOps] = chunkSlot; + ceScatterOps.dsts[ceScatterOps.numOps] = (void*)(scatterMcBase + winOffset); + ceScatterOps.sizes[ceScatterOps.numOps] = subBytes; + ceScatterOps.numOps++; + } else { + for (int r = 1; r < lsaSize; r++) { + ceScatterOps.srcs[ceScatterOps.numOps] = chunkSlot; + ceScatterOps.dsts[ceScatterOps.numOps] = (void*)(scatterPeerBases[r - 1] + winOffset); + ceScatterOps.sizes[ceScatterOps.numOps] = subBytes; + ceScatterOps.numOps++; + } + } + } + } + if (ccwBytes > 0) { + for (int ci = 0; ci < ccwCtxOps[k]; ci++) { + int c = ccwChunksByCtx[k][ci]; + size_t subBytes = ccwChunkPlan.chunkBytes[c]; + size_t off = ccwChunkPlan.chunkOff[c]; + uint8_t* chunkSlot = (uint8_t*)recvbuff + meta->originRankRecvCcw * perRankBytes + cwBytes + off; + size_t winOffset = chunkSlot - (uint8_t*)recvWin->userPtr; + + if (agUseMulticast) { + ceScatterOps.srcs[ceScatterOps.numOps] = chunkSlot; + ceScatterOps.dsts[ceScatterOps.numOps] = (void*)(scatterMcBase + winOffset); + ceScatterOps.sizes[ceScatterOps.numOps] = subBytes; + ceScatterOps.numOps++; + } else { + for (int r = 1; r < lsaSize; r++) { + ceScatterOps.srcs[ceScatterOps.numOps] = chunkSlot; + ceScatterOps.dsts[ceScatterOps.numOps] = (void*)(scatterPeerBases[r - 1] + winOffset); + ceScatterOps.sizes[ceScatterOps.numOps] = subBytes; + ceScatterOps.numOps++; + } + } + } + } + } + + if (ceScatterOps.numOps > 0) { + NCCLCHECKGOTO(ncclCeLaunchBatchOps(comm, &ceScatterOps, stream, args), ret, fail); + } + + NCCLCHECKGOTO(ncclCuStreamBatchMemOp(stream, nActiveCtx * doneOps, groupDoneParams[slot]), ret, fail); + } + } + + // ==================================================================== + // Phase 5: Final intra-node barrier + // ==================================================================== + NCCLCHECKGOTO(ncclMemOpSync(comm, stream, args), ret, fail); + +exit: + ncclCeFreeBatchOpsParams(&ceBcastOps); + ncclCeFreeBatchOpsParams(&ceScatterOps); + if (groupDesc) { + for (int k = 0; k < numCtx; k++) { + if (groupDesc[k] != nullptr) (void)ncclRmaProxyDestroyDesc(comm, &groupDesc[k]); + } + } + if (groupOps) { + for (int k = 0; k < numCtx; k++) free(groupOps[k]); + } + free(groupOps); + for (int i = 0; i < 2; i++) { + free(groupStartParams[i]); + free(groupDoneParams[i]); + } + free(groupDesc); + free(ctxOps); + free(cwCtxOps); + free(ccwCtxOps); + free(ctxFill); + free(cwChunkFill); + free(ccwChunkFill); + free(activeCtxs); + free(scatterPeerBases); + free(roundMeta); + if (waitPeersByActiveCtx) { + for (int k = 0; k < numCtx; k++) free(waitPeersByActiveCtx[k]); + } + if (waitSignalsByActiveCtx) { + for (int k = 0; k < numCtx; k++) free(waitSignalsByActiveCtx[k]); + } + free(waitNPeersByActiveCtx); + free(waitPeersByActiveCtx); + free(waitSignalsByActiveCtx); + if (waitDesc) { + for (int k = 0; k < numCtx; k++) { + if (waitDesc[k] != nullptr) (void)ncclRmaProxyDestroyDesc(comm, &waitDesc[k]); + } + } + free(waitDesc); + free(waitBatch); + if (cwChunksByCtx) { + for (int k = 0; k < numCtx; k++) free(cwChunksByCtx[k]); + } + if (ccwChunksByCtx) { + for (int k = 0; k < numCtx; k++) free(ccwChunksByCtx[k]); + } + free(cwChunksByCtx); + free(ccwChunksByCtx); + ncclHierCollFreeChunkPlan(&cwChunkPlan); + ncclHierCollFreeChunkPlan(&ccwChunkPlan); + return ret; +fail: + goto exit; +} + +ncclResult_t ncclHierCeAllGather(struct ncclComm* comm, struct ncclKernelPlan* plan, cudaStream_t stream) { + size_t perRankBytes = plan->ceCollArgs->nElts * plan->ceCollArgs->eltSize; + bool useRing = ncclParamHierCeCollAgRailRingEnable() > 0; + const char* railAlgo = useRing ? "Ring" : "Direct"; + if (comm->rank == 0) + INFO(NCCL_TUNING, "AllGather [Hierarchical CE]: %zu Bytes -> Rail %s RMA proxy + CE", perRankBytes, railAlgo); + + if (useRing) return ncclHierCeAllGatherRing(comm, plan, stream); + return ncclHierCeAllGatherDirect(comm, plan, stream); +} + // Hierarchical AlltoAll: alltoall inter-node + intra-node CE alltoall. // DAG on the user stream: // RailSync // rail-only entry barrier @@ -1885,22 +2614,22 @@ ncclResult_t ncclHierCeAlltoAll(struct ncclComm* comm, struct ncclKernelPlan* pl // null the slots they consume. int startOps = ncclRmaProxyPutGroupStartNumOps(persistent); int doneOps = ncclRmaProxyPutGroupDoneNumOps(persistent); - int* ctxOps = nullptr; // [numCtx] ops assigned per ctx - int* ctxFill = nullptr; // [numCtx] running fill index - struct ncclRmaProxyDesc** groupDesc = nullptr; // [numCtx] - struct ncclRmaPutSignalOp** groupOps = nullptr; // [numCtx] + int* ctxOps = nullptr; // [numCtx] ops assigned per ctx + int* ctxFill = nullptr; // [numCtx] running fill index + struct ncclRmaProxyDesc** groupDesc = nullptr; // [numCtx] + struct ncclRmaPutSignalOp** groupOps = nullptr; // [numCtx] // Start/done memop params for all active contexts, one contiguous slice per // context, so each phase fires a single stream batch instead of one per ctx. int nActiveCtx = 0; - CUstreamBatchMemOpParams* groupStartParams = nullptr; // [nActiveCtx * startOps] - CUstreamBatchMemOpParams* groupDoneParams = nullptr; // [nActiveCtx * doneOps] + CUstreamBatchMemOpParams* groupStartParams = nullptr; // [nActiveCtx * startOps] + CUstreamBatchMemOpParams* groupDoneParams = nullptr; // [nActiveCtx * doneOps] // Per-context inbound wait descriptors (each covers the peers/counts whose chunks // landed on that context); their stream memops are likewise fired as one batch. - int** waitPeers = nullptr; // [numCtx][] - int** waitSigCounts = nullptr; // [numCtx][] - int** waitSignalIdxs = nullptr; // [numCtx][] all-zero (hier-CE uses signal 0) - struct ncclRmaProxyDesc** waitDesc = nullptr; // [numCtx] - CUstreamBatchMemOpParams* waitBatch = nullptr; // [sum of per-ctx wait ops] + int** waitPeers = nullptr; // [numCtx][] + int** waitSigCounts = nullptr; // [numCtx][] + int** waitSignalIdxs = nullptr; // [numCtx][] all-zero (hier-CE uses signal 0) + struct ncclRmaProxyDesc** waitDesc = nullptr; // [numCtx] + CUstreamBatchMemOpParams* waitBatch = nullptr; // [sum of per-ctx wait ops] // Intra-node alltoall scratch. struct ncclCeBatchOpsParams ceLocalA2A = {}; @@ -1949,7 +2678,7 @@ ncclResult_t ncclHierCeAlltoAll(struct ncclComm* comm, struct ncclKernelPlan* pl NCCLCHECKGOTO(ncclCalloc(&groupDoneParams, (size_t)nActiveCtx * doneOps), ret, fail); // Pass 2: build each chunk's put op into its context's ops array. - int p = 0; // chunk plan slot index + int p = 0; // chunk plan slot index for (int s = 1; s < nNodes; s++) { int n = (myNode + s) % nNodes; for (int lr = 0; lr < localRanks; lr++) { @@ -2536,10 +3265,12 @@ ncclResult_t scheduleCeCollTaskToPlan(struct ncclComm* comm, struct ncclKernelPl if (comm->rank == 0) { // Same predicate ncclLaunchCeColl dispatches on, so the marker cannot claim - // a path the launch did not take. + // a path the launch did not take. AllGather logs its own marker (with the + // rail algorithm) from ncclHierCeAllGather. if (ncclHierCeDispatch(comm)) { - INFO(NCCL_TUNING, "%s " RCCL_CE_HIER_SELECTED_TAG ": %ld Bytes -> RMA proxy + CE", ncclFuncToString(task->func), - task->count * ncclTypeSize(task->datatype)); + if (task->func != ncclFuncAllGather) + INFO(NCCL_TUNING, "%s " RCCL_CE_HIER_SELECTED_TAG ": %ld Bytes -> RMA proxy + CE", ncclFuncToString(task->func), + task->count * ncclTypeSize(task->datatype)); } else { // Matches the useMCSync predicate in ncclMemOpSync. const char* nvlsSync = comm->symkState.hasLsaMultimem ? "; CE synchronization with NVLS" : ""; diff --git a/projects/rccl/src/cft_dev_runtime.cc b/projects/rccl/src/cft_dev_runtime.cc index 51636367d17a..812522a018bc 100644 --- a/projects/rccl/src/cft_dev_runtime.cc +++ b/projects/rccl/src/cft_dev_runtime.cc @@ -5,6 +5,7 @@ * See LICENSE.txt for more license information *************************************************************************/ +#include #include "dev_runtime_internal.h" #include "comm.h" #include "nccl_device/core.h" @@ -15,20 +16,33 @@ NCCL_PARAM(CftEnable, "CFT_ENABLE", 1); -ncclResult_t ncclGpuCftSupport(struct ncclComm* comm, int* gpuCftSupport) { +ncclResult_t ncclGpuCftSupport(struct ncclComm* comm, int* gpuCftSupport, bool* gpuCftMulticastSupport, + bool* gpuCftCountedSupport) { *gpuCftSupport = 0; + *gpuCftMulticastSupport = false; + *gpuCftCountedSupport = false; #if CUDA_VERSION >= 13030 if (ncclParamCftEnable()) { int driverVersion; NCCLCHECK(ncclCudaDriverVersion(&driverVersion)); - int unicastSupported = 0, multicastSupported = 0; + int unicastSupported = 0, multicastSupported = 0, countedSupported = 0; if (driverVersion >= 13030) { CUCHECK(cuDeviceGetAttribute(&unicastSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_UNICAST_SUPPORTED, (CUdevice)comm->cudaDev)); CUCHECK(cuDeviceGetAttribute(&multicastSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_MULTICAST_SUPPORTED, (CUdevice)comm->cudaDev)); } - if (unicastSupported && multicastSupported) *gpuCftSupport = std::min(CUDA_VERSION, driverVersion); + if (unicastSupported) { + *gpuCftSupport = std::min(CUDA_VERSION, driverVersion); + } + if (multicastSupported) { + *gpuCftMulticastSupport = (bool)multicastSupported; + } + if (unicastSupported || multicastSupported) { + CUCHECK(cuDeviceGetAttribute(&countedSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_COUNTED_OPS_SUPPORTED, + (CUdevice)comm->cudaDev)); + *gpuCftCountedSupport = (bool)countedSupported; + } } #endif return ncclSuccess; @@ -49,7 +63,8 @@ int computeCftMcSize(struct ncclComm* comm) { } ncclResult_t symBindTeamLe(struct ncclComm* comm, struct ncclDevrMemory* mem, ncclCftLeId le) { - if (le != NCCL_LE_ID_INVALID && !mem->globalHasSysmemSegment) { + if (ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterCft) && le != NCCL_LE_ID_INVALID && + !mem->globalHasSysmemSegment) { #if CUDA_VERSION >= 13030 CUCHECK(cuLogicalEndpointBindAddr(le, (CUdevice)comm->cudaDev, mem->bigOffset, mem->primaryAddr, mem->lsaMinSize, 0)); @@ -59,7 +74,8 @@ ncclResult_t symBindTeamLe(struct ncclComm* comm, struct ncclDevrMemory* mem, nc } ncclResult_t symUnbindTeamLe(struct ncclComm* comm, struct ncclDevrMemory* mem, ncclCftLeId le) { - if (le != NCCL_LE_ID_INVALID && !mem->globalHasSysmemSegment) { + if (ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterCft) && le != NCCL_LE_ID_INVALID && + !mem->globalHasSysmemSegment) { #if CUDA_VERSION >= 13030 CUCHECKIGNORE(cuLogicalEndpointUnbind(le, (CUdevice)comm->cudaDev, mem->bigOffset, mem->lsaMinSize)); #endif @@ -96,37 +112,49 @@ static ncclResult_t waitLeReady(ncclCftLeId le, uint32_t leCount) { } ncclResult_t symTeamObtainUcLe(struct ncclComm* comm, struct ncclDevrTeam* t, struct ncclDevrState* devr, - bool* /*needBarrier*/) { + bool* /*needBarrier*/, bool counted) { // No need to request a barrier: local completion of waitLeReady guarantees that the remote LE is ready. // Remote completion of the binding is guaranteed by the bootstrapAllGather. ncclResult_t ret = ncclSuccess; - struct ncclDevrStateCftUc* cftUc = &devr->le; // Always create the flat LE team to be stored in the devrState. ncclTeam_t flatTeam = ncclTeamCft(comm); int importedCount = 0, bindCount = 0, releaseCount = 0; + // Counted LEs are limited to 256GB in size + const size_t leSize = counted ? std::min(size_t(256ULL << 30), devr->bigSize) : devr->bigSize; ncclCftLeId leUcSelf = NCCL_LE_ID_INVALID; CUlogicalEndpointFabricHandle* leUcHandles = nullptr; int* cftRankList = nullptr; - if (cftUc->baseId == NCCL_LE_ID_INVALID) { + struct ncclDevrStateCftUc* cftUc = devr->le; + if (cftUc[counted].baseId == NCCL_LE_ID_INVALID) { CUlogicalEndpointProp prop = {}; prop.type = CU_LOGICAL_ENDPOINT_TYPE_UNICAST; - prop.flags = CU_LOGICAL_ENDPOINT_FLAG_NONE; + prop.flags = counted ? CU_LOGICAL_ENDPOINT_FLAG_COUNTED_OPS : CU_LOGICAL_ENDPOINT_FLAG_NONE; prop.unicast.device = (CUdevice)comm->cudaDev; prop.ipcHandleTypes = CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC; - prop.size = devr->bigSize; - NCCLCHECK(checkLeSize(prop, devr->bigSize)); + prop.size = leSize; + NCCLCHECK(checkLeSize(prop, leSize)); NCCLCHECKGOTO(ncclCalloc(&leUcHandles, flatTeam.nRanks), ret, fail); NCCLCHECKGOTO(ncclCalloc(&cftRankList, flatTeam.nRanks), ret, fail); - CUCHECKGOTO(cuLogicalEndpointIdReserve(&cftUc->baseId, flatTeam.nRanks), ret, fail); - releaseCount = flatTeam.nRanks; - CUCHECKGOTO(cuLogicalEndpointCreate(cftUc->baseId + flatTeam.rank, &prop), ret, fail); - // leUcSelf is set if creation was succesfull - leUcSelf = cftUc->baseId + flatTeam.rank; + // UC LD IDs are reserved in contiguous pairs, such that in devComm: counted ucLeId = devComm->ucLeId + nRanks + // Check if the other counted configuration was created. + bool ucLeIdsCreated = cftUc[!counted].baseId != NCCL_LE_ID_INVALID; + if (ucLeIdsCreated) { + cftUc[counted].baseId = counted ? cftUc[0].baseId + flatTeam.nRanks : cftUc[1].baseId - flatTeam.nRanks; + } else { + ncclCftLeId ucLeIdBase; + CUCHECKGOTO(cuLogicalEndpointIdReserve(&ucLeIdBase, flatTeam.nRanks * 2), ret, fail); + cftUc[counted].baseId = ucLeIdBase + (counted ? flatTeam.nRanks : 0); + releaseCount = flatTeam.nRanks * 2; + } + CUCHECKGOTO(cuLogicalEndpointCreate(cftUc[counted].baseId + flatTeam.rank, &prop), ret, fail); + // leUcSelf is set if creation was successful + leUcSelf = cftUc[counted].baseId + flatTeam.rank; NCCLCHECKGOTO(waitLeReady(leUcSelf, /*count=*/1), ret, fail); // bind all the windows already existing in the comm for (struct ncclDevrMemory* mem = devr->memHead; mem != nullptr; mem = mem->next) { + if ((bool)(mem->winFlags & NCCL_WIN_CFT_COUNTED) != counted) continue; NCCLCHECKGOTO(symBindTeamLe(comm, mem, leUcSelf), ret, fail); bindCount++; } @@ -140,16 +168,16 @@ ncclResult_t symTeamObtainUcLe(struct ncclComm* comm, struct ncclDevrTeam* t, st ret, fail); for (int r = 0; r < flatTeam.nRanks; r++) { if (r == flatTeam.rank) continue; - CUCHECKGOTO(cuLogicalEndpointImport(cftUc->baseId + r, &leUcHandles[r], + CUCHECKGOTO(cuLogicalEndpointImport(cftUc[counted].baseId + r, &leUcHandles[r], CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC), ret, fail); importedCount++; } - NCCLCHECKGOTO(waitLeReady(cftUc->baseId, flatTeam.nRanks), ret, fail); + NCCLCHECKGOTO(waitLeReady(cftUc[counted].baseId, flatTeam.nRanks), ret, fail); // no need of barrier, see above. } // Devr team stores the base LE for that specific team - t->ucLeId = cftUc->baseId + flatTeam.rank - t->team.rank * t->team.stride; + t->ucLeId[counted] = cftUc[counted].baseId + flatTeam.rank - t->team.rank * t->team.stride; exit: free(cftRankList); @@ -158,45 +186,56 @@ ncclResult_t symTeamObtainUcLe(struct ncclComm* comm, struct ncclDevrTeam* t, st fail: for (int r = 0; r < flatTeam.nRanks && importedCount > 0; r++) { if (r == flatTeam.rank) continue; - CUCHECKIGNORE(cuLogicalEndpointDestroy(cftUc->baseId + r)); + CUCHECKIGNORE(cuLogicalEndpointDestroy(cftUc[counted].baseId + r)); importedCount--; } for (struct ncclDevrMemory* mem = devr->memHead; mem != nullptr && bindCount > 0; mem = mem->next) { - symUnbindTeamLe(comm, mem, leUcSelf); + NCCLCHECKIGNORE(symUnbindTeamLe(comm, mem, leUcSelf), ret); bindCount--; } if (leUcSelf != NCCL_LE_ID_INVALID) CUCHECKIGNORE(cuLogicalEndpointDestroy(leUcSelf)); - if (releaseCount > 0) CUCHECKIGNORE(cuLogicalEndpointIdRelease(cftUc->baseId, releaseCount)); - cftUc->baseId = NCCL_LE_ID_INVALID; + if (releaseCount > 0) + CUCHECKIGNORE(cuLogicalEndpointIdRelease(cftUc[counted].baseId - ncclCftLeId(counted) * flatTeam.nRanks, + releaseCount)); + cftUc[counted].baseId = NCCL_LE_ID_INVALID; goto exit; } ncclResult_t symTeamObtainMcLe(struct ncclComm* comm, struct ncclDevrTeam* t, struct ncclDevrState* devr, - bool* needBarrier) { + bool* needBarrier, bool counted) { if (!comm->nvlsSupport) { WARN("CFT multicast support requested, but NVLS is disabled or unsupported."); return ncclInvalidArgument; } ncclResult_t ret = ncclSuccess; - ncclCftLeId leMcId = NCCL_LE_ID_INVALID; + ncclCftLeId mcLeIdBase = NCCL_LE_ID_INVALID; int bindCount = 0, releaseCount = 0; + // Counted LEs are limited to 256GB in size + const size_t leSize = counted ? std::min(size_t(256ULL << 30), devr->bigSize) : devr->bigSize; CUlogicalEndpointProp prop = {}; prop.type = CU_LOGICAL_ENDPOINT_TYPE_MULTICAST; - prop.flags = CU_LOGICAL_ENDPOINT_FLAG_NONE; + prop.flags = counted ? CU_LOGICAL_ENDPOINT_FLAG_COUNTED_OPS : CU_LOGICAL_ENDPOINT_FLAG_NONE; prop.multicast.numDevices = t->team.nRanks; prop.ipcHandleTypes = CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC; - prop.size = devr->bigSize; - NCCLCHECK(checkLeSize(prop, devr->bigSize)); - CUCHECKGOTO(cuLogicalEndpointIdReserve(&leMcId, 1), ret, fail); - releaseCount = 1; + prop.size = leSize; + NCCLCHECK(checkLeSize(prop, leSize)); + // MC LD IDs are reserved in contiguous pairs, such that in devComm: counted mcLeId = devComm->mcLeId + 1 + bool mcLeIdsCreated = t->mcLeId[!counted] != NCCL_LE_ID_INVALID; + if (mcLeIdsCreated) { + mcLeIdBase = counted ? t->mcLeId[0] + 1 : t->mcLeId[1] - 1; + } else { + CUCHECKGOTO(cuLogicalEndpointIdReserve(&mcLeIdBase, 2), ret, fail); + mcLeIdBase += ncclCftLeId(counted); + releaseCount = 2; + } CUlogicalEndpointFabricHandle leMcHandle; if (t->team.rank == 0) { - CUCHECKGOTO(cuLogicalEndpointCreate(leMcId, &prop), ret, fail); - t->mcLeId = leMcId; // t->mcLeId is only set after creation is succesfull - CUCHECKGOTO(cuLogicalEndpointExport(&leMcHandle, leMcId, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC), ret, fail); + CUCHECKGOTO(cuLogicalEndpointCreate(mcLeIdBase, &prop), ret, fail); + CUCHECKGOTO(cuLogicalEndpointExport(&leMcHandle, mcLeIdBase, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC), ret, + fail); NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, t->worldRankList, t->team.rank, t->team.nRanks, 0, &leMcHandle, sizeof(CUlogicalEndpointFabricHandle)), ret, fail); @@ -204,43 +243,46 @@ ncclResult_t symTeamObtainMcLe(struct ncclComm* comm, struct ncclDevrTeam* t, st NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, t->worldRankList, t->team.rank, t->team.nRanks, 0, &leMcHandle, sizeof(CUlogicalEndpointFabricHandle)), ret, fail); - CUCHECKGOTO(cuLogicalEndpointImport(leMcId, &leMcHandle, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC), ret, fail); - t->mcLeId = leMcId; // t->mcLeId is only set after import is succesfull + CUCHECKGOTO(cuLogicalEndpointImport(mcLeIdBase, &leMcHandle, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC), ret, + fail); } // LE is ready when all devices have joined the LE, it acts as a barrier - CUCHECKGOTO(cuLogicalEndpointAddDevice(leMcId, (CUdevice)comm->cudaDev), ret, fail); - NCCLCHECKGOTO(waitLeReady(leMcId, /*count=*/1), ret, fail); + CUCHECKGOTO(cuLogicalEndpointAddDevice(mcLeIdBase, (CUdevice)comm->cudaDev), ret, fail); + NCCLCHECKGOTO(waitLeReady(mcLeIdBase, /*count=*/1), ret, fail); for (struct ncclDevrMemory* mem = devr->memHead; mem != nullptr; mem = mem->next) { - NCCLCHECKGOTO(symBindTeamLe(comm, mem, leMcId), ret, fail); + if ((bool)(mem->winFlags & NCCL_WIN_CFT_COUNTED) != counted) continue; + NCCLCHECKGOTO(symBindTeamLe(comm, mem, mcLeIdBase), ret, fail); bindCount++; } // Barrier needed to guarantee that no peer will start using the MC LE before binding completes. if (needBarrier) *needBarrier = true; + t->mcLeId[counted] = mcLeIdBase; // t->mcLeId is only set after import is successful exit: return ret; fail: for (struct ncclDevrMemory* mem = devr->memHead; mem != nullptr && bindCount > 0; mem = mem->next) { - symUnbindTeamLe(comm, mem, leMcId); + if ((bool)(mem->winFlags & NCCL_WIN_CFT_COUNTED) != counted) continue; + NCCLCHECKIGNORE(symUnbindTeamLe(comm, mem, mcLeIdBase), ret); bindCount--; } - if (t->mcLeId != NCCL_LE_ID_INVALID) CUCHECKIGNORE(cuLogicalEndpointDestroy(t->mcLeId)); - if (releaseCount > 0) CUCHECKIGNORE(cuLogicalEndpointIdRelease(leMcId, releaseCount)); - t->mcLeId = NCCL_LE_ID_INVALID; + if (mcLeIdBase != NCCL_LE_ID_INVALID) CUCHECKIGNORE(cuLogicalEndpointDestroy(mcLeIdBase)); + if (releaseCount > 0) CUCHECKIGNORE(cuLogicalEndpointIdRelease(mcLeIdBase - ncclCftLeId(counted), releaseCount)); + t->mcLeId[counted] = NCCL_LE_ID_INVALID; goto exit; } #else // CUDA_VERSION >= 13030 ncclResult_t symTeamObtainUcLe(struct ncclComm* comm, struct ncclDevrTeam* t, struct ncclDevrState* devr, - bool* /*needBarrier*/) { + bool* /*needBarrier*/, bool /*counted*/) { WARN("CFT unicast logical endpoints are not supported on this platform."); return ncclInvalidUsage; } ncclResult_t symTeamObtainMcLe(struct ncclComm* comm, struct ncclDevrTeam* t, struct ncclDevrState* devr, - bool* /*needBarrier*/) { + bool* /*needBarrier*/, bool /*counted*/) { WARN("CFT multicast logical endpoints are not supported on this platform."); return ncclInvalidUsage; } @@ -257,6 +299,10 @@ ncclResult_t ncclGetMultimemDeviceLeInfo(ncclWindow_t window, size_t offset, ncc ncclComm_t comm = nullptr; struct ncclDevrWindow* winHost = nullptr; NCCLCHECK(findCommAndHostWindowFromDeviceWindow(window, &comm, &winHost)); + if (!ncclDevrWinRegEnabled(winHost->winFlags, ncclDevrRegisterCft)) { + WARN("CFT access is disabled because the window was not registered for CFT."); + return ncclInvalidUsage; + } if (comm->gpuCftSupport == 0) { WARN("Using CFT query function without CFT support in the communicator."); return ncclInvalidArgument; @@ -264,12 +310,13 @@ ncclResult_t ncclGetMultimemDeviceLeInfo(ncclWindow_t window, size_t offset, ncc struct ncclDevrTeam* tm; bool needBarrier = false; + bool counted = winHost->winFlags & NCCL_WIN_CFT_COUNTED; ncclTeam flatTeam = ncclTeamCftMultimem(comm); - NCCLCHECK(symTeamObtain(comm, flatTeam, /*multimem=*/false, /*uc=*/false, /*mc=*/true, &tm, &needBarrier)); + NCCLCHECK(symTeamObtain(comm, flatTeam, /*multimem=*/false, counted, /*uc=*/false, /*mc=*/true, &tm, &needBarrier)); if (needBarrier) { NCCLCHECK(bootstrapIntraNodeBarrier(comm->bootstrap, tm->worldRankList, flatTeam.rank, flatTeam.nRanks, 0xbeef)); } - *leId = tm->mcLeId; + *leId = tm->mcLeId[counted]; *leOffset = winHost->bigOffset + offset; return ncclSuccess; @@ -287,13 +334,18 @@ ncclResult_t ncclGetCftDeviceLeInfo(ncclWindow_t window, size_t offset, int peer ncclComm_t comm = nullptr; struct ncclDevrWindow* winHost = nullptr; NCCLCHECK(findCommAndHostWindowFromDeviceWindow(window, &comm, &winHost)); + if (!ncclDevrWinRegEnabled(winHost->winFlags, ncclDevrRegisterCft)) { + WARN("CFT access is disabled because the window was not registered for CFT."); + return ncclInvalidUsage; + } if (comm->gpuCftSupport == 0) { WARN("Using CFT query function without CFT support in the communicator."); return ncclInvalidArgument; } - ncclCftLeId baseLeId = comm->devrState.le.baseId; + bool counted = winHost->winFlags & NCCL_WIN_CFT_COUNTED; + ncclCftLeId baseLeId = comm->devrState.le[counted].baseId; if (baseLeId == NCCL_LE_ID_INVALID) { WARN("Querying CFT LE before LE creation. Create a CFT-enabled devComm first or enable hostCftMode."); return ncclInvalidUsage; @@ -318,6 +370,10 @@ ncclResult_t ncclGetPeerDeviceLeInfo(ncclWindow_t window, size_t offset, int pee ncclComm_t comm = nullptr; struct ncclDevrWindow* winHost = nullptr; NCCLCHECK(findCommAndHostWindowFromDeviceWindow(window, &comm, &winHost)); + if (!ncclDevrWinRegEnabled(winHost->winFlags, ncclDevrRegisterCft)) { + WARN("CFT access is disabled because the window was not registered for CFT."); + return ncclInvalidUsage; + } if (comm->gpuCftSupport == 0) { WARN("Using CFT query function without CFT support in the communicator."); @@ -332,7 +388,8 @@ ncclResult_t ncclGetPeerDeviceLeInfo(ncclWindow_t window, size_t offset, int pee return ncclInvalidArgument; } - ncclCftLeId baseLeId = comm->devrState.le.baseId; + bool counted = winHost->winFlags & NCCL_WIN_CFT_COUNTED; + ncclCftLeId baseLeId = comm->devrState.le[counted].baseId; if (baseLeId == NCCL_LE_ID_INVALID) { WARN("Querying CFT LE before LE creation. Create a CFT-enabled devComm first or enable hostCftMode."); return ncclInvalidUsage; diff --git a/projects/rccl/src/collectives.cc b/projects/rccl/src/collectives.cc index 7fa04fa9ca9e..73d56a074e97 100644 --- a/projects/rccl/src/collectives.cc +++ b/projects/rccl/src/collectives.cc @@ -26,6 +26,9 @@ #include "ce_coll.h" #include "alltoallv_meta.h" #include "strongstream.h" +#include "mem_manager.h" + +#include #ifdef ENABLE_ROCSHMEM #include @@ -256,6 +259,46 @@ bool rcclAllReduceShouldTakeDdaPath(const ncclComm* comm, size_t count, ncclData return result; } +// Addon backends (DDA / CE 2-shot / GIN-SDMA) return from the collective impl +// without ncclEnqueueCheck, which is where NCCL_CHECK_MODE pointer checks and the +// suspend-while-collective guard live. Divert those calls through enqueue so the +// guards still fire; debug-mode collectives then use the native/CE path. +bool ncclCommIsSuspended(ncclComm* comm) { +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + if (comm == nullptr || comm->memManager == nullptr) return false; + return ncclIntruQueueEmpty(&comm->resumeTaskQueue) && + (!ncclIntruQueueEmpty(&comm->suspendTaskQueue) || + __atomic_load_n(&comm->memManager->released, __ATOMIC_ACQUIRE)); +#else + (void)comm; + return false; +#endif +} + +bool rcclCollectiveMustUseEnqueuePath(ncclComm* comm) { + if (comm == nullptr) return true; + if (comm->checkMode != ncclCheckModeDefault) return true; + if (ncclCommIsSuspended(comm)) return true; + return false; +} + +// Direct AllGather sets useDirect, and AlltoAll pivot/GDA enqueue a different +// ncclInfo, before calling ncclEnqueueCheck. Diverting those here drops the +// algorithm. Every other selected path either already enqueues that same info +// or skips enqueue entirely (DDA, hier, GIN, CE 2-shot). +static bool rcclEnqueueSetupNeedsSwitch(int algo) { + switch (algo) { + case RCCL_DIRECT_ALLGATHER: + case RCCL_A2A_PIVOT: +#ifdef ENABLE_ROCSHMEM + case RCCL_A2A_GDA: +#endif + return true; + default: + return false; + } +} + bool rcclAlltoAllShouldTakeDdaPath(const ncclComm* comm, size_t totalBytes, bool ceAlltoAllAllowed) { // AlltoAll has no symmetric kernel, so DDA must yield here or registered-window // CE never dispatches. Full contract is on the declaration in rccl_common.h. @@ -369,8 +412,10 @@ static ncclResult_t ncclHierarchicalAllGather_Impl(const void* sendbuff, void* r // Step 3: Shuffle tempBuffer (local-rank-major) -> recvbuff (node-major). size_t totalAGBytes = (size_t)nNodes * localRanks * rankOffset; int numBlocks = hierarchicalShuffleNumBlocks(totalAGBytes); - hierarchicalShuffle<<>>((const char*)tempBuffer, (char*)recvbuff, - rankOffset, nNodes, localRanks); + const hipEvent_t stopEvent = rcclTakeAddonStopEvent(comm); + hipExtLaunchKernelGGL(hierarchicalShuffle, numBlocks, HIERARCHICAL_SHUFFLE_THREADS, 0, stream, /*startEvent=*/nullptr, + stopEvent, /*flags=*/0, (const char*)tempBuffer, (char*)recvbuff, rankOffset, nNodes, + localRanks); CUDACHECK(hipGetLastError()); return ncclSuccess; @@ -407,10 +452,12 @@ ncclResult_t ncclAllGather_impl(const void* sendbuff, void* recvbuff, size_t sen info.decision = decision; info.decisionValid = true; - // Canonical selection line for addon backends (CE / DDA / Direct / Hier / - // symmetric). Native kernels report via the enqueue.cc channel{Lo..Hi} tuning - // line instead; this names the addon RCCL runs so rcclGetCollImplInfo can be - // checked against it. + if (rcclCollectiveMustUseEnqueuePath(comm) && !rcclEnqueueSetupNeedsSwitch(decision.algo)) { + return ncclEnqueueCheck(&info); + } + + // Name the addon only after the check-mode divert. Logging first would claim + // DDA / hier / GIN for a call that ncclEnqueueCheck is about to run instead. if (comm->rank == 0 && decision.algo >= NCCL_NUM_ALGORITHMS) { const char* an = nullptr; rcclGetAlgoName(decision.algo, &an); @@ -422,24 +469,30 @@ ncclResult_t ncclAllGather_impl(const void* sendbuff, void* recvbuff, size_t sen INFO(NCCL_COLL, "AllGather: taking DDA fabric LL path: nRanks=%d nNodes=%d sendcount=%zu datatype=%d totalBytes=%zu", comm->nRanks, comm->nNodes, sendcount, (int)datatype, msgSize); - NCCLCHECK(ncclAllGatherDdaFabricLL(sendbuff, recvbuff, sendcount, datatype, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllGatherDdaFabricLL(sendbuff, recvbuff, sendcount, datatype, comm, stream); + }); case RCCL_DDA_FABRIC_LL128: INFO(NCCL_COLL, "AllGather: taking DDA fabric LL128 path: nRanks=%d nNodes=%d sendcount=%zu datatype=%d totalBytes=%zu", comm->nRanks, comm->nNodes, sendcount, (int)datatype, msgSize); - NCCLCHECK(ncclAllGatherDdaFabricLL128(sendbuff, recvbuff, sendcount, datatype, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllGatherDdaFabricLL128(sendbuff, recvbuff, sendcount, datatype, comm, stream); + }); case RCCL_DDA_FABRIC_VMM: INFO(NCCL_COLL, "AllGather: taking DDA fabric (VMM) path: nRanks=%d nNodes=%d sendcount=%zu datatype=%d bytes=%zu", comm->nRanks, comm->nNodes, sendcount, (int)datatype, sendcount * ncclTypeSize(datatype)); - NCCLCHECK(ncclAllGatherDdaFabric(sendbuff, recvbuff, sendcount, datatype, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllGatherDdaFabric(sendbuff, recvbuff, sendcount, datatype, comm, stream); + }); case RCCL_DDA_IPC: - NCCLCHECK(ncclAllGatherDdaIpc(sendbuff, recvbuff, sendcount, datatype, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllGatherDdaIpc(sendbuff, recvbuff, sendcount, datatype, comm, stream); + }); case RCCL_HIERARCHICAL_ALLGATHER: - return ncclHierarchicalAllGather_Impl(sendbuff, recvbuff, sendcount, datatype, comm, stream); + return rcclAddonLaunch(comm, stream, [&] { + return ncclHierarchicalAllGather_Impl(sendbuff, recvbuff, sendcount, datatype, comm, stream); + }); case RCCL_DIRECT_ALLGATHER: INFO(NCCL_TUNING, "RCCL DIRECT ALLGATHER count = %zu, msgSize = %zu, comm = %p, stream = %p, rank = %d, sendbuff = %p, recvbuff " @@ -463,24 +516,6 @@ ncclResult_t ncclAllGather_impl(const void* sendbuff, void* recvbuff, size_t sen RCCL_PARAM(AlltoAllPivotEnable, "ALL_TO_ALL_PIVOT_ENABLE", 0); -// Window/graph prologue, then ncclCeAlltoAllEligible. CTA_POLICY_ZERO is also -// checked by the caller so the default AlltoAll path skips this lookup. -static ncclResult_t alltoAllRegisteredCeAllowed(ncclComm* comm, const void* sendbuff, void* recvbuff, - ncclDataType_t datatype, cudaStream_t stream, bool* allowed) { - *allowed = false; - struct ncclDevrWindow* sendWin = nullptr; - struct ncclDevrWindow* recvWin = nullptr; - NCCLCHECK(ncclDevrFindWindow(comm, sendbuff, &sendWin)); - NCCLCHECK(ncclDevrFindWindow(comm, recvbuff, &recvWin)); - const bool hasSysmemSegment = ncclDevrWindowHasSysmemSegment(sendWin) || ncclDevrWindowHasSysmemSegment(recvWin); - ncclSymRegType_t winRegType; - NCCLCHECK(ncclGetSymRegType(sendWin, recvWin, &winRegType)); - struct ncclCudaGraph ceGraph; - NCCLCHECK(ncclCudaGetCapturingGraph(&ceGraph, stream, comm->config.graphUsageMode)); - *allowed = ncclCeAlltoAllEligible(comm, datatype, winRegType, hasSysmemSegment, ncclCudaGraphValid(ceGraph)); - return ncclSuccess; -} - NCCL_API(ncclResult_t, ncclAlltoAll, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm* comm, cudaStream_t stream); ncclResult_t ncclAlltoAll_impl(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, @@ -494,10 +529,18 @@ ncclResult_t ncclAlltoAll_impl(const void* sendbuff, void* recvbuff, size_t coun NCCLCHECK(rcclSelectAlltoAll(comm, sendbuff, recvbuff, count, datatype, stream, /*query=*/false, /*graphCapturingHint=*/false, &decision)); - // Canonical selection line for addon backends (CE / DDA / Pivot / GDA / - // Direct). Native kernels report via the enqueue.cc channel{Lo..Hi} tuning - // line instead; this names the addon RCCL runs so rcclGetCollImplInfo can be - // checked against it. DDA launchers already log grid/block at launch. + struct ncclInfo info = { + ncclFuncAlltoAll, "AlltoAll", sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, + ALLTOALL_CHUNKSTEPS, ALLTOALL_SLICESTEPS + }; + info.decision = decision; + info.decisionValid = true; + if (rcclCollectiveMustUseEnqueuePath(comm) && !rcclEnqueueSetupNeedsSwitch(decision.algo)) { + return ncclEnqueueCheck(&info); + } + + // Name the addon only after the check-mode divert. DDA launchers also log + // grid/block at launch. if (comm->rank == 0 && decision.algo >= NCCL_NUM_ALGORITHMS) { const char* an = nullptr; rcclGetAlgoName(decision.algo, &an); @@ -506,34 +549,43 @@ ncclResult_t ncclAlltoAll_impl(const void* sendbuff, void* recvbuff, size_t coun switch (decision.algo) { case RCCL_A2A_PIVOT: { - struct ncclInfo info = {ncclFuncAlltoAllPivot, "AlltoAllPivot", + struct ncclInfo pivotInfo = {ncclFuncAlltoAllPivot, "AlltoAllPivot", sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, ALLTOALL_PIVOT_CHUNKSTEPS, ALLTOALL_PIVOT_SLICESTEPS, nullptr}; - return ncclEnqueueCheck(&info); + return ncclEnqueueCheck(&pivotInfo); } #ifdef ENABLE_ROCSHMEM case RCCL_A2A_GDA: { - struct ncclInfo info = {ncclFuncAlltoAllGda, "AlltoAllGda", + struct ncclInfo gdaInfo = {ncclFuncAlltoAllGda, "AlltoAllGda", sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, ALLTOALL_PIVOT_CHUNKSTEPS, ALLTOALL_PIVOT_SLICESTEPS, nullptr}; - return ncclEnqueueCheck(&info); + return ncclEnqueueCheck(&gdaInfo); } #endif #if defined(ENABLE_ROCSHMEM_GIN) case RCCL_A2A_GIN_SDMA: INFO(NCCL_COLL, "AllToAll: taking GIN-SDMA path: nRanks=%d count=%zu datatype=%d bytes=%zu inPlace=%d", comm->nRanks, count, (int)datatype, count * ncclTypeSize(datatype), sendbuff == recvbuff ? 1 : 0); - NCCLCHECK(ncclAllToAllGinSdma(sendbuff, recvbuff, count, datatype, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllToAllGinSdma(sendbuff, recvbuff, count, datatype, comm, stream); + }); #endif case RCCL_DDA_FABRIC_LL: - return ncclAllToAllDdaFabricLL(sendbuff, recvbuff, count, datatype, comm, stream); + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllToAllDdaFabricLL(sendbuff, recvbuff, count, datatype, comm, stream); + }); case RCCL_DDA_FABRIC_LL128: - return ncclAllToAllDdaFabricLL128(sendbuff, recvbuff, count, datatype, comm, stream); + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllToAllDdaFabricLL128(sendbuff, recvbuff, count, datatype, comm, stream); + }); case RCCL_DDA_FABRIC_VMM: - return ncclAllToAllDdaFabric(sendbuff, recvbuff, count, datatype, comm, stream); + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllToAllDdaFabric(sendbuff, recvbuff, count, datatype, comm, stream); + }); case RCCL_DDA_IPC: - return ncclAllToAllDdaIpc(sendbuff, recvbuff, count, datatype, comm, stream); + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllToAllDdaIpc(sendbuff, recvbuff, count, datatype, comm, stream); + }); case RCCL_CE_REGISTERED: case RCCL_CE_SCRATCH: case RCCL_DIRECT_ALLTOALL: @@ -543,12 +595,6 @@ ncclResult_t ncclAlltoAll_impl(const void* sendbuff, void* recvbuff, size_t coun // CE and Direct (per-peer Send/Recv) share this enqueue; taskAppend honors // info.decision instead of re-selecting. - struct ncclInfo info = { - ncclFuncAlltoAll, "AlltoAll", sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, - ALLTOALL_CHUNKSTEPS, ALLTOALL_SLICESTEPS - }; - info.decision = decision; - info.decisionValid = true; return ncclEnqueueCheck(&info); } @@ -733,9 +779,11 @@ ncclResult_t ncclAllReduce_impl(const void* sendbuff, void* recvbuff, size_t cou info.decision = decision; info.decisionValid = true; - // Canonical selection line for addon backends (GIN / CE / DDA / symmetric). Native - // kernels report via the enqueue.cc channel{Lo..Hi} tuning line instead; this - // names the addon RCCL runs so rcclGetCollImplInfo can be checked against it. + if (rcclCollectiveMustUseEnqueuePath(comm) && !rcclEnqueueSetupNeedsSwitch(decision.algo)) { + return ncclEnqueueCheck(&info); + } + + // Name the addon only after the check-mode divert. if (comm->rank == 0 && decision.algo >= NCCL_NUM_ALGORITHMS) { const char* an = nullptr; rcclGetAlgoName(decision.algo, &an); @@ -747,8 +795,9 @@ ncclResult_t ncclAllReduce_impl(const void* sendbuff, void* recvbuff, size_t cou case RCCL_GIN_SDMA: INFO(NCCL_COLL, "AllReduce: taking GIN SDMA path: nRanks=%d count=%zu bytes=%zu", comm->nRanks, count, count * ncclTypeSize(datatype)); - NCCLCHECK(ncclAllReduceGinSdma(sendbuff, recvbuff, count, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllReduceGinSdma(sendbuff, recvbuff, count, datatype, op, comm, stream); + }); #endif case RCCL_CE_2SHOT: { if (count == 0) return ncclSuccess; @@ -766,28 +815,34 @@ ncclResult_t ncclAllReduce_impl(const void* sendbuff, void* recvbuff, size_t cou // No CollApi parent: this path returns before ncclEnqueueCheck, so the // thread-local handle still names a previous collective. NCCLCHECK(ncclProfilerStartCeCollEvent(comm, &ceArgs, stream)); - ncclResult_t ceRet = ncclCeAllReduce(comm, sendbuff, recvbuff, count, datatype, op, stream, nullptr, &ceArgs); + ncclResult_t ceRet = rcclAddonLaunch(comm, stream, [&] { + return ncclCeAllReduce(comm, sendbuff, recvbuff, count, datatype, op, stream, nullptr, &ceArgs); + }); ncclProfilerStopCeCollEvent(comm, &ceArgs, stream); return ceRet; } case RCCL_DDA_FABRIC_LL: INFO(NCCL_COLL, "AllReduce: taking DDA fabric LL path: nRanks=%d nNodes=%d count=%zu datatype=%d bytes=%zu", comm->nRanks, comm->nNodes, count, (int)datatype, count * ncclTypeSize(datatype)); - NCCLCHECK(ncclAllReduceDdaFabricLL(sendbuff, recvbuff, count, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllReduceDdaFabricLL(sendbuff, recvbuff, count, datatype, op, comm, stream); + }); case RCCL_DDA_FABRIC_LL128: INFO(NCCL_COLL, "AllReduce: taking DDA fabric LL128 path: nRanks=%d nNodes=%d count=%zu datatype=%d bytes=%zu", comm->nRanks, comm->nNodes, count, (int)datatype, count * ncclTypeSize(datatype)); - NCCLCHECK(ncclAllReduceDdaFabricLL128(sendbuff, recvbuff, count, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllReduceDdaFabricLL128(sendbuff, recvbuff, count, datatype, op, comm, stream); + }); case RCCL_DDA_FABRIC_VMM: INFO(NCCL_COLL, "AllReduce: taking DDA fabric (VMM) path: nRanks=%d nNodes=%d count=%zu datatype=%d bytes=%zu", comm->nRanks, comm->nNodes, count, (int)datatype, count * ncclTypeSize(datatype)); - NCCLCHECK(ncclAllReduceDdaFabric(sendbuff, recvbuff, count, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllReduceDdaFabric(sendbuff, recvbuff, count, datatype, op, comm, stream); + }); case RCCL_DDA_IPC: - NCCLCHECK(ncclAllReduceDdaIpc(sendbuff, recvbuff, count, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclAllReduceDdaIpc(sendbuff, recvbuff, count, datatype, op, comm, stream); + }); default: // RCCL_SYMMETRIC / RCCL_CE_REGISTERED / native kernel algorithms all go // through the standard enqueue path; taskAppend() honors info->decision. @@ -1140,7 +1195,11 @@ ncclResult_t ncclReduceScatter_impl(const void* sendbuff, void* recvbuff, size_t info.decision = decision; info.decisionValid = true; - // Canonical line for addon backends; native kernels report via the enqueue tuning line. + if (rcclCollectiveMustUseEnqueuePath(comm) && !rcclEnqueueSetupNeedsSwitch(decision.algo)) { + return ncclEnqueueCheck(&info); + } + + // Name the addon only after the check-mode divert. if (comm->rank == 0 && decision.algo >= NCCL_NUM_ALGORITHMS) { const char* an = nullptr; rcclGetAlgoName(decision.algo, &an); @@ -1151,25 +1210,31 @@ ncclResult_t ncclReduceScatter_impl(const void* sendbuff, void* recvbuff, size_t case RCCL_DDA_FABRIC_LL: INFO(NCCL_COLL, "ReduceScatter: taking DDA fabric LL path: nRanks=%d nNodes=%d recvcount=%zu datatype=%d bytes=%zu", comm->nRanks, comm->nNodes, recvcount, (int)datatype, recvcount * ncclTypeSize(datatype)); - NCCLCHECK(ncclReduceScatterDdaFabricLL(sendbuff, recvbuff, recvcount, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclReduceScatterDdaFabricLL(sendbuff, recvbuff, recvcount, datatype, op, comm, stream); + }); case RCCL_DDA_FABRIC_LL128: INFO(NCCL_COLL, "ReduceScatter: taking DDA fabric LL128 path: nRanks=%d nNodes=%d recvcount=%zu datatype=%d bytes=%zu", comm->nRanks, comm->nNodes, recvcount, (int)datatype, recvcount * ncclTypeSize(datatype)); - NCCLCHECK(ncclReduceScatterDdaFabricLL128(sendbuff, recvbuff, recvcount, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclReduceScatterDdaFabricLL128(sendbuff, recvbuff, recvcount, datatype, op, comm, stream); + }); case RCCL_DDA_FABRIC_VMM: INFO(NCCL_COLL, "ReduceScatter: taking DDA fabric (VMM) path: nRanks=%d nNodes=%d recvcount=%zu datatype=%d bytes=%zu", comm->nRanks, comm->nNodes, recvcount, (int)datatype, recvcount * ncclTypeSize(datatype)); - NCCLCHECK(ncclReduceScatterDdaFabric(sendbuff, recvbuff, recvcount, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclReduceScatterDdaFabric(sendbuff, recvbuff, recvcount, datatype, op, comm, stream); + }); case RCCL_DDA_IPC: - NCCLCHECK(ncclReduceScatterDdaIpc(sendbuff, recvbuff, recvcount, datatype, op, comm, stream)); - return ncclSuccess; + return rcclAddonLaunch(comm, stream, [&] { + return ncclReduceScatterDdaIpc(sendbuff, recvbuff, recvcount, datatype, op, comm, stream); + }); case RCCL_HIERARCHICAL_REDUCESCATTER: - return ncclHierarchicalReduceScatter_Impl(sendbuff, recvbuff, recvcount, datatype, op, comm, stream); + return rcclAddonLaunch(comm, stream, [&] { + return ncclHierarchicalReduceScatter_Impl(sendbuff, recvbuff, recvcount, datatype, op, comm, stream); + }); case RCCL_DIRECT_REDUCESCATTER: INFO(NCCL_TUNING, "RCCL DIRECT REDUCE-SCATTER recvcount=%zu msgSize=%zu rank=%d nRanks=%d nNodes=%d comm=%p stream=%p " diff --git a/projects/rccl/src/config/collconfig.cc b/projects/rccl/src/config/collconfig.cc index 1372004908ca..e0a083a1e552 100644 --- a/projects/rccl/src/config/collconfig.cc +++ b/projects/rccl/src/config/collconfig.cc @@ -17,6 +17,10 @@ bool ncclCollConfigHasAlgSelection(const ncclCollConfig_t* config) { return config->algSelection != NCCL_CONFIG_UNDEF_PTR && config->algSelection[0] != '\0'; } +cudaEvent_t ncclCollConfigGetLaunchCompletionEvent(const ncclCollConfig_t* c) { + return (c == nullptr || c->size == 0) ? nullptr : c->launchCompletionEvent; +} + // Helper function to set aggregation isolation bool ncclCollConfigNeedAggIsolate(const ncclCollConfig_t* config) { if (config->size == 0) { @@ -30,6 +34,7 @@ bool ncclCollConfigNeedAggIsolate(const ncclCollConfig_t* config) { // cap, so it does not force aggregation isolation; it is applied to the group as-is. // algorithm selection need aggregation isolation so that selection does not spill to other collectives in the group. if (ncclCollConfigHasAlgSelection(config)) return true; + if (ncclCollConfigGetLaunchCompletionEvent(config) != nullptr) return true; // CTAPolicy is resolved in place (env > per-call > comm) before this runs, so its isolation is // decided by the caller comparing the resolved value against the comm default. return false; @@ -42,25 +47,18 @@ bool ncclCollConfigNeedAggIsolate(const ncclCollConfig_t* config) { // consumers via resolveCollRes. ncclResult_t ncclParseCollConfig(const ncclCollConfig_t* config, ncclCollConfig_t* internal_config) { *internal_config = NCCL_COLLCONFIG_INITIALIZER; + internal_config->magic = 0; if (config != nullptr) { - // ncclCollConfig_t is append-only (size only grows, fields never move). The smallest valid - // struct is the v23100 one, used as the minimal size check. - if (config->size < sizeof(struct ncclCollConfig_v23100)) { - WARN("ncclCollConfig_t size %zu too small (expected >= %zu)", config->size, sizeof(ncclCollConfig_t)); + // Match ncclConfig_t parsing: callers provide a valid initialized struct prefix. + // Bound the copy by the current layout; initializer defaults remain for appended fields. + size_t realSize; + memcpy(&realSize, config, sizeof(realSize)); + realSize = realSize > sizeof(*internal_config) ? sizeof(*internal_config) : realSize; + memcpy(internal_config, config, realSize); + if (internal_config->magic != NCCL_API_MAGIC) { + WARN("ncclCollConfig_t magic mismatch (got 0x%x, expected 0x%x)", internal_config->magic, NCCL_API_MAGIC); return ncclInvalidArgument; } - if (config->magic != NCCL_API_MAGIC) { - WARN("ncclCollConfig_t magic mismatch (got 0x%x, expected 0x%x)", config->magic, NCCL_API_MAGIC); - return ncclInvalidArgument; - } - // copy user config to internal_config for safe access later - if (config->version <= NCCL_VERSION_CODE) { - // A same/older user config struct is smaller than (or equal to) current library's struct. - memcpy(internal_config, config, config->size); - } else { - // A newer user config struct is larger than current library's struct - memcpy(internal_config, config, internal_config->size); - } // forceAlgSelection is boolean and only accepts 0 or 1 if (internal_config->forceAlgSelection != 1 && internal_config->forceAlgSelection != 0) { diff --git a/projects/rccl/src/debug.cc b/projects/rccl/src/debug.cc index 2911e08bd289..a725bf7569cf 100644 --- a/projects/rccl/src/debug.cc +++ b/projects/rccl/src/debug.cc @@ -22,9 +22,7 @@ #include "env.h" #include -#define NCCL_DEBUG_RESET_TRIGGERED (-2) - -__attribute__((visibility("default"))) int ncclDebugLevel = -1; +__attribute__((visibility("default"))) uint32_t ncclDebugLevelMask = NCCL_DEBUG_LEVEL_MASK_UNINITIALIZED; static uint32_t ncclDebugTimestampLevels = 0; // bitmaps of levels that have timestamps turned on static char ncclDebugTimestampFormat[256]; // with space for subseconds static int ncclDebugTimestampSubsecondsStart; // index where the subseconds starts @@ -46,12 +44,27 @@ static thread_local int tid = -1; DEFINE_NCCL_PARAM(ncclParamDebugLevel, ncclDebugLogLevel, NCCL_DEBUG, NCCL_LOG_NONE, NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, ncclParamOneOf(makeOptions( - makeOption("VERSION", NCCL_LOG_VERSION, "Prints the NCCL version info only"), - makeOption("WARN", NCCL_LOG_WARN, "Prints only messages indicating a fatal error."), - makeOption("INFO", NCCL_LOG_INFO, "Prints debug message"), + makeOption("VERSION", NCCL_LOG_VERSION, "Prints NCCL version information only."), + makeOption("WARN", NCCL_LOG_WARN, "Prints error messages."), + makeOption("ATTN", NCCL_LOG_ATTN, "Prints error messages plus informational notices."), + makeOption("INFO", NCCL_LOG_INFO, "Prints debug information."), makeOption("ABORT", NCCL_LOG_ABORT, ""), - makeOption("TRACE", NCCL_LOG_TRACE, "Prints replayable trace info on all calls") - )), "Set debug output level, the option is inclusive for any level that is less verbose than the set value"); + makeOption("TRACE", NCCL_LOG_TRACE, "Prints replayable trace information on all calls.") + )), "Set the debug output level. Each level includes less-verbose levels."); + +DEFINE_NCCL_PARAM(ncclParamDebugLevels, uint32_t, NCCL_DEBUG_LEVELS, 0, + NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, + ncclParamBitsetOf(makeOptions( + makeOption("VERSION", (1u << NCCL_LOG_VERSION), "Prints NCCL version information only."), + makeOption("WARN", (1u << NCCL_LOG_WARN), "Prints error messages."), + makeOption("ATTN", (1u << NCCL_LOG_ATTN), "Prints error messages plus informational notices."), + makeOption("INFO", (1u << NCCL_LOG_INFO), "Prints debug information."), + makeOption("ABORT", (1u << NCCL_LOG_ABORT), ""), + makeOption("TRACE", (1u << NCCL_LOG_TRACE), "Prints replayable trace information on all calls."), + makeOption("ALL", (1u << NCCL_LOG_VERSION | 1u << NCCL_LOG_WARN | 1u << NCCL_LOG_ATTN | + 1u << NCCL_LOG_INFO | 1u << NCCL_LOG_ABORT | 1u << NCCL_LOG_TRACE), + "Prints all debug messages")), ',', true), + "Add comma-separated debug levels to NCCL_DEBUG."); DEFINE_NCCL_PARAM(ncclParamDebugSubsys, uint64_t, NCCL_DEBUG_SUBSYS, NCCL_INIT | NCCL_BOOTSTRAP | NCCL_ENV, @@ -81,21 +94,22 @@ DEFINE_NCCL_PARAM(ncclParamDebugSubsys, uint64_t, NCCL_DEBUG_SUBSYS, DEFINE_NCCL_PARAM(ncclParamWarnEnableDebugInfo, bool, NCCL_WARN_ENABLE_DEBUG_INFO, false, NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, NCCL_PARAM_DEFAULT, "If enabled, the debug level will be set to INFO after a WARN level debug message is logged."); -// clang-format on -DEFINE_NCCL_PARAM(ncclParamDebugTimestampLevel, uint32_t, NCCL_DEBUG_TIMESTAMP_LEVELS, (1u << NCCL_LOG_WARN), +DEFINE_NCCL_PARAM(ncclParamDebugTimestampLevel, uint32_t, NCCL_DEBUG_TIMESTAMP_LEVELS, + (1u << NCCL_LOG_WARN) | (1u << NCCL_LOG_ATTN), NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, - ncclParamBitsetOf( - makeOptions(makeOption("VERSION", (1u << NCCL_LOG_VERSION), "on NCCL version info message"), - makeOption("WARN", (1u << NCCL_LOG_WARN), "on Explicit error message"), - makeOption("INFO", (1u << NCCL_LOG_INFO), "on Debug message"), - makeOption("ABORT", (1u << NCCL_LOG_ABORT), ""), - makeOption("TRACE", (1u << NCCL_LOG_TRACE), "on Replayable trace message"), - makeOption("ALL", - (1u << NCCL_LOG_VERSION | 1u << NCCL_LOG_WARN | 1u << NCCL_LOG_INFO | - 1u << NCCL_LOG_ABORT | 1u << NCCL_LOG_TRACE), - "on All messages"))), - "Set which log lines get a timestamp depending upon the level of the log"); + ncclParamBitsetOf(makeOptions( + makeOption("VERSION", (1u << NCCL_LOG_VERSION), "NCCL version information"), + makeOption("WARN", (1u << NCCL_LOG_WARN), "Error messages"), + makeOption("ATTN", (1u << NCCL_LOG_ATTN), "Informational notices"), + makeOption("INFO", (1u << NCCL_LOG_INFO), "Debug messages"), + makeOption("ABORT", (1u << NCCL_LOG_ABORT), ""), + makeOption("TRACE", (1u << NCCL_LOG_TRACE), "Replayable trace messages"), + makeOption("ALL", (1u << NCCL_LOG_VERSION | 1u << NCCL_LOG_WARN | 1u << NCCL_LOG_ATTN | + 1u << NCCL_LOG_INFO | 1u << NCCL_LOG_ABORT | 1u << NCCL_LOG_TRACE), + "All messages") + )), "Set the log levels that include timestamps."); +// clang-format on DEFINE_NCCL_PARAM(ncclParamDebugTsFormat, const char*, NCCL_DEBUG_TIMESTAMP_FORMAT, "[%F %T] ", NCCL_PARAM_FLAG_PUBLISHED | NCCL_PARAM_FLAG_NO_ENVPLUGIN_INIT, NCCL_PARAM_DEFAULT, @@ -129,13 +143,28 @@ static ncclResult_t getHostNameForLog(char* hostname, int maxlen, const char del return ncclSuccess; } +// Convert the legacy scalar setting to its logical inclusive level set. +static uint32_t ncclDebugLevelToMask(ncclDebugLogLevel level) { + if (level < NCCL_LOG_VERSION) return 0; + uint32_t mask = 1u << NCCL_LOG_VERSION; + if (level >= NCCL_LOG_INFO) mask |= 1u << NCCL_LOG_ATTN; + if (level == NCCL_LOG_ATTN) level = NCCL_LOG_WARN; + if (level >= NCCL_LOG_WARN) mask |= 1u << NCCL_LOG_WARN; + if (level >= NCCL_LOG_INFO) mask |= 1u << NCCL_LOG_INFO; + if (level >= NCCL_LOG_ABORT) mask |= 1u << NCCL_LOG_ABORT; + if (level >= NCCL_LOG_TRACE) mask |= 1u << NCCL_LOG_TRACE; + return mask; +} + // This function must be called with ncclDebugLock locked! static void ncclDebugInit() { ncclGetEnvFunc_t getEnvFunc = ncclEnvPluginInitialized() ? ncclGetEnv : (ncclGetEnvFunc_t)std::getenv; const char* nccl_debug = getEnvFunc("NCCL_DEBUG"); int tempNcclDebugLevel = -1; + uint32_t tempNcclDebugLevelMask = 0; uint64_t tempNcclDebugMask = NCCL_INIT | NCCL_BOOTSTRAP | NCCL_ENV; // Default debug sub-system mask - if (ncclDebugLevel == NCCL_DEBUG_RESET_TRIGGERED && ncclDebugFile != stdout) { + if (COMPILER_ATOMIC_LOAD(&ncclDebugLevelMask, std::memory_order_relaxed) == NCCL_DEBUG_LEVEL_MASK_RESET_TRIGGERED && + ncclDebugFile != stdout) { // Finish the reset initiated via ncclResetDebugInit(). fclose(ncclDebugFile); ncclDebugFile = stdout; @@ -149,6 +178,8 @@ static void ncclDebugInit() { tempNcclDebugLevel = NCCL_LOG_VERSION; } else if (strcasecmp(nccl_debug, "WARN") == 0) { tempNcclDebugLevel = NCCL_LOG_WARN; + } else if (strcasecmp(nccl_debug, "ATTN") == 0) { + tempNcclDebugLevel = NCCL_LOG_ATTN; } else if (strcasecmp(nccl_debug, "INFO") == 0) { tempNcclDebugLevel = NCCL_LOG_INFO; } else if (strcasecmp(nccl_debug, "ABORT") == 0) { @@ -221,6 +252,44 @@ static void ncclDebugInit() { free(ncclDebugSubsys); } + // Map the scalar NCCL_DEBUG level to its inclusive set of levels. + // An unrecognized NCCL_DEBUG value (-1) enables no levels. + if (tempNcclDebugLevel >= 0) tempNcclDebugLevelMask = ncclDebugLevelToMask((ncclDebugLogLevel)tempNcclDebugLevel); + // [RCCL] ERROR (fatal) messages are shown at every level except NONE. + if (tempNcclDebugLevel >= NCCL_LOG_ERROR) tempNcclDebugLevelMask |= (1u << NCCL_LOG_ERROR); + + /* Parse the NCCL_DEBUG_LEVELS env var: a comma separated list of levels + * (e.g. WARN,ATTN) added on top of NCCL_DEBUG. Unknown entries are ignored. + */ + const char* ncclDebugLevelsEnv = getEnvFunc("NCCL_DEBUG_LEVELS"); + if (ncclDebugLevelsEnv != NULL) { + char* ncclDebugLevels = strdup(ncclDebugLevelsEnv); + char* level = strtok(ncclDebugLevels, ","); + while (level != NULL) { + if (strcasecmp(level, "ERROR") == 0) { + tempNcclDebugLevelMask |= (1u << NCCL_LOG_ERROR); + } else if (strcasecmp(level, "VERSION") == 0) { + tempNcclDebugLevelMask |= (1u << NCCL_LOG_VERSION); + } else if (strcasecmp(level, "WARN") == 0) { + tempNcclDebugLevelMask |= (1u << NCCL_LOG_WARN); + } else if (strcasecmp(level, "ATTN") == 0) { + tempNcclDebugLevelMask |= (1u << NCCL_LOG_ATTN); + } else if (strcasecmp(level, "INFO") == 0) { + tempNcclDebugLevelMask |= (1u << NCCL_LOG_INFO); + } else if (strcasecmp(level, "ABORT") == 0) { + tempNcclDebugLevelMask |= (1u << NCCL_LOG_ABORT); + } else if (strcasecmp(level, "TRACE") == 0) { + tempNcclDebugLevelMask |= (1u << NCCL_LOG_TRACE); + } else if (strcasecmp(level, "ALL") == 0) { + tempNcclDebugLevelMask |= (1u << NCCL_LOG_ERROR | 1u << NCCL_LOG_VERSION | 1u << NCCL_LOG_WARN | + 1u << NCCL_LOG_ATTN | 1u << NCCL_LOG_INFO | 1u << NCCL_LOG_ABORT | + 1u << NCCL_LOG_TRACE); + } + level = strtok(NULL, ","); + } + free(ncclDebugLevels); + } + const char* ncclWarnSetDebugInfoEnv = getEnvFunc("NCCL_WARN_ENABLE_DEBUG_INFO"); if (ncclWarnSetDebugInfoEnv != NULL && strlen(ncclWarnSetDebugInfoEnv) > 0) { int64_t value; @@ -320,10 +389,11 @@ static void ncclDebugInit() { /* Parse and expand the NCCL_DEBUG_FILE path and * then create the debug file. But don't bother unless the - * NCCL_DEBUG level is > VERSION + * effective debug levels include more than VERSION. */ const char* ncclDebugFileEnv = getEnvFunc("NCCL_DEBUG_FILE"); - if (tempNcclDebugLevel > NCCL_LOG_VERSION && ncclDebugFileEnv != NULL) { + if ((tempNcclDebugLevelMask & ~((1u << NCCL_LOG_ERROR) | (1u << NCCL_LOG_VERSION))) != 0 && + ncclDebugFileEnv != NULL) { int c = 0; char debugFn[PATH_MAX + 1] = ""; char* dfn = debugFn; @@ -372,23 +442,22 @@ static void ncclDebugInit() { ncclEpoch = std::chrono::steady_clock::now(); ncclDebugMask = tempNcclDebugMask; - COMPILER_ATOMIC_STORE(&ncclDebugLevel, tempNcclDebugLevel, std::memory_order_release); + COMPILER_ATOMIC_STORE(&ncclDebugLevelMask, tempNcclDebugLevelMask, std::memory_order_release); } -/* Common logging function used by the INFO, WARN and TRACE macros +/* Common logging function used by the INFO, WARN, ATTN and TRACE macros * Also exported to the dynamically loadable Net transport modules so * they can share the debugging mechanisms and output files */ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char* filefunc, int line, const char* fmt, ...) { bool locked = false; // Keeps track of the ncclDebugLock state. - int gotLevel = COMPILER_ATOMIC_LOAD(&ncclDebugLevel, std::memory_order_acquire); if (ncclDebugNoWarn != 0 && level == NCCL_LOG_WARN) { level = NCCL_LOG_INFO; flags = ncclDebugNoWarn; } - // Save the last error (WARN) as a human readable string + // Save the last error (WARN) as a human readable string. ATTN does not set lastError. if (level == NCCL_LOG_WARN) { pthread_mutex_lock(&ncclDebugLock); locked = true; @@ -398,7 +467,7 @@ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char* file va_end(vargs); } - if (gotLevel >= 0 && (gotLevel < level || (flags & ncclDebugMask) == 0)) { + if (!ncclDebugShouldLog(level, flags, ncclDebugMask)) { if (locked) pthread_mutex_unlock(&ncclDebugLock); return; } @@ -408,8 +477,10 @@ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char* file locked = true; } // From this point on ncclDebugLock is always locked so we don't need to check "locked" anymore. - if (ncclDebugLevel < 0) ncclDebugInit(); - if (ncclDebugLevel < level || ((flags & ncclDebugMask) == 0)) { + uint32_t levelMask = COMPILER_ATOMIC_LOAD(&ncclDebugLevelMask, std::memory_order_relaxed); + if (levelMask == NCCL_DEBUG_LEVEL_MASK_UNINITIALIZED || levelMask == NCCL_DEBUG_LEVEL_MASK_RESET_TRIGGERED) + ncclDebugInit(); + if (!ncclDebugShouldLog(level, flags, ncclDebugMask)) { pthread_mutex_unlock(&ncclDebugLock); return; } @@ -421,8 +492,8 @@ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char* file char buffer[1024]; size_t len = 0; - // WARNs come with an extra newline at the beginning. - if (level == NCCL_LOG_WARN) { + // WARN and ATTN messages come with an extra newline at the beginning. + if (level == NCCL_LOG_WARN || level == NCCL_LOG_ATTN) { buffer[len++] = '\n'; } @@ -474,8 +545,13 @@ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char* file // Add level specific formatting. if (level == NCCL_LOG_WARN) { len += snprintf(buffer + len, sizeof(buffer) - len, "[%d] %s:%d NCCL WARN ", cudaDev, filefunc, line); - if (ncclWarnSetDebugInfo) - COMPILER_ATOMIC_STORE(&ncclDebugLevel, static_cast(NCCL_LOG_INFO), std::memory_order_release); + if (ncclWarnSetDebugInfo) { + uint32_t levelMask = COMPILER_ATOMIC_LOAD(&ncclDebugLevelMask, std::memory_order_relaxed); + COMPILER_ATOMIC_STORE(&ncclDebugLevelMask, levelMask | ncclDebugLevelToMask(NCCL_LOG_INFO), + std::memory_order_release); + } + } else if (level == NCCL_LOG_ATTN) { + len += snprintf(buffer + len, sizeof(buffer) - len, "[%d] %s:%d NCCL ATTN ", cudaDev, filefunc, line); } else if (level == NCCL_LOG_INFO) { len += snprintf(buffer + len, sizeof(buffer) - len, "[%d] NCCL INFO ", cudaDev); } else if (level == NCCL_LOG_TRACE && flags == NCCL_CALL) { @@ -516,7 +592,7 @@ extern "C" // Use this after changing NCCL_DEBUG and related parameters in the environment. pthread_mutex_lock(&ncclDebugLock); // Let ncclDebugInit() know to complete the reset. - COMPILER_ATOMIC_STORE(&ncclDebugLevel, static_cast(NCCL_DEBUG_RESET_TRIGGERED), std::memory_order_release); + COMPILER_ATOMIC_STORE(&ncclDebugLevelMask, NCCL_DEBUG_LEVEL_MASK_RESET_TRIGGERED, std::memory_order_release); pthread_mutex_unlock(&ncclDebugLock); } diff --git a/projects/rccl/src/dev_runtime.cc b/projects/rccl/src/dev_runtime.cc index 65fa6d049da0..f5376919b787 100644 --- a/projects/rccl/src/dev_runtime.cc +++ b/projects/rccl/src/dev_runtime.cc @@ -62,6 +62,23 @@ extern struct ncclDevCommCompat ncclDevCommCompat_v22902, ncclDevCommCompat_v229 static struct ncclDevCommCompat* devCommCompat[] = {&ncclDevCommCompat_v22902, &ncclDevCommCompat_v22907, &ncclDevCommCompat_v23000, &ncclDevCommCompat_v23100, nullptr}; +// Flags that restrict which window-registration capabilities are enabled. Keep this mask and +// ncclDevrRegisterMaskFromWinFlags in sync with all public NCCL_WIN_*_ONLY flags. +static constexpr int ncclDevrWinCapRestrictionMask = NCCL_WIN_GIN_ONLY; + +static int ncclDevrRegisterMaskFromWinFlags(int winFlags) { + int requested = winFlags & ncclDevrWinCapRestrictionMask; + if (requested == 0) return ncclDevrRegisterAll; + + int registerMask = 0; + if (requested & NCCL_WIN_GIN_ONLY) registerMask |= ncclDevrRegisterGin; + return registerMask; +} + +bool ncclDevrWinRegEnabled(int winFlags, enum ncclDevrRegisterCapability capability) { + return (ncclDevrRegisterMaskFromWinFlags(winFlags) & capability) != 0; +} + // Returns ncclInvalidUsage if the compiled version is greater than the runtime version // and NCCL_ENABLE_VERSION_CHECK=0 is not set static ncclResult_t getNcclVersionCompat(int version, struct ncclDevCommCompat** devCompatPtr) { @@ -179,7 +196,7 @@ ncclResult_t ncclDevrInitOnce(struct ncclComm* comm) { devr->cftMcSize = computeCftMcSize(comm); devr->cftMcSelf = comm->rank % devr->cftMcSize; - devr->le.baseId = NCCL_LE_ID_INVALID; + devr->le[0].baseId = devr->le[1].baseId = NCCL_LE_ID_INVALID; devr->lsaRankList = (int*)malloc(devr->lsaSize * sizeof(int)); for (int i = 0; i < devr->lsaSize; i++) { @@ -257,13 +274,20 @@ ncclResult_t ncclDevrInitOnce(struct ncclComm* comm) { return ret; } -static void symTeamDestroyAll(struct ncclComm* comm); // Further down +static ncclResult_t symTeamDestroyAll(struct ncclComm* comm); // Further down static void symMemoryDestroy(struct ncclComm* comm, struct ncclDevrMemory* mem); // Further down #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) static ncclResult_t windowDeregisterNonSym(struct ncclComm* comm, struct ncclWindow_vidmem* winDev); // Further down #endif +static bool symHasCountedCftMemory(struct ncclDevrState* devr) { + for (struct ncclDevrMemory* mem = devr->memHead; mem != nullptr; mem = mem->next) { + if (mem->winFlags & NCCL_WIN_CFT_COUNTED) return true; + } + return false; +} + ncclResult_t ncclDevrFinalize(struct ncclComm* comm) { struct ncclDevrState* devr = &comm->devrState; cudaStream_t stream; @@ -308,7 +332,7 @@ ncclResult_t ncclDevrFinalize(struct ncclComm* comm) { } if (comm->symmetricSupport) { - symTeamDestroyAll(comm); + NCCLCHECKIGNORE(symTeamDestroyAll(comm), ret); { // delete windowTable struct ncclDevCommWindowTable* tableDev = devr->windowTable; while (tableDev != nullptr) { @@ -403,12 +427,12 @@ static ncclResult_t symMemoryImportAndMapSegmentHandle(struct ncclComm* comm, in impHandle = memHandle; } else { if (ncclCuMemHandleType == CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR) { - int fd = -1; + ncclIpcFd fd = NCCL_INVALID_IPC_FD; NCCLCHECKGOTO(ncclProxyClientGetFdBlocking(comm, devr->lsaRankList[r], msg, &fd), ret, fail); CUCHECKGOTO(cuMemImportFromShareableHandle(&impHandle, reinterpret_cast((uintptr_t)fd), ncclCuMemHandleType), ret, fail); - SYSCHECKGOTO(close(fd), "close", ret, fail); + SYSCHECKGOTO(ncclIpcFdClose(fd), "close", ret, fail); } else { CUCHECKGOTO(cuMemImportFromShareableHandle(&impHandle, (void*)&msg->fabricHandle, ncclCuMemHandleType), ret, fail); @@ -447,10 +471,12 @@ static ncclResult_t symMemoryMapLsaTeam(struct ncclComm* comm, struct ncclDevrMe ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; symLsaMessage* messages = nullptr; - int* segmentCounts = mem->lsaNumSegments; // filled from global allgather in symMemoryObtain + int* segmentCounts = mem->lsaNumSegments; // filled from global allgather in symMemoryObtain int maxSegments = 0; const int numSegments = mem->numSegments; size_t* segmentSizes = mem->segmentSizes; + // When LSA registration is not requested, do not import or map peer handles. + const bool registerLsa = ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterLsa); for (int rank = 0; rank < devr->lsaSize; rank++) { maxSegments = std::max(maxSegments, segmentCounts[rank]); @@ -460,14 +486,22 @@ static ncclResult_t symMemoryMapLsaTeam(struct ncclComm* comm, struct ncclDevrMe for (int segment = 0; segment < numSegments; segment++) { symLsaMessage* msg = messages + devr->lsaSelf * maxSegments + segment; - NCCLCHECKGOTO(symMemoryExportSegmentHandle(comm, msg, mem->memHandles[segment], segmentSizes[segment]), ret, fail); - INFO(NCCL_REG, "[%d] Segment %d, Type : %d, numSegments : %d, Segment size : %ld, memHandle : %lld", devr->lsaSelf, - segment, msg->type, numSegments, msg->segmentSize, msg->memHandle); + if (!registerLsa) { + // The local handle is reused directly, so only its segment size is needed. + msg->segmentSize = segmentSizes[segment]; + } else { + NCCLCHECKGOTO(symMemoryExportSegmentHandle(comm, msg, mem->memHandles[segment], segmentSizes[segment]), ret, + fail); + INFO(NCCL_REG, "[%d] Segment %d, Type : %d, numSegments : %d, Segment size : %ld, memHandle : %lld", + devr->lsaSelf, segment, msg->type, numSegments, msg->segmentSize, msg->memHandle); + } } - NCCLCHECKGOTO(bootstrapIntraNodeAllGather(comm->bootstrap, devr->lsaRankList, devr->lsaSelf, devr->lsaSize, messages, - sizeof(symLsaMessage) * maxSegments), - ret, fail); + if (registerLsa) { + NCCLCHECKGOTO(bootstrapIntraNodeAllGather(comm->bootstrap, devr->lsaRankList, devr->lsaSelf, devr->lsaSize, + messages, sizeof(symLsaMessage) * maxSegments), + ret, fail); + } if (devr->lsaFlatBase == nullptr) { // Create on first need. @@ -476,14 +510,20 @@ static ncclResult_t symMemoryMapLsaTeam(struct ncclComm* comm, struct ncclDevrMe devr->lsaFlatBase = reinterpret_cast(addr); } - for (int r = 0; r < devr->lsaSize; r++) { - NCCLCHECKGOTO(symMemoryImportAndMapSegmentsForRank(comm, r, messages, maxSegments, segmentCounts[r], + if (!registerLsa) { + NCCLCHECKGOTO(symMemoryImportAndMapSegmentsForRank(comm, devr->lsaSelf, messages, maxSegments, numSegments, mem->memHandles, mem->bigOffset), ret, fail); + } else { + for (int r = 0; r < devr->lsaSize; r++) { + NCCLCHECKGOTO(symMemoryImportAndMapSegmentsForRank(comm, r, messages, maxSegments, segmentCounts[r], + mem->memHandles, mem->bigOffset), + ret, fail); + } + // Ensure everyone has imported my mem handles. + NCCLCHECKGOTO(bootstrapIntraNodeBarrier(comm->bootstrap, devr->lsaRankList, devr->lsaSelf, devr->lsaSize, 0xbeef), + ret, fail); } - // Ensure everyone has imported my mem handles. - NCCLCHECKGOTO(bootstrapIntraNodeBarrier(comm->bootstrap, devr->lsaRankList, devr->lsaSelf, devr->lsaSize, 0xbeef), - ret, fail); leave: free(messages); return ret; @@ -492,7 +532,7 @@ static ncclResult_t symMemoryMapLsaTeam(struct ncclComm* comm, struct ncclDevrMe } static ncclResult_t symBindTeamMemory(struct ncclComm* comm, struct ncclDevrTeam* tm, struct ncclDevrMemory* mem) { - if (comm->nvlsSupport && tm->mcBasePtr != nullptr) { + if (ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterLsa) && comm->nvlsSupport && tm->mcBasePtr != nullptr) { #if CUDART_VERSION >= 12010 // Multimem teams are currently unsupported for memory containing CPU-backed physical segments if (mem->globalHasSysmemSegment) { @@ -510,7 +550,8 @@ static ncclResult_t symBindTeamMemory(struct ncclComm* comm, struct ncclDevrTeam } static ncclResult_t symUnbindTeamMemory(struct ncclComm* comm, struct ncclDevrTeam* tm, struct ncclDevrMemory* mem) { - if (comm->nvlsSupport && tm->mcBasePtr != nullptr && !mem->globalHasSysmemSegment) { + if (ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterLsa) && comm->nvlsSupport && tm->mcBasePtr != nullptr && + !mem->globalHasSysmemSegment) { #if CUDART_VERSION >= 12010 CUCHECK(cuMulticastUnbind(tm->mcHandle, comm->cudaDev, mem->bigOffset, mem->size)); #endif @@ -519,8 +560,8 @@ static ncclResult_t symUnbindTeamMemory(struct ncclComm* comm, struct ncclDevrTe } // Caller must barrier the team afterward unless *needBarrier == false on return. -ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool multimem, bool cftUc, bool cftMc, - struct ncclDevrTeam** outTeam, bool* needBarrier) { +ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool multimem, bool counted, bool cftUc, + bool cftMc, struct ncclDevrTeam** outTeam, bool* needBarrier) { ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; struct ncclDevrTeam* t = devr->teamHead; @@ -534,15 +575,16 @@ ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool mul t->team = team; t->mcHandle = 0x0; t->mcBasePtr = nullptr; - t->ucLeId = NCCL_LE_ID_INVALID; - t->mcLeId = NCCL_LE_ID_INVALID; + t->ucLeId[0] = t->ucLeId[1] = NCCL_LE_ID_INVALID; + t->mcLeId[0] = t->mcLeId[1] = NCCL_LE_ID_INVALID; for (int i = 0; i < team.nRanks; i++) { t->worldRankList[i] = comm->rank + (i - team.rank) * team.stride; } break; } else if (t->team.rank == team.rank && t->team.nRanks == team.nRanks && t->team.stride == team.stride) { bool needsMultimem = multimem && t->mcBasePtr == nullptr; - bool needsCft = (cftUc && t->ucLeId == NCCL_LE_ID_INVALID) || (cftMc && t->mcLeId == NCCL_LE_ID_INVALID); + bool needsCft = + (cftUc && t->ucLeId[counted] == NCCL_LE_ID_INVALID) || (cftMc && t->mcLeId[counted] == NCCL_LE_ID_INVALID); if (!needsMultimem && !needsCft) { if (outTeam) *outTeam = t; return ncclSuccess; @@ -563,15 +605,14 @@ ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool mul CUmemGenericAllocationHandle mcHandle = 0; CUdeviceptr mcAddr = 0; CUmulticastObjectProp mcProp = {}; - char shareableHandle[NVLS_HANDLE_SIZE]; + char shareableHandle[NVLS_HANDLE_SIZE] = {}; mcProp.numDevices = team.nRanks; mcProp.handleTypes = ncclCuMemHandleType; mcProp.flags = 0; mcProp.size = devr->bigSize; if (team.rank == 0) { - NCCLCHECKGOTO(ncclNvlsGroupCreate(comm, &mcProp, team.rank, team.nRanks, &mcHandle, shareableHandle), ret, - fail); + NCCLCHECKGOTO(ncclMcCreate(comm, &mcProp, team.rank, team.nRanks, &mcHandle, shareableHandle), ret, fail); NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, t->worldRankList, team.rank, team.nRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail_mcHandle); @@ -579,11 +620,14 @@ ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool mul NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, t->worldRankList, team.rank, team.nRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail); - NCCLCHECKGOTO(ncclNvlsGroupConnect(comm, shareableHandle, t->worldRankList[0], &mcHandle), ret, fail); + NCCLCHECKGOTO(ncclMcImport(comm, shareableHandle, t->worldRankList[0], &mcHandle), ret, fail); } CUCHECKGOTO(cuMulticastAddDevice(mcHandle, comm->cudaDev), ret, fail_mcHandle); CUCHECKGOTO(cuMemAddressReserve(&mcAddr, devr->bigSize, NCCL_MAX_PAGE_SIZE, 0, 0), ret, fail_mcHandle); + // NOTE(preexisting): cuMemMap blocks until every device has been added, and no + // abort-aware barrier precedes it, so a peer failing before its addDevice can + // strand survivors here during abort. ncclMcGroupBuildPartitions guards its map with one. CUCHECKGOTO(cuMemMap(mcAddr, devr->bigSize, 0, mcHandle, 0), ret, fail_mcHandle_mcAddr); { CUmemAccessDesc accessDesc = {}; @@ -623,15 +667,15 @@ ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool mul } } - if (cftUc && t->ucLeId == NCCL_LE_ID_INVALID) { - NCCLCHECKGOTO(symTeamObtainUcLe(comm, t, devr, needBarrier), ret, fail); + if (cftUc && t->ucLeId[counted] == NCCL_LE_ID_INVALID) { + NCCLCHECKGOTO(symTeamObtainUcLe(comm, t, devr, needBarrier, counted), ret, fail); } - if (cftMc && t->mcLeId == NCCL_LE_ID_INVALID) { - NCCLCHECKGOTO(symTeamObtainMcLe(comm, t, devr, needBarrier), ret, fail); + if (cftMc && t->mcLeId[counted] == NCCL_LE_ID_INVALID) { + NCCLCHECKGOTO(symTeamObtainMcLe(comm, t, devr, needBarrier, counted), ret, fail); } if (teamIsNew) { - // Add to list + // Add to list t->next = devr->teamHead; devr->teamHead = t; } @@ -643,16 +687,20 @@ ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool mul return ret; } -static void symTeamDestroyAll(struct ncclComm* comm) { +static ncclResult_t symTeamDestroyAll(struct ncclComm* comm) { + ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; while (devr->teamHead != nullptr) { struct ncclDevrTeam* t = devr->teamHead; devr->teamHead = t->next; - bool hasLe = t->mcLeId != NCCL_LE_ID_INVALID; + bool hasLe = false; + hasLe |= t->mcLeId[0] != NCCL_LE_ID_INVALID || t->mcLeId[1] != NCCL_LE_ID_INVALID; if (t->mcBasePtr != nullptr || hasLe) { for (struct ncclDevrMemory* m = devr->memHead; m != nullptr; m = m->next) { symUnbindTeamMemory(comm, t, m); - symUnbindTeamLe(comm, m, t->mcLeId); + for (int i = 0; i <= 1; i++) { + NCCLCHECKIGNORE(symUnbindTeamLe(comm, m, t->mcLeId[i]), ret); + } } } if (t->mcBasePtr != nullptr) { @@ -662,29 +710,48 @@ static void symTeamDestroyAll(struct ncclComm* comm) { CUCHECKIGNORE(cuMemRelease(t->mcHandle)); } #if CUDA_VERSION >= 13030 - if (t->mcLeId != NCCL_LE_ID_INVALID) { - // Each rank owns its own local MC reference (created by rank 0, imported by others). - CUCHECKIGNORE(cuLogicalEndpointDestroy(t->mcLeId)); - CUCHECKIGNORE(cuLogicalEndpointIdRelease(t->mcLeId, 1)); + bool relMcLeIds = false; + for (int i = 0; i <= 1; i++) { + if (t->mcLeId[i] != NCCL_LE_ID_INVALID) { + // Each rank owns its own local MC reference (created by rank 0, imported by others). + CUCHECKIGNORE(cuLogicalEndpointDestroy(t->mcLeId[i])); + relMcLeIds = true; + } + } + if (relMcLeIds) { + ncclCftLeId mcLeIdBase = t->mcLeId[0] != NCCL_LE_ID_INVALID ? t->mcLeId[0] : t->mcLeId[1] - 1; + CUCHECKIGNORE(cuLogicalEndpointIdRelease(mcLeIdBase, 2)); } #endif free(t); } - struct ncclDevrStateCftUc* cftUc = &devr->le; - if (cftUc->baseId != NCCL_LE_ID_INVALID) { - ncclCftLeId leUcSelf = cftUc->baseId + devr->cftSelf; - for (struct ncclDevrMemory* m = devr->memHead; m != nullptr; m = m->next) { - symUnbindTeamLe(comm, m, leUcSelf); - } + bool relUcLeIds = false; + for (int i = 0; i <= 1; i++) { + struct ncclDevrStateCftUc* cftUc = &devr->le[i]; + if (cftUc->baseId != NCCL_LE_ID_INVALID) { + ncclCftLeId leUcSelf = cftUc->baseId + devr->cftSelf; + for (struct ncclDevrMemory* m = devr->memHead; m != nullptr; m = m->next) { + NCCLCHECKIGNORE(symUnbindTeamLe(comm, m, leUcSelf), ret); + } #if CUDA_VERSION >= 13030 - for (int flatRank = 0; flatRank < devr->cftSize; flatRank++) { - CUCHECKIGNORE(cuLogicalEndpointDestroy(cftUc->baseId + flatRank)); + for (int flatRank = 0; flatRank < devr->cftSize; flatRank++) { + CUCHECKIGNORE(cuLogicalEndpointDestroy(cftUc->baseId + flatRank)); + } +#endif + relUcLeIds = true; } - CUCHECKIGNORE(cuLogicalEndpointIdRelease(cftUc->baseId, devr->cftSize)); + } + if (relUcLeIds) { +#if CUDA_VERSION >= 13030 + ncclCftLeId ucLeIdBase = + devr->le[0].baseId != NCCL_LE_ID_INVALID ? devr->le[0].baseId : devr->le[1].baseId - devr->cftSize; + CUCHECKIGNORE(cuLogicalEndpointIdRelease(ucLeIdBase, devr->cftSize * 2)); #endif - cftUc->baseId = NCCL_LE_ID_INVALID; + devr->le[0].baseId = NCCL_LE_ID_INVALID; + devr->le[1].baseId = NCCL_LE_ID_INVALID; } + return ret; } static ncclResult_t symMemoryRegisterGin(struct ncclComm* comm, struct ncclDevrMemory* mem) { @@ -692,6 +759,11 @@ static ncclResult_t symMemoryRegisterGin(struct ncclComm* comm, struct ncclDevrM int numSegmentsRegistered = 0; size_t offset = 0; + if (!ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterGin)) { + mem->numGinSegments = 1; + return ncclSuccess; + } + NCCLCHECKGOTO(ncclDevrVerifySegmentLayouts(mem, comm), ret, fail); NCCLCHECKGOTO(ncclDevrBuildGinSegmentInfos(mem), ret, fail); @@ -731,6 +803,7 @@ static ncclResult_t symMemoryRegisterGin(struct ncclComm* comm, struct ncclDevrM } static ncclResult_t symMemoryRegisterRma(struct ncclComm* comm, struct ncclDevrMemory* mem) { + if (!ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterRma)) return ncclSuccess; NCCLCHECK(ncclRmaProxyConnectOnce(comm)); if (mem->maxGlobalNumSegments == 1) { NCCLCHECK(ncclRmaProxyRegister(comm, mem->primaryAddr, mem->size, mem->rmaHostWins)); @@ -748,29 +821,38 @@ static void symMemoryUnregister(struct ncclComm* comm, struct ncclDevrMemory* me // rmaHostWins[0] is a reliable witness that register completed (same pattern // as windowRegisterNonSym / ncclCommWindowDeregister). Skip if connect/register // failed partway so deregister does not walk an unbound rmaCommCount. - if (devr->rmaProxyEnabled && mem->maxGlobalNumSegments == 1 && - mem->rmaHostWins[0] != nullptr) { + if (devr->rmaProxyEnabled && ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterRma) && + mem->maxGlobalNumSegments == 1 && mem->rmaHostWins[0] != nullptr) { (void)ncclRmaProxyDeregister(comm, mem->rmaHostWins); } } +static void symMemoryUnmapLsaRank(struct ncclDevrState* devr, struct ncclDevrMemory* mem, int rank) { + uintptr_t base = reinterpret_cast(devr->lsaFlatBase); + uintptr_t addr = base + rank * devr->bigSize + mem->bigOffset; + for (int idx = 0; idx < mem->lsaNumSegments[rank]; idx++) { + CUdeviceptr tmpBase = nullptr; + size_t tmpBaseSize = 0; + CUCHECKIGNORE(cuMemGetAddressRange(&tmpBase, &tmpBaseSize, reinterpret_cast(addr))); + CUCHECKIGNORE(cuMemUnmap(reinterpret_cast(addr), tmpBaseSize)); + addr = addr + tmpBaseSize; + } +} + // Unmap LSA-flat slices created by symMemoryMapLsaTeam. Must run before // ncclSpaceFree so a later obtain can remap the same bigOffset, and so // ncclDevrFinalize's cuMemAddressFree does not see leftover mappings. +// Without LSA registration only our own rank's slice was mapped. static void symMemoryUnmapLsaTeam(struct ncclComm* comm, struct ncclDevrMemory* mem) { if (comm == nullptr || mem == nullptr || mem->lsaNumSegments == nullptr) return; struct ncclDevrState* devr = &comm->devrState; if (devr->lsaFlatBase == nullptr) return; - for (int r = 0; r < devr->lsaSize; r++) { - uintptr_t base = reinterpret_cast(devr->lsaFlatBase); - uintptr_t addr = base + r * devr->bigSize + mem->bigOffset; - for (int idx = 0; idx < mem->lsaNumSegments[r]; idx++) { - CUdeviceptr tmpBase = nullptr; - size_t tmpBaseSize = 0; - CUCHECKIGNORE(cuMemGetAddressRange(&tmpBase, &tmpBaseSize, reinterpret_cast(addr))); - CUCHECKIGNORE(cuMemUnmap(reinterpret_cast(addr), tmpBaseSize)); - addr = addr + tmpBaseSize; + if (ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterLsa)) { + for (int r = 0; r < devr->lsaSize; r++) { + symMemoryUnmapLsaRank(devr, mem, r); } + } else { + symMemoryUnmapLsaRank(devr, mem, devr->lsaSelf); } } @@ -786,48 +868,102 @@ static ncclResult_t symMemoryObtain(struct ncclComm* comm, CUmemGenericAllocatio struct segmentInfo { int numSegments; bool hasSysmemSegment; + int registerMask; size_t totalSize; int hostCftMode; + uint64_t candidateRegistryId; // invalidRegistryId if this rank has no compatible registration }; struct segmentInfo* globalSegmentInfo = nullptr; const int globalLsaTeamBaseIdx = devr->lsaSize * (comm->rank / devr->lsaSize); bool ucBound = false; + bool counted = winFlags & NCCL_WIN_CFT_COUNTED; - struct ncclDevrMemory* mem = nullptr; - // New memory. - NCCLCHECKGOTO(ncclCalloc(&mem, 1), ret, fail_mem); - NCCLCHECKGOTO(ncclCalloc(&mem->memHandles, numSegments), ret, fail_mem); - memcpy(mem->memHandles, memHandles, sizeof(*mem->memHandles) * numSegments); - mem->primaryAddr = memAddr; - mem->size = size; - mem->winFlags = winFlags; - mem->hasSysmemSegment = hasSysmemSegment; - mem->numSegments = numSegments; + if (counted && size > (256ULL << 30)) { + WARN("Window size exceeded limit of 256GB for CFT Counted."); + return ncclInvalidUsage; + } - NCCLCHECKGOTO(ncclCalloc(&mem->segmentSizes, numSegments), ret, fail_mem); - NCCLCHECKGOTO(ncclCalloc(&globalSegmentInfo, comm->nRanks), ret, fail_mem); + struct ncclDevrMemory* mem = nullptr; + constexpr uint64_t invalidRegistryId = UINT64_MAX; + uint64_t candidateRegistryId = invalidRegistryId; + uint64_t registryId = invalidRegistryId; + int maxGlobalNumSegments = 0; + bool globalHasSysmemSegment = false; + + // Look for an existing registration of the same backing allocation with compatible flags. memHead + // is head-pushed, so the first match is the most recent one, which is the choice most likely to be + // shared by the other ranks. This is only a candidate: reuse requires all ranks to agree below. + for (struct ncclDevrMemory* m = devr->memHead; m != nullptr; m = m->next) { + if (m->primaryAddr != memAddr || m->size != size || m->numSegments != numSegments || m->winFlags != winFlags) { + continue; + } + // Check if all memHandles that [memAddr, memAddr + size] spans also match + if (std::equal(m->memHandles, m->memHandles + numSegments, memHandles)) { + mem = m; + candidateRegistryId = m->registryId; + break; + } + } - NCCLCHECKGOTO(ncclDevrPopulateSegmentSizes(mem, numSegments), ret, fail_mem); + NCCLCHECKGOTO(ncclCalloc(&globalSegmentInfo, comm->nRanks), ret, fail); // We need max segments and global sysmem info to selectively disable some features globalSegmentInfo[comm->rank].numSegments = numSegments; globalSegmentInfo[comm->rank].hasSysmemSegment = hasSysmemSegment; + globalSegmentInfo[comm->rank].registerMask = ncclDevrRegisterMaskFromWinFlags(winFlags); globalSegmentInfo[comm->rank].totalSize = size; globalSegmentInfo[comm->rank].hostCftMode = comm->config.hostCftMode; - NCCLCHECKGOTO(bootstrapAllGather(comm->bootstrap, globalSegmentInfo, sizeof(*globalSegmentInfo)), ret, fail_mem); - mem->globalHasSysmemSegment = false; + globalSegmentInfo[comm->rank].candidateRegistryId = candidateRegistryId; + NCCLCHECKGOTO(bootstrapAllGather(comm->bootstrap, globalSegmentInfo, sizeof(*globalSegmentInfo)), ret, + fail_global_segment_info); for (int r = 0; r < comm->nRanks; r++) { - if (mem->maxGlobalNumSegments < globalSegmentInfo[r].numSegments) { - mem->maxGlobalNumSegments = globalSegmentInfo[r].numSegments; - } - if (globalSegmentInfo[r].hasSysmemSegment) mem->globalHasSysmemSegment = true; + if (globalSegmentInfo[r].registerMask != globalSegmentInfo[comm->rank].registerMask) { + WARN("Window registration capabilities disagree between rank %d and rank %d", comm->rank, r); + ret = ncclInvalidUsage; + goto fail_global_segment_info; + } + if (maxGlobalNumSegments < globalSegmentInfo[r].numSegments) { + maxGlobalNumSegments = globalSegmentInfo[r].numSegments; + } + if (globalSegmentInfo[r].hasSysmemSegment) globalHasSysmemSegment = true; + // Reuse only if every rank found the same registration. A rank which found none has + // mem == nullptr already, so its invalidRegistryId only has to veto reuse on the others. + if (globalSegmentInfo[r].candidateRegistryId != candidateRegistryId) mem = nullptr; if (globalSegmentInfo[r].hostCftMode != comm->config.hostCftMode) { WARN("Communicator ranks have mismatched hostCftMode configuration."); ret = ncclInvalidArgument; - goto fail_mem; + goto fail_global_segment_info; + } + } + + if (mem != nullptr) { + // The decision is unanimous, so all ranks skip the create path's LSA-team collectives. + for (int segment = 0; segment < numSegments; segment++) { + CUCHECKIGNORE(cuMemRelease(memHandles[segment])); } + goto exit; } + // Consumed before any fallible create work so that all ranks taking this branch consume the same + // id, keeping the counter in lockstep even if a rank fails below. + registryId = devr->nextRegistryId++; + + // New memory. + NCCLCHECKGOTO(ncclCalloc(&mem, 1), ret, fail_mem); + NCCLCHECKGOTO(ncclCalloc(&mem->memHandles, numSegments), ret, fail_mem); + memcpy(mem->memHandles, memHandles, sizeof(*mem->memHandles) * numSegments); + mem->registryId = registryId; + mem->primaryAddr = memAddr; + mem->size = size; + mem->winFlags = winFlags; + mem->hasSysmemSegment = hasSysmemSegment; + mem->numSegments = numSegments; + mem->maxGlobalNumSegments = maxGlobalNumSegments; + mem->globalHasSysmemSegment = globalHasSysmemSegment; + + NCCLCHECKGOTO(ncclCalloc(&mem->segmentSizes, numSegments), ret, fail_mem); + NCCLCHECKGOTO(ncclDevrPopulateSegmentSizes(mem, numSegments), ret, fail_mem); + NCCLCHECKGOTO(ncclCalloc(&mem->lsaNumSegments, devr->lsaSize), ret, fail_mem); mem->lsaMinSize = size; mem->lsaMaxSize = size; @@ -851,12 +987,13 @@ static ncclResult_t symMemoryObtain(struct ncclComm* comm, CUmemGenericAllocatio mem->primaryAddr = (char*)devr->lsaFlatBase + devr->lsaSelf * devr->bigSize + mem->bigOffset; } - if (comm->gpuCftSupport > 0) { + if (ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterCft) && comm->gpuCftSupport > 0) { ncclTeam_t ucTeam = ncclTeamCft(comm), mcTeam = ncclTeamCftMultimem(comm); if (comm->config.hostCftMode != ncclHostCftDisable) { // Add the UC and MC team to the teamHead list, and create the corresponding LEs. // In case of failure, "enable" will report the error, fallback will skip it. - NOWARN(ret = symTeamObtain(comm, ucTeam, /*multimem=*/false, /*uc=*/true, /*mc=*/false, nullptr, nullptr), + NOWARN(ret = symTeamObtain(comm, ucTeam, /*multimem=*/false, counted, /*uc=*/true, /*mc=*/false, + /*outTeam=*/nullptr, /*needBarrier=*/nullptr), NCCL_INIT); if (ret != ncclSuccess && comm->config.hostCftMode == ncclHostCftEnable) { WARN("Failed to obtain the UC team"); @@ -865,7 +1002,8 @@ static ncclResult_t symMemoryObtain(struct ncclComm* comm, CUmemGenericAllocatio ret = ncclSuccess; if (comm->nvlsSupport) { - NOWARN(ret = symTeamObtain(comm, mcTeam, /*multimem=*/false, /*uc=*/false, /*mc=*/true, nullptr, nullptr), + NOWARN(ret = symTeamObtain(comm, mcTeam, /*multimem=*/false, /*counted=*/false, /*uc=*/false, /*mc=*/true, + /*outTeam=*/nullptr, /*needBarrier=*/nullptr), NCCL_INIT); if (ret != ncclSuccess && comm->config.hostCftMode == ncclHostCftEnable) { WARN("Failed to obtain the MC team"); @@ -879,16 +1017,16 @@ static ncclResult_t symMemoryObtain(struct ncclComm* comm, CUmemGenericAllocatio // Bind new memory with each existing team. for (struct ncclDevrTeam* t = devr->teamHead; t != nullptr; t = t->next) { NCCLCHECKGOTO(symBindTeamMemory(comm, t, mem), ret, fail_mem_space_teams); - if (comm->gpuCftSupport) { + if (ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterCft) && comm->gpuCftSupport) { // Only bind to the UC once, all teams share the same UC LE. - if (t->ucLeId != NCCL_LE_ID_INVALID && !ucBound) { - ncclCftLeId_t le = t->ucLeId + t->team.rank * t->team.stride; + if (t->ucLeId[counted] != NCCL_LE_ID_INVALID && !ucBound) { + ncclCftLeId_t le = t->ucLeId[counted] + t->team.rank * t->team.stride; NCCLCHECKGOTO(symBindTeamLe(comm, mem, le), ret, fail_mem_space_teams); ucBound = true; } // Each team has a different MC LE, bind to all of them. - if (t->mcLeId != NCCL_LE_ID_INVALID) { - NCCLCHECKGOTO(symBindTeamLe(comm, mem, t->mcLeId), ret, fail_mem_space_teams); + if (t->mcLeId[counted] != NCCL_LE_ID_INVALID) { + NCCLCHECKGOTO(symBindTeamLe(comm, mem, t->mcLeId[counted]), ret, fail_mem_space_teams); } } } @@ -915,6 +1053,8 @@ static ncclResult_t symMemoryObtain(struct ncclComm* comm, CUmemGenericAllocatio NCCLCHECKGOTO(symMemoryRegisterRma(comm, mem), ret, fail_mem_space_teams); } +exit: + mem->refCount += 1; *outMem = mem; free(globalSegmentInfo); return ret; @@ -928,12 +1068,13 @@ static ncclResult_t symMemoryObtain(struct ncclComm* comm, CUmemGenericAllocatio } for (struct ncclDevrTeam* t = devr->teamHead; t != nullptr; t = t->next) { symUnbindTeamMemory(comm, t, mem); - if (ucBound && t->ucLeId != NCCL_LE_ID_INVALID) { - ncclCftLeId_t le = t->ucLeId + t->team.rank * t->team.stride; + if (ucBound) { + ncclCftLeId_t baseLe = t->ucLeId[counted]; + ncclCftLeId_t le = baseLe == NCCL_LE_ID_INVALID ? NCCL_LE_ID_INVALID : baseLe + t->team.rank * t->team.stride; symUnbindTeamLe(comm, mem, le); ucBound = false; } - symUnbindTeamLe(comm, mem, t->mcLeId); + symUnbindTeamLe(comm, mem, t->mcLeId[counted]); } fail_mem_space: symMemoryUnmapLsaTeam(comm, mem); @@ -946,8 +1087,9 @@ static ncclResult_t symMemoryObtain(struct ncclComm* comm, CUmemGenericAllocatio free(mem->lsaNumSegments); } free(mem); +fail_global_segment_info: free(globalSegmentInfo); -// fail: +fail: return ret; } @@ -960,17 +1102,22 @@ static void symMemoryDestroy(struct ncclComm* comm, struct ncclDevrMemory* mem) while (*memLink != nullptr && *memLink != mem) { memLink = &(*memLink)->next; } - // Idempotent: a window pair or finalize drain may already have destroyed this mem. + // Membership check before any field access. The caller's error-path + // fallthrough passes mem == nullptr, which already returned above. This + // guards an explicit second destroy of a stale pointer: do not walk off a + // drained memHead or repeat unmap/release/free. if (*memLink != mem) { return; } symMemoryUnregister(comm, mem); - ncclCftLeId leUcSelf = devr->le.baseId == NCCL_LE_ID_INVALID ? NCCL_LE_ID_INVALID : devr->le.baseId + devr->cftSelf; + bool counted = mem->winFlags & NCCL_WIN_CFT_COUNTED; + ncclCftLeId leUcSelf = + devr->le[counted].baseId == NCCL_LE_ID_INVALID ? NCCL_LE_ID_INVALID : devr->le[counted].baseId + devr->cftSelf; symUnbindTeamLe(comm, mem, leUcSelf); for (struct ncclDevrTeam* t = devr->teamHead; t != nullptr; t = t->next) { symUnbindTeamMemory(comm, t, mem); - symUnbindTeamLe(comm, mem, t->mcLeId); + symUnbindTeamLe(comm, mem, t->mcLeId[counted]); } symMemoryUnmapLsaTeam(comm, mem); @@ -988,6 +1135,13 @@ static void symMemoryDestroy(struct ncclComm* comm, struct ncclDevrMemory* mem) free(mem); } +// Backing registrations are shared by windows over the same allocation; the last +// reference tears the registration down. ncclDevrFinalize calls symMemoryDestroy +// directly to force-drain registrations regardless of outstanding references. +static void symMemoryDropRef(struct ncclComm* comm, struct ncclDevrMemory* mem) { + if (mem != nullptr && 0 == --mem->refCount) symMemoryDestroy(comm, mem); +} + static ncclResult_t symWindowTableInitOnce(struct ncclComm* comm, cudaStream_t stream) { struct ncclDevrState* devr = &comm->devrState; struct ncclDevCommWindowTable* tableDev = devr->windowTable; @@ -1002,6 +1156,7 @@ static ncclResult_t symWindowTableInitOnce(struct ncclComm* comm, cudaStream_t s static ncclResult_t symWindowInitGin(struct ncclDevrState* devr, struct ncclDevrMemory* mem, struct ncclWindow_vidmem* winDevHost, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; + if (!ncclDevrWinRegEnabled(mem->winFlags, ncclDevrRegisterGin)) return ret; struct ncclSegmentWindow* segmentWindowsDev; NCCLCHECK(ncclDevrAllocAndPopulateSegmentWindows(devr, mem, stream, &segmentWindowsDev)); winDevHost->ginMultiSegmentWins = segmentWindowsDev; @@ -1042,11 +1197,13 @@ static ncclResult_t symWindowCreate(struct ncclComm* comm, struct ncclDevrMemory winDevHost->lsaFlatBase = (char*)devr->lsaFlatBase + win->bigOffset; winDevHost->mcOffset4K = win->bigOffset >> 12; winDevHost->stride4G = devr->bigSize >> 32; + winDevHost->winFlags = winFlags; winDevHost->lsaRank = devr->lsaSelf; winDevHost->cftFlatRank = devr->cftSelf; winDevHost->worldRank = comm->rank; winDevHost->winHost = (void*)win; winDevHost->ginOffset4K = memOffset >> 12; + winDevHost->ucLeIdBase = winFlags & NCCL_WIN_CFT_COUNTED ? devr->le[1].baseId : devr->le[0].baseId; if (devr->ginEnabled) { NCCLCHECK(symWindowInitGin(devr, mem, winDevHost, stream)); } @@ -1098,7 +1255,7 @@ static ncclResult_t symWindowDestroy(struct ncclComm* comm, struct ncclWindow_vi NCCLCHECKGOTO(ncclShadowPoolToHost(&devr->shadows, winDev, &winDevHost), ret, fail); winHost = (struct ncclDevrWindow*)winDevHost->winHost; - symMemoryDestroy(comm, winHost->memory); + symMemoryDropRef(comm, winHost->memory); { struct ncclDevCommWindowTable* tableDev = devr->windowTable; @@ -1124,19 +1281,36 @@ static ncclResult_t symWindowDestroy(struct ncclComm* comm, struct ncclWindow_vi NCCLCHECKGOTO(ncclShadowPoolFree(&devr->shadows, winDev, stream), ret, remove_winSorted); - NCCLCHECKGOTO(ncclCommDeregister(comm, winHost->localRegHandle), ret, remove_winSorted); - remove_winSorted: + // Every checked call above jumps here, then winHost is freed. Deregister + // first so those exits still release the registration the caller handed off. + NCCLCHECKIGNORE(ncclCommDeregister(comm, winHost->localRegHandle), ret); { - int i = listFindSortedLub(&ncclDevrWindowSorted::userAddr, devr->winSorted, devr->winSortedCount, - reinterpret_cast(winHost->userPtr)); + uintptr_t userAddr = (uintptr_t)winHost->userPtr; + int i = listFindSortedLub(&ncclDevrWindowSorted::userAddr, devr->winSorted, devr->winSortedCount, userAddr); i -= 1; // least upper bound is just after ours. - listRemove(devr->winSorted, &devr->winSortedCount, i); + + // The same address may be registered by several windows, and their entries are contiguous + // below the least upper bound, so scan that block for the window being destroyed. + bool found = false; + for (; i >= 0; --i) { + if (devr->winSorted[i].userAddr != userAddr) break; + if (devr->winSorted[i].win != winHost) continue; + + listRemove(devr->winSorted, &devr->winSortedCount, i); + found = true; + break; + } + + if (!found) { + WARN("Window %p not found in sorted window list.", winHost); + if (ret == ncclSuccess) ret = ncclInternalError; + } } // Remove the just deallocated window from the table storing the communicator pointer { std::lock_guard lock(ncclWindowMapMutex); - NCCLCHECKGOTO(ncclIntruAddressMapRemove(&ncclWindowMap, winDev), ret, fail); + NCCLCHECKIGNORE(ncclIntruAddressMapRemove(&ncclWindowMap, winDev), ret); } free(winHost); @@ -1159,8 +1333,8 @@ static void windowCloseIpcPeers(struct ncclComm* comm, struct ncclDevrWindow* wi // RCCL: register a non-sym window. Mirrors upstream sym's two-stage shape: // (1) intra-node mapping via cudaIpcOpenMemHandle peer pointers, when // lsaSize > 1. -// (2) inter-node MR via ncclRmaProxyRegister, when hostRmaSupport. Same -// call upstream issues from symMemoryRegisterRma. +// (2) inter-node MR via ncclRmaProxyRegister, when hostRmaSupport and not +// NCCL_RMA_DISABLE. Same call upstream issues from symMemoryRegisterRma. static ncclResult_t windowRegisterNonSym(struct ncclComm* comm, void* userPtr, size_t userSize, int winFlags, void* localRegHandle, ncclWindow_t* outWinDev) { struct ExchangeEntry { @@ -1180,7 +1354,7 @@ static ncclResult_t windowRegisterNonSym(struct ncclComm* comm, void* userPtr, s int* teamRankList = devr->lsaRankList; bool doIpc = teamSize > 1; - bool doRma = comm->hostRmaSupport && (teamSize < comm->nRanks); + bool doRma = comm->hostRmaSupport && (teamSize < comm->nRanks) && !ncclParamRMADisable(); struct ncclDevrWindow* win = nullptr; cudaStream_t stream = nullptr; @@ -1388,6 +1562,12 @@ ncclResult_t ncclDevrWindowRegisterInGroup(struct ncclComm* comm, void* userPtr, CUDACHECK(cudaThreadExchangeStreamCaptureMode(&captureMode)); + if ((winFlags & NCCL_WIN_CFT_COUNTED) && !comm->gpuCftCountedSupport) { + WARN("User requested COUNTED but the communicator does not support it"); + ret = ncclInvalidArgument; + goto fail; + } + NCCLCHECKGOTO(ncclCommRegister(comm, userPtr, userSize, &localRegHandle), ret, fail); #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) @@ -1421,7 +1601,7 @@ ncclResult_t ncclDevrWindowRegisterInGroup(struct ncclComm* comm, void* userPtr, goto fail_locReg; } #if CUDART_VERSION >= 12080 - else if (comm->MNNVL) { + else if (comm->MNNVL && ncclDevrWinRegEnabled(winFlags, ncclDevrRegisterLsa)) { int multiNodeLsaSupported = 0; CUCHECKGOTO(cuDeviceGetAttribute(&multiNodeLsaSupported, CU_DEVICE_ATTRIBUTE_HOST_NUMA_MULTINODE_IPC_SUPPORTED, comm->cudaDev), @@ -1470,12 +1650,13 @@ ncclResult_t ncclDevrWindowRegisterInGroup(struct ncclComm* comm, void* userPtr, ret, fail_locReg_memHandle); memset(memHandles, 0, numSegments * sizeof(*memHandles)); // symMemoryObtain took our reference - CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), ret, fail); + CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), ret, fail_locReg_memHandle_mem_stream); NCCLCHECKGOTO(symWindowCreate(comm, mem, memOffset, userPtr, userSize, winFlags, localRegHandle, outWinDev, &winHost, stream), ret, fail_locReg_memHandle_mem_stream); mem = nullptr; // symWindowCreate took our reference + localRegHandle = nullptr; // window owns the registration; destroy will deregister it CUDACHECKGOTO(cudaStreamSynchronize(stream), ret, fail_locReg_memHandle_mem_stream_win); @@ -1502,8 +1683,12 @@ ncclResult_t ncclDevrWindowRegisterInGroup(struct ncclComm* comm, void* userPtr, *outWinDev = nullptr; CUDACHECKIGNORE(cudaStreamSynchronize(stream)); fail_locReg_memHandle_mem_stream: - CUDACHECKIGNORE(cudaStreamDestroy(stream)); - symMemoryDestroy(comm, mem); + // Stream create jumps here before assigning stream. Destroying a null + // handle is the same call windowRegisterNonSym skips. + if (stream != nullptr) { + CUDACHECKIGNORE(cudaStreamDestroy(stream)); + } + symMemoryDropRef(comm, mem); fail_locReg_memHandle: for (int idx = 0; idx < numSegments; idx++) { if (memHandles[idx] != 0x0ULL) { @@ -1512,7 +1697,7 @@ ncclResult_t ncclDevrWindowRegisterInGroup(struct ncclComm* comm, void* userPtr, } free(memHandles); fail_locReg: - ncclCommDeregister(comm, localRegHandle); + if (localRegHandle != nullptr) ncclCommDeregister(comm, localRegHandle); fail: CUDACHECKIGNORE(cudaThreadExchangeStreamCaptureMode(&captureMode)); *outWinDev = nullptr; @@ -1662,6 +1847,8 @@ ncclResult_t ncclDevrCommCreateInternal(struct ncclComm* comm, struct ncclDevCom struct ncclTeam world = ncclTeamWorld(comm); struct ncclTeam lsa = ncclTeamInnerFactor(world, devr->lsaSize); bool ginActivated = false; + bool cftUcActivated = false; + bool hasCountedCftMemory = symHasCountedCftMemory(devr); struct ncclDevrTeam* tmLsa; ncclTeam_t ucTeam, mcTeam; size_t bufSizeTotal; @@ -1769,7 +1956,9 @@ ncclResult_t ncclDevrCommCreateInternal(struct ncclComm* comm, struct ncclDevCom if (isInternal) outDevComm->abortFlag = comm->abortFlagDev; // needBarrier is ignored, the bootstrapBarrier below will barrier regardless - NCCLCHECKGOTO(symTeamObtain(comm, lsa, reqs->lsaMultimem, /*uc=*/false, /*mc=*/false, &tmLsa, nullptr), ret, fail); + NCCLCHECKGOTO(symTeamObtain(comm, lsa, reqs->lsaMultimem, /*counted=*/false, /*uc=*/false, /*mc=*/false, &tmLsa, + /*needBarrier=*/nullptr), + ret, fail); outDevComm->lsaMultimem.mcBasePtr = tmLsa->mcBasePtr; ucTeam = ncclTeamCft(comm); @@ -1781,19 +1970,35 @@ ncclResult_t ncclDevrCommCreateInternal(struct ncclComm* comm, struct ncclDevCom outDevComm->cftMultimemSize_rcp32 = idivRcp32(devr->cftMcSize); if (comm->gpuCftSupport && (reqs->cftCaps & NCCL_CFT)) { + cftUcActivated = devr->le[0].baseId == NCCL_LE_ID_INVALID; // needBarrier = nullptr, the bootstrapBarrier below will barrier regardless struct ncclDevrTeam* tmCft = nullptr; - NCCLCHECKGOTO(symTeamObtain(comm, ucTeam, /*multimem=*/false, /*uc=*/true, /*mc=*/false, &tmCft, nullptr), ret, - fail); - if (tmCft != nullptr) outDevComm->ucLeId = tmCft->ucLeId; + NCCLCHECKGOTO(symTeamObtain(comm, ucTeam, /*multimem=*/false, /*counted=*/false, /*uc=*/true, /*mc=*/false, &tmCft, + /*needBarrier=*/nullptr), + ret, fail); + if (tmCft != nullptr) outDevComm->ucLeId = tmCft->ucLeId[0]; + + if (hasCountedCftMemory) { + cftUcActivated |= devr->le[1].baseId == NCCL_LE_ID_INVALID; + NCCLCHECKGOTO(symTeamObtain(comm, ucTeam, /*multimem=*/false, /*counted=*/true, /*uc=*/true, /*mc=*/false, &tmCft, + /*needBarrier=*/nullptr), + ret, fail); + } } if (comm->gpuCftSupport && comm->nvlsSupport && (reqs->cftCaps & NCCL_CFT_MULTIMEM)) { // needBarrier = nullptr, the bootstrapBarrier below will barrier regardless struct ncclDevrTeam* tmCft = nullptr; - NCCLCHECKGOTO(symTeamObtain(comm, mcTeam, /*multimem=*/false, /*uc=*/false, /*mc=*/true, &tmCft, nullptr), ret, - fail); - if (tmCft != nullptr) outDevComm->mcLeId = tmCft->mcLeId; + NCCLCHECKGOTO(symTeamObtain(comm, mcTeam, /*multimem=*/false, /*counted=*/false, /*uc=*/false, /*mc=*/true, &tmCft, + /*needBarrier=*/nullptr), + ret, fail); + if (tmCft != nullptr) outDevComm->mcLeId = tmCft->mcLeId[0]; + + if (hasCountedCftMemory) { + NCCLCHECKGOTO(symTeamObtain(comm, mcTeam, /*multimem=*/false, /*counted=*/true, /*uc=*/false, /*mc=*/true, &tmCft, + /*needBarrier=*/nullptr), + ret, fail); + } } { @@ -1802,7 +2007,9 @@ ncclResult_t ncclDevrCommCreateInternal(struct ncclComm* comm, struct ncclDevCom if (tr->multimem) { struct ncclDevrTeam* tm = nullptr; // needBarrier is ignored, the bootstrapBarrier below will barrier regardless - NCCLCHECKGOTO(symTeamObtain(comm, tr->team, tr->multimem, /*uc=*/false, /*mc=*/false, &tm, nullptr), ret, fail); + NCCLCHECKGOTO(symTeamObtain(comm, tr->team, tr->multimem, /*counted=*/false, /*uc=*/false, /*mc=*/false, &tm, + /*needBarrier=*/nullptr), + ret, fail); if (tr->outMultimemHandle != nullptr) tr->outMultimemHandle->mcBasePtr = tm->mcBasePtr; } tr = tr->next; @@ -1888,7 +2095,7 @@ ncclResult_t ncclDevrCommCreateInternal(struct ncclComm* comm, struct ncclDevCom bufSizeTotal = alignUp(bufSizeTotal, devr->granularity); } - if (devr->ginEnabled) { + if (requestedConnectionType != NCCL_GIN_CONNECTION_NONE) { reqs->ginSignalCount = ginSignalTotal; reqs->ginCounterCount = ginCounterTotal; NCCLCHECKGOTO(ncclGinDevCommSetup(comm, reqs, outDevComm, deviceCodeVersion), ret, fail); @@ -1898,6 +2105,20 @@ ncclResult_t ncclDevrCommCreateInternal(struct ncclComm* comm, struct ncclDevCom CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), ret, fail); + if (cftUcActivated) { + for (int i = 0; i < devr->winSortedCount; i++) { + struct ncclDevrWindow* win = devr->winSorted[i].win; + if (!ncclDevrWinRegEnabled(win->winFlags, ncclDevrRegisterCft)) continue; + struct ncclWindow_vidmem* winHost; + NCCLCHECKGOTO(ncclShadowPoolToHost(&devr->shadows, win->vidmem, &winHost), ret, fail_stream); + bool counted = win->winFlags & NCCL_WIN_CFT_COUNTED; + winHost->ucLeIdBase = devr->le[counted].baseId; + CUDACHECKGOTO(cudaMemcpyAsync(win->vidmem, winHost, sizeof(struct ncclWindow_vidmem), cudaMemcpyHostToDevice, + stream), + ret, fail_stream); + } + } + if (ginActivated) { // Now update the GIN handles in all existing windows. Registration of memories happened above. for (int i = 0; i < devr->winSortedCount; i++) { @@ -1976,13 +2197,15 @@ ncclResult_t ncclDevrCommCreateInternal(struct ncclComm* comm, struct ncclDevCom return ret; fail_stream_mem_win: - symWindowDestroy(comm, win->vidmem, stream); - CUDACHECKIGNORE(cudaStreamSynchronize(stream)); + if (win != nullptr) { + symWindowDestroy(comm, win->vidmem, stream); + CUDACHECKIGNORE(cudaStreamSynchronize(stream)); + } fail_stream_mem: if (memHandle != 0x0) { CUCHECKIGNORE(cuMemRelease(memHandle)); } - symMemoryDestroy(comm, mem); + symMemoryDropRef(comm, mem); fail_stream: CUDACHECKIGNORE(cudaStreamDestroy(stream)); fail: @@ -2064,6 +2287,10 @@ ncclResult_t ncclCommWindowRegister_impl(struct ncclComm* comm, void* userPtr, s WARN("%s: invalid pointer %p / size %zu", __func__, userPtr, userSize); return ncclInvalidArgument; } + if ((winFlags & NCCL_WIN_COLL_SYMMETRIC) && (winFlags & ncclDevrWinCapRestrictionMask)) { + WARN("NCCL_WIN_COLL_SYMMETRIC cannot be combined with window capability-restriction flags"); + return ncclInvalidArgument; + } // RCCL: NCCL_WIN_ENABLE=0 opts out of symmetric-window paths, and the // non-sym fallback cannot back cuMem/VMM buffers (cudaIpcGetMemHandle rejects @@ -2123,8 +2350,8 @@ ncclResult_t ncclCommWindowRegister_impl(struct ncclComm* comm, void* userPtr, s #else const bool ceInitNeeded = comm->hostRmaSupport; #endif - if (ceInitNeeded && !comm->rmaState.rmaCeState.initialized && - ncclIntruQueueEmpty(&comm->rmaCeInitTaskQueue)) { + if (ncclDevrWinRegEnabled(winFlags, ncclDevrRegisterRma) && ceInitNeeded && + !comm->rmaState.rmaCeState.initialized && ncclIntruQueueEmpty(&comm->rmaCeInitTaskQueue)) { NCCLCHECKGOTO(ncclCalloc(&ceTask, 1), ret, fail); ceTask->comm = comm; ncclIntruQueueEnqueue(&comm->rmaCeInitTaskQueue, ceTask); @@ -2295,10 +2522,7 @@ ncclResult_t ncclCommQueryProperties(ncclComm_t comm, ncclCommProperties_t* prop NCCLCHECK(ncclGetGinType(comm, &props->ginType)); NCCLCHECK(ncclGetRailedGinType(comm, &props->railedGinType)); - // Preferring to call ncclDevrInitOnce directly instead to calling ncclTeam* functions because - // we can propagate the result of ncclDevrInitOnce back to the caller. - NCCLCHECK(ncclDevrInitOnce(comm)); - props->nLsaTeams = comm->devrState.nLsaTeams; + props->nLsaTeams = comm->nRanks / computeLsaSize(comm); } if (props->version >= NCCL_VERSION(2, 31, 0)) { @@ -2319,6 +2543,12 @@ ncclResult_t ncclCommQueryProperties(ncclComm_t comm, ncclCommProperties_t* prop if (devCompat->commPropertiesFilter) { NCCLCHECK(devCompat->commPropertiesFilter(comm, props)); } + + if (props->version >= NCCL_VERSION(2, 32, 0)) { + props->cftSupport = comm->gpuCftSupport >= 13030; + props->cftMulticastSupport = comm->gpuCftMulticastSupport; + props->cftCountedSupport = comm->gpuCftCountedSupport; + } return ncclSuccess; } @@ -2525,6 +2755,11 @@ ncclResult_t ncclDevrGetLsaRankPtr(struct ncclComm* comm, struct ncclDevrWindow* NCCLCHECK(CommCheck(comm, __func__, "comm")); NCCLCHECK(PtrCheck(outPtr, __func__, "outPtr")); + if (!ncclDevrWinRegEnabled(winHost->winFlags, ncclDevrRegisterLsa)) { + WARN("LSA pointer access is disabled because the window was not registered for LSA"); + return ncclInvalidUsage; + } + struct ncclDevrState* devr = &comm->devrState; if (offset >= winHost->size) { @@ -2597,6 +2832,10 @@ size_t ncclDevrGetWinOffset(struct ncclDevrWindow* winHost) { ncclResult_t ncclDevrGetLsaTeamPtrMC(struct ncclComm* comm, struct ncclDevrWindow* winHost, size_t offset, struct ncclTeam lsaTeam, void** outPtr) { if (winHost == nullptr || outPtr == nullptr) return ncclInternalError; + if (!ncclDevrWinRegEnabled(winHost->winFlags, ncclDevrRegisterLsa)) { + WARN("Multimem pointer access is disabled because the window was not registered for LSA"); + return ncclInvalidUsage; + } if (!comm->nvlsSupport) { WARN("Multimem pointer requested but system does not support multimem."); @@ -2605,7 +2844,8 @@ ncclResult_t ncclDevrGetLsaTeamPtrMC(struct ncclComm* comm, struct ncclDevrWindo bool needBarrier = false; struct ncclDevrTeam* tm; - NCCLCHECK(symTeamObtain(comm, lsaTeam, /*multimem=*/true, /*uc=*/false, /*mc=*/false, &tm, &needBarrier)); + NCCLCHECK(symTeamObtain(comm, lsaTeam, /*multimem=*/true, /*counted=*/false, /*uc=*/false, /*mc=*/false, &tm, + &needBarrier)); if (needBarrier) { struct ncclDevrState* devr = &comm->devrState; NCCLCHECK(bootstrapIntraNodeBarrier(comm->bootstrap, devr->lsaRankList, devr->lsaSelf, devr->lsaSize, 0xbeef)); @@ -2648,6 +2888,10 @@ ncclResult_t ncclGetMultimemDevicePointer(ncclWindow_t window, size_t offset, nc struct ncclDevrWindow* winHost = nullptr; NCCLCHECK(findCommAndHostWindowFromDeviceWindow(window, &comm, &winHost)); + if (!ncclDevrWinRegEnabled(winHost->winFlags, ncclDevrRegisterLsa)) { + WARN("Multimem pointer access is disabled because the window was not registered for LSA"); + return ncclInvalidUsage; + } if (!comm->nvlsSupport) { *outPtr = nullptr; diff --git a/projects/rccl/src/dev_runtime_internal.h b/projects/rccl/src/dev_runtime_internal.h index 422e0900144f..ee69cc37b474 100644 --- a/projects/rccl/src/dev_runtime_internal.h +++ b/projects/rccl/src/dev_runtime_internal.h @@ -23,15 +23,19 @@ struct ncclDevrTeam { struct ncclTeam team; CUmemGenericAllocationHandle mcHandle; void* mcBasePtr; - ncclCftLeId ucLeId; - ncclCftLeId mcLeId; + ncclCftLeId ucLeId[2]; // 0: UC LE ID, 1: counted UC LE ID + ncclCftLeId mcLeId[2]; // 0: MC LE ID, 1: counted MC LE ID +#if defined(NCCL_OS_WINDOWS) + int worldRankList[1]; // Variable length. [1] for MSVC. +#else int worldRankList[]; +#endif }; // Non-static functions in dev_runtime.cc also called from cft_dev_runtime.cc: int computeLsaSize(struct ncclComm* comm); -ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool multimem, bool wantsLeUc, bool wantsLeMc, - struct ncclDevrTeam** outTeam, bool* needBarrier); +ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool multimem, bool counted, bool wantsLeUc, + bool wantsLeMc, struct ncclDevrTeam** outTeam, bool* needBarrier); ncclResult_t findCommAndHostWindowFromDeviceWindow(ncclWindow_t devWindow, ncclComm_t* foundComm, struct ncclDevrWindow** hostWindow); @@ -41,9 +45,9 @@ int computeCftMcSize(struct ncclComm* comm); ncclResult_t symBindTeamLe(struct ncclComm* comm, struct ncclDevrMemory* mem, ncclCftLeId le); ncclResult_t symUnbindTeamLe(struct ncclComm* comm, struct ncclDevrMemory* mem, ncclCftLeId le); ncclResult_t symTeamObtainUcLe(struct ncclComm* comm, struct ncclDevrTeam* t, struct ncclDevrState* devr, - bool* needBarrier); + bool* needBarrier, bool counted); ncclResult_t symTeamObtainMcLe(struct ncclComm* comm, struct ncclDevrTeam* t, struct ncclDevrState* devr, - bool* needBarrier); + bool* needBarrier, bool counted); struct ncclDevrGinSegmentInfo { void* ginHostWins[NCCL_GIN_MAX_CONNECTIONS * NCCL_GIN_MAX_ACTIVE_BACKENDS]; @@ -55,6 +59,9 @@ struct ncclDevrGinSegmentInfo { // Complete type for src/include/dev_runtime.h's forward declaration. struct ncclDevrMemory { int refCount; + // Communicator-wide identity of this backing registration: backing registrations are created + // collectively in the same order on every rank, so equal ids denote the same registration. + uint64_t registryId; struct ncclDevrMemory* next; CUmemGenericAllocationHandle* memHandles; void* primaryAddr; // What we hope is the VA of this memory's first mapping. @@ -84,7 +91,7 @@ struct ncclDevrMemory { ncclResult_t ncclDevrPopulateSegmentSizes(struct ncclDevrMemory* mem, int numSegments); ncclResult_t ncclDevrCheckRegistrationSupport(void* userPtr, size_t userSize, struct ncclComm* comm, - bool hasSysmemSegment); + bool hasSysmemSegment, int winFlags); ncclResult_t ncclDevrValidateHandleLocationType(CUmemGenericAllocationHandle memHandle, int segment); diff --git a/projects/rccl/src/dev_runtime_segments.cc b/projects/rccl/src/dev_runtime_segments.cc index 3ae064740f35..d7922bc97f95 100644 --- a/projects/rccl/src/dev_runtime_segments.cc +++ b/projects/rccl/src/dev_runtime_segments.cc @@ -42,7 +42,7 @@ ncclResult_t ncclDevrPopulateSegmentSizes(struct ncclDevrMemory* mem, int numSeg } ncclResult_t ncclDevrCheckRegistrationSupport(void* userPtr, size_t userSize, struct ncclComm* comm, - bool hasSysmemSegment) { + bool hasSysmemSegment, int winFlags) { ncclResult_t ret = ncclSuccess; if (hasSysmemSegment) { if (!ncclParamElasticBufferRegister()) { @@ -54,7 +54,7 @@ ncclResult_t ncclDevrCheckRegistrationSupport(void* userPtr, size_t userSize, st goto exit; } #if CUDART_VERSION >= 12080 - else if (comm->MNNVL) { + else if (comm->MNNVL && ncclDevrWinRegEnabled(winFlags, ncclDevrRegisterLsa)) { int multiNodeLsaSupported = 0; CUCHECKGOTO(cuDeviceGetAttribute(&multiNodeLsaSupported, CU_DEVICE_ATTRIBUTE_HOST_NUMA_MULTINODE_IPC_SUPPORTED, comm->cudaDev), diff --git a/projects/rccl/src/device/all_gather.h b/projects/rccl/src/device/all_gather.h index c89bf7a9f0bd..82d3a175a53c 100644 --- a/projects/rccl/src/device/all_gather.h +++ b/projects/rccl/src/device/all_gather.h @@ -11,7 +11,7 @@ namespace { template -#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && !defined(__gfx1250__) +#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && (!defined(__gfx1250__) && !defined(__gfx1250_strict__)) __device__ void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { #else __device__ __attribute__((noinline)) void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { @@ -178,8 +178,8 @@ struct RunWorkCollisOneRPN ? NCCL_PAT_NWORKERS : NCCL_PAT_MULTI_RPN_NWORKERS; + const int nGatherWorkers = NCCL_MAX_NTHREADS - nworkers - WARP_SIZE; struct ncclPatShmem* shmem = (struct ncclPatShmem*)ncclScratchForWarp(0); uint64_t pollCount = 0; (void)pollCount; // unused variable - compiler warning @@ -203,7 +203,7 @@ struct RunWorkCollpatSteps + (step % NCCL_SHMEM_PAT_STEPS); int* poll = &ps->flags; - while (__hip_atomic_load(poll, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_WORKGROUP) != 0) { + while (__scoped_atomic_load_n(poll, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WRKGRP) != 0) { pollCount++; // Wait for workers to be done with step 'step-NCCL_SHMEM_PAT_STEPS' } patAlgo.getNextOp(ps); @@ -231,14 +231,14 @@ struct RunWorkCollpatSteps + (step % NCCL_SHMEM_PAT_STEPS); int* poll = &ps->flags; - while (__hip_atomic_load(poll, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_WORKGROUP) == 0) { + while (__scoped_atomic_load_n(poll, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WRKGRP) == 0) { pollCount++; // Wait for compute thread } int last = ps->last; prims.patCopy(ps, shmem); if (tidInGroup == 0) - __hip_atomic_store(poll, 0, __ATOMIC_RELEASE, - __HIP_MEMORY_SCOPE_WORKGROUP); // Return element to compute thread + __scoped_atomic_store_n(poll, 0, __ATOMIC_RELEASE, + __MEMORY_SCOPE_WRKGRP); // Return element to compute thread if (last) break; step += nGroups; } @@ -249,7 +249,7 @@ struct RunWorkColl patAlgo(chunkCount * sizeof(T), NCCL_STEPS, NCCL_PAT_NWORKERS / WARP_SIZE, channelOffset, + PatAGAlgorithm patAlgo(chunkCount * sizeof(T), NCCL_STEPS, nworkers / WARP_SIZE, channelOffset, channelOffset + channelCount, patCount, chunkCount, node, nNodes, ncclShmem.comm.patSharedQps); shmem->parallelFactor = patAlgo.getParallelFactor(); @@ -257,9 +257,9 @@ struct RunWorkCollpatSteps + (step % NCCL_SHMEM_PAT_STEPS); int* schedulerStep = &ps->step; - while (__hip_atomic_load(schedulerStep, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_WORKGROUP) != -1) pollCount++; + while (__scoped_atomic_load_n(schedulerStep, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WRKGRP) != -1) pollCount++; patAlgo.getNextOp(ps); - __hip_atomic_store(schedulerStep, step, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_WORKGROUP); + __scoped_atomic_store_n(schedulerStep, step, __ATOMIC_RELEASE, __MEMORY_SCOPE_WRKGRP); int last = ps->last; step++; if (last == 2) break; @@ -285,7 +285,7 @@ struct RunWorkCollpatSteps + (step % NCCL_SHMEM_PAT_STEPS); int* schedulerStep = &ps->step; - while (__hip_atomic_load(schedulerStep, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_WORKGROUP) != step) pollCount++; + while (__scoped_atomic_load_n(schedulerStep, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WRKGRP) != step) pollCount++; int last = ps->last; prims.template patCopy(ps, shmem); if (last) break; @@ -298,7 +298,7 @@ struct RunWorkColl, /*Direct=*/0, Proto, 0> prims( gatherTid, nGatherWorkers, nvls->up, nullptr, inputBuf, outputBuf, work->redOpArg, - /*group=*/1, /*connIndexRecv=*/1, /*connIndexSend=*/0); + /*group=*/NCCL_PAT_MULTI_RPN_NWORKERS / WARP_SIZE, /*connIndexRecv=*/1, /*connIndexSend=*/0); int parallelFactor = 0; volatile int* pfPtr = &shmem->parallelFactor; @@ -307,7 +307,7 @@ struct RunWorkCollpatSteps + (step % NCCL_SHMEM_PAT_STEPS); int* schedulerStep = &ps->step; - while (__hip_atomic_load(schedulerStep, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_WORKGROUP) != step) pollCount++; + while (__scoped_atomic_load_n(schedulerStep, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WRKGRP) != step) pollCount++; int last = ps->last; prims.patGather(ps, shmem, step, parallelFactor, count); int localIdx = step % parallelFactor; @@ -317,7 +317,7 @@ struct RunWorkCollpatSteps + ((waveStart + i) % NCCL_SHMEM_PAT_STEPS); - __hip_atomic_store(&donePs->step, -1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_WORKGROUP); + __scoped_atomic_store_n(&donePs->step, -1, __ATOMIC_RELEASE, __MEMORY_SCOPE_WRKGRP); } } if (last == 2) break; diff --git a/projects/rccl/src/device/all_reduce.h b/projects/rccl/src/device/all_reduce.h index 4f1964063f5e..58c4b2cbed94 100644 --- a/projects/rccl/src/device/all_reduce.h +++ b/projects/rccl/src/device/all_reduce.h @@ -15,7 +15,7 @@ namespace { // template header. UserRegMode selects the LL128 user-buffer access path at // compile time (see prims_ll128.h): 0=runtime, 1=registered, 2=non-registered. template -#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && !defined(__gfx1250__) +#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && (!defined(__gfx1250__) && !defined(__gfx1250_strict__)) __device__ void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { #else __device__ __attribute__((noinline)) void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { @@ -116,7 +116,7 @@ __device__ __attribute__((noinline)) void runRing(int tid, int nthreads, struct } template -#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && !defined(__gfx1250__) +#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && (!defined(__gfx1250__) && !defined(__gfx1250_strict__)) __device__ void runTreeUpDown(int tid, int nthreads, struct ncclDevWorkColl* work) { #else __device__ __attribute__((noinline)) void runTreeUpDown(int tid, int nthreads, struct ncclDevWorkColl* work) { @@ -186,7 +186,7 @@ __device__ __attribute__((noinline)) void runTreeUpDown(int tid, int nthreads, s } template -#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && !defined(__gfx1250__) +#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && (!defined(__gfx1250__) && !defined(__gfx1250_strict__)) __device__ void runTreeSplit(int tid, int nthreads, struct ncclDevWorkColl* work) { #else __device__ __attribute__((noinline)) void runTreeSplit(int tid, int nthreads, struct ncclDevWorkColl* work) { diff --git a/projects/rccl/src/device/broadcast.h b/projects/rccl/src/device/broadcast.h index 2760e483c074..2a1b3f5426cd 100644 --- a/projects/rccl/src/device/broadcast.h +++ b/projects/rccl/src/device/broadcast.h @@ -11,7 +11,7 @@ namespace { template -#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && !defined(__gfx1250__) +#if defined(USE_INDIRECT_FUNCTION_CALL) && !defined(__gfx942__) && !defined(__gfx950__) && (!defined(__gfx1250__) && !defined(__gfx1250_strict__)) __device__ void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { #else __device__ __attribute__((noinline)) void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { diff --git a/projects/rccl/src/device/common.cu b/projects/rccl/src/device/common.cu index 37091a2ceb1f..eb7ceaacd9a4 100644 --- a/projects/rccl/src/device/common.cu +++ b/projects/rccl/src/device/common.cu @@ -55,3 +55,8 @@ __device__ __attribute__((noinline)) void ncclDevFunc_Nop(); // per-impl/specialized files, so the definition must live here (as in upstream and // the v2.29.7-1 base). The generated ncclDevFuncTable references this symbol. __device__ void ncclDevFunc_Nop() {} + +// Capture %globaltimer for optional CPU/GPU clock calibration in init.cc. +__global__ void ncclProgressCounterCaptureGpuTime(uint64_t* out) { + if (threadIdx.x == 0) *out = globaltimer(); +} diff --git a/projects/rccl/src/device/common.h b/projects/rccl/src/device/common.h index a78ac1997122..05d3098e4828 100644 --- a/projects/rccl/src/device/common.h +++ b/projects/rccl/src/device/common.h @@ -84,6 +84,8 @@ struct ncclShmemData { int workSize; uint64_t workCounter; bool profilerEnabled; + // ncclFunc_t this batch's completions are counted under. + uint8_t func; struct ncclShmemGroup groups[NCCL_MAX_GROUPS]; alignas(16) char workStorage[ncclMaxDevWorkBatchBytes()]; @@ -337,6 +339,7 @@ __device__ __forceinline__ void loadWorkBatchToShmem(int tid, int tn, struct ncc ncclShmem.workType = (enum ncclDevWorkType)batch.workType; ncclShmem.nWorks = workCursor; ncclShmem.funcId = batch.funcId; + ncclShmem.func = batch.func; } break; } @@ -378,21 +381,16 @@ struct RunWorkBatch; #define STOP 1 #define FINI 2 +// These kernels record only KernelCh start/stop. KernelPhase instrumentation is +// symmetric-only: the open/close barriers it measures exist in the symmetric +// kernels, so stamping them here would report boundaries that carry no meaning. +// Items are strided by workSize, not sizeof(ncclDevWorkColl), which is smaller for +// registered collectives. Bcast work carries no profiling bit. __device__ __forceinline__ bool profilerEnabled(int workItemIdx) { - return (ncclShmem.workType == ncclDevWorkTypeP2p) ? - ((struct ncclDevWorkP2p*)ncclShmem.workStorage)[workItemIdx].profilerEnabled : - ((struct ncclDevWorkColl*)ncclShmem.workStorage)[workItemIdx].profilerEnabled; -} - -__device__ __forceinline__ void profilerPhase(int phaseId) { - uint64_t ts = globaltimer(); - int idx = 0; - uint64_t wc = ncclShmem.channel.workCounter + 1; - for (; wc <= ncclShmem.channel.workCounter + ncclShmem.nWorks; wc++) { - if (!profilerEnabled(idx++)) continue; - int slot = wc % MAX_PROFILER_EVENTS_PER_CHANNEL; - ncclShmem.comm.workPhases[ncclShmem.channelId].data[slot].timestamps[phaseId] = ts; - } + char* work = ncclShmem.workStorage + workItemIdx * ncclShmem.workSize; + if (ncclShmem.workType == ncclDevWorkTypeP2p) return ((struct ncclDevWorkP2p*)work)->profilerEnabled; + if (ncclShmem.workType == ncclDevWorkTypeBcast) return false; + return ((struct ncclDevWorkColl*)work)->profilerEnabled; } // Specialized here for non-P2p (Coll and CollReg) @@ -416,11 +414,6 @@ struct RunWorkBatch { __syncthreads(); } - // Block-uniform gate so a profiler-off launch keeps the baseline cost (no extra - // sync/stamps). profilerEnabled(0) is uniform; nWorks>0 guards workStorage[0]. - bool profOn = (ncclShmem.nWorks > 0) && profilerEnabled(0); - if (profOn && threadIdx.x == 0) profilerPhase(NCCL_KERNEL_PHASE_AFTER_OPEN); // end of initial sync - #pragma unroll 1 for (int w = 0; w < ncclShmem.nWorks; w++) { struct ncclDevWorkColl* work = (struct ncclDevWorkColl*)(ncclShmem.workStorage + w * ncclShmem.workSize); @@ -445,53 +438,88 @@ struct RunWorkBatch { if (tid < subtn) RunWorkColl().run(tid, subtn, work); #endif } - // End of compute. Sync so thread 0's stamp reflects the last worker finishing. - if (profOn) { - __syncthreads(); - if (threadIdx.x == 0) profilerPhase(NCCL_KERNEL_PHASE_BEFORE_CLOSE); - } } }; +static_assert(NCCL_NUM_PROGRESS_COUNTERS <= 64, "Progress-counter slots must fit in collOpActive's bitmask"); + +// Return this batch's progress-counter slot, or -1 for an empty batch. +// START and STOP see the same batch until loadWorkBatchToShmem runs. +__device__ __forceinline__ int progressCounterBatchSlot() { + if (ncclShmem.nWorks == 0) return -1; + return (int)ncclShmem.func; +} + +// Count completed work owned by this channel at STOP/FINI while workStorage is stable. +__device__ __forceinline__ uint32_t progressCounterScanBatch() { + uint32_t count = 0; + if (ncclShmem.workType == ncclDevWorkTypeBcast) { + // Batched Broadcast completion is owned by channel 0. + return ncclShmem.channelId == 0 ? (uint32_t)ncclShmem.nWorks : 0; + } + if (ncclShmem.workType == ncclDevWorkTypeP2p) { + for (int w = 0; w < ncclShmem.nWorks; w++) { + struct ncclDevWorkP2p* work = (struct ncclDevWorkP2p*)(ncclShmem.workStorage + w * ncclShmem.workSize); + if ((int)ncclShmem.channelId == (int)work->channelBase) count++; + } + } else { + for (int w = 0; w < ncclShmem.nWorks; w++) { + struct ncclDevWorkColl* work = (struct ncclDevWorkColl*)(ncclShmem.workStorage + w * ncclShmem.workSize); + if ((int)ncclShmem.channelId == (int)work->channelLo) count++; + } + } + return count; +} + +__device__ __forceinline__ void progressCounterPublishCompletedSlot(int slot, uint32_t count) { + struct ncclProgressCountersBlock* counters = ncclShmem.comm.progressCounters; + if (count == 0 || counters == nullptr) return; + counters->completedTimeNs[slot] = globaltimer(); + // Publish the count after the timestamp. Future consumers that require + // stronger snapshot ordering may need a device fence here. +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + __scoped_atomic_fetch_add(&counters->completedWorkCount[slot], (uint64_t)count, + __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +#else + asm volatile("red.global.add.u64 [%0], %1;" : : "l"(&counters->completedWorkCount[slot]), "l"((uint64_t)count)); +#endif +} + __device__ __forceinline__ void profiler(int action) { if (threadIdx.x == 0) { int idx = 0; uint64_t wc = ncclShmem.channel.workCounter + 1; if (action == START) { - // workStarted timestamp+counter share one 16B slot (single cache line), so no - // fence is needed; the BEGIN phase stamp is ordered by STOP's fence below. + // timestamp+counter share one 16B slot (single cache line), so publishing + // them together needs no fence. + // Resolve this batch's progress-counter slot and mark the channel active. + if (ncclShmem.comm.progressCounters != nullptr) { + int slot = progressCounterBatchSlot(); + if (slot >= 0) ncclShmem.comm.progressCounters->collOpActive[ncclShmem.channelId] = 1ull << slot; + } for (; wc <= ncclShmem.channel.workCounter + ncclShmem.nWorks; wc++) { if (!profilerEnabled(idx++)) continue; uint64_t ts = globaltimer(); int slot = wc % MAX_PROFILER_EVENTS_PER_CHANNEL; - ncclShmem.comm.workPhases[ncclShmem.channelId].data[slot].timestamps[NCCL_KERNEL_PHASE_BEGIN] = ts; ncclShmem.comm.workStarted[ncclShmem.channelId].data[slot].timestamp = ts; ncclShmem.comm.workStarted[ncclShmem.channelId].data[slot].counter = wc; } } else { - bool fenceNeeded = false; for (; wc <= ncclShmem.channel.workCounter + ncclShmem.nWorks; wc++) { if (!profilerEnabled(idx++)) continue; uint64_t ts = globaltimer(); int slot = wc % MAX_PROFILER_EVENTS_PER_CHANNEL; ncclShmem.comm.workCompleted[ncclShmem.channelId].data[slot].timestamp = ts; - ncclShmem.comm.workPhases[ncclShmem.channelId].data[slot].timestamps[NCCL_KERNEL_PHASE_END] = ts; - fenceNeeded = true; - } - // workPhases stamps span the kernel and straddle cache lines, so fence once - // before publishing the counters to order all phase stamps ahead of them. - if (fenceNeeded) { - __threadfence_system(); - idx = 0; - for (uint64_t wc2 = ncclShmem.channel.workCounter + 1; wc2 <= ncclShmem.channel.workCounter + ncclShmem.nWorks; - wc2++) { - if (!profilerEnabled(idx++)) continue; - int slot = wc2 % MAX_PROFILER_EVENTS_PER_CHANNEL; - ncclShmem.comm.workPhases[ncclShmem.channelId].data[slot].counter = wc2; - ncclShmem.comm.workCompleted[ncclShmem.channelId].data[slot].counter = wc2; - } + ncclShmem.comm.workCompleted[ncclShmem.channelId].data[slot].counter = wc; } ncclShmem.channel.workCounter += ncclShmem.nWorks; + // Do not report aborted work as completed; leave its active mask set. + if (ncclShmem.aborted == 0 && ncclShmem.comm.progressCounters != nullptr) { + int slot = progressCounterBatchSlot(); + if (slot >= 0) ncclShmem.comm.progressCounters->collOpActive[ncclShmem.channelId] = 0; + progressCounterPublishCompletedSlot(slot, progressCounterScanBatch()); + // No system fence; counter-monitor snapshots are diagnostic and may be stale. + } if (action == FINI) ((ncclKernelCommAndChannels*)ncclShmem.args.comm)->channels[ncclShmem.channelId].workCounter = ncclShmem.channel.workCounter; @@ -712,9 +740,13 @@ __device__ __forceinline__ void ncclKernelMain(struct ncclDevKernelArgs const* a break; } profiler(STOP); + // Keep workStorage stable until thread 0 finishes counting. + if (ncclShmem.comm.progressCounters != nullptr) __syncthreads(); loadWorkBatchToShmem(tid % WARP_SIZE, tn, args, batchIx); __syncthreads(); } + // Publish progress-counter completion for the last batch only after every thread has left the work body. + if (ncclShmem.comm.progressCounters != nullptr) __syncthreads(); profiler(FINI); } diff --git a/projects/rccl/src/device/common_kernel.h b/projects/rccl/src/device/common_kernel.h index df624f2d4f89..984c5f265b2f 100644 --- a/projects/rccl/src/device/common_kernel.h +++ b/projects/rccl/src/device/common_kernel.h @@ -606,7 +606,7 @@ __device__ __forceinline__ void reduceCopy(int thread, int nThreads, uint64_t re // alignment. constexpr int AlignedPathPackSize = 4; constexpr int NativeScalarUnroll = Unroll * (16 / sizeof(T)) / 2; -#if defined(__gfx1250__) +#if defined(__gfx1250__) || defined(__gfx1250_strict__) // Larger pack loops make LLVM register allocation scale pathologically on gfx1250. // Limit the helper's work quantum; its outer loop still consumes all requested bytes. constexpr int ScalarUnroll = (useAcc && NativeScalarUnroll > 64) ? 64 : NativeScalarUnroll; diff --git a/projects/rccl/src/device/generate.py b/projects/rccl/src/device/generate.py index dcc5b458935d..ad1d3e55ac8d 100755 --- a/projects/rccl/src/device/generate.py +++ b/projects/rccl/src/device/generate.py @@ -51,8 +51,8 @@ def reg_values_of(coll, proto): # SendRecv is generated as two latency-protocol kernel variants, selected on the # host by ncclDevFuncId_P2p(useLL128) (see src/enqueue.cc / src/include/device.h): # reg "0" = legacy LL latency path (built on every arch; the default) - # reg "1" = LL128 latency path (activated on gfx942/gfx950 only; used when - # NCCL_ALLOC_P2P_NET_LL_BUFFERS=1). The reg value is threaded into the + # reg "1" = LL128 latency path (gfx942/gfx950 via NCCL_ALLOC_P2P_NET_LL_BUFFERS, + # gfx1250 via NCCL_P2P_LL128_ENABLE). The reg value is threaded into the # SendRecv RunWorkBatch specialization as UserRegMode to pick LL vs LL128. if coll == "SendRecv": return ["0", "1"] @@ -293,7 +293,7 @@ def calc_unroll_and_pipeline_for_local_arch(): return (["1", "2"], ["0"]) # Disable pipelining for gfx950 elif "gfx908" == gfx_name or ("gfx942" == gfx_name and cu_count > 80): return (["2"], all_pipelines) - elif "gfx1250" == gfx_name: + elif gfx_name.startswith("gfx1250"): # gfx1250 (MI450/MI455) runs unroll 32. Unroll 8 was once noted as required for the FP8 # launch; nothing in src/ depends on it now. Use --all_unrolls to build 8 and 16. return (unrolls_requiring_arch("gfx1250"), all_pipelines) @@ -447,19 +447,21 @@ def get_arch_guard(fn): cond = None if fn.coll == "SendRecv" and fn.reg == "1": - # LL128 SendRecv kernel. Only gfx942/gfx950 activate it, but gfx1250 builds it too - # or its unroll-32 slot would be a nullptr in the one table gfx1250 indexes. - cond = "(defined(__gfx942__) || defined(__gfx950__) || defined(__gfx1250__)) && defined(ENABLE_LL128)" + # LL128 SendRecv kernel. gfx942/gfx950 activate it via ALLOC_P2P_NET_LL_BUFFERS; + # gfx1250 activates it via NCCL_P2P_LL128_ENABLE. Built for all three so the + # unroll-32 slot is not a nullptr in the table gfx1250 indexes. + cond = "(defined(__gfx942__) || defined(__gfx950__) || (defined(__gfx1250__) || defined(__gfx1250_strict__))) && defined(ENABLE_LL128)" elif fn.unroll in unroll_arch_requirement: - cond = "defined(__%s__)" % unroll_arch_requirement[fn.unroll] + arch = unroll_arch_requirement[fn.unroll] + cond = "(defined(__%s__) || defined(__%s_strict__))" % (arch, arch) if fn.proto == "LL128": cond += " && defined(ENABLE_LL128)" elif fn.proto == "LL128" and fn.acc == "1": - cond = "(defined(__gfx942__) || defined(__gfx950__) || defined(__gfx1250__)) && defined(ENABLE_LL128)" + cond = "(defined(__gfx942__) || defined(__gfx950__) || (defined(__gfx1250__) || defined(__gfx1250_strict__))) && defined(ENABLE_LL128)" elif fn.proto == "LL128": - cond = "(defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__) || defined(__gfx1250__)) && defined(ENABLE_LL128)" + cond = "(defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__) || (defined(__gfx1250__) || defined(__gfx1250_strict__))) && defined(ENABLE_LL128)" elif fn.acc == "1": - cond = "defined(__gfx942__) || defined(__gfx950__) || defined(__gfx1250__)" + cond = "defined(__gfx942__) || defined(__gfx950__) || (defined(__gfx1250__) || defined(__gfx1250_strict__))" return cond # Build the mangled function symbol suffix. The user-buffer registration mode is @@ -573,6 +575,14 @@ def fn_sym(fn): out = f.write out('#include "device.h"\n') out("\n") + + # funcId travels in an NCCL_DEV_WORK_BATCH_FUNC_ID_BITS-wide ncclDevWorkBatch + # field. RCCL indexes a per-unroll table, so the largest per-unroll function + # count must fit. + max_funcs_per_unroll = max([sum(1 for fn in primary_funcs if fn.unroll == u) for u in all_unrolls] + [0]) + out("static_assert(%d <= (1 << NCCL_DEV_WORK_BATCH_FUNC_ID_BITS), " % max_funcs_per_unroll) + out('"Device function IDs must fit in ncclDevWorkBatch");\n') + out("\n") out("// The key for the ncclDevFuncNameToId map is a 64-bit unsigned integer.\n") out("// Each field (coll, algo, proto, redop, ty, acc, pipeline) is packed into 4 bits,\n") out("// This allows up to 16 unique values per field. The layout is:\n") diff --git a/projects/rccl/src/device/onerank.cu b/projects/rccl/src/device/onerank.cu index 814564ba7d78..742f831d9c98 100644 --- a/projects/rccl/src/device/onerank.cu +++ b/projects/rccl/src/device/onerank.cu @@ -7,6 +7,7 @@ #include "alloc.h" #include "collectives.h" +#include "cudawrap.h" #include "common_kernel.h" #include @@ -19,10 +20,33 @@ #endif namespace { -template +// rccl_float8 and rccl_bfloat8 are different types in the host pass and the gfx942 device +// pass (rccl_float8.h), so a kernel templated on them is registered under a name the gfx942 +// code object does not contain. The fp8 kernels are instantiated on these stand-ins instead, +// and the element type is resolved in the kernel body, which only the device pass compiles. +struct OneRankFloat8e4m3 {}; +struct OneRankFloat8e5m2 {}; + +template +struct OneRankEltType { + using Type = T; +}; +#if defined(RCCL_FLOAT8) +template <> +struct OneRankEltType { + using Type = rccl_float8; +}; +template <> +struct OneRankEltType { + using Type = rccl_bfloat8; +}; +#endif + +template